Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

aria2 下载增加做种状态 #1422

Merged
merged 3 commits into from
Sep 29, 2022
Merged
Show file tree
Hide file tree
Changes from 1 commit
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
refactor: refactor recycle aria2 temp file
  • Loading branch information
XYenon committed Aug 27, 2022
commit b7c6155d3d70022af4033667f52c2db6ecb231cc
4 changes: 2 additions & 2 deletions pkg/aria2/common/common.go
Original file line number Diff line number Diff line change
Expand Up @@ -38,8 +38,6 @@ const (
Downloading
// Paused 暂停中
Paused
// Seeding 做种中
Seeding
// Error 出错
Error
// Complete 完成
Expand All @@ -48,6 +46,8 @@ const (
Canceled
// Unknown 未知状态
Unknown
// Seeding 做种中
Seeding
)

var (
Expand Down
2 changes: 1 addition & 1 deletion pkg/aria2/monitor/monitor.go
Original file line number Diff line number Diff line change
Expand Up @@ -251,7 +251,7 @@ func (monitor *Monitor) Complete(pool task.Pool) bool {

// 转存完成,回收下载目录
if transferTask.Type == task.TransferTaskType && transferTask.Status >= task.Error {
job, err := task.NewRecycleTask(monitor.Task.UserID, monitor.Task.Parent, monitor.node.ID())
job, err := task.NewRecycleTask(monitor.Task)
if err != nil {
monitor.setErrorStatus(err)
monitor.RemoveTempFolder()
Expand Down
39 changes: 0 additions & 39 deletions pkg/filesystem/driver/shadow/slaveinmaster/handler.go
Original file line number Diff line number Diff line change
Expand Up @@ -122,42 +122,3 @@ func (d *Driver) List(ctx context.Context, path string, recursive bool) ([]respo
func (d *Driver) CancelToken(ctx context.Context, uploadSession *serializer.UploadSession) error {
return nil
}

func (d *Driver) Recycle(ctx context.Context, path string) error {
req := serializer.SlaveRecycleReq{
Path: path,
}

body, err := json.Marshal(req)
if err != nil {
return err
}

// 订阅回收结果
resChan := mq.GlobalMQ.Subscribe(req.Hash(model.GetSettingByName("siteID")), 0)
defer mq.GlobalMQ.Unsubscribe(req.Hash(model.GetSettingByName("siteID")), resChan)

res, err := d.client.Request("PUT", "task/recycle", bytes.NewReader(body)).
CheckHTTPResponse(200).
DecodeResponse()
if err != nil {
return err
}

if res.Code != 0 {
return serializer.NewErrorFromResponse(res)
}

// 等待回收结果或者超时
waitTimeout := model.GetIntSetting("slave_transfer_timeout", 172800)
select {
case <-time.After(time.Duration(waitTimeout) * time.Second):
return ErrWaitResultTimeout
case msg := <-resChan:
if msg.Event != serializer.SlaveRecycleSuccess {
return errors.New(msg.Content.(serializer.SlaveRecycleResult).Error)
}
}

return nil
}
20 changes: 0 additions & 20 deletions pkg/serializer/slave.go
Original file line number Diff line number Diff line change
Expand Up @@ -54,35 +54,15 @@ func (s *SlaveTransferReq) Hash(id string) string {
return fmt.Sprintf("%x", bs)
}

// SlaveRecycleReq 从机回收任务创建请求
type SlaveRecycleReq struct {
Path string `json:"path"`
}

// Hash 返回创建请求的唯一标识,保持创建请求幂等
func (s *SlaveRecycleReq) Hash(id string) string {
h := sha1.New()
h.Write([]byte(fmt.Sprintf("transfer-%s-%s", id, s.Path)))
bs := h.Sum(nil)
return fmt.Sprintf("%x", bs)
}

const (
SlaveTransferSuccess = "success"
SlaveTransferFailed = "failed"
SlaveRecycleSuccess = "success"
SlaveRecycleFailed = "failed"
)

type SlaveTransferResult struct {
Error string
}

type SlaveRecycleResult struct {
Error string
}

func init() {
gob.Register(SlaveTransferResult{})
gob.Register(SlaveRecycleResult{})
}
71 changes: 23 additions & 48 deletions pkg/task/recycle.go
Original file line number Diff line number Diff line change
@@ -1,14 +1,10 @@
package task

import (
"context"
"encoding/json"
"os"

model "github.com/cloudreve/Cloudreve/v3/models"
"github.com/cloudreve/Cloudreve/v3/pkg/cluster"
"github.com/cloudreve/Cloudreve/v3/pkg/filesystem"
"github.com/cloudreve/Cloudreve/v3/pkg/filesystem/driver/shadow/slaveinmaster"
"github.com/cloudreve/Cloudreve/v3/pkg/util"
)

Expand All @@ -22,10 +18,8 @@ type RecycleTask struct {

// RecycleProps 回收任务属性
type RecycleProps struct {
// 回收目录
Path string `json:"path"`
// 负责处理回收任务的节点ID
NodeID uint `json:"node_id"`
// 下载任务 GID
DownloadGID string `json:"download_gid"`
}

// Props 获取任务属性
Expand Down Expand Up @@ -59,7 +53,6 @@ func (job *RecycleTask) SetError(err *JobError) {
job.Err = err
res, _ := json.Marshal(job.Err)
job.TaskModel.SetError(string(res))

}

// SetErrorMsg 设定任务失败信息
Expand All @@ -78,51 +71,33 @@ func (job *RecycleTask) GetError() *JobError {

// Do 开始执行任务
func (job *RecycleTask) Do() {
if job.TaskProps.NodeID == 1 {
err := os.RemoveAll(job.TaskProps.Path)
if err != nil {
util.Log().Warning("无法删除中转临时目录[%s], %s", job.TaskProps.Path, err)
job.SetErrorMsg("文件回收失败", err)
}
} else {
// 指定为从机回收

// 创建文件系统
fs, err := filesystem.NewFileSystem(job.User)
if err != nil {
job.SetErrorMsg(err.Error(), nil)
return
}

// 获取从机节点
node := cluster.Default.GetNodeByID(job.TaskProps.NodeID)
if node == nil {
job.SetErrorMsg("从机节点不可用", nil)
}

// 切换为从机节点处理回收
fs.SwitchToSlaveHandler(node)
handler := fs.Handler.(*slaveinmaster.Driver)
err = handler.Recycle(context.Background(), job.TaskProps.Path)
if err != nil {
util.Log().Warning("无法删除中转临时目录[%s], %s", job.TaskProps.Path, err)
job.SetErrorMsg("文件回收失败", err)
}
download, err := model.GetDownloadByGid(job.TaskProps.DownloadGID, job.User.ID)
if err != nil {
util.Log().Warning("回收任务 %d 找不到下载记录", job.TaskModel.ID)
job.SetErrorMsg("无法找到下载任务", err)
return
}
}

// NewRecycleTask 新建回收任务
func NewRecycleTask(user uint, path string, node uint) (Job, error) {
creator, err := model.GetActiveUserByID(user)
nodeID := download.GetNodeID()
node := cluster.Default.GetNodeByID(nodeID)
if node == nil {
util.Log().Warning("回收任务 %d 找不到节点", job.TaskModel.ID)
job.SetErrorMsg("从机节点不可用", nil)
return
}
err = node.GetAria2Instance().DeleteTempFile(download)
if err != nil {
return nil, err
util.Log().Warning("无法删除中转临时目录[%s], %s", download.Parent, err)
job.SetErrorMsg("文件回收失败", err)
return
}
}

// NewRecycleTask 新建回收任务
func NewRecycleTask(download *model.Download) (Job, error) {
newTask := &RecycleTask{
User: &creator,
User: download.GetOwner(),
TaskProps: RecycleProps{
Path: path,
NodeID: node,
DownloadGID: download.GID,
},
}

Expand Down
42 changes: 14 additions & 28 deletions pkg/task/recycle_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -54,32 +54,6 @@ func TestRecycleTask_SetError(t *testing.T) {
asserts.Equal("error", task.GetError().Msg)
}

func TestRecycleTask_Do(t *testing.T) {
asserts := assert.New(t)
task := &RecycleTask{
TaskModel: &model.Task{
Model: gorm.Model{ID: 1},
},
}

// 目录不存在
{
task.TaskProps.Path = "test/not_exist"
task.User = &model.User{
Policy: model.Policy{
Type: "unknown",
},
}
mock.ExpectBegin()
mock.ExpectExec("UPDATE(.+)").WillReturnResult(sqlmock.NewResult(1,
1))
mock.ExpectCommit()
task.Do()
asserts.NoError(mock.ExpectationsWereMet())
asserts.NotEmpty(task.GetError().Msg)
}
}

func TestNewRecycleTask(t *testing.T) {
asserts := assert.New(t)

Expand All @@ -89,7 +63,13 @@ func TestNewRecycleTask(t *testing.T) {
mock.ExpectBegin()
mock.ExpectExec("INSERT(.+)").WillReturnResult(sqlmock.NewResult(1, 1))
mock.ExpectCommit()
job, err := NewRecycleTask(1, "/", 0)
job, err := NewRecycleTask(&model.Download{
Model: gorm.Model{ID: 1},
GID: "test_g_id",
Parent: "/",
UserID: 1,
NodeID: 1,
})
asserts.NoError(mock.ExpectationsWereMet())
asserts.NotNil(job)
asserts.NoError(err)
Expand All @@ -101,7 +81,13 @@ func TestNewRecycleTask(t *testing.T) {
mock.ExpectBegin()
mock.ExpectExec("INSERT(.+)").WillReturnError(errors.New("error"))
mock.ExpectRollback()
job, err := NewRecycleTask(1, "test/not_exist", 0)
job, err := NewRecycleTask(&model.Download{
Model: gorm.Model{ID: 1},
GID: "test_g_id",
Parent: "test/not_exist",
UserID: 1,
NodeID: 1,
})
asserts.NoError(mock.ExpectationsWereMet())
asserts.Nil(job)
asserts.Error(err)
Expand Down
95 changes: 0 additions & 95 deletions pkg/task/slavetask/recycle.go

This file was deleted.

11 changes: 0 additions & 11 deletions routers/controllers/slave.go
Original file line number Diff line number Diff line change
Expand Up @@ -212,17 +212,6 @@ func SlaveCreateTransferTask(c *gin.Context) {
}
}

// SlaveCreateRecycleTask 从机创建回收任务
func SlaveCreateRecycleTask(c *gin.Context) {
var service serializer.SlaveRecycleReq
if err := c.ShouldBindJSON(&service); err == nil {
res := explorer.CreateRecycleTask(c, &service)
c.JSON(200, res)
} else {
c.JSON(200, ErrorResponse(err))
}
}

// SlaveNotificationPush 处理从机发送的消息推送
func SlaveNotificationPush(c *gin.Context) {
var service node.SlaveNotificationService
Expand Down
1 change: 0 additions & 1 deletion routers/router.go
Original file line number Diff line number Diff line change
Expand Up @@ -88,7 +88,6 @@ func InitSlaveRouter() *gin.Engine {
task := v3.Group("task")
{
task.PUT("transfer", controllers.SlaveCreateTransferTask)
task.PUT("recycle", controllers.SlaveCreateRecycleTask)
}
}
return r
Expand Down
Loading