feat(dashboard): cleanup tasks and events (#2368)

* fix(db): 将MariaDB数据库类型映射到MySQL类型

* Update client.go

README.md

@@ -38,7 +38,7 @@
 
 ## :sparkles: Features
 
-- :cloud: Support storing files into Local, Remote node, OneDrive, S3 compatible API, Qiniu, Aliyun OSS, Tencent COS, Upyun, OneDrive.
+- :cloud: Support storing files into Local, Remote node, OneDrive, S3 compatible API, Qiniu, Aliyun OSS, Tencent COS, Upyun.
 - :outbox_tray: Upload/Download in directly transmission from client to storage providers.
 - 💾 Integrate with Aria2/qBittorrent to download files in background, use multiple download nodes to share the load.
 - 📚 Compress/Extract files, download files in batch.

go.mod

@@ -16,7 +16,7 @@ require (
 	github.com/dsoprea/go-png-image-structure v0.0.0-20210512210324-29b889a6093d
 	github.com/dsoprea/go-tiff-image-structure v0.0.0-20221003165014-8ecc4f52edca
 	github.com/dsoprea/go-utility v0.0.0-20200711062821-fab8125e9bdf
-	github.com/fatih/color v1.9.0
+	github.com/fatih/color v1.18.0
 	github.com/gin-contrib/cors v1.3.0
 	github.com/gin-contrib/sessions v1.0.2
 	github.com/gin-contrib/static v0.0.0-20191128031702-f81c604d8ac2
@@ -110,7 +110,7 @@ require (
 	github.com/klauspost/pgzip v1.2.5 // indirect
 	github.com/kr/pretty v0.3.1 // indirect
 	github.com/leodido/go-urn v1.4.0 // indirect
-	github.com/mattn/go-colorable v0.1.6 // indirect
+	github.com/mattn/go-colorable v0.1.13 // indirect
 	github.com/mattn/go-isatty v0.0.20 // indirect
 	github.com/mitchellh/go-wordwrap v0.0.0-20150314170334-ad45545899c7 // indirect
 	github.com/mitchellh/mapstructure v1.5.0 // indirect

go.sum

@@ -276,6 +276,8 @@ github.com/etcd-io/gofail v0.0.0-20190801230047-ad7f989257ca/go.mod h1:49H/RkXP8
 github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4=
 github.com/fatih/color v1.9.0 h1:8xPHl4/q1VyqGIPif1F+1V3Y3lSmrq01EabUW3CoW5s=
 github.com/fatih/color v1.9.0/go.mod h1:eQcE1qtQxscV5RaZvpXrrb8Drkc3/DdQ+uUYCNjL+zU=
+github.com/fatih/color v1.18.0 h1:S8gINlzdQ840/4pfAwic/ZE0djQEH3wM94VfqLTZcOM=
+github.com/fatih/color v1.18.0/go.mod h1:4FelSpRwEGDpQ12mAdzqdOukCy4u8WUtOY6lkT/6HfU=
 github.com/flynn/go-shlex v0.0.0-20150515145356-3f9db97f8568/go.mod h1:xEzjJPgXI435gkrCt3MPfRiAkVrwSbHsst4LCFVfpJc=
 github.com/form3tech-oss/jwt-go v3.2.2+incompatible/go.mod h1:pbq4aXjuKjdthFRnoDwaVPLA+WlJuPGy+QneDUgJi2k=
 github.com/form3tech-oss/jwt-go v3.2.3+incompatible/go.mod h1:pbq4aXjuKjdthFRnoDwaVPLA+WlJuPGy+QneDUgJi2k=
@@ -660,6 +662,8 @@ github.com/mattn/go-colorable v0.1.2/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVc
 github.com/mattn/go-colorable v0.1.4/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE=
 github.com/mattn/go-colorable v0.1.6 h1:6Su7aK7lXmJ/U79bYtBjLNaha4Fs1Rg9plHpcH+vvnE=
 github.com/mattn/go-colorable v0.1.6/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc=
+github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA=
+github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg=
 github.com/mattn/go-ieproxy v0.0.0-20190610004146-91bb50d98149/go.mod h1:31jz6HNzdxOmlERGGEc4v/dMssOfmp2p5bT/okiKFFc=
 github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4=
 github.com/mattn/go-isatty v0.0.4/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4=
@@ -669,6 +673,7 @@ github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hd
 github.com/mattn/go-isatty v0.0.9/go.mod h1:YNRxwqDuOph6SZLI9vUUz6OYw3QyUt7WiY2yME+cCiQ=
 github.com/mattn/go-isatty v0.0.11/go.mod h1:PhnuNfih5lzO57/f3n+odYbM4JtupLOxQOAqxQCu2WE=
 github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU=
+github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM=
 github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY=
 github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
 github.com/mattn/go-runewidth v0.0.2/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU=
@@ -1248,6 +1253,7 @@ golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBc
 golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220728004956-3c1f35247d10/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220928140112-f11e5e49a4ec/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=

inventory/client.go

@@ -52,6 +52,9 @@ func NewRawEntClient(l logging.Logger, config conf.ConfigProvider) (*ent.Client,
 	if confDBType == conf.SQLite3DB || confDBType == "" {
 		confDBType = conf.SQLiteDB
 	}
+	if confDBType == conf.MariaDB {
+		confDBType = conf.MySqlDB
+	}
 
 	var (
 		err    error

inventory/group.go

@@ -85,8 +85,8 @@ func (c *groupClient) Upsert(ctx context.Context, group *ent.Group) (*ent.Group,
 			SetPermissions(group.Permissions).
 			SetSettings(group.Settings)
 
-		if group.StoragePolicyID > 0 {
-			stm.SetStoragePolicyID(group.StoragePolicyID)
+		if group.Edges.StoragePolicies != nil && group.Edges.StoragePolicies.ID > 0 {
+			stm.SetStoragePolicyID(group.Edges.StoragePolicies.ID)
 		}
 
 		return stm.Save(ctx)

inventory/policy.go

@@ -27,6 +27,7 @@ type (
 	SkipStoragePolicyCache struct{}
 
 	StoragePolicyClient interface {
+		TxOperator
 		// GetByGroup returns the storage policies of the group.
 		GetByGroup(ctx context.Context, group *ent.Group) (*ent.StoragePolicy, error)
 		// GetPolicyByID returns the storage policy by id.
@@ -64,6 +65,14 @@ type storagePolicyClient struct {
 	cache  cache.Driver
 }
 
+func (c *storagePolicyClient) SetClient(newClient *ent.Client) TxOperator {
+	return &storagePolicyClient{client: newClient, cache: c.cache}
+}
+
+func (c *storagePolicyClient) GetClient() *ent.Client {
+	return c.client
+}
+
 func (c *storagePolicyClient) Delete(ctx context.Context, policy *ent.StoragePolicy) error {
 	if err := c.client.StoragePolicy.DeleteOne(policy).Exec(ctx); err != nil {
 		return fmt.Errorf("failed to delete storage policy: %w", err)

inventory/setting.go

@@ -395,6 +395,7 @@ var DefaultSettings = map[string]string{
 	"captcha_cap_instance_url":                   "",
 	"captcha_cap_site_key":                       "",
 	"captcha_cap_secret_key":                     "",
+	"captcha_cap_asset_server":                   "jsdelivr",
 	"thumb_width":                                "400",
 	"thumb_height":                               "300",
 	"thumb_entity_suffix":                        "._thumb",

inventory/task.go

@@ -3,6 +3,7 @@ package inventory
 import (
 	"context"
 	"fmt"
+	"time"
 
 	"entgo.io/ent/dialect/sql"
 	"github.com/cloudreve/Cloudreve/v4/ent"
@@ -44,6 +45,8 @@ type TaskClient interface {
 	List(ctx context.Context, args *ListTaskArgs) (*ListTaskResult, error)
 	// DeleteByIDs deletes the tasks with the given IDs.
 	DeleteByIDs(ctx context.Context, ids ...int) error
+	// DeleteBy deletes the tasks with the given args.
+	DeleteBy(ctx context.Context, args *DeleteTaskArgs) error
 }
 
 type (
@@ -59,6 +62,12 @@ type (
 		*PaginationResults
 		Tasks []*ent.Task
 	}
+
+	DeleteTaskArgs struct {
+		NotAfter time.Time
+		Types    []string
+		Status   []task.Status
+	}
 )
 
 func NewTaskClient(client *ent.Client, dbType conf.DBType, hasher hashid.Encoder) TaskClient {
@@ -113,6 +122,23 @@ func (c *taskClient) DeleteByIDs(ctx context.Context, ids ...int) error {
 	return err
 }
 
+func (c *taskClient) DeleteBy(ctx context.Context, args *DeleteTaskArgs) error {
+	query := c.client.Task.
+		Delete().
+		Where(task.CreatedAtLTE(args.NotAfter))
+
+	if len(args.Status) > 0 {
+		query.Where(task.StatusIn(args.Status...))
+	}
+
+	if len(args.Types) > 0 {
+		query.Where(task.TypeIn(args.Types...))
+	}
+
+	_, err := query.Exec(ctx)
+	return err
+}
+
 func (c *taskClient) Update(ctx context.Context, task *ent.Task, args *TaskArgs) (*ent.Task, error) {
 	stm := c.client.Task.UpdateOne(task).
 		SetPublicState(args.PublicState)

inventory/types/types.go

@@ -90,6 +90,8 @@ type (
 		UseCname bool `json:"use_cname,omitempty"`
 		// CDN domain does not need to be signed.
 		SourceAuth bool `json:"source_auth,omitempty"`
+		// QiniuUploadCdn whether to use CDN for Qiniu upload.
+		QiniuUploadCdn bool `json:"qiniu_upload_cdn,omitempty"`
 	}
 
 	FileType         int
@@ -177,6 +179,8 @@ type (
 	ShareProps struct {
 		// Whether to share view setting from owner
 		ShareView bool `json:"share_view,omitempty"`
+		// Whether to automatically show readme file in share view
+		ShowReadMe bool `json:"show_read_me,omitempty"`
 	}
 
 	FileTypeIconSetting struct {
@@ -286,6 +290,7 @@ type Viewer struct {
 	MaxSize     int64                              `json:"max_size,omitempty"`
 	Disabled    bool                               `json:"disabled,omitempty"`
 	Templates   []NewFileTemplate                  `json:"templates,omitempty"`
+	Platform    string                             `json:"platform,omitempty"`
 }
 
 type ViewerGroup struct {

pkg/auth/auth.go

@@ -220,5 +220,8 @@ func getUrlSignContent(ctx context.Context, url *url.URL) string {
 	// host = strings.TrimSuffix(host, "/")
 	// // remove port if it exists
 	// host = strings.Split(host, ":")[0]
+	if url.Path == "" {
+		return "/"
+	}
 	return url.Path
 }

pkg/conf/types.go

@@ -10,6 +10,7 @@ var (
 	MySqlDB    DBType = "mysql"
 	MsSqlDB    DBType = "mssql"
 	PostgresDB DBType = "postgres"
+	MariaDB    DBType = "mariadb"
 )
 
 // Database 数据库

pkg/filemanager/driver/qiniu/qiniu.go

@@ -67,7 +67,10 @@ func New(ctx context.Context, policy *ent.StoragePolicy, settings setting.Provid
 	}
 
 	mac := qbox.NewMac(policy.AccessKey, policy.SecretKey)
-	cfg := &storage.Config{UseHTTPS: true}
+	cfg := &storage.Config{
+		UseHTTPS:      true,
+		UseCdnDomains: policy.Settings.QiniuUploadCdn,
+	}
 
 	driver := &Driver{
 		policy:     policy,

pkg/filemanager/manager/manager.go

@@ -115,6 +115,7 @@ type (
 		RemainDownloads int
 		Expire          *time.Time
 		ShareView       bool
+		ShowReadMe      bool
 	}
 )
 

pkg/filemanager/manager/operation.go

@@ -267,7 +267,8 @@ func (l *manager) CreateOrUpdateShare(ctx context.Context, path *fs.URI, args *C
 	}
 
 	props := &types.ShareProps{
-		ShareView: args.ShareView,
+		ShareView:  args.ShareView,
+		ShowReadMe: args.ShowReadMe,
 	}
 
 	share, err := shareClient.Upsert(ctx, &inventory.CreateShareParams{

pkg/setting/provider.go

@@ -671,6 +671,7 @@ func (s *settingProvider) CapCaptcha(ctx context.Context) *Cap {
 		InstanceURL: s.getString(ctx, "captcha_cap_instance_url", ""),
 		SiteKey:     s.getString(ctx, "captcha_cap_site_key", ""),
 		SecretKey:   s.getString(ctx, "captcha_cap_secret_key", ""),
+		AssetServer: s.getString(ctx, "captcha_cap_asset_server", "jsdelivr"),
 	}
 }
 

pkg/setting/types.go

@@ -52,6 +52,7 @@ type Cap struct {
 	InstanceURL string
 	SiteKey     string
 	SecretKey   string
+	AssetServer string
 }
 
 type SMTP struct {

routers/controllers/admin.go

@@ -518,6 +518,17 @@ func AdminBatchDeleteEntity(c *gin.Context) {
 	}
 }
 
+func AdminCleanupTask(c *gin.Context) {
+	service := ParametersFromContext[*admin.CleanupTaskService](c, admin.CleanupTaskParameterCtx{})
+	err := service.CleanupTask(c)
+	if err != nil {
+		c.JSON(200, serializer.Err(c, err))
+		return
+	}
+
+	c.JSON(200, serializer.Response{})
+}
+
 func AdminListTasks(c *gin.Context) {
 	service := ParametersFromContext[*admin.AdminListService](c, admin.AdminListServiceParamsCtx{})
 	res, err := service.Tasks(c)

routers/router.go

@@ -854,6 +854,11 @@ func initMasterRouter(dep dependency.Dep) *gin.Engine {
 						controllers.FromJSON[adminsvc.BatchTaskService](adminsvc.BatchTaskParamCtx{}),
 						controllers.AdminBatchDeleteTask,
 					)
+					// Cleanup tasks
+					queue.POST("cleanup",
+						controllers.FromJSON[adminsvc.CleanupTaskService](adminsvc.CleanupTaskParameterCtx{}),
+						controllers.AdminCleanupTask,
+					)
 					// // 列出任务
 					// queue.POST("list", controllers.AdminListTask)
 					// // 新建文件导入任务

service/admin/policy.go

@@ -294,11 +294,22 @@ func (service *UpdateStoragePolicyService) Update(c *gin.Context) (*GetStoragePo
 	}
 
 	service.Policy.ID = idInt
-	_, err = storagePolicyClient.Upsert(c, service.Policy)
+
+	sc, tx, ctx, err := inventory.WithTx(c, storagePolicyClient)
 	if err != nil {
+		return nil, serializer.NewError(serializer.CodeDBError, "Failed to create transaction", err)
+	}
+
+	_, err = sc.Upsert(ctx, service.Policy)
+	if err != nil {
+		_ = inventory.Rollback(tx)
 		return nil, serializer.NewError(serializer.CodeDBError, "Failed to update policy", err)
 	}
 
+	if err := inventory.Commit(tx); err != nil {
+		return nil, serializer.NewError(serializer.CodeDBError, "Failed to commit transaction", err)
+	}
+
 	_ = dep.KV().Delete(manager.EntityUrlCacheKeyPrefix)
 
 	s := SingleStoragePolicyService{ID: idInt}

service/admin/task.go

@@ -3,6 +3,7 @@ package admin
 import (
 	"context"
 	"strconv"
+	"time"
 
 	"github.com/cloudreve/Cloudreve/v4/application/dependency"
 	"github.com/cloudreve/Cloudreve/v4/ent"
@@ -251,3 +252,31 @@ func (s *BatchTaskService) Delete(c *gin.Context) error {
 
 	return nil
 }
+
+type (
+	CleanupTaskService struct {
+		NotAfter time.Time     `json:"not_after" binding:"required"`
+		Types    []string      `json:"types"`
+		Status   []task.Status `json:"status"`
+	}
+	CleanupTaskParameterCtx struct{}
+)
+
+func (s *CleanupTaskService) CleanupTask(c *gin.Context) error {
+	dep := dependency.FromContext(c)
+	taskClient := dep.TaskClient()
+
+	if len(s.Status) == 0 {
+		s.Status = []task.Status{task.StatusCanceled, task.StatusCompleted, task.StatusError}
+	}
+
+	if err := taskClient.DeleteBy(c, &inventory.DeleteTaskArgs{
+		NotAfter: s.NotAfter,
+		Types:    s.Types,
+		Status:   s.Status,
+	}); err != nil {
+		return serializer.NewError(serializer.CodeDBError, "Failed to cleanup tasks", err)
+	}
+
+	return nil
+}

service/basic/site.go

@@ -31,6 +31,7 @@ type SiteConfig struct {
 	TurnstileSiteID  string              `json:"turnstile_site_id,omitempty"`
 	CapInstanceURL   string              `json:"captcha_cap_instance_url,omitempty"`
 	CapSiteKey       string              `json:"captcha_cap_site_key,omitempty"`
+	CapAssetServer   string              `json:"captcha_cap_asset_server,omitempty"`
 	RegisterEnabled  bool                `json:"register_enabled,omitempty"`
 	TosUrl           string              `json:"tos_url,omitempty"`
 	PrivacyPolicyUrl string              `json:"privacy_policy_url,omitempty"`
@@ -138,6 +139,7 @@ func (s *GetSettingService) GetSiteConfig(c *gin.Context) (*SiteConfig, error) {
 		ReCaptchaKey:    reCaptcha.Key,
 		CapInstanceURL:  capCaptcha.InstanceURL,
 		CapSiteKey:      capCaptcha.SiteKey,
+		CapAssetServer:  capCaptcha.AssetServer,
 		AppPromotion:    appSetting.Promotion,
 	}, nil
 }

service/explorer/response.go

@@ -280,6 +280,7 @@ type Share struct {
 	CreatedAt       time.Time       `json:"created_at,omitempty"`
 	Expired         bool            `json:"expired"`
 	Url             string          `json:"url"`
+	ShowReadMe      bool            `json:"show_readme,omitempty"`
 
 	// Only viewable by owner
 	IsPrivate bool   `json:"is_private,omitempty"`
@@ -313,6 +314,7 @@ func BuildShare(s *ent.Share, base *url.URL, hasher hashid.Encoder, requester *e
 		res.Downloaded = s.Downloads
 		res.Expires = s.Expires
 		res.Password = s.Password
+		res.ShowReadMe = s.Props != nil && s.Props.ShowReadMe
 	}
 
 	if requester.ID == owner.ID {

service/share/manage.go

@@ -24,6 +24,7 @@ type (
 		RemainDownloads int    `json:"downloads"`
 		Expire          int    `json:"expire"`
 		ShareView       bool   `json:"share_view"`
+		ShowReadMe      bool   `json:"show_readme"`
 	}
 	ShareCreateParamCtx struct{}
 )
@@ -58,6 +59,7 @@ func (service *ShareCreateService) Upsert(c *gin.Context, existed int) (string,
 		Expire:          expires,
 		ExistedShareID:  existed,
 		ShareView:       service.ShareView,
+		ShowReadMe:      service.ShowReadMe,
 	})
 	if err != nil {
 		return "", err