Compare commits

...

53 Commits
3.7.1 ... 3.8.2

Author SHA1 Message Date
Aaron Liu
ce832bf13d release: 3.8.2 2023-08-07 20:09:23 +08:00
Aaron Liu
5642dd3b66 feat(webdav): support setting download proxy 2023-07-29 08:58:14 +08:00
Aaron Liu
a1747073df feat(webdav): support setting download proxy 2023-07-29 08:53:26 +08:00
WeidiDeng
ad6c6bcd93 feat(webdav): supoort rename in copy and move (#1774) 2023-07-18 15:27:56 +08:00
WeidiDeng
f4a04ce3c3 fix webdav proppatch (#1771) 2023-07-18 15:25:43 +08:00
Aaron Liu
247e31079c fix(thumb): cannot generate thumb using ffmpeg for specific format (#1756) 2023-07-18 15:18:54 +08:00
Darren Yu
a26893aabc Add: thumb quality for 3rd storage policy (#1763)
Add thumb quality for third party storage policy.
2023-07-05 22:13:24 +08:00
初雪
ce759c02b1 feat(redis): support confiuring username (#1752)
替换Golang Redis依赖: redigo的版本至当前最新版1.8.9
(v2.0.0被标记为已撤回,且长期未更新)

Redis 6 及以上版本均可配置为使用username+password认证的ACL,故作此变更。
2023-07-05 22:12:33 +08:00
Aaron Liu
9f6f9adc89 Merge remote-tracking branch 'origin/master' 2023-06-25 18:53:54 +08:00
Aaron Liu
91025b9f24 fix(thumb): cannot generate thumbnails in slave mode 2023-06-25 18:53:37 +08:00
hallucination
a9bee3e638 Update docker-compose.yml (#1727)
add  Redis retains login sessions after restarting
2023-06-19 12:38:29 +08:00
Aaron Liu
243c312066 fix: failed UT 2023-06-11 09:50:57 +08:00
Aaron Liu
1d52ddd93a release: 3.8.0 2023-06-11 09:45:27 +08:00
Aaron Liu
cbc549229b fix(wopi): anonymous users cannot preview files 2023-06-11 09:45:06 +08:00
Aaron Liu
173ca6cdf8 fix(preview): use absolute URL for local storage policy 2023-06-11 09:44:43 +08:00
Aaron Liu
fb166fb3e4 release: 3.8.0-beta1 2023-05-27 14:06:19 +08:00
Aaron Liu
b1344616b8 test: fix failed ut 2023-05-27 10:44:28 +08:00
Aaron Liu
89ee147961 feat(upload): detect and specify mime type for files uploaded to S3 and OSS (fix#1681) 2023-05-25 19:51:51 +08:00
Aaron Liu
4aafe1dc7a enhance(download): Use just-in-time host in download URl, instead of SiteURL in site settings 2023-05-25 19:49:32 +08:00
Aaron Liu
4c834e75fa adhoc: commit todo changes related to google drive 2023-05-25 19:46:05 +08:00
Aaron Liu
31d4a3445d fix(cache): panic if redis connection fails 2023-05-25 19:44:59 +08:00
Aaron Liu
37926e3133 feat(policy): add Google Drive Oauth client 2023-05-24 14:39:54 +08:00
WeidiDeng
4c18e5acd1 webdav兼容nextcloud propset设置修改时间 (#1710) 2023-05-24 12:10:03 +08:00
Arkylin
6358740cc9 modified: models/policy.go (#1718)
modified:   models/policy_test.go
2023-05-24 12:09:24 +08:00
Aaron Liu
00d56d6d07 test: fix failed ut 2023-04-16 09:25:57 +08:00
Aaron Liu
b9143b53f6 chore: update runner to ubuntu-latest 2023-04-16 09:20:30 +08:00
Aaron Liu
b9d9e036c9 feat(kv): persist cache and session into disk before shutdown 2023-04-16 09:17:06 +08:00
Aaron Liu
4d131db504 test(hook): NewWebdavAfterUploadHook 2023-04-15 09:21:29 +08:00
Aaron Liu
c5ffdbfcfb Merge remote-tracking branch 'origin/master' 2023-04-13 19:39:22 +08:00
Aaron Liu
8e2fc1a8f6 test(thumb): new changes in filesystem pkg 2023-04-13 19:39:12 +08:00
AaronLiu
ce579d387a Merge pull request #1690 from cloudreve/webdav-checksum
webdav兼容rclone的nextcloud选项(修改日期和checksum)
2023-04-08 10:10:46 +08:00
AaronLiu
f1e7af67bc Merge branch 'master' into webdav-checksum 2023-04-08 10:09:55 +08:00
AaronLiu
98788dc72b Merge pull request #1679 from xkeyC/master
feat(Webdav): Add overwrite support for moveFiles and copyFiles
2023-04-08 10:08:13 +08:00
Weidi Deng
1b4eff624d webdav兼容rclone的nextcloud选项(修改日期和checksum) 2023-04-07 22:16:11 +08:00
Aaron Liu
408733a974 test(thumb): new changes in models/cache pkg 2023-04-07 20:33:05 +08:00
Aaron Liu
c8b736bd8f fix(dashboard): add missing utils for thumb setting 2023-04-07 19:42:23 +08:00
Aaron Liu
cf03206283 feat(thumb): generator settings and test button 2023-04-07 19:33:02 +08:00
Aaron Liu
ac536408c6 feat(thumb): use libreoffice to generate thumb 2023-04-07 19:31:43 +08:00
Aaron Liu
98b86b37de feat(thumb): use ffmpeg to generate thumb 2023-04-07 19:30:41 +08:00
Aaron Liu
b55344459d feat(thumb): use libvips to generate thumb 2023-04-07 19:30:10 +08:00
Aaron Liu
bde4459519 feat(thumb): add ext whitelist for all policy types 2023-04-07 19:29:43 +08:00
Aaron Liu
f5a21a7e6f feat(thumb): set size limit for original file 2023-04-07 19:28:39 +08:00
Aaron Liu
b910254cc5 feat(thumb): delete generated thumb file
fix(s3): return empty list of file failed to be deleted
2023-04-07 19:27:57 +08:00
Aaron Liu
e115497dfe feat(thumb): generate thumb for OneDrive files 2023-04-07 19:27:31 +08:00
Aaron Liu
62b73b577b feat(thumb): generate and return sidecar thumb 2023-04-07 19:26:39 +08:00
Aaron Liu
7cb5e68b78 refactor(thumb): thumb logic for slave policy 2023-04-07 19:25:29 +08:00
Aaron Liu
ae118c337e refactor(thumb): reset thumb status after renaming a file with no thumb available 2023-04-07 19:09:13 +08:00
Aaron Liu
f36e39991d refactor(thumb): new thumb pipeline model to generate thumb on-demand 2023-04-07 19:08:54 +08:00
Aaron Liu
da1eaf2d1f fix(wopi): cannot set preferred language for LibreOffice online 2023-04-07 19:06:46 +08:00
xkeyC
42f7613bfa moveFiles 修改回无条件 overwrite (Move 或 Rename 都会处罚冲突问题) 2023-03-29 20:16:09 +08:00
xkeyC
e8e38029ca fix:error code 2023-03-28 00:19:18 +08:00
xkeyC
cd9e9e25b9 fix:仅在需要移动时 overwrite 2023-03-28 00:06:10 +08:00
xkeyC
ca7b21dc3e feat(Webdav):Add overwrite support for moveFiles and copyFiles 2023-03-27 22:55:20 +08:00
101 changed files with 2817 additions and 902 deletions

View File

@@ -5,7 +5,7 @@ on: workflow_dispatch
jobs:
build:
name: Build
runs-on: ubuntu-18.04
runs-on: ubuntu-latest
steps:
- name: Set up Go 1.20
uses: actions/setup-go@v2

View File

@@ -10,12 +10,12 @@ on:
jobs:
test:
name: Test
runs-on: ubuntu-18.04
runs-on: ubuntu-latest
steps:
- name: Set up Go 1.18
- name: Set up Go 1.20
uses: actions/setup-go@v2
with:
go-version: "1.18"
go-version: "1.20"
id: go
- name: Check out code into the Go module directory

2
assets

Submodule assets updated: 9f847f466c...b993b4283e

View File

@@ -15,6 +15,7 @@ import (
"github.com/cloudreve/Cloudreve/v3/pkg/wopi"
"github.com/gin-gonic/gin"
"io/fs"
"path/filepath"
)
// Init 初始化启动
@@ -39,7 +40,19 @@ func Init(path string, statics fs.FS) {
{
"both",
func() {
cache.Init(conf.SystemConfig.Mode == "slave")
cache.Init()
},
},
{
"slave",
func() {
model.InitSlaveDefaults()
},
},
{
"slave",
func() {
cache.InitSlaveOverwrites()
},
},
{
@@ -48,6 +61,12 @@ func Init(path string, statics fs.FS) {
model.Init()
},
},
{
"both",
func() {
cache.Restore(filepath.Join(model.GetSettingByName("temp_path"), cache.DefaultCacheFile))
},
},
{
"both",
func() {

View File

@@ -1,5 +1,14 @@
version: "3.8"
services:
redis:
container_name: redis
image: bitnami/redis:latest
restart: unless-stopped
environment:
- ALLOW_EMPTY_PASSWORD=yes
volumes:
- redis_data:/bitnami/redis/data
cloudreve:
container_name: cloudreve
image: cloudreve/cloudreve:latest
@@ -14,6 +23,7 @@ services:
- ./cloudreve/avatar:/cloudreve/avatar
depends_on:
- aria2
aria2:
container_name: aria2
image: p3terx/aria2-pro # third party image, please keep notice what you are doing
@@ -25,9 +35,11 @@ services:
- ./aria2/config:/config
- temp_data:/data
volumes:
redis_data:
driver: local
temp_data:
driver: local
driver_opts:
type: none
device: $PWD/data
o: bind
o: bind

17
go.mod
View File

@@ -20,6 +20,8 @@ require (
github.com/gofrs/uuid v4.0.0+incompatible
github.com/gomodule/redigo v2.0.0+incompatible
github.com/google/go-querystring v1.0.0
github.com/gorilla/securecookie v1.1.1
github.com/gorilla/sessions v1.2.1
github.com/gorilla/websocket v1.4.2
github.com/hashicorp/go-version v1.3.0
github.com/jinzhu/gorm v1.9.11
@@ -31,6 +33,7 @@ require (
github.com/qiniu/go-sdk/v7 v7.11.1
github.com/rafaeljusto/redigomock v0.0.0-20191117212112-00b2509252a1
github.com/robfig/cron/v3 v3.0.1
github.com/samber/lo v1.38.1
github.com/speps/go-hashids v2.0.0+incompatible
github.com/stretchr/testify v1.7.2
github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/captcha v1.0.393
@@ -40,6 +43,7 @@ require (
github.com/upyun/go-sdk v2.1.0+incompatible
golang.org/x/image v0.0.0-20211028202545-6944b10bf410
golang.org/x/time v0.0.0-20210220033141-f8bda1e9f3ba
google.golang.org/api v0.45.0
)
require (
@@ -48,7 +52,6 @@ require (
github.com/baiyubin/aliyun-sts-go-sdk v0.0.0-20180326062324-cfa1a18b161f // indirect
github.com/beorn7/perks v1.0.1 // indirect
github.com/bgentry/speakeasy v0.1.0 // indirect
github.com/boj/redistore v0.0.0-20180917114910-cd5dcc76aeff // indirect
github.com/boombuler/barcode v1.0.1-0.20190219062509-6c824513bacc // indirect
github.com/census-instrumentation/opencensus-proto v0.3.0 // indirect
github.com/cespare/xxhash/v2 v2.1.1 // indirect
@@ -82,9 +85,8 @@ require (
github.com/google/certificate-transparency-go v1.1.2-0.20210511102531-373a877eec92 // indirect
github.com/google/go-cmp v0.5.9 // indirect
github.com/google/uuid v1.3.0 // indirect
github.com/googleapis/gax-go/v2 v2.0.5 // indirect
github.com/gorilla/context v1.1.1 // indirect
github.com/gorilla/securecookie v1.1.1 // indirect
github.com/gorilla/sessions v1.2.1 // indirect
github.com/grpc-ecosystem/go-grpc-middleware v1.3.0 // indirect
github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 // indirect
github.com/grpc-ecosystem/grpc-gateway v1.16.0 // indirect
@@ -115,7 +117,6 @@ require (
github.com/prometheus/client_model v0.2.0 // indirect
github.com/prometheus/common v0.24.0 // indirect
github.com/prometheus/procfs v0.6.0 // indirect
github.com/quasoft/memstore v0.0.0-20191010062613-2bce066d2b0b // indirect
github.com/remyoudompheng/bigfft v0.0.0-20230126093431-47fa9a501578 // indirect
github.com/rivo/uniseg v0.2.0 // indirect
github.com/russross/blackfriday/v2 v2.1.0 // indirect
@@ -142,17 +143,19 @@ require (
go.etcd.io/etcd/server/v3 v3.5.0-alpha.0 // indirect
go.etcd.io/etcd/tests/v3 v3.5.0-alpha.0 // indirect
go.etcd.io/etcd/v3 v3.5.0-alpha.0 // indirect
go.opencensus.io v0.23.0 // indirect
go.uber.org/atomic v1.7.0 // indirect
go.uber.org/multierr v1.7.0 // indirect
go.uber.org/zap v1.16.0 // indirect
golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d // indirect
golang.org/x/mod v0.4.2 // indirect
golang.org/x/exp v0.0.0-20220303212507-bbda1eaf7a17 // indirect
golang.org/x/mod v0.6.0-dev.0.20211013180041-c96bc1413d57 // indirect
golang.org/x/net v0.0.0-20220630215102-69896b714898 // indirect
golang.org/x/oauth2 v0.0.0-20210427180440-81ed05c6b58c // indirect
golang.org/x/sync v0.0.0-20210220032951-036812b2e83c // indirect
golang.org/x/sys v0.4.0 // indirect
golang.org/x/text v0.3.7 // indirect
golang.org/x/tools v0.1.0 // indirect
golang.org/x/tools v0.1.8-0.20211029000441-d6a9af8af023 // indirect
golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 // indirect
google.golang.org/appengine v1.6.7 // indirect
google.golang.org/genproto v0.0.0-20210510173355-fb37daa5cd7a // indirect
@@ -170,3 +173,5 @@ require (
sigs.k8s.io/yaml v1.2.0 // indirect
)
replace github.com/gomodule/redigo v2.0.0+incompatible => github.com/gomodule/redigo v1.8.9

22
go.sum
View File

@@ -131,8 +131,6 @@ github.com/bgentry/speakeasy v0.1.0 h1:ByYyxL9InA1OWqxJqqp2A5pYHUrCiAL6K3J+LKSsQ
github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs=
github.com/bketelsen/crypt v0.0.3-0.20200106085610-5cbc8cc4026c/go.mod h1:MKsuJmJgSg28kpZDP6UIiPt0e0Oz0kqKNGyRaWEPv84=
github.com/blakesmith/ar v0.0.0-20190502131153-809d4375e1fb/go.mod h1:PkYb9DJNAwrSvRx5DYA+gUcOIgTGVMNkfSCbZM8cWpI=
github.com/boj/redistore v0.0.0-20180917114910-cd5dcc76aeff h1:RmdPFa+slIr4SCBg4st/l/vZWVe9QJKMXGO60Bxbe04=
github.com/boj/redistore v0.0.0-20180917114910-cd5dcc76aeff/go.mod h1:+RTT1BOk5P97fT2CiHkbFQwkK3mjsFAP6zCYV2aXtjw=
github.com/boombuler/barcode v1.0.1-0.20190219062509-6c824513bacc h1:biVzkmvwrH8WK8raXaxBx6fRVTlJILwEwQGL1I/ByEI=
github.com/boombuler/barcode v1.0.1-0.20190219062509-6c824513bacc/go.mod h1:paBWMcWSl3LHKBqUq+rly7CNSldXjb2rDl3JlRe0mD8=
github.com/caarlos0/ctrlc v1.0.0/go.mod h1:CdXpj4rmq0q/1Eb44M9zi2nKB0QraNKuRGYGrrHhcQw=
@@ -364,8 +362,8 @@ github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiu
github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM=
github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
github.com/gomodule/redigo v2.0.0+incompatible h1:K/R+8tc58AaqLkqG2Ol3Qk+DR/TlNuhuh457pBFPtt0=
github.com/gomodule/redigo v2.0.0+incompatible/go.mod h1:B4C85qUVwatsJoIUNIfCRsp7qO0iAmpGFZ4EELWSbC4=
github.com/gomodule/redigo v1.8.9 h1:Sl3u+2BI/kk+VEatbj0scLdrFhjPmbxOc1myhDP41ws=
github.com/gomodule/redigo v1.8.9/go.mod h1:7ArFNvsTjH8GMMzB4uy1snslv2BwmginuMs06a1uzZE=
github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
github.com/google/btree v1.0.1 h1:gK4Kx5IaGY9CD5sPJ36FHiBJ6ZXl0kilRiiCj+jdYp4=
@@ -426,6 +424,7 @@ github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+
github.com/google/wire v0.3.0/go.mod h1:i1DMg/Lu8Sz5yYl25iOdmc5CT5qusaa+zmRWs16741s=
github.com/googleapis/gax-go v2.0.2+incompatible/go.mod h1:SFVmujtThgffbyetf+mdk2eWhX2bMyUtNHzFKcPA9HY=
github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg=
github.com/googleapis/gax-go/v2 v2.0.5 h1:sjZBwGj9Jlw33ImPtvFviGYvseOtDM7hkSKB7+Tv3SM=
github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk=
github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1 h1:EGx4pi6eqNxGaHF6qqu48+N2wcFQ5qg5FXgOdqsJ5d8=
github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY=
@@ -438,7 +437,6 @@ github.com/gorilla/mux v1.6.2/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2z
github.com/gorilla/mux v1.7.3/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs=
github.com/gorilla/securecookie v1.1.1 h1:miw7JPhV+b/lAHSXz4qd/nN9jRiAFV5FwjeKyCS8BvQ=
github.com/gorilla/securecookie v1.1.1/go.mod h1:ra0sb63/xPlUeL+yeDciTfxMRAA+MP+HVt/4epWDjd4=
github.com/gorilla/sessions v1.1.1/go.mod h1:8KCfur6+4Mqcc6S0FEfKuN15Vl5MgXW92AE8ovaJD0w=
github.com/gorilla/sessions v1.2.1 h1:DHd3rPN5lE3Ts3D8rKkQ8x/0kqfeNmBAaiSi+o7FsgI=
github.com/gorilla/sessions v1.2.1/go.mod h1:dk2InVEVJ0sfLlnXv9EAgkf6ecYs/i80K/zI+bUmuGM=
github.com/gorilla/websocket v0.0.0-20170926233335-4201258b820c/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ=
@@ -750,8 +748,6 @@ github.com/qiniu/dyn v1.3.0/go.mod h1:E8oERcm8TtwJiZvkQPbcAh0RL8jO1G0VXJMW3FAWdk
github.com/qiniu/go-sdk/v7 v7.11.1 h1:/LZ9rvFS4p6SnszhGv11FNB1+n4OZvBCwFg7opH5Ovs=
github.com/qiniu/go-sdk/v7 v7.11.1/go.mod h1:btsaOc8CA3hdVloULfFdDgDc+g4f3TDZEFsDY0BLE+w=
github.com/qiniu/x v1.10.5/go.mod h1:03Ni9tj+N2h2aKnAz+6N0Xfl8FwMEDRC2PAlxekASDs=
github.com/quasoft/memstore v0.0.0-20191010062613-2bce066d2b0b h1:aUNXCGgukb4gtY99imuIeoh8Vr0GSwAlYxPAhqZrpFc=
github.com/quasoft/memstore v0.0.0-20191010062613-2bce066d2b0b/go.mod h1:wTPjTepVu7uJBYgZ0SdWHQlIas582j6cn2jgk4DDdlg=
github.com/rafaeljusto/redigomock v0.0.0-20191117212112-00b2509252a1 h1:leEwA4MD1ew0lNgzz6Q4G76G3AEfeci+TMggN6WuFRs=
github.com/rafaeljusto/redigomock v0.0.0-20191117212112-00b2509252a1/go.mod h1:JaY6n2sDr+z2WTsXkOmNRUfDy6FN0L6Nk7x06ndm4tY=
github.com/rcrowley/go-metrics v0.0.0-20181016184325-3113b8401b8a/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4=
@@ -776,6 +772,8 @@ github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQD
github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk=
github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts=
github.com/samber/lo v1.38.1 h1:j2XEAqXKb09Am4ebOg31SpvzUTTs6EN3VfgeLUhPdXM=
github.com/samber/lo v1.38.1/go.mod h1:+m/ZKRl6ClXCE2Lgf3MsQlWfh4bn1bz6CXEOxnEXnEA=
github.com/samuel/go-zookeeper v0.0.0-20190923202752-2cc03de413da/go.mod h1:gi+0XIa01GRL2eRQVjQkKGqKF3SF9vZR/HnPullcV2E=
github.com/sassoftware/go-rpmutils v0.0.0-20190420191620-a8f1baeba37b/go.mod h1:am+Fp8Bt506lA3Rk3QCmSqmYmLMnPDhdDUcosQCAx+I=
github.com/satori/go.uuid v1.2.0 h1:0uYX9dsZ2yD7q2RtLRtPSdGDWzjeM3TbMJP9utgA0ww=
@@ -935,6 +933,7 @@ go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
go.opencensus.io v0.22.5/go.mod h1:5pWMHQbX5EPX2/62yrJeAkowc+lfs/XD7Uxpq3pI6kk=
go.opencensus.io v0.23.0 h1:gqCw0LfLxScz8irSi8exQc7fyQ0fKQU/qnC/X8+V/1M=
go.opencensus.io v0.23.0/go.mod h1:XItmlyltB5F7CS4xOC1DcqMoFqwtC6OG2xF7mCv7P7E=
go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE=
go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE=
@@ -988,6 +987,8 @@ golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u0
golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM=
golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU=
golang.org/x/exp v0.0.0-20200331195152-e8c3332aa8e5/go.mod h1:4M0jN8W1tt0AVLNr8HDosyJCDCDuyL9N9+3m7wDWgKw=
golang.org/x/exp v0.0.0-20220303212507-bbda1eaf7a17 h1:3MTrJm4PyNL9NBqvYDSj3DHl46qQakyfqfWo4jgfaEM=
golang.org/x/exp v0.0.0-20220303212507-bbda1eaf7a17/go.mod h1:lgLbSvA5ygNOMpwM/9anMpWVlVJ7Z+cHWq/eFuinpGE=
golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js=
golang.org/x/image v0.0.0-20190501045829-6d32002ffd75/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js=
golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0=
@@ -1016,8 +1017,9 @@ golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
golang.org/x/mod v0.4.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
golang.org/x/mod v0.4.1/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
golang.org/x/mod v0.4.2 h1:Gz96sIWK3OalVv/I/qNygP42zyoKp3xptRVCWRFEBvo=
golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
golang.org/x/mod v0.6.0-dev.0.20211013180041-c96bc1413d57 h1:LQmS1nU0twXLA96Kt7U9qtHJEbBk3z6Q0V4UXjZkpr4=
golang.org/x/mod v0.6.0-dev.0.20211013180041-c96bc1413d57/go.mod h1:3p9vT2HGsQu2K1YbXdKPJLVgG5VJdoTa1poYQBtP1AY=
golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
@@ -1252,8 +1254,9 @@ golang.org/x/tools v0.0.0-20201201161351-ac6f37ff4c2a/go.mod h1:emZCQorbCU4vsT4f
golang.org/x/tools v0.0.0-20201208233053-a543418bbed2/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
golang.org/x/tools v0.0.0-20210105154028-b0ab187a4818/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
golang.org/x/tools v0.1.0 h1:po9/4sTYwZU9lPhi1tOrb4hCv3qrhiQ77LZfGa2OjwY=
golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0=
golang.org/x/tools v0.1.8-0.20211029000441-d6a9af8af023 h1:0c3L82FDQ5rt1bjTBlchS8t6RQ6299/+5bWMnRLh+uI=
golang.org/x/tools v0.1.8-0.20211029000441-d6a9af8af023/go.mod h1:nABZi5QlRsZVlzPpHl034qft6wpY4eDcsTt5AaioBiU=
golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
@@ -1284,6 +1287,7 @@ google.golang.org/api v0.36.0/go.mod h1:+z5ficQTmoYpPn8LCUNVpK5I7hwkpjbcgqA7I34q
google.golang.org/api v0.40.0/go.mod h1:fYKFpnQN0DsDSKRVRcQSDQNtqWPfM9i+zNPxepjRCQ8=
google.golang.org/api v0.41.0/go.mod h1:RkxM5lITDfTzmyKFPt+wGrCJbVfniCr2ool8kTBzRTU=
google.golang.org/api v0.43.0/go.mod h1:nQsDGjRXMo4lvh5hP0TKqF244gqhGcr/YSIykhUk/94=
google.golang.org/api v0.45.0 h1:pqMffJFLBVUDIoYsHcqtxgQVTsmxMDpYLOc5MT4Jrww=
google.golang.org/api v0.45.0/go.mod h1:ISLIJCedJolbZvDfAk+Ctuq5hf+aJ33WgtUsfyFoLXA=
google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
google.golang.org/appengine v1.2.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=

42
main.go
View File

@@ -9,11 +9,13 @@ import (
"net/http"
"os"
"os/signal"
"path/filepath"
"syscall"
"time"
"github.com/cloudreve/Cloudreve/v3/bootstrap"
model "github.com/cloudreve/Cloudreve/v3/models"
"github.com/cloudreve/Cloudreve/v3/pkg/cache"
"github.com/cloudreve/Cloudreve/v3/pkg/conf"
"github.com/cloudreve/Cloudreve/v3/pkg/util"
"github.com/cloudreve/Cloudreve/v3/routers"
@@ -67,20 +69,10 @@ func main() {
// 收到信号后关闭服务器
sigChan := make(chan os.Signal, 1)
signal.Notify(sigChan, os.Interrupt, syscall.SIGTERM, syscall.SIGHUP, syscall.SIGQUIT)
go func() {
sig := <-sigChan
util.Log().Info("Signal %s received, shutting down server...", sig)
ctx := context.Background()
if conf.SystemConfig.GracePeriod != 0 {
var cancel context.CancelFunc
ctx, cancel = context.WithTimeout(ctx, time.Duration(conf.SystemConfig.GracePeriod)*time.Second)
defer cancel()
}
go shutdown(sigChan, server)
err := server.Shutdown(ctx)
if err != nil {
util.Log().Error("Failed to shutdown server: %s", err)
}
defer func() {
<-sigChan
}()
// 如果启用了SSL
@@ -140,3 +132,27 @@ func RunUnix(server *http.Server) error {
return server.Serve(listener)
}
func shutdown(sigChan chan os.Signal, server *http.Server) {
sig := <-sigChan
util.Log().Info("Signal %s received, shutting down server...", sig)
ctx := context.Background()
if conf.SystemConfig.GracePeriod != 0 {
var cancel context.CancelFunc
ctx, cancel = context.WithTimeout(ctx, time.Duration(conf.SystemConfig.GracePeriod)*time.Second)
defer cancel()
}
// Shutdown http server
err := server.Shutdown(ctx)
if err != nil {
util.Log().Error("Failed to shutdown server: %s", err)
}
// Persist in-memory cache
if err := cache.Store.Persist(filepath.Join(model.GetSettingByName("temp_path"), cache.DefaultCacheFile)); err != nil {
util.Log().Warning("Failed to persist cache: %s", err)
}
close(sigChan)
}

View File

@@ -116,6 +116,11 @@ func WebDAVAuth() gin.HandlerFunc {
return
}
// 用户组已启用WebDAV代理
if !expectedUser.Group.OptionsSerialized.WebDAVProxy {
webdav.UseProxy = false
}
c.Set("user", &expectedUser)
c.Set("webdav", webdav)
c.Next()

View File

@@ -1,6 +1,8 @@
package middleware
import (
"github.com/cloudreve/Cloudreve/v3/pkg/cache"
"github.com/cloudreve/Cloudreve/v3/pkg/sessionstore"
"net/http"
"strings"
@@ -8,28 +10,16 @@ import (
"github.com/cloudreve/Cloudreve/v3/pkg/serializer"
"github.com/cloudreve/Cloudreve/v3/pkg/util"
"github.com/gin-contrib/sessions"
"github.com/gin-contrib/sessions/memstore"
"github.com/gin-contrib/sessions/redis"
"github.com/gin-gonic/gin"
)
// Store session存储
var Store memstore.Store
var Store sessions.Store
// Session 初始化session
func Session(secret string) gin.HandlerFunc {
// Redis设置不为空且非测试模式时使用Redis
if conf.RedisConfig.Server != "" && gin.Mode() != gin.TestMode {
var err error
Store, err = redis.NewStoreWithDB(10, conf.RedisConfig.Network, conf.RedisConfig.Server, conf.RedisConfig.Password, conf.RedisConfig.DB, []byte(secret))
if err != nil {
util.Log().Panic("Failed to connect to Redis%s", err)
}
util.Log().Info("Connect to Redis server %q.", conf.RedisConfig.Server)
} else {
Store = memstore.NewStore([]byte(secret))
}
Store = sessionstore.NewStore(cache.Store, []byte(secret))
sameSiteMode := http.SameSiteDefaultMode
switch strings.ToLower(conf.CORSConfig.SameSite) {

View File

@@ -5,7 +5,6 @@ import (
"net/http/httptest"
"testing"
"github.com/cloudreve/Cloudreve/v3/pkg/conf"
"github.com/cloudreve/Cloudreve/v3/pkg/util"
"github.com/gin-gonic/gin"
"github.com/stretchr/testify/assert"
@@ -20,14 +19,6 @@ func TestSession(t *testing.T) {
asserts.NotNil(Store)
asserts.IsType(emptyFunc(), handler)
}
{
conf.RedisConfig.Server = "123"
asserts.Panics(func() {
Session("2333")
})
conf.RedisConfig.Server = ""
}
}
func emptyFunc() gin.HandlerFunc {

View File

@@ -1,6 +1,7 @@
package model
import (
"github.com/cloudreve/Cloudreve/v3/pkg/cache"
"github.com/cloudreve/Cloudreve/v3/pkg/conf"
"github.com/cloudreve/Cloudreve/v3/pkg/util"
"github.com/gofrs/uuid"
@@ -106,6 +107,20 @@ Neue',Helvetica,Arial,sans-serif; box-sizing: border-box; font-size: 14px; verti
{Name: "thumb_encode_method", Value: "jpg", Type: "thumb"},
{Name: "thumb_gc_after_gen", Value: "0", Type: "thumb"},
{Name: "thumb_encode_quality", Value: "85", Type: "thumb"},
{Name: "thumb_builtin_enabled", Value: "1", Type: "thumb"},
{Name: "thumb_vips_enabled", Value: "0", Type: "thumb"},
{Name: "thumb_ffmpeg_enabled", Value: "0", Type: "thumb"},
{Name: "thumb_vips_path", Value: "vips", Type: "thumb"},
{Name: "thumb_vips_exts", Value: "csv,mat,img,hdr,pbm,pgm,ppm,pfm,pnm,svg,svgz,j2k,jp2,jpt,j2c,jpc,gif,png,jpg,jpeg,jpe,webp,tif,tiff,fits,fit,fts,exr,jxl,pdf,heic,heif,avif,svs,vms,vmu,ndpi,scn,mrxs,svslide,bif,raw", Type: "thumb"},
{Name: "thumb_ffmpeg_seek", Value: "00:00:01.00", Type: "thumb"},
{Name: "thumb_ffmpeg_path", Value: "ffmpeg", Type: "thumb"},
{Name: "thumb_ffmpeg_exts", Value: "3g2,3gp,asf,asx,avi,divx,flv,m2ts,m2v,m4v,mkv,mov,mp4,mpeg,mpg,mts,mxf,ogv,rm,swf,webm,wmv", Type: "thumb"},
{Name: "thumb_libreoffice_path", Value: "soffice", Type: "thumb"},
{Name: "thumb_libreoffice_enabled", Value: "0", Type: "thumb"},
{Name: "thumb_libreoffice_exts", Value: "md,ods,ots,fods,uos,xlsx,xml,xls,xlt,dif,dbf,html,slk,csv,xlsm,docx,dotx,doc,dot,rtf,xlsm,xlst,xls,xlw,xlc,xlt,pptx,ppsx,potx,pomx,ppt,pps,ppm,pot,pom", Type: "thumb"},
{Name: "thumb_proxy_enabled", Value: "0", Type: "thumb"},
{Name: "thumb_proxy_policy", Value: "[]", Type: "thumb"},
{Name: "thumb_max_src_size", Value: "31457280", Type: "thumb"},
{Name: "pwa_small_icon", Value: "/static/img/favicon.ico", Type: "pwa"},
{Name: "pwa_medium_icon", Value: "/static/img/logo192.png", Type: "pwa"},
{Name: "pwa_large_icon", Value: "/static/img/logo512.png", Type: "pwa"},
@@ -120,3 +135,9 @@ Neue',Helvetica,Arial,sans-serif; box-sizing: border-box; font-size: 14px; verti
{Name: "wopi_max_size", Value: "52428800", Type: "wopi"},
{Name: "wopi_session_timeout", Value: "36000", Type: "wopi"},
}
func InitSlaveDefaults() {
for _, setting := range defaultSettings {
cache.Set("setting_"+setting.Name, setting.Value, -1)
}
}

View File

@@ -6,6 +6,8 @@ import (
"errors"
"fmt"
"path"
"path/filepath"
"strings"
"time"
"github.com/cloudreve/Cloudreve/v3/pkg/util"
@@ -34,6 +36,18 @@ type File struct {
MetadataSerialized map[string]string `gorm:"-"`
}
// Thumb related metadata
const (
ThumbStatusNotExist = ""
ThumbStatusExist = "exist"
ThumbStatusNotAvailable = "not_available"
ThumbStatusMetadataKey = "thumb_status"
ThumbSidecarMetadataKey = "thumb_sidecar"
ChecksumMetadataKey = "webdav_checksum"
)
func init() {
// 注册缓存用到的复杂结构
gob.Register(File{})
@@ -64,6 +78,8 @@ func (file *File) AfterFind() (err error) {
// 反序列化文件元数据
if file.Metadata != "" {
err = json.Unmarshal([]byte(file.Metadata), &file.MetadataSerialized)
} else {
file.MetadataSerialized = make(map[string]string)
}
return
@@ -71,9 +87,13 @@ func (file *File) AfterFind() (err error) {
// BeforeSave Save策略前的钩子
func (file *File) BeforeSave() (err error) {
metaValue, err := json.Marshal(&file.MetadataSerialized)
file.Metadata = string(metaValue)
return err
if len(file.MetadataSerialized) > 0 {
metaValue, err := json.Marshal(&file.MetadataSerialized)
file.Metadata = string(metaValue)
return err
}
return nil
}
// GetChildFile 查找目录下名为name的子文件
@@ -279,7 +299,19 @@ func GetFilesByUploadSession(sessionID string, uid uint) (*File, error) {
// Rename 重命名文件
func (file *File) Rename(new string) error {
return DB.Model(&file).UpdateColumn("name", new).Error
if file.MetadataSerialized[ThumbStatusMetadataKey] == ThumbStatusNotAvailable {
if !strings.EqualFold(filepath.Ext(new), filepath.Ext(file.Name)) {
// Reset thumb status for new ext name.
if err := file.resetThumb(); err != nil {
return err
}
}
}
return DB.Model(&file).Set("gorm:association_autoupdate", false).Updates(map[string]interface{}{
"name": new,
"metadata": file.Metadata,
}).Error
}
// UpdatePicInfo 更新文件的图像信息
@@ -287,6 +319,23 @@ func (file *File) UpdatePicInfo(value string) error {
return DB.Model(&file).Set("gorm:association_autoupdate", false).UpdateColumns(File{PicInfo: value}).Error
}
// UpdateMetadata 新增或修改文件的元信息
func (file *File) UpdateMetadata(data map[string]string) error {
if file.MetadataSerialized == nil {
file.MetadataSerialized = make(map[string]string)
}
for k, v := range data {
file.MetadataSerialized[k] = v
}
metaValue, err := json.Marshal(&file.MetadataSerialized)
if err != nil {
return err
}
return DB.Model(&file).Set("gorm:association_autoupdate", false).UpdateColumns(File{Metadata: string(metaValue)}).Error
}
// UpdateSize 更新文件的大小信息
// TODO: 全局锁
func (file *File) UpdateSize(value uint64) error {
@@ -302,10 +351,18 @@ func (file *File) UpdateSize(value uint64) error {
sizeDelta = file.Size - value
}
if err := file.resetThumb(); err != nil {
tx.Rollback()
return err
}
if res := tx.Model(&file).
Where("size = ?", file.Size).
Set("gorm:association_autoupdate", false).
Update("size", value); res.Error != nil {
Updates(map[string]interface{}{
"size": value,
"metadata": file.Metadata,
}); res.Error != nil {
tx.Rollback()
return res.Error
}
@@ -321,7 +378,14 @@ func (file *File) UpdateSize(value uint64) error {
// UpdateSourceName 更新文件的源文件名
func (file *File) UpdateSourceName(value string) error {
return DB.Model(&file).Set("gorm:association_autoupdate", false).Update("source_name", value).Error
if err := file.resetThumb(); err != nil {
return err
}
return DB.Model(&file).Set("gorm:association_autoupdate", false).Updates(map[string]interface{}{
"source_name": value,
"metadata": file.Metadata,
}).Error
}
func (file *File) PopChunkToFile(lastModified *time.Time, picInfo string) error {
@@ -361,6 +425,17 @@ func (file *File) CreateOrGetSourceLink() (*SourceLink, error) {
return res, nil
}
func (file *File) resetThumb() error {
if _, ok := file.MetadataSerialized[ThumbStatusMetadataKey]; !ok {
return nil
}
delete(file.MetadataSerialized, ThumbStatusMetadataKey)
metaValue, err := json.Marshal(&file.MetadataSerialized)
file.Metadata = string(metaValue)
return err
}
/*
实现 webdav.FileInfo 接口
*/
@@ -383,3 +458,15 @@ func (file *File) IsDir() bool {
func (file *File) GetPosition() string {
return file.Position
}
// ShouldLoadThumb returns if file explorer should try to load thumbnail for this file.
// `True` does not guarantee the load request will success in next step, but the client
// should try to load and fallback to default placeholder in case error returned.
func (file *File) ShouldLoadThumb() bool {
return file.MetadataSerialized[ThumbStatusMetadataKey] != ThumbStatusNotAvailable
}
// return sidecar thumb file name
func (file *File) ThumbFile() string {
return file.SourceName + GetSettingByNameWithDefault("thumb_file_suffix", "._thumb")
}

View File

@@ -52,26 +52,54 @@ func TestFile_Create(t *testing.T) {
func TestFile_AfterFind(t *testing.T) {
a := assert.New(t)
file := File{
Name: "123",
Metadata: "{\"name\":\"123\"}",
// metadata not empty
{
file := File{
Name: "123",
Metadata: "{\"name\":\"123\"}",
}
a.NoError(file.AfterFind())
a.Equal("123", file.MetadataSerialized["name"])
}
a.NoError(file.AfterFind())
a.Equal("123", file.MetadataSerialized["name"])
// metadata empty
{
file := File{
Name: "123",
Metadata: "",
}
a.Nil(file.MetadataSerialized)
a.NoError(file.AfterFind())
a.NotNil(file.MetadataSerialized)
}
}
func TestFile_BeforeSave(t *testing.T) {
a := assert.New(t)
file := File{
Name: "123",
MetadataSerialized: map[string]string{
"name": "123",
},
// metadata not empty
{
file := File{
Name: "123",
MetadataSerialized: map[string]string{
"name": "123",
},
}
a.NoError(file.BeforeSave())
a.Equal("{\"name\":\"123\"}", file.Metadata)
}
a.NoError(file.BeforeSave())
a.Equal("{\"name\":\"123\"}", file.Metadata)
// metadata empty
{
file := File{
Name: "123",
}
a.NoError(file.BeforeSave())
a.Equal("", file.Metadata)
}
}
func TestFolder_GetChildFile(t *testing.T) {
@@ -468,12 +496,45 @@ func TestFile_Updates(t *testing.T) {
// rename
{
mock.ExpectBegin()
mock.ExpectExec("UPDATE(.+)files(.+)SET(.+)").WithArgs("newName", 1).WillReturnResult(sqlmock.NewResult(1, 1))
mock.ExpectCommit()
err := file.Rename("newName")
asserts.NoError(mock.ExpectationsWereMet())
asserts.NoError(err)
// not reset thumb
{
file := File{Model: gorm.Model{ID: 1}}
mock.ExpectBegin()
mock.ExpectExec("UPDATE(.+)files(.+)SET(.+)").WithArgs("", "newName", sqlmock.AnyArg(), 1).WillReturnResult(sqlmock.NewResult(1, 1))
mock.ExpectCommit()
err := file.Rename("newName")
asserts.NoError(mock.ExpectationsWereMet())
asserts.NoError(err)
}
// thumb not available, rename base name only
{
file := File{Model: gorm.Model{ID: 1}, Name: "1.txt", MetadataSerialized: map[string]string{
ThumbStatusMetadataKey: ThumbStatusNotAvailable,
},
Metadata: "{}"}
mock.ExpectBegin()
mock.ExpectExec("UPDATE(.+)files(.+)SET(.+)").WithArgs("{}", "newName.txt", sqlmock.AnyArg(), 1).WillReturnResult(sqlmock.NewResult(1, 1))
mock.ExpectCommit()
err := file.Rename("newName.txt")
asserts.NoError(mock.ExpectationsWereMet())
asserts.NoError(err)
asserts.Equal(ThumbStatusNotAvailable, file.MetadataSerialized[ThumbStatusMetadataKey])
}
// thumb not available, rename base name only
{
file := File{Model: gorm.Model{ID: 1}, Name: "1.txt", MetadataSerialized: map[string]string{
ThumbStatusMetadataKey: ThumbStatusNotAvailable,
}}
mock.ExpectBegin()
mock.ExpectExec("UPDATE(.+)files(.+)SET(.+)").WithArgs("{}", "newName.jpg", sqlmock.AnyArg(), 1).WillReturnResult(sqlmock.NewResult(1, 1))
mock.ExpectCommit()
err := file.Rename("newName.jpg")
asserts.NoError(mock.ExpectationsWereMet())
asserts.NoError(err)
asserts.Empty(file.MetadataSerialized[ThumbStatusMetadataKey])
}
}
// UpdatePicInfo
@@ -489,7 +550,7 @@ func TestFile_Updates(t *testing.T) {
// UpdateSourceName
{
mock.ExpectBegin()
mock.ExpectExec("UPDATE(.+)").WithArgs("newName", sqlmock.AnyArg(), 1).WillReturnResult(sqlmock.NewResult(1, 1))
mock.ExpectExec("UPDATE(.+)").WithArgs("", "newName", sqlmock.AnyArg(), 1).WillReturnResult(sqlmock.NewResult(1, 1))
mock.ExpectCommit()
err := file.UpdateSourceName("newName")
asserts.NoError(mock.ExpectationsWereMet())
@@ -504,7 +565,7 @@ func TestFile_UpdateSize(t *testing.T) {
{
file := File{Size: 10}
mock.ExpectBegin()
mock.ExpectExec("UPDATE(.+)files(.+)").WithArgs(11, sqlmock.AnyArg(), 10).WillReturnResult(sqlmock.NewResult(1, 1))
mock.ExpectExec("UPDATE(.+)files(.+)").WithArgs("", 11, sqlmock.AnyArg(), 10).WillReturnResult(sqlmock.NewResult(1, 1))
mock.ExpectExec("UPDATE(.+)storage(.+)+(.+)").WithArgs(uint64(1), sqlmock.AnyArg()).WillReturnResult(sqlmock.NewResult(1, 1))
mock.ExpectCommit()
@@ -516,7 +577,7 @@ func TestFile_UpdateSize(t *testing.T) {
{
file := File{Size: 10}
mock.ExpectBegin()
mock.ExpectExec("UPDATE(.+)files(.+)").WithArgs(8, sqlmock.AnyArg(), 10).WillReturnResult(sqlmock.NewResult(1, 1))
mock.ExpectExec("UPDATE(.+)files(.+)").WithArgs("", 8, sqlmock.AnyArg(), 10).WillReturnResult(sqlmock.NewResult(1, 1))
mock.ExpectExec("UPDATE(.+)storage(.+)-(.+)").WithArgs(uint64(2), sqlmock.AnyArg()).WillReturnResult(sqlmock.NewResult(1, 1))
mock.ExpectCommit()
@@ -528,7 +589,7 @@ func TestFile_UpdateSize(t *testing.T) {
{
file := File{Size: 10}
mock.ExpectBegin()
mock.ExpectExec("UPDATE(.+)files(.+)").WithArgs(8, sqlmock.AnyArg(), 10).WillReturnError(errors.New("error"))
mock.ExpectExec("UPDATE(.+)files(.+)").WithArgs("", 8, sqlmock.AnyArg(), 10).WillReturnError(errors.New("error"))
mock.ExpectRollback()
a.Error(file.UpdateSize(8))
@@ -539,7 +600,7 @@ func TestFile_UpdateSize(t *testing.T) {
{
file := File{Size: 10}
mock.ExpectBegin()
mock.ExpectExec("UPDATE(.+)files(.+)").WithArgs(8, sqlmock.AnyArg(), 10).WillReturnResult(sqlmock.NewResult(1, 1))
mock.ExpectExec("UPDATE(.+)files(.+)").WithArgs("", 8, sqlmock.AnyArg(), 10).WillReturnResult(sqlmock.NewResult(1, 1))
mock.ExpectExec("UPDATE(.+)storage(.+)-(.+)").WithArgs(uint64(2), sqlmock.AnyArg()).WillReturnError(errors.New("error"))
mock.ExpectRollback()
@@ -665,3 +726,60 @@ func TestFile_CreateOrGetSourceLink(t *testing.T) {
a.NoError(mock.ExpectationsWereMet())
}
}
func TestFile_UpdateMetadata(t *testing.T) {
a := assert.New(t)
file := &File{}
file.ID = 1
// 更新失败
{
expectedErr := errors.New("error")
mock.ExpectBegin()
mock.ExpectExec("UPDATE(.+)files(.+)").WithArgs(sqlmock.AnyArg(), 1).WillReturnError(expectedErr)
mock.ExpectRollback()
a.ErrorIs(file.UpdateMetadata(map[string]string{"1": "1"}), expectedErr)
a.NoError(mock.ExpectationsWereMet())
}
// 成功
{
mock.ExpectBegin()
mock.ExpectExec("UPDATE(.+)files(.+)").WithArgs(sqlmock.AnyArg(), 1).WillReturnResult(sqlmock.NewResult(1, 1))
mock.ExpectCommit()
a.NoError(file.UpdateMetadata(map[string]string{"1": "1"}))
a.NoError(mock.ExpectationsWereMet())
a.Equal("1", file.MetadataSerialized["1"])
}
}
func TestFile_ShouldLoadThumb(t *testing.T) {
a := assert.New(t)
file := &File{
MetadataSerialized: map[string]string{},
}
file.ID = 1
// 无缩略图
{
file.MetadataSerialized[ThumbStatusMetadataKey] = ThumbStatusNotAvailable
a.False(file.ShouldLoadThumb())
}
// 有缩略图
{
file.MetadataSerialized[ThumbStatusMetadataKey] = ThumbStatusExist
a.True(file.ShouldLoadThumb())
}
}
func TestFile_ThumbFile(t *testing.T) {
a := assert.New(t)
file := &File{
SourceName: "test",
MetadataSerialized: map[string]string{},
}
file.ID = 1
a.Equal("test._thumb", file.ThumbFile())
}

View File

@@ -18,7 +18,8 @@ type Folder struct {
OwnerID uint `gorm:"index:owner_id"`
// 数据库忽略字段
Position string `gorm:"-"`
Position string `gorm:"-"`
WebdavDstName string `gorm:"-"`
}
// Create 创建目录
@@ -169,6 +170,11 @@ func (folder *Folder) MoveOrCopyFileTo(files []uint, dstFolder *Folder, isCopy b
oldFile.FolderID = dstFolder.ID
oldFile.UserID = dstFolder.OwnerID
// webdav目标名重置
if dstFolder.WebdavDstName != "" {
oldFile.Name = dstFolder.WebdavDstName
}
if err := DB.Create(&oldFile).Error; err != nil {
return copiedSize, err
}
@@ -177,6 +183,14 @@ func (folder *Folder) MoveOrCopyFileTo(files []uint, dstFolder *Folder, isCopy b
}
} else {
var updates = map[string]interface{}{
"folder_id": dstFolder.ID,
}
// webdav目标名重置
if dstFolder.WebdavDstName != "" {
updates["name"] = dstFolder.WebdavDstName
}
// 更改顶级要移动文件的父目录指向
err := DB.Model(File{}).Where(
"id in (?) and user_id = ? and folder_id = ?",
@@ -184,9 +198,7 @@ func (folder *Folder) MoveOrCopyFileTo(files []uint, dstFolder *Folder, isCopy b
folder.OwnerID,
folder.ID,
).
Update(map[string]interface{}{
"folder_id": dstFolder.ID,
}).
Update(updates).
Error
if err != nil {
return 0, err
@@ -221,6 +233,10 @@ func (folder *Folder) CopyFolderTo(folderID uint, dstFolder *Folder) (size uint6
// 顶级目录直接指向新的目的目录
if folder.ID == folderID {
newID = dstFolder.ID
// webdav目标名重置
if dstFolder.WebdavDstName != "" {
folder.Name = dstFolder.WebdavDstName
}
} else if IDCache, ok := newIDCache[*folder.ParentID]; ok {
newID = IDCache
} else {
@@ -282,15 +298,21 @@ func (folder *Folder) MoveFolderTo(dirs []uint, dstFolder *Folder) error {
return errors.New("cannot move a folder into itself")
}
var updates = map[string]interface{}{
"parent_id": dstFolder.ID,
}
// webdav目标名重置
if dstFolder.WebdavDstName != "" {
updates["name"] = dstFolder.WebdavDstName
}
// 更改顶级要移动目录的父目录指向
err := DB.Model(Folder{}).Where(
"id in (?) and owner_id = ? and parent_id = ?",
dirs,
folder.OwnerID,
folder.ID,
).Update(map[string]interface{}{
"parent_id": dstFolder.ID,
}).Error
).Update(updates).Error
return err

View File

@@ -35,6 +35,7 @@ type GroupOption struct {
RedirectedSource bool `json:"redirected_source,omitempty"`
Aria2BatchSize int `json:"aria2_batch,omitempty"`
AdvanceDelete bool `json:"advance_delete,omitempty"`
WebDAVProxy bool `json:"webdav_proxy,omitempty"`
}
// GetGroupByID 用ID获取用户组

View File

@@ -4,6 +4,7 @@ import (
"encoding/gob"
"encoding/json"
"github.com/gofrs/uuid"
"github.com/samber/lo"
"path"
"path/filepath"
"strconv"
@@ -47,8 +48,8 @@ type PolicyOption struct {
FileType []string `json:"file_type"`
// MimeType
MimeType string `json:"mimetype"`
// OdRedirect Onedrive 重定向地址
OdRedirect string `json:"od_redirect,omitempty"`
// OauthRedirect Oauth 重定向地址
OauthRedirect string `json:"od_redirect,omitempty"`
// OdProxy Onedrive 反代地址
OdProxy string `json:"od_proxy,omitempty"`
// OdDriver OneDrive 驱动器定位符
@@ -68,18 +69,8 @@ type PolicyOption struct {
// Set this to `true` to force the request to use path-style addressing,
// i.e., `http://s3.amazonaws.com/BUCKET/KEY `
S3ForcePathStyle bool `json:"s3_path_style"`
}
// thumbSuffix 支持缩略图处理的文件扩展名
var thumbSuffix = map[string][]string{
"local": {},
"qiniu": {".psd", ".jpg", ".jpeg", ".png", ".gif", ".webp", ".tiff", ".bmp"},
"oss": {".jpg", ".jpeg", ".png", ".gif", ".webp", ".tiff", ".bmp"},
"cos": {".jpg", ".jpeg", ".png", ".gif", ".webp", ".tiff", ".bmp"},
"upyun": {".svg", ".jpg", ".jpeg", ".png", ".gif", ".webp", ".tiff", ".bmp"},
"s3": {},
"remote": {},
"onedrive": {"*"},
// File extensions that support thumbnail generation using native policy API.
ThumbExts []string `json:"thumb_exts,omitempty"`
}
func init() {
@@ -165,22 +156,23 @@ func (policy *Policy) GenerateFileName(uid uint, origin string) string {
fileRule := policy.FileNameRule
replaceTable := map[string]string{
"{randomkey16}": util.RandStringRunes(16),
"{randomkey8}": util.RandStringRunes(8),
"{timestamp}": strconv.FormatInt(time.Now().Unix(), 10),
"{timestamp_nano}": strconv.FormatInt(time.Now().UnixNano(), 10),
"{uid}": strconv.Itoa(int(uid)),
"{datetime}": time.Now().Format("20060102150405"),
"{date}": time.Now().Format("20060102"),
"{year}": time.Now().Format("2006"),
"{month}": time.Now().Format("01"),
"{day}": time.Now().Format("02"),
"{hour}": time.Now().Format("15"),
"{minute}": time.Now().Format("04"),
"{second}": time.Now().Format("05"),
"{originname}": origin,
"{ext}": filepath.Ext(origin),
"{uuid}": uuid.Must(uuid.NewV4()).String(),
"{randomkey16}": util.RandStringRunes(16),
"{randomkey8}": util.RandStringRunes(8),
"{timestamp}": strconv.FormatInt(time.Now().Unix(), 10),
"{timestamp_nano}": strconv.FormatInt(time.Now().UnixNano(), 10),
"{uid}": strconv.Itoa(int(uid)),
"{datetime}": time.Now().Format("20060102150405"),
"{date}": time.Now().Format("20060102"),
"{year}": time.Now().Format("2006"),
"{month}": time.Now().Format("01"),
"{day}": time.Now().Format("02"),
"{hour}": time.Now().Format("15"),
"{minute}": time.Now().Format("04"),
"{second}": time.Now().Format("05"),
"{originname}": origin,
"{ext}": filepath.Ext(origin),
"{originname_without_ext}": strings.TrimSuffix(origin, filepath.Ext(origin)),
"{uuid}": uuid.Must(uuid.NewV4()).String(),
}
fileRule = util.Replace(replaceTable, fileRule)
@@ -192,17 +184,6 @@ func (policy *Policy) IsDirectlyPreview() bool {
return policy.Type == "local"
}
// IsThumbExist 给定文件名,返回此存储策略下是否可能存在缩略图
func (policy *Policy) IsThumbExist(name string) bool {
if list, ok := thumbSuffix[policy.Type]; ok {
if len(list) == 1 && list[0] == "*" {
return true
}
return util.ContainsString(list, strings.ToLower(filepath.Ext(name)))
}
return false
}
// IsTransitUpload 返回此策略上传给定size文件时是否需要服务端中转
func (policy *Policy) IsTransitUpload(size uint64) bool {
return policy.Type == "local"
@@ -249,3 +230,14 @@ func (policy *Policy) UpdateAccessKeyAndClearCache(s string) error {
func (policy *Policy) ClearCache() {
cache.Deletes([]string{strconv.FormatUint(uint64(policy.ID), 10)}, "policy_")
}
// CouldProxyThumb return if proxy thumbs is allowed for this policy.
func (policy *Policy) CouldProxyThumb() bool {
if policy.Type == "local" || !IsTrueVal(GetSettingByName("thumb_proxy_enabled")) {
return false
}
allowed := make([]uint, 0)
_ = json.Unmarshal([]byte(GetSettingByName("thumb_proxy_policy")), &allowed)
return lo.Contains[uint](allowed, policy.ID)
}

View File

@@ -25,7 +25,7 @@ func TestGetPolicyByID(t *testing.T) {
asserts.NoError(err)
asserts.NoError(mock.ExpectationsWereMet())
asserts.Equal("默认存储策略", policy.Name)
asserts.Equal("123", policy.OptionsSerialized.OdRedirect)
asserts.Equal("123", policy.OptionsSerialized.OauthRedirect)
rows = sqlmock.NewRows([]string{"name", "type", "options"})
mock.ExpectQuery("^SELECT(.+)").WillReturnRows(rows)
@@ -39,7 +39,7 @@ func TestGetPolicyByID(t *testing.T) {
policy, err := GetPolicyByID(uint(22))
asserts.NoError(err)
asserts.Equal("默认存储策略", policy.Name)
asserts.Equal("123", policy.OptionsSerialized.OdRedirect)
asserts.Equal("123", policy.OptionsSerialized.OauthRedirect)
}
@@ -50,7 +50,7 @@ func TestPolicy_BeforeSave(t *testing.T) {
testPolicy := Policy{
OptionsSerialized: PolicyOption{
OdRedirect: "123",
OauthRedirect: "123",
},
}
expected, _ := json.Marshal(testPolicy.OptionsSerialized)
@@ -134,6 +134,12 @@ func TestPolicy_GenerateFileName(t *testing.T) {
testPolicy.FileNameRule = "123{date}ss{datetime}"
asserts.Len(testPolicy.GenerateFileName(1, "123.txt"), 27)
testPolicy.FileNameRule = "{originname_without_ext}"
asserts.Len(testPolicy.GenerateFileName(1, "123.txt"), 3)
testPolicy.FileNameRule = "{originname_without_ext}_{randomkey8}{ext}"
asserts.Len(testPolicy.GenerateFileName(1, "123.txt"), 16)
// 支持{originname}的策略
testPolicy.Type = "local"
testPolicy.FileNameRule = "123{originname}"
@@ -212,57 +218,6 @@ func TestPolicy_Props(t *testing.T) {
asserts.True(policy.IsUploadPlaceholderWithSize())
}
func TestPolicy_IsThumbExist(t *testing.T) {
asserts := assert.New(t)
testCases := []struct {
name string
expect bool
policy string
}{
{
"1.png",
false,
"unknown",
},
{
"1.png",
false,
"local",
},
{
"1.png",
true,
"cos",
},
{
"1",
false,
"cos",
},
{
"1.txt.png",
true,
"cos",
},
{
"1.png.txt",
false,
"cos",
},
{
"1",
true,
"onedrive",
},
}
for _, testCase := range testCases {
policy := Policy{Type: testCase.policy}
asserts.Equal(testCase.expect, policy.IsThumbExist(testCase.name))
}
}
func TestPolicy_UpdateAccessKeyAndClearCache(t *testing.T) {
a := assert.New(t)
cache.Set("policy_1331", Policy{}, 3600)
@@ -277,3 +232,38 @@ func TestPolicy_UpdateAccessKeyAndClearCache(t *testing.T) {
_, ok := cache.Get("policy_1331")
a.False(ok)
}
func TestPolicy_CouldProxyThumb(t *testing.T) {
a := assert.New(t)
p := &Policy{Type: "local"}
// local policy
{
a.False(p.CouldProxyThumb())
}
// feature not enabled
{
p.Type = "remote"
cache.Set("setting_thumb_proxy_enabled", "0", 0)
a.False(p.CouldProxyThumb())
}
// list not contain current policy
{
p.ID = 2
cache.Set("setting_thumb_proxy_enabled", "1", 0)
cache.Set("setting_thumb_proxy_policy", "[1]", 0)
a.False(p.CouldProxyThumb())
}
// enabled
{
p.ID = 2
cache.Set("setting_thumb_proxy_enabled", "1", 0)
cache.Set("setting_thumb_proxy_policy", "[2]", 0)
a.True(p.CouldProxyThumb())
}
cache.Deletes([]string{"thumb_proxy_enabled", "thumb_proxy_policy"}, "setting_")
}

View File

@@ -12,6 +12,7 @@ type Webdav struct {
UserID uint `gorm:"unique_index:password_only_on"` // 用户ID
Root string `gorm:"type:text"` // 根目录
Readonly bool `gorm:"type:bool"` // 是否只读
UseProxy bool `gorm:"type:bool"` // 是否进行反代
}
// Create 创建账户
@@ -41,7 +42,7 @@ func DeleteWebDAVAccountByID(id, uid uint) {
DB.Where("user_id = ? and id = ?", uid, id).Delete(&Webdav{})
}
// UpdateWebDAVAccountReadonlyByID 根据账户ID和UID更新账户的只读性
func UpdateWebDAVAccountReadonlyByID(id, uid uint, readonly bool) {
DB.Model(&Webdav{Model: gorm.Model{ID: id}, UserID: uid}).UpdateColumn("readonly", readonly)
// UpdateWebDAVAccountByID 根据账户ID和UID更新账户
func UpdateWebDAVAccountByID(id, uid uint, updates map[string]interface{}) {
DB.Model(&Webdav{Model: gorm.Model{ID: id}, UserID: uid}).Updates(updates)
}

31
pkg/cache/driver.go vendored
View File

@@ -1,31 +1,44 @@
package cache
import (
"encoding/gob"
"github.com/cloudreve/Cloudreve/v3/pkg/conf"
"github.com/cloudreve/Cloudreve/v3/pkg/util"
"github.com/gin-gonic/gin"
)
func init() {
gob.Register(map[string]itemWithTTL{})
}
// Store 缓存存储器
var Store Driver = NewMemoStore()
// Init 初始化缓存
func Init(isSlave bool) {
func Init() {
if conf.RedisConfig.Server != "" && gin.Mode() != gin.TestMode {
Store = NewRedisStore(
10,
conf.RedisConfig.Network,
conf.RedisConfig.Server,
conf.RedisConfig.User,
conf.RedisConfig.Password,
conf.RedisConfig.DB,
)
}
}
if isSlave {
err := Store.Sets(conf.OptionOverwrite, "setting_")
if err != nil {
util.Log().Warning("Failed to overwrite database setting: %s", err)
}
// Restore restores cache from given disk file
func Restore(persistFile string) {
if err := Store.Restore(persistFile); err != nil {
util.Log().Warning("Failed to restore cache from disk: %s", err)
}
}
func InitSlaveOverwrites() {
err := Store.Sets(conf.OptionOverwrite, "setting_")
if err != nil {
util.Log().Warning("Failed to overwrite database setting: %s", err)
}
}
@@ -45,6 +58,12 @@ type Driver interface {
// 删除值
Delete(keys []string, prefix string) error
// Save in-memory cache to disk
Persist(path string) error
// Restore cache from disk
Restore(path string) error
}
// Set 设置缓存值

View File

@@ -56,10 +56,14 @@ func TestInit(t *testing.T) {
asserts := assert.New(t)
asserts.NotPanics(func() {
Init(false)
})
asserts.NotPanics(func() {
Init(true)
Init()
})
}
func TestInitSlaveOverwrites(t *testing.T) {
asserts := assert.New(t)
asserts.NotPanics(func() {
InitSlaveOverwrites()
})
}

79
pkg/cache/memo.go vendored
View File

@@ -1,6 +1,9 @@
package cache
import (
"encoding/gob"
"fmt"
"os"
"sync"
"time"
@@ -14,18 +17,20 @@ type MemoStore struct {
// item 存储的对象
type itemWithTTL struct {
expires int64
value interface{}
Expires int64
Value interface{}
}
const DefaultCacheFile = "cache_persist.bin"
func newItem(value interface{}, expires int) itemWithTTL {
expires64 := int64(expires)
if expires > 0 {
expires64 = time.Now().Unix() + expires64
}
return itemWithTTL{
value: value,
expires: expires64,
Value: value,
Expires: expires64,
}
}
@@ -40,11 +45,11 @@ func getValue(item interface{}, ok bool) (interface{}, bool) {
return item, true
}
if itemObj.expires > 0 && itemObj.expires < time.Now().Unix() {
if itemObj.Expires > 0 && itemObj.Expires < time.Now().Unix() {
return nil, false
}
return itemObj.value, ok
return itemObj.Value, ok
}
@@ -52,7 +57,7 @@ func getValue(item interface{}, ok bool) (interface{}, bool) {
func (store *MemoStore) GarbageCollect() {
store.Store.Range(func(key, value interface{}) bool {
if item, ok := value.(itemWithTTL); ok {
if item.expires > 0 && item.expires < time.Now().Unix() {
if item.Expires > 0 && item.Expires < time.Now().Unix() {
util.Log().Debug("Cache %q is garbage collected.", key.(string))
store.Store.Delete(key)
}
@@ -98,7 +103,7 @@ func (store *MemoStore) Gets(keys []string, prefix string) (map[string]interface
// Sets 批量设置值
func (store *MemoStore) Sets(values map[string]interface{}, prefix string) error {
for key, value := range values {
store.Store.Store(prefix+key, value)
store.Store.Store(prefix+key, newItem(value, 0))
}
return nil
}
@@ -110,3 +115,61 @@ func (store *MemoStore) Delete(keys []string, prefix string) error {
}
return nil
}
// Persist write memory store into cache
func (store *MemoStore) Persist(path string) error {
persisted := make(map[string]itemWithTTL)
store.Store.Range(func(key, value interface{}) bool {
v, ok := store.Store.Load(key)
if _, ok := getValue(v, ok); ok {
persisted[key.(string)] = v.(itemWithTTL)
}
return true
})
res, err := serializer(persisted)
if err != nil {
return fmt.Errorf("failed to serialize cache: %s", err)
}
err = os.WriteFile(path, res, 0644)
return err
}
// Restore memory cache from disk file
func (store *MemoStore) Restore(path string) error {
if !util.Exists(path) {
return nil
}
f, err := os.Open(path)
if err != nil {
return fmt.Errorf("failed to read cache file: %s", err)
}
defer func() {
f.Close()
os.Remove(path)
}()
persisted := &item{}
dec := gob.NewDecoder(f)
if err := dec.Decode(&persisted); err != nil {
return fmt.Errorf("unknown cache file format: %s", err)
}
items := persisted.Value.(map[string]itemWithTTL)
loaded := 0
for k, v := range items {
if _, ok := getValue(v, true); ok {
loaded++
store.Store.Store(k, v)
} else {
util.Log().Debug("Persisted cache %q is expired.", k)
}
}
util.Log().Info("Restored %d items from %q into memory cache.", loaded, path)
return nil
}

View File

@@ -2,6 +2,7 @@ package cache
import (
"github.com/stretchr/testify/assert"
"path/filepath"
"testing"
"time"
)
@@ -23,7 +24,7 @@ func TestMemoStore_Set(t *testing.T) {
val, ok := store.Store.Load("KEY")
asserts.True(ok)
asserts.Equal("vAL", val.(itemWithTTL).value)
asserts.Equal("vAL", val.(itemWithTTL).Value)
}
func TestMemoStore_Get(t *testing.T) {
@@ -145,3 +146,46 @@ func TestMemoStore_GarbageCollect(t *testing.T) {
_, ok := store.Get("test")
asserts.False(ok)
}
func TestMemoStore_PersistFailed(t *testing.T) {
a := assert.New(t)
store := NewMemoStore()
type testStruct struct{ v string }
store.Set("test", 1, 0)
store.Set("test2", testStruct{v: "test"}, 0)
err := store.Persist(filepath.Join(t.TempDir(), "TestMemoStore_PersistFailed"))
a.Error(err)
}
func TestMemoStore_PersistAndRestore(t *testing.T) {
a := assert.New(t)
store := NewMemoStore()
store.Set("test", 1, 0)
// already expired
store.Store.Store("test2", itemWithTTL{Value: "test", Expires: 1})
// expired after persist
store.Set("test3", 1, 1)
temp := filepath.Join(t.TempDir(), "TestMemoStore_PersistFailed")
// Persist
err := store.Persist(temp)
a.NoError(err)
a.FileExists(temp)
time.Sleep(2 * time.Second)
// Restore
store2 := NewMemoStore()
err = store2.Restore(temp)
a.NoError(err)
test, testOk := store2.Get("test")
a.EqualValues(1, test)
a.True(testOk)
test2, test2Ok := store2.Get("test2")
a.Nil(test2)
a.False(test2Ok)
test3, test3Ok := store2.Get("test3")
a.Nil(test3)
a.False(test3Ok)
a.NoFileExists(temp)
}

16
pkg/cache/redis.go vendored
View File

@@ -44,7 +44,7 @@ func deserializer(value []byte) (interface{}, error) {
}
// NewRedisStore 创建新的redis存储
func NewRedisStore(size int, network, address, password, database string) *RedisStore {
func NewRedisStore(size int, network, address, user, password, database string) *RedisStore {
return &RedisStore{
pool: &redis.Pool{
MaxIdle: size,
@@ -63,11 +63,11 @@ func NewRedisStore(size int, network, address, password, database string) *Redis
network,
address,
redis.DialDatabase(db),
redis.DialUsername(user),
redis.DialPassword(password),
)
if err != nil {
util.Log().Warning("Failed to create Redis connection: %s", err)
return nil, err
util.Log().Panic("Failed to create Redis connection: %s", err)
}
return c, nil
},
@@ -215,3 +215,13 @@ func (store *RedisStore) DeleteAll() error {
return err
}
// Persist Dummy implementation
func (store *RedisStore) Persist(path string) error {
return nil
}
// Restore dummy implementation
func (store *RedisStore) Restore(path string) error {
return nil
}

View File

@@ -13,16 +13,16 @@ import (
func TestNewRedisStore(t *testing.T) {
asserts := assert.New(t)
store := NewRedisStore(10, "tcp", "", "", "0")
store := NewRedisStore(10, "tcp", "", "", "", "0")
asserts.NotNil(store)
conn, err := store.pool.Dial()
asserts.Nil(conn)
asserts.Error(err)
asserts.Panics(func() {
store.pool.Dial()
})
testConn := redigomock.NewConn()
cmd := testConn.Command("PING").Expect("PONG")
err = store.pool.TestOnBorrow(testConn, time.Now())
err := store.pool.TestOnBorrow(testConn, time.Now())
if testConn.Stats(cmd) != 1 {
fmt.Println("Command was not used")
return

View File

@@ -35,8 +35,8 @@ type Controller interface {
// Get master node info
GetMasterInfo(string) (*MasterInfo, error)
// Get master OneDrive policy credential
GetOneDriveToken(string, uint) (string, error)
// Get master Oauth based policy credential
GetPolicyOauthToken(string, uint) (string, error)
}
type slaveController struct {
@@ -181,8 +181,8 @@ func (c *slaveController) GetMasterInfo(id string) (*MasterInfo, error) {
return nil, ErrMasterNotFound
}
// GetOneDriveToken 获取主机OneDrive凭证
func (c *slaveController) GetOneDriveToken(id string, policyID uint) (string, error) {
// GetPolicyOauthToken 获取主机存储策略 Oauth 凭证
func (c *slaveController) GetPolicyOauthToken(id string, policyID uint) (string, error) {
c.lock.RLock()
if node, ok := c.masters[id]; ok {
@@ -190,7 +190,7 @@ func (c *slaveController) GetOneDriveToken(id string, policyID uint) (string, er
res, err := node.Client.Request(
"GET",
fmt.Sprintf("/api/v3/slave/credential/onedrive/%d", policyID),
fmt.Sprintf("/api/v3/slave/credential/%d", policyID),
nil,
).CheckHTTPResponse(200).DecodeResponse()
if err != nil {

View File

@@ -320,7 +320,7 @@ func TestSlaveController_GetOneDriveToken(t *testing.T) {
// node not exit
{
res, err := c.GetOneDriveToken("2", 1)
res, err := c.GetPolicyOauthToken("2", 1)
a.Equal(ErrMasterNotFound, err)
a.Empty(res)
}
@@ -328,7 +328,7 @@ func TestSlaveController_GetOneDriveToken(t *testing.T) {
// return none 200
{
mockRequest := &requestMock{}
mockRequest.On("Request", "GET", "/api/v3/slave/credential/onedrive/1", testMock.Anything, testMock.Anything).Return(&request.Response{
mockRequest.On("Request", "GET", "/api/v3/slave/credential/1", testMock.Anything, testMock.Anything).Return(&request.Response{
Response: &http.Response{StatusCode: http.StatusConflict},
})
c := &slaveController{
@@ -336,7 +336,7 @@ func TestSlaveController_GetOneDriveToken(t *testing.T) {
"1": {Client: mockRequest},
},
}
res, err := c.GetOneDriveToken("1", 1)
res, err := c.GetPolicyOauthToken("1", 1)
a.Error(err)
a.Empty(res)
mockRequest.AssertExpectations(t)
@@ -345,7 +345,7 @@ func TestSlaveController_GetOneDriveToken(t *testing.T) {
// master return error
{
mockRequest := &requestMock{}
mockRequest.On("Request", "GET", "/api/v3/slave/credential/onedrive/1", testMock.Anything, testMock.Anything).Return(&request.Response{
mockRequest.On("Request", "GET", "/api/v3/slave/credential/1", testMock.Anything, testMock.Anything).Return(&request.Response{
Response: &http.Response{
StatusCode: 200,
Body: ioutil.NopCloser(strings.NewReader("{\"code\":1}")),
@@ -356,7 +356,7 @@ func TestSlaveController_GetOneDriveToken(t *testing.T) {
"1": {Client: mockRequest},
},
}
res, err := c.GetOneDriveToken("1", 1)
res, err := c.GetPolicyOauthToken("1", 1)
a.Equal(1, err.(serializer.AppError).Code)
a.Empty(res)
mockRequest.AssertExpectations(t)
@@ -365,7 +365,7 @@ func TestSlaveController_GetOneDriveToken(t *testing.T) {
// success
{
mockRequest := &requestMock{}
mockRequest.On("Request", "GET", "/api/v3/slave/credential/onedrive/1", testMock.Anything, testMock.Anything).Return(&request.Response{
mockRequest.On("Request", "GET", "/api/v3/slave/credential/1", testMock.Anything, testMock.Anything).Return(&request.Response{
Response: &http.Response{
StatusCode: 200,
Body: ioutil.NopCloser(strings.NewReader("{\"data\":\"expected\"}")),
@@ -376,7 +376,7 @@ func TestSlaveController_GetOneDriveToken(t *testing.T) {
"1": {Client: mockRequest},
},
}
res, err := c.GetOneDriveToken("1", 1)
res, err := c.GetPolicyOauthToken("1", 1)
a.NoError(err)
a.Equal("expected", res)
mockRequest.AssertExpectations(t)

View File

@@ -53,6 +53,7 @@ type slave struct {
type redis struct {
Network string
Server string
User string
Password string
DB string
}

View File

@@ -1,13 +1,13 @@
package conf
// BackendVersion 当前后端版本号
var BackendVersion = "3.7.1"
var BackendVersion = "3.8.2"
// RequiredDBVersion 与当前版本匹配的数据库版本
var RequiredDBVersion = "3.7.1"
var RequiredDBVersion = "3.8.1"
// RequiredStaticVersion 与当前版本匹配的静态资源版本
var RequiredStaticVersion = "3.7.1"
var RequiredStaticVersion = "3.8.1"
// IsPro 是否为Pro版本
var IsPro = "false"

View File

@@ -17,10 +17,12 @@ import (
"time"
model "github.com/cloudreve/Cloudreve/v3/models"
"github.com/cloudreve/Cloudreve/v3/pkg/filesystem/driver"
"github.com/cloudreve/Cloudreve/v3/pkg/filesystem/fsctx"
"github.com/cloudreve/Cloudreve/v3/pkg/filesystem/response"
"github.com/cloudreve/Cloudreve/v3/pkg/request"
"github.com/cloudreve/Cloudreve/v3/pkg/serializer"
"github.com/cloudreve/Cloudreve/v3/pkg/util"
"github.com/google/go-querystring/query"
cossdk "github.com/tencentyun/cos-go-sdk-v5"
)
@@ -148,14 +150,7 @@ func (handler Driver) CORS() error {
// Get 获取文件
func (handler Driver) Get(ctx context.Context, path string) (response.RSCloser, error) {
// 获取文件源地址
downloadURL, err := handler.Source(
ctx,
path,
url.URL{},
int64(model.GetIntSetting("preview_timeout", 60)),
false,
0,
)
downloadURL, err := handler.Source(ctx, path, int64(model.GetIntSetting("preview_timeout", 60)), false, 0)
if err != nil {
return nil, err
}
@@ -222,19 +217,33 @@ func (handler Driver) Delete(ctx context.Context, files []string) ([]string, err
}
// Thumb 获取文件缩略图
func (handler Driver) Thumb(ctx context.Context, path string) (*response.ContentResponse, error) {
func (handler Driver) Thumb(ctx context.Context, file *model.File) (*response.ContentResponse, error) {
// quick check by extension name
// https://cloud.tencent.com/document/product/436/44893
supported := []string{"png", "jpg", "jpeg", "gif", "bmp", "webp", "heif", "heic"}
if len(handler.Policy.OptionsSerialized.ThumbExts) > 0 {
supported = handler.Policy.OptionsSerialized.ThumbExts
}
if !util.IsInExtensionList(supported, file.Name) || file.Size > (32<<(10*2)) {
return nil, driver.ErrorThumbNotSupported
}
var (
thumbSize = [2]uint{400, 300}
ok = false
)
if thumbSize, ok = ctx.Value(fsctx.ThumbSizeCtx).([2]uint); !ok {
return nil, errors.New("无法获取缩略图尺寸设置")
return nil, errors.New("failed to get thumbnail size")
}
thumbParam := fmt.Sprintf("imageMogr2/thumbnail/%dx%d", thumbSize[0], thumbSize[1])
thumbEncodeQuality := model.GetIntSetting("thumb_encode_quality", 85)
thumbParam := fmt.Sprintf("imageMogr2/thumbnail/%dx%d/quality/%d", thumbSize[0], thumbSize[1], thumbEncodeQuality)
source, err := handler.signSourceURL(
ctx,
path,
file.SourceName,
int64(model.GetIntSetting("preview_timeout", 60)),
&urlOption{},
)
@@ -254,14 +263,7 @@ func (handler Driver) Thumb(ctx context.Context, path string) (*response.Content
}
// Source 获取外链URL
func (handler Driver) Source(
ctx context.Context,
path string,
baseURL url.URL,
ttl int64,
isDownload bool,
speed int,
) (string, error) {
func (handler Driver) Source(ctx context.Context, path string, ttl int64, isDownload bool, speed int) (string, error) {
// 尝试从上下文获取文件名
fileName := ""
if file, ok := ctx.Value(fsctx.FileModelCtx).(model.File); ok {

View File

@@ -0,0 +1,73 @@
package googledrive
import (
"errors"
model "github.com/cloudreve/Cloudreve/v3/models"
"github.com/cloudreve/Cloudreve/v3/pkg/cluster"
"github.com/cloudreve/Cloudreve/v3/pkg/request"
"google.golang.org/api/drive/v3"
)
// Client Google Drive client
type Client struct {
Endpoints *Endpoints
Policy *model.Policy
Credential *Credential
ClientID string
ClientSecret string
Redirect string
Request request.Client
ClusterController cluster.Controller
}
// Endpoints OneDrive客户端相关设置
type Endpoints struct {
UserConsentEndpoint string // OAuth认证的基URL
TokenEndpoint string // OAuth token 基URL
EndpointURL string // 接口请求的基URL
}
const (
TokenCachePrefix = "googledrive_"
oauthEndpoint = "https://oauth2.googleapis.com/token"
userConsentBase = "https://accounts.google.com/o/oauth2/auth"
v3DriveEndpoint = "https://www.googleapis.com/drive/v3"
)
var (
// Defualt required scopes
RequiredScope = []string{
drive.DriveScope,
"openid",
"profile",
"https://www.googleapis.com/auth/userinfo.profile",
}
// ErrInvalidRefreshToken 上传策略无有效的RefreshToken
ErrInvalidRefreshToken = errors.New("no valid refresh token in this policy")
)
// NewClient 根据存储策略获取新的client
func NewClient(policy *model.Policy) (*Client, error) {
client := &Client{
Endpoints: &Endpoints{
TokenEndpoint: oauthEndpoint,
UserConsentEndpoint: userConsentBase,
EndpointURL: v3DriveEndpoint,
},
Credential: &Credential{
RefreshToken: policy.AccessKey,
},
Policy: policy,
ClientID: policy.BucketName,
ClientSecret: policy.SecretKey,
Redirect: policy.OptionsSerialized.OauthRedirect,
Request: request.NewClient(),
ClusterController: cluster.DefaultController,
}
return client, nil
}

View File

@@ -0,0 +1,65 @@
package googledrive
import (
"context"
model "github.com/cloudreve/Cloudreve/v3/models"
"github.com/cloudreve/Cloudreve/v3/pkg/filesystem/driver"
"github.com/cloudreve/Cloudreve/v3/pkg/filesystem/fsctx"
"github.com/cloudreve/Cloudreve/v3/pkg/filesystem/response"
"github.com/cloudreve/Cloudreve/v3/pkg/request"
"github.com/cloudreve/Cloudreve/v3/pkg/serializer"
)
// Driver Google Drive 适配器
type Driver struct {
Policy *model.Policy
HTTPClient request.Client
}
// NewDriver 从存储策略初始化新的Driver实例
func NewDriver(policy *model.Policy) (driver.Handler, error) {
return &Driver{
Policy: policy,
HTTPClient: request.NewClient(),
}, nil
}
func (d *Driver) Put(ctx context.Context, file fsctx.FileHeader) error {
//TODO implement me
panic("implement me")
}
func (d *Driver) Delete(ctx context.Context, files []string) ([]string, error) {
//TODO implement me
panic("implement me")
}
func (d *Driver) Get(ctx context.Context, path string) (response.RSCloser, error) {
//TODO implement me
panic("implement me")
}
func (d *Driver) Thumb(ctx context.Context, file *model.File) (*response.ContentResponse, error) {
//TODO implement me
panic("implement me")
}
func (d *Driver) Source(ctx context.Context, path string, ttl int64, isDownload bool, speed int) (string, error) {
//TODO implement me
panic("implement me")
}
func (d *Driver) Token(ctx context.Context, ttl int64, uploadSession *serializer.UploadSession, file fsctx.FileHeader) (*serializer.UploadCredential, error) {
//TODO implement me
panic("implement me")
}
func (d *Driver) CancelToken(ctx context.Context, uploadSession *serializer.UploadSession) error {
//TODO implement me
panic("implement me")
}
func (d *Driver) List(ctx context.Context, path string, recursive bool) ([]response.Object, error) {
//TODO implement me
panic("implement me")
}

View File

@@ -0,0 +1,154 @@
package googledrive
import (
"context"
"encoding/json"
"github.com/cloudreve/Cloudreve/v3/pkg/cache"
"github.com/cloudreve/Cloudreve/v3/pkg/filesystem/oauth"
"github.com/cloudreve/Cloudreve/v3/pkg/request"
"github.com/cloudreve/Cloudreve/v3/pkg/util"
"io"
"net/http"
"net/url"
"strings"
"time"
)
// OAuthURL 获取OAuth认证页面URL
func (client *Client) OAuthURL(ctx context.Context, scope []string) string {
query := url.Values{
"client_id": {client.ClientID},
"scope": {strings.Join(scope, " ")},
"response_type": {"code"},
"redirect_uri": {client.Redirect},
"access_type": {"offline"},
"prompt": {"consent"},
}
u, _ := url.Parse(client.Endpoints.UserConsentEndpoint)
u.RawQuery = query.Encode()
return u.String()
}
// ObtainToken 通过code或refresh_token兑换token
func (client *Client) ObtainToken(ctx context.Context, code, refreshToken string) (*Credential, error) {
body := url.Values{
"client_id": {client.ClientID},
"redirect_uri": {client.Redirect},
"client_secret": {client.ClientSecret},
}
if code != "" {
body.Add("grant_type", "authorization_code")
body.Add("code", code)
} else {
body.Add("grant_type", "refresh_token")
body.Add("refresh_token", refreshToken)
}
strBody := body.Encode()
res := client.Request.Request(
"POST",
client.Endpoints.TokenEndpoint,
io.NopCloser(strings.NewReader(strBody)),
request.WithHeader(http.Header{
"Content-Type": {"application/x-www-form-urlencoded"}},
),
request.WithContentLength(int64(len(strBody))),
)
if res.Err != nil {
return nil, res.Err
}
respBody, err := res.GetResponse()
if err != nil {
return nil, err
}
var (
errResp OAuthError
credential Credential
decodeErr error
)
if res.Response.StatusCode != 200 {
decodeErr = json.Unmarshal([]byte(respBody), &errResp)
} else {
decodeErr = json.Unmarshal([]byte(respBody), &credential)
}
if decodeErr != nil {
return nil, decodeErr
}
if errResp.ErrorType != "" {
return nil, errResp
}
return &credential, nil
}
// UpdateCredential 更新凭证,并检查有效期
func (client *Client) UpdateCredential(ctx context.Context, isSlave bool) error {
if isSlave {
return client.fetchCredentialFromMaster(ctx)
}
oauth.GlobalMutex.Lock(client.Policy.ID)
defer oauth.GlobalMutex.Unlock(client.Policy.ID)
// 如果已存在凭证
if client.Credential != nil && client.Credential.AccessToken != "" {
// 检查已有凭证是否过期
if client.Credential.ExpiresIn > time.Now().Unix() {
// 未过期,不要更新
return nil
}
}
// 尝试从缓存中获取凭证
if cacheCredential, ok := cache.Get(TokenCachePrefix + client.ClientID); ok {
credential := cacheCredential.(Credential)
if credential.ExpiresIn > time.Now().Unix() {
client.Credential = &credential
return nil
}
}
// 获取新的凭证
if client.Credential == nil || client.Credential.RefreshToken == "" {
// 无有效的RefreshToken
util.Log().Error("Failed to refresh credential for policy %q, please login your Google account again.", client.Policy.Name)
return ErrInvalidRefreshToken
}
credential, err := client.ObtainToken(ctx, "", client.Credential.RefreshToken)
if err != nil {
return err
}
// 更新有效期为绝对时间戳
expires := credential.ExpiresIn - 60
credential.ExpiresIn = time.Now().Add(time.Duration(expires) * time.Second).Unix()
// refresh token for Google Drive does not expire in production
credential.RefreshToken = client.Credential.RefreshToken
client.Credential = credential
// 更新缓存
cache.Set(TokenCachePrefix+client.ClientID, *credential, int(expires))
return nil
}
func (client *Client) AccessToken() string {
return client.Credential.AccessToken
}
// UpdateCredential 更新凭证,并检查有效期
func (client *Client) fetchCredentialFromMaster(ctx context.Context) error {
res, err := client.ClusterController.GetPolicyOauthToken(client.Policy.MasterID, client.Policy.ID)
if err != nil {
return err
}
client.Credential = &Credential{AccessToken: res}
return nil
}

View File

@@ -0,0 +1,43 @@
package googledrive
import "encoding/gob"
// RespError 接口返回错误
type RespError struct {
APIError APIError `json:"error"`
}
// APIError 接口返回的错误内容
type APIError struct {
Code string `json:"code"`
Message string `json:"message"`
}
// Error 实现error接口
func (err RespError) Error() string {
return err.APIError.Message
}
// Credential 获取token时返回的凭证
type Credential struct {
ExpiresIn int64 `json:"expires_in"`
Scope string `json:"scope"`
AccessToken string `json:"access_token"`
RefreshToken string `json:"refresh_token"`
UserID string `json:"user_id"`
}
// OAuthError OAuth相关接口的错误响应
type OAuthError struct {
ErrorType string `json:"error"`
ErrorDescription string `json:"error_description"`
}
// Error 实现error接口
func (err OAuthError) Error() string {
return err.ErrorDescription
}
func init() {
gob.Register(Credential{})
}

View File

@@ -2,10 +2,16 @@ package driver
import (
"context"
"fmt"
model "github.com/cloudreve/Cloudreve/v3/models"
"github.com/cloudreve/Cloudreve/v3/pkg/filesystem/fsctx"
"github.com/cloudreve/Cloudreve/v3/pkg/filesystem/response"
"github.com/cloudreve/Cloudreve/v3/pkg/serializer"
"net/url"
)
var (
ErrorThumbNotExist = fmt.Errorf("thumb not exist")
ErrorThumbNotSupported = fmt.Errorf("thumb not supported")
)
// Handler 存储策略适配器
@@ -22,12 +28,15 @@ type Handler interface {
// 获取缩略图可直接在ContentResponse中返回文件数据流也可指
// 定为重定向
Thumb(ctx context.Context, path string) (*response.ContentResponse, error)
// 如果缩略图不存在, 且需要 Cloudreve 代理生成并上传,应返回 ErrorThumbNotExist
// 成的缩略图文件存储规则与本机策略一致。
// 如果不支持此文件的缩略图,并且不希望后续继续请求此缩略图,应返回 ErrorThumbNotSupported
Thumb(ctx context.Context, file *model.File) (*response.ContentResponse, error)
// 获取外链/下载地址,
// url - 站点本身地址,
// isDownload - 是否直接下载
Source(ctx context.Context, path string, url url.URL, ttl int64, isDownload bool, speed int) (string, error)
Source(ctx context.Context, path string, ttl int64, isDownload bool, speed int) (string, error)
// Token 获取有效期为ttl的上传凭证和签名
Token(ctx context.Context, ttl int64, uploadSession *serializer.UploadSession, file fsctx.FileHeader) (*serializer.UploadCredential, error)

View File

@@ -12,6 +12,8 @@ import (
model "github.com/cloudreve/Cloudreve/v3/models"
"github.com/cloudreve/Cloudreve/v3/pkg/auth"
"github.com/cloudreve/Cloudreve/v3/pkg/cache"
"github.com/cloudreve/Cloudreve/v3/pkg/conf"
"github.com/cloudreve/Cloudreve/v3/pkg/filesystem/driver"
"github.com/cloudreve/Cloudreve/v3/pkg/filesystem/fsctx"
"github.com/cloudreve/Cloudreve/v3/pkg/filesystem/response"
"github.com/cloudreve/Cloudreve/v3/pkg/serializer"
@@ -194,39 +196,43 @@ func (handler Driver) Delete(ctx context.Context, files []string) ([]string, err
}
// Thumb 获取文件缩略图
func (handler Driver) Thumb(ctx context.Context, path string) (*response.ContentResponse, error) {
file, err := handler.Get(ctx, path+model.GetSettingByNameWithDefault("thumb_file_suffix", "._thumb"))
func (handler Driver) Thumb(ctx context.Context, file *model.File) (*response.ContentResponse, error) {
// Quick check thumb existence on master.
if conf.SystemConfig.Mode == "master" && file.MetadataSerialized[model.ThumbStatusMetadataKey] == model.ThumbStatusNotExist {
// Tell invoker to generate a thumb
return nil, driver.ErrorThumbNotExist
}
thumbFile, err := handler.Get(ctx, file.ThumbFile())
if err != nil {
if errors.Is(err, os.ErrNotExist) {
err = fmt.Errorf("thumb not exist: %w (%w)", err, driver.ErrorThumbNotExist)
}
return nil, err
}
return &response.ContentResponse{
Redirect: false,
Content: file,
Content: thumbFile,
}, nil
}
// Source 获取外链URL
func (handler Driver) Source(
ctx context.Context,
path string,
baseURL url.URL,
ttl int64,
isDownload bool,
speed int,
) (string, error) {
func (handler Driver) Source(ctx context.Context, path string, ttl int64, isDownload bool, speed int) (string, error) {
file, ok := ctx.Value(fsctx.FileModelCtx).(model.File)
if !ok {
return "", errors.New("failed to read file model context")
}
var baseURL *url.URL
// 是否启用了CDN
if handler.Policy.BaseURL != "" {
cdnURL, err := url.Parse(handler.Policy.BaseURL)
if err != nil {
return "", err
}
baseURL = *cdnURL
baseURL = cdnURL
}
var (
@@ -260,7 +266,11 @@ func (handler Driver) Source(
return "", serializer.NewError(serializer.CodeEncryptError, "Failed to sign url", err)
}
finalURL := baseURL.ResolveReference(signedURI).String()
finalURL := signedURI.String()
if baseURL != nil {
finalURL = baseURL.ResolveReference(signedURI).String()
}
return finalURL, nil
}

View File

@@ -4,13 +4,13 @@ import (
"context"
model "github.com/cloudreve/Cloudreve/v3/models"
"github.com/cloudreve/Cloudreve/v3/pkg/auth"
"github.com/cloudreve/Cloudreve/v3/pkg/filesystem/driver"
"github.com/cloudreve/Cloudreve/v3/pkg/filesystem/fsctx"
"github.com/cloudreve/Cloudreve/v3/pkg/serializer"
"github.com/cloudreve/Cloudreve/v3/pkg/util"
"github.com/jinzhu/gorm"
"github.com/stretchr/testify/assert"
"io"
"net/url"
"os"
"strings"
"testing"
@@ -141,17 +141,34 @@ func TestHandler_Thumb(t *testing.T) {
asserts.NoError(err)
file.Close()
f := &model.File{
SourceName: "TestHandler_Thumb",
MetadataSerialized: map[string]string{
model.ThumbStatusMetadataKey: model.ThumbStatusExist,
},
}
// 正常
{
thumb, err := handler.Thumb(ctx, "TestHandler_Thumb")
thumb, err := handler.Thumb(ctx, f)
asserts.NoError(err)
asserts.NotNil(thumb.Content)
}
// 不存在
// file 不存在
{
_, err := handler.Thumb(ctx, "not_exist")
f.SourceName = "not_exist"
_, err := handler.Thumb(ctx, f)
asserts.Error(err)
asserts.ErrorIs(err, driver.ErrorThumbNotExist)
}
// thumb not exist
{
f.MetadataSerialized[model.ThumbStatusMetadataKey] = model.ThumbStatusNotExist
_, err := handler.Thumb(ctx, f)
asserts.Error(err)
asserts.ErrorIs(err, driver.ErrorThumbNotExist)
}
}
@@ -172,13 +189,10 @@ func TestHandler_Source(t *testing.T) {
Name: "test.jpg",
}
ctx := context.WithValue(ctx, fsctx.FileModelCtx, file)
baseURL, err := url.Parse("https://cloudreve.org")
asserts.NoError(err)
sourceURL, err := handler.Source(ctx, "", *baseURL, 0, false, 0)
sourceURL, err := handler.Source(ctx, "", 0, false, 0)
asserts.NoError(err)
asserts.NotEmpty(sourceURL)
asserts.Contains(sourceURL, "sign=")
asserts.Contains(sourceURL, "https://cloudreve.org")
}
// 下载
@@ -190,21 +204,16 @@ func TestHandler_Source(t *testing.T) {
Name: "test.jpg",
}
ctx := context.WithValue(ctx, fsctx.FileModelCtx, file)
baseURL, err := url.Parse("https://cloudreve.org")
asserts.NoError(err)
sourceURL, err := handler.Source(ctx, "", *baseURL, 0, true, 0)
sourceURL, err := handler.Source(ctx, "", 0, true, 0)
asserts.NoError(err)
asserts.NotEmpty(sourceURL)
asserts.Contains(sourceURL, "sign=")
asserts.Contains(sourceURL, "download")
asserts.Contains(sourceURL, "https://cloudreve.org")
}
// 无法获取上下文
{
baseURL, err := url.Parse("https://cloudreve.org")
asserts.NoError(err)
sourceURL, err := handler.Source(ctx, "", *baseURL, 0, false, 0)
sourceURL, err := handler.Source(ctx, "", 0, false, 0)
asserts.Error(err)
asserts.Empty(sourceURL)
}
@@ -219,9 +228,7 @@ func TestHandler_Source(t *testing.T) {
Name: "test.jpg",
}
ctx := context.WithValue(ctx, fsctx.FileModelCtx, file)
baseURL, err := url.Parse("https://cloudreve.org")
asserts.NoError(err)
sourceURL, err := handler.Source(ctx, "", *baseURL, 0, false, 0)
sourceURL, err := handler.Source(ctx, "", 0, false, 0)
asserts.NoError(err)
asserts.NotEmpty(sourceURL)
asserts.Contains(sourceURL, "sign=")
@@ -238,9 +245,7 @@ func TestHandler_Source(t *testing.T) {
Name: "test.jpg",
}
ctx := context.WithValue(ctx, fsctx.FileModelCtx, file)
baseURL, err := url.Parse("https://cloudreve.org")
asserts.NoError(err)
sourceURL, err := handler.Source(ctx, "", *baseURL, 0, false, 0)
sourceURL, err := handler.Source(ctx, "", 0, false, 0)
asserts.Error(err)
asserts.Empty(sourceURL)
}
@@ -261,19 +266,14 @@ func TestHandler_GetDownloadURL(t *testing.T) {
Name: "test.jpg",
}
ctx := context.WithValue(ctx, fsctx.FileModelCtx, file)
baseURL, err := url.Parse("https://cloudreve.org")
asserts.NoError(err)
downloadURL, err := handler.Source(ctx, "", *baseURL, 10, true, 0)
downloadURL, err := handler.Source(ctx, "", 10, true, 0)
asserts.NoError(err)
asserts.Contains(downloadURL, "sign=")
asserts.Contains(downloadURL, "https://cloudreve.org")
}
// 无法获取上下文
{
baseURL, err := url.Parse("https://cloudreve.org")
asserts.NoError(err)
downloadURL, err := handler.Source(ctx, "", *baseURL, 10, true, 0)
downloadURL, err := handler.Source(ctx, "", 10, true, 0)
asserts.Error(err)
asserts.Empty(downloadURL)
}

View File

@@ -3,7 +3,6 @@ package onedrive
import (
"context"
"encoding/json"
"errors"
"fmt"
"github.com/cloudreve/Cloudreve/v3/pkg/conf"
"io"
@@ -32,6 +31,8 @@ const (
// ListRetry 列取请求重试次数
ListRetry = 1
chunkRetrySleep = time.Second * 5
notFoundError = "itemNotFound"
)
// GetSourcePath 获取文件的绝对路径
@@ -438,7 +439,7 @@ func (client *Client) GetThumbURL(ctx context.Context, dst string, w, h uint) (s
}
}
return "", errors.New("failed to generate thumb")
return "", ErrThumbSizeNotFound
}
// MonitorUpload 监控客户端分片上传进度
@@ -469,7 +470,7 @@ func (client *Client) MonitorUpload(uploadURL, callbackKey, path string, size ui
if err != nil {
if resErr, ok := err.(*RespError); ok {
if resErr.APIError.Code == "itemNotFound" {
if resErr.APIError.Code == notFoundError {
util.Log().Debug("Upload completed, will check upload callback later.")
select {
case <-time.After(time.Duration(interval) * time.Second):

View File

@@ -17,6 +17,8 @@ var (
ErrDeleteFile = errors.New("cannot delete file")
// ErrClientCanceled 客户端取消操作
ErrClientCanceled = errors.New("client canceled")
// Desired thumb size not available
ErrThumbSizeNotFound = errors.New("thumb size not found")
)
// Client OneDrive客户端
@@ -56,7 +58,7 @@ func NewClient(policy *model.Policy) (*Client, error) {
Policy: policy,
ClientID: policy.BucketName,
ClientSecret: policy.SecretKey,
Redirect: policy.OptionsSerialized.OdRedirect,
Redirect: policy.OptionsSerialized.OauthRedirect,
Request: request.NewClient(),
ClusterController: cluster.DefaultController,
}

View File

@@ -91,7 +91,6 @@ func (handler Driver) Get(ctx context.Context, path string) (response.RSCloser,
downloadURL, err := handler.Source(
ctx,
path,
url.URL{},
60,
false,
0,
@@ -136,7 +135,7 @@ func (handler Driver) Delete(ctx context.Context, files []string) ([]string, err
}
// Thumb 获取文件缩略图
func (handler Driver) Thumb(ctx context.Context, path string) (*response.ContentResponse, error) {
func (handler Driver) Thumb(ctx context.Context, file *model.File) (*response.ContentResponse, error) {
var (
thumbSize = [2]uint{400, 300}
ok = false
@@ -145,13 +144,15 @@ func (handler Driver) Thumb(ctx context.Context, path string) (*response.Content
return nil, errors.New("failed to get thumbnail size")
}
res, err := handler.Client.GetThumbURL(ctx, path, thumbSize[0], thumbSize[1])
res, err := handler.Client.GetThumbURL(ctx, file.SourceName, thumbSize[0], thumbSize[1])
if err != nil {
// 如果出现异常就清空文件的pic_info
if file, ok := ctx.Value(fsctx.FileModelCtx).(model.File); ok {
file.UpdatePicInfo("")
var apiErr *RespError
if errors.As(err, &apiErr); err == ErrThumbSizeNotFound || (apiErr != nil && apiErr.APIError.Code == notFoundError) {
// OneDrive cannot generate thumbnail for this file
return nil, driver.ErrorThumbNotSupported
}
}
return &response.ContentResponse{
Redirect: true,
URL: res,
@@ -162,7 +163,6 @@ func (handler Driver) Thumb(ctx context.Context, path string) (*response.Content
func (handler Driver) Source(
ctx context.Context,
path string,
baseURL url.URL,
ttl int64,
isDownload bool,
speed int,

View File

@@ -9,7 +9,6 @@ import (
"io"
"io/ioutil"
"net/http"
"net/url"
"strings"
"testing"
"time"
@@ -106,7 +105,7 @@ func TestDriver_Source(t *testing.T) {
// 失败
{
res, err := handler.Source(context.Background(), "123.jpg", url.URL{}, 1, true, 0)
res, err := handler.Source(context.Background(), "123.jpg", 1, true, 0)
asserts.Error(err)
asserts.Empty(res)
}
@@ -116,7 +115,7 @@ func TestDriver_Source(t *testing.T) {
handler.Client.Credential.ExpiresIn = time.Now().Add(time.Duration(100) * time.Hour).Unix()
handler.Client.Credential.AccessToken = "1"
cache.Set("onedrive_source_0_123.jpg", "res", 1)
res, err := handler.Source(context.Background(), "123.jpg", url.URL{}, 0, true, 0)
res, err := handler.Source(context.Background(), "123.jpg", 0, true, 0)
cache.Deletes([]string{"0_123.jpg"}, "onedrive_source_")
asserts.NoError(err)
asserts.Equal("res", res)
@@ -131,7 +130,7 @@ func TestDriver_Source(t *testing.T) {
handler.Client.Credential.ExpiresIn = time.Now().Add(time.Duration(100) * time.Hour).Unix()
handler.Client.Credential.AccessToken = "1"
cache.Set(fmt.Sprintf("onedrive_source_file_%d_1", file.UpdatedAt.Unix()), "res", 0)
res, err := handler.Source(ctx, "123.jpg", url.URL{}, 1, true, 0)
res, err := handler.Source(ctx, "123.jpg", 1, true, 0)
cache.Deletes([]string{"0_123.jpg"}, "onedrive_source_")
asserts.NoError(err)
asserts.Equal("res", res)
@@ -156,7 +155,7 @@ func TestDriver_Source(t *testing.T) {
})
handler.Client.Request = clientMock
handler.Client.Credential.AccessToken = "1"
res, err := handler.Source(context.Background(), "123.jpg", url.URL{}, 1, true, 0)
res, err := handler.Source(context.Background(), "123.jpg", 1, true, 0)
asserts.NoError(err)
asserts.Equal("123321", res)
}
@@ -246,19 +245,19 @@ func TestDriver_Thumb(t *testing.T) {
}
handler.Client, _ = NewClient(&model.Policy{})
handler.Client.Credential.ExpiresIn = time.Now().Add(time.Duration(100) * time.Hour).Unix()
file := &model.File{PicInfo: "1,1", Model: gorm.Model{ID: 1}}
// 失败
{
ctx := context.WithValue(context.Background(), fsctx.ThumbSizeCtx, [2]uint{10, 20})
ctx = context.WithValue(ctx, fsctx.FileModelCtx, model.File{PicInfo: "1,1", Model: gorm.Model{ID: 1}})
res, err := handler.Thumb(ctx, "123.jpg")
res, err := handler.Thumb(ctx, file)
asserts.Error(err)
asserts.Empty(res.URL)
}
// 上下文错误
{
_, err := handler.Thumb(context.Background(), "123.jpg")
_, err := handler.Thumb(context.Background(), file)
asserts.Error(err)
}
}

View File

@@ -10,6 +10,7 @@ import (
"time"
"github.com/cloudreve/Cloudreve/v3/pkg/cache"
"github.com/cloudreve/Cloudreve/v3/pkg/filesystem/oauth"
"github.com/cloudreve/Cloudreve/v3/pkg/request"
"github.com/cloudreve/Cloudreve/v3/pkg/util"
)
@@ -128,8 +129,8 @@ func (client *Client) UpdateCredential(ctx context.Context, isSlave bool) error
return client.fetchCredentialFromMaster(ctx)
}
GlobalMutex.Lock(client.Policy.ID)
defer GlobalMutex.Unlock(client.Policy.ID)
oauth.GlobalMutex.Lock(client.Policy.ID)
defer oauth.GlobalMutex.Unlock(client.Policy.ID)
// 如果已存在凭证
if client.Credential != nil && client.Credential.AccessToken != "" {
@@ -175,9 +176,13 @@ func (client *Client) UpdateCredential(ctx context.Context, isSlave bool) error
return nil
}
func (client *Client) AccessToken() string {
return client.Credential.AccessToken
}
// UpdateCredential 更新凭证,并检查有效期
func (client *Client) fetchCredentialFromMaster(ctx context.Context) error {
res, err := client.ClusterController.GetOneDriveToken(client.Policy.MasterID, client.Policy.ID)
res, err := client.ClusterController.GetPolicyOauthToken(client.Policy.MasterID, client.Policy.ID)
if err != nil {
return err
}

View File

@@ -368,7 +368,7 @@ func TestClient_UpdateCredential(t *testing.T) {
// slave failed
{
mockController := &controllermock.SlaveControllerMock{}
mockController.On("GetOneDriveToken", testMock.Anything, testMock.Anything).Return("", errors.New("error"))
mockController.On("GetPolicyOauthToken", testMock.Anything, testMock.Anything).Return("", errors.New("error"))
client.ClusterController = mockController
err := client.UpdateCredential(context.Background(), true)
asserts.Error(err)
@@ -377,7 +377,7 @@ func TestClient_UpdateCredential(t *testing.T) {
// slave success
{
mockController := &controllermock.SlaveControllerMock{}
mockController.On("GetOneDriveToken", testMock.Anything, testMock.Anything).Return("AccessToken3", nil)
mockController.On("GetPolicyOauthToken", testMock.Anything, testMock.Anything).Return("AccessToken3", nil)
client.ClusterController = mockController
err := client.UpdateCredential(context.Background(), true)
asserts.NoError(err)

View File

@@ -17,6 +17,7 @@ import (
model "github.com/cloudreve/Cloudreve/v3/models"
"github.com/cloudreve/Cloudreve/v3/pkg/filesystem/chunk"
"github.com/cloudreve/Cloudreve/v3/pkg/filesystem/chunk/backoff"
"github.com/cloudreve/Cloudreve/v3/pkg/filesystem/driver"
"github.com/cloudreve/Cloudreve/v3/pkg/filesystem/fsctx"
"github.com/cloudreve/Cloudreve/v3/pkg/filesystem/response"
"github.com/cloudreve/Cloudreve/v3/pkg/request"
@@ -193,14 +194,7 @@ func (handler *Driver) Get(ctx context.Context, path string) (response.RSCloser,
ctx = context.WithValue(ctx, fsctx.ForceUsePublicEndpointCtx, false)
// 获取文件源地址
downloadURL, err := handler.Source(
ctx,
path,
url.URL{},
int64(model.GetIntSetting("preview_timeout", 60)),
false,
0,
)
downloadURL, err := handler.Source(ctx, path, int64(model.GetIntSetting("preview_timeout", 60)), false, 0)
if err != nil {
return nil, err
}
@@ -293,7 +287,18 @@ func (handler *Driver) Delete(ctx context.Context, files []string) ([]string, er
}
// Thumb 获取文件缩略图
func (handler *Driver) Thumb(ctx context.Context, path string) (*response.ContentResponse, error) {
func (handler *Driver) Thumb(ctx context.Context, file *model.File) (*response.ContentResponse, error) {
// quick check by extension name
// https://help.aliyun.com/document_detail/183902.html
supported := []string{"png", "jpg", "jpeg", "gif", "bmp", "webp", "heic", "tiff", "avif"}
if len(handler.Policy.OptionsSerialized.ThumbExts) > 0 {
supported = handler.Policy.OptionsSerialized.ThumbExts
}
if !util.IsInExtensionList(supported, file.Name) || file.Size > (20<<(10*2)) {
return nil, driver.ErrorThumbNotSupported
}
// 初始化客户端
if err := handler.InitOSSClient(true); err != nil {
return nil, err
@@ -307,12 +312,14 @@ func (handler *Driver) Thumb(ctx context.Context, path string) (*response.Conten
return nil, errors.New("failed to get thumbnail size")
}
thumbParam := fmt.Sprintf("image/resize,m_lfit,h_%d,w_%d", thumbSize[1], thumbSize[0])
thumbEncodeQuality := model.GetIntSetting("thumb_encode_quality", 85)
thumbParam := fmt.Sprintf("image/resize,m_lfit,h_%d,w_%d/quality,q_%d", thumbSize[1], thumbSize[0], thumbEncodeQuality)
ctx = context.WithValue(ctx, fsctx.ThumbSizeCtx, thumbParam)
thumbOption := []oss.Option{oss.Process(thumbParam)}
thumbURL, err := handler.signSourceURL(
ctx,
path,
file.SourceName,
int64(model.GetIntSetting("preview_timeout", 60)),
thumbOption,
)
@@ -327,14 +334,7 @@ func (handler *Driver) Thumb(ctx context.Context, path string) (*response.Conten
}
// Source 获取外链URL
func (handler *Driver) Source(
ctx context.Context,
path string,
baseURL url.URL,
ttl int64,
isDownload bool,
speed int,
) (string, error) {
func (handler *Driver) Source(ctx context.Context, path string, ttl int64, isDownload bool, speed int) (string, error) {
// 初始化客户端
usePublicEndpoint := true
if forceUsePublicEndpoint, ok := ctx.Value(fsctx.ForceUsePublicEndpointCtx).(bool); ok {
@@ -435,6 +435,7 @@ func (handler *Driver) Token(ctx context.Context, ttl int64, uploadSession *seri
options := []oss.Option{
oss.Expires(time.Now().Add(time.Duration(ttl) * time.Second)),
oss.ForbidOverWrite(true),
oss.ContentType(fileInfo.DetectMimeType()),
}
imur, err := handler.bucket.InitiateMultipartUpload(fileInfo.SavePath, options...)
if err != nil {

View File

@@ -13,10 +13,12 @@ import (
"time"
model "github.com/cloudreve/Cloudreve/v3/models"
"github.com/cloudreve/Cloudreve/v3/pkg/filesystem/driver"
"github.com/cloudreve/Cloudreve/v3/pkg/filesystem/fsctx"
"github.com/cloudreve/Cloudreve/v3/pkg/filesystem/response"
"github.com/cloudreve/Cloudreve/v3/pkg/request"
"github.com/cloudreve/Cloudreve/v3/pkg/serializer"
"github.com/cloudreve/Cloudreve/v3/pkg/util"
"github.com/qiniu/go-sdk/v7/auth/qbox"
"github.com/qiniu/go-sdk/v7/storage"
)
@@ -117,14 +119,7 @@ func (handler *Driver) Get(ctx context.Context, path string) (response.RSCloser,
path = fmt.Sprintf("%s?v=%d", path, time.Now().UnixNano())
// 获取文件源地址
downloadURL, err := handler.Source(
ctx,
path,
url.URL{},
int64(model.GetIntSetting("preview_timeout", 60)),
false,
0,
)
downloadURL, err := handler.Source(ctx, path, int64(model.GetIntSetting("preview_timeout", 60)), false, 0)
if err != nil {
return nil, err
}
@@ -230,35 +225,41 @@ func (handler *Driver) Delete(ctx context.Context, files []string) ([]string, er
}
// Thumb 获取文件缩略图
func (handler *Driver) Thumb(ctx context.Context, path string) (*response.ContentResponse, error) {
func (handler *Driver) Thumb(ctx context.Context, file *model.File) (*response.ContentResponse, error) {
// quick check by extension name
// https://developer.qiniu.com/dora/api/basic-processing-images-imageview2
supported := []string{"png", "jpg", "jpeg", "gif", "bmp", "webp", "tiff", "avif", "psd"}
if len(handler.Policy.OptionsSerialized.ThumbExts) > 0 {
supported = handler.Policy.OptionsSerialized.ThumbExts
}
if !util.IsInExtensionList(supported, file.Name) || file.Size > (20<<(10*2)) {
return nil, driver.ErrorThumbNotSupported
}
var (
thumbSize = [2]uint{400, 300}
ok = false
)
if thumbSize, ok = ctx.Value(fsctx.ThumbSizeCtx).([2]uint); !ok {
return nil, errors.New("无法获取缩略图尺寸设置")
return nil, errors.New("failed to get thumbnail size")
}
path = fmt.Sprintf("%s?imageView2/1/w/%d/h/%d", path, thumbSize[0], thumbSize[1])
thumbEncodeQuality := model.GetIntSetting("thumb_encode_quality", 85)
thumb := fmt.Sprintf("%s?imageView2/1/w/%d/h/%d/q/%d", file.SourceName, thumbSize[0], thumbSize[1], thumbEncodeQuality)
return &response.ContentResponse{
Redirect: true,
URL: handler.signSourceURL(
ctx,
path,
thumb,
int64(model.GetIntSetting("preview_timeout", 60)),
),
}, nil
}
// Source 获取外链URL
func (handler *Driver) Source(
ctx context.Context,
path string,
baseURL url.URL,
ttl int64,
isDownload bool,
speed int,
) (string, error) {
func (handler *Driver) Source(ctx context.Context, path string, ttl int64, isDownload bool, speed int) (string, error) {
// 尝试从上下文获取文件名
fileName := ""
if file, ok := ctx.Value(fsctx.FileModelCtx).(model.File); ok {

View File

@@ -8,15 +8,18 @@ import (
"fmt"
"net/url"
"path"
"path/filepath"
"strings"
"time"
model "github.com/cloudreve/Cloudreve/v3/models"
"github.com/cloudreve/Cloudreve/v3/pkg/auth"
"github.com/cloudreve/Cloudreve/v3/pkg/filesystem/driver"
"github.com/cloudreve/Cloudreve/v3/pkg/filesystem/fsctx"
"github.com/cloudreve/Cloudreve/v3/pkg/filesystem/response"
"github.com/cloudreve/Cloudreve/v3/pkg/request"
"github.com/cloudreve/Cloudreve/v3/pkg/serializer"
"github.com/cloudreve/Cloudreve/v3/pkg/util"
)
// Driver 远程存储策略适配器
@@ -121,7 +124,7 @@ func (handler *Driver) Get(ctx context.Context, path string) (response.RSCloser,
}
// 获取文件源地址
downloadURL, err := handler.Source(ctx, path, url.URL{}, 0, true, speedLimit)
downloadURL, err := handler.Source(ctx, path, 0, true, speedLimit)
if err != nil {
return nil, err
}
@@ -204,9 +207,19 @@ func (handler *Driver) Delete(ctx context.Context, files []string) ([]string, er
}
// Thumb 获取文件缩略图
func (handler *Driver) Thumb(ctx context.Context, path string) (*response.ContentResponse, error) {
sourcePath := base64.RawURLEncoding.EncodeToString([]byte(path))
thumbURL := handler.getAPIUrl("thumb") + "/" + sourcePath
func (handler *Driver) Thumb(ctx context.Context, file *model.File) (*response.ContentResponse, error) {
// quick check by extension name
supported := []string{"png", "jpg", "jpeg", "gif"}
if len(handler.Policy.OptionsSerialized.ThumbExts) > 0 {
supported = handler.Policy.OptionsSerialized.ThumbExts
}
if !util.IsInExtensionList(supported, file.Name) {
return nil, driver.ErrorThumbNotSupported
}
sourcePath := base64.RawURLEncoding.EncodeToString([]byte(file.SourceName))
thumbURL := fmt.Sprintf("%s/%s/%s", handler.getAPIUrl("thumb"), sourcePath, filepath.Ext(file.Name))
ttl := model.GetIntSetting("preview_timeout", 60)
signedThumbURL, err := auth.SignURI(handler.AuthInstance, thumbURL, int64(ttl))
if err != nil {
@@ -220,14 +233,7 @@ func (handler *Driver) Thumb(ctx context.Context, path string) (*response.Conten
}
// Source 获取外链URL
func (handler *Driver) Source(
ctx context.Context,
path string,
baseURL url.URL,
ttl int64,
isDownload bool,
speed int,
) (string, error) {
func (handler *Driver) Source(ctx context.Context, path string, ttl int64, isDownload bool, speed int) (string, error) {
// 尝试从上下文获取文件名
fileName := "file"
if file, ok := ctx.Value(fsctx.FileModelCtx).(model.File); ok {

View File

@@ -3,12 +3,12 @@ package remote
import (
"context"
"errors"
"github.com/cloudreve/Cloudreve/v3/pkg/filesystem/driver"
"github.com/cloudreve/Cloudreve/v3/pkg/mocks/remoteclientmock"
"github.com/cloudreve/Cloudreve/v3/pkg/serializer"
"io"
"io/ioutil"
"net/http"
"net/url"
"strings"
"testing"
@@ -50,7 +50,7 @@ func TestHandler_Source(t *testing.T) {
AuthInstance: auth.HMACAuth{},
}
ctx := context.Background()
res, err := handler.Source(ctx, "", url.URL{}, 0, true, 0)
res, err := handler.Source(ctx, "", 0, true, 0)
asserts.NoError(err)
asserts.NotEmpty(res)
}
@@ -65,7 +65,7 @@ func TestHandler_Source(t *testing.T) {
SourceName: "1.txt",
}
ctx := context.WithValue(context.Background(), fsctx.FileModelCtx, file)
res, err := handler.Source(ctx, "", url.URL{}, 10, true, 0)
res, err := handler.Source(ctx, "", 10, true, 0)
asserts.NoError(err)
asserts.Contains(res, "api/v3/slave/download/0")
}
@@ -80,7 +80,7 @@ func TestHandler_Source(t *testing.T) {
SourceName: "1.txt",
}
ctx := context.WithValue(context.Background(), fsctx.FileModelCtx, file)
res, err := handler.Source(ctx, "", url.URL{}, 10, true, 0)
res, err := handler.Source(ctx, "", 10, true, 0)
asserts.NoError(err)
asserts.Contains(res, "api/v3/slave/download/0")
asserts.Contains(res, "https://cqu.edu.cn")
@@ -96,7 +96,7 @@ func TestHandler_Source(t *testing.T) {
SourceName: "1.txt",
}
ctx := context.WithValue(context.Background(), fsctx.FileModelCtx, file)
res, err := handler.Source(ctx, "", url.URL{}, 10, true, 0)
res, err := handler.Source(ctx, "", 10, true, 0)
asserts.Error(err)
asserts.Empty(res)
}
@@ -111,7 +111,7 @@ func TestHandler_Source(t *testing.T) {
SourceName: "1.txt",
}
ctx := context.WithValue(context.Background(), fsctx.FileModelCtx, file)
res, err := handler.Source(ctx, "", url.URL{}, 10, false, 0)
res, err := handler.Source(ctx, "", 10, false, 0)
asserts.NoError(err)
asserts.Contains(res, "api/v3/slave/source/0")
}
@@ -373,14 +373,33 @@ func TestHandler_Thumb(t *testing.T) {
Type: "remote",
SecretKey: "test",
Server: "http://test.com",
OptionsSerialized: model.PolicyOption{
ThumbExts: []string{"txt"},
},
},
AuthInstance: auth.HMACAuth{},
}
file := &model.File{
Name: "1.txt",
SourceName: "1.txt",
}
ctx := context.Background()
asserts.NoError(cache.Set("setting_preview_timeout", "60", 0))
resp, err := handler.Thumb(ctx, "/1.txt")
asserts.NoError(err)
asserts.True(resp.Redirect)
// no error
{
resp, err := handler.Thumb(ctx, file)
asserts.NoError(err)
asserts.True(resp.Redirect)
}
// ext not support
{
file.Name = "1.jpg"
resp, err := handler.Thumb(ctx, file)
asserts.ErrorIs(err, driver.ErrorThumbNotSupported)
asserts.Nil(resp)
}
}
func TestHandler_Token(t *testing.T) {

View File

@@ -4,6 +4,7 @@ import (
"context"
"errors"
"fmt"
"github.com/cloudreve/Cloudreve/v3/pkg/filesystem/driver"
"io"
"net/http"
"net/url"
@@ -163,14 +164,7 @@ func (handler *Driver) List(ctx context.Context, base string, recursive bool) ([
// Get 获取文件
func (handler *Driver) Get(ctx context.Context, path string) (response.RSCloser, error) {
// 获取文件源地址
downloadURL, err := handler.Source(
ctx,
path,
url.URL{},
int64(model.GetIntSetting("preview_timeout", 60)),
false,
0,
)
downloadURL, err := handler.Source(ctx, path, int64(model.GetIntSetting("preview_timeout", 60)), false, 0)
if err != nil {
return nil, err
}
@@ -257,26 +251,19 @@ func (handler *Driver) Delete(ctx context.Context, files []string) ([]string, er
for _, deleteRes := range res.Deleted {
deleted = append(deleted, *deleteRes.Key)
}
failed = util.SliceDifference(failed, deleted)
failed = util.SliceDifference(files, deleted)
return failed, nil
}
// Thumb 获取文件缩略图
func (handler *Driver) Thumb(ctx context.Context, path string) (*response.ContentResponse, error) {
return nil, errors.New("未实现")
func (handler *Driver) Thumb(ctx context.Context, file *model.File) (*response.ContentResponse, error) {
return nil, driver.ErrorThumbNotSupported
}
// Source 获取外链URL
func (handler *Driver) Source(
ctx context.Context,
path string,
baseURL url.URL,
ttl int64,
isDownload bool,
speed int,
) (string, error) {
func (handler *Driver) Source(ctx context.Context, path string, ttl int64, isDownload bool, speed int) (string, error) {
// 尝试从上下文获取文件名
fileName := ""
@@ -339,9 +326,10 @@ func (handler *Driver) Token(ctx context.Context, ttl int64, uploadSession *seri
// 创建分片上传
expires := time.Now().Add(time.Duration(ttl) * time.Second)
res, err := handler.svc.CreateMultipartUpload(&s3.CreateMultipartUploadInput{
Bucket: &handler.Policy.BucketName,
Key: &fileInfo.SavePath,
Expires: &expires,
Bucket: &handler.Policy.BucketName,
Key: &fileInfo.SavePath,
Expires: &expires,
ContentType: aws.String(fileInfo.DetectMimeType()),
})
if err != nil {
return nil, fmt.Errorf("failed to create multipart upload: %w", err)

View File

@@ -8,7 +8,6 @@ import (
"github.com/cloudreve/Cloudreve/v3/pkg/filesystem/fsctx"
"github.com/cloudreve/Cloudreve/v3/pkg/filesystem/response"
"github.com/cloudreve/Cloudreve/v3/pkg/serializer"
"net/url"
)
// Driver 影子存储策略,用于在从机端上传文件
@@ -39,11 +38,11 @@ func (d *Driver) Get(ctx context.Context, path string) (response.RSCloser, error
return nil, ErrNotImplemented
}
func (d *Driver) Thumb(ctx context.Context, path string) (*response.ContentResponse, error) {
func (d *Driver) Thumb(ctx context.Context, file *model.File) (*response.ContentResponse, error) {
return nil, ErrNotImplemented
}
func (d *Driver) Source(ctx context.Context, path string, url url.URL, ttl int64, isDownload bool, speed int) (string, error) {
func (d *Driver) Source(ctx context.Context, path string, ttl int64, isDownload bool, speed int) (string, error) {
return "", ErrNotImplemented
}

View File

@@ -102,11 +102,11 @@ func (d *Driver) Get(ctx context.Context, path string) (response.RSCloser, error
return nil, ErrNotImplemented
}
func (d *Driver) Thumb(ctx context.Context, path string) (*response.ContentResponse, error) {
func (d *Driver) Thumb(ctx context.Context, file *model.File) (*response.ContentResponse, error) {
return nil, ErrNotImplemented
}
func (d *Driver) Source(ctx context.Context, path string, url url.URL, ttl int64, isDownload bool, speed int) (string, error) {
func (d *Driver) Source(ctx context.Context, path string, ttl int64, isDownload bool, speed int) (string, error) {
return "", ErrNotImplemented
}

View File

@@ -18,10 +18,12 @@ import (
"time"
model "github.com/cloudreve/Cloudreve/v3/models"
"github.com/cloudreve/Cloudreve/v3/pkg/filesystem/driver"
"github.com/cloudreve/Cloudreve/v3/pkg/filesystem/fsctx"
"github.com/cloudreve/Cloudreve/v3/pkg/filesystem/response"
"github.com/cloudreve/Cloudreve/v3/pkg/request"
"github.com/cloudreve/Cloudreve/v3/pkg/serializer"
"github.com/cloudreve/Cloudreve/v3/pkg/util"
"github.com/upyun/go-sdk/upyun"
)
@@ -105,14 +107,7 @@ func (handler Driver) List(ctx context.Context, base string, recursive bool) ([]
// Get 获取文件
func (handler Driver) Get(ctx context.Context, path string) (response.RSCloser, error) {
// 获取文件源地址
downloadURL, err := handler.Source(
ctx,
path,
url.URL{},
int64(model.GetIntSetting("preview_timeout", 60)),
false,
0,
)
downloadURL, err := handler.Source(ctx, path, int64(model.GetIntSetting("preview_timeout", 60)), false, 0)
if err != nil {
return nil, err
}
@@ -220,7 +215,18 @@ func (handler Driver) Delete(ctx context.Context, files []string) ([]string, err
}
// Thumb 获取文件缩略图
func (handler Driver) Thumb(ctx context.Context, path string) (*response.ContentResponse, error) {
func (handler Driver) Thumb(ctx context.Context, file *model.File) (*response.ContentResponse, error) {
// quick check by extension name
// https://help.upyun.com/knowledge-base/image/
supported := []string{"png", "jpg", "jpeg", "gif", "bmp", "webp", "svg"}
if len(handler.Policy.OptionsSerialized.ThumbExts) > 0 {
supported = handler.Policy.OptionsSerialized.ThumbExts
}
if !util.IsInExtensionList(supported, file.Name) {
return nil, driver.ErrorThumbNotSupported
}
var (
thumbSize = [2]uint{400, 300}
ok = false
@@ -229,15 +235,10 @@ func (handler Driver) Thumb(ctx context.Context, path string) (*response.Content
return nil, errors.New("failed to get thumbnail size")
}
thumbParam := fmt.Sprintf("!/fwfh/%dx%d", thumbSize[0], thumbSize[1])
thumbURL, err := handler.Source(
ctx,
path+thumbParam,
url.URL{},
int64(model.GetIntSetting("preview_timeout", 60)),
false,
0,
)
thumbEncodeQuality := model.GetIntSetting("thumb_encode_quality", 85)
thumbParam := fmt.Sprintf("!/fwfh/%dx%d/quality/%d", thumbSize[0], thumbSize[1], thumbEncodeQuality)
thumbURL, err := handler.Source(ctx, file.SourceName+thumbParam, int64(model.GetIntSetting("preview_timeout", 60)), false, 0)
if err != nil {
return nil, err
}
@@ -249,14 +250,7 @@ func (handler Driver) Thumb(ctx context.Context, path string) (*response.Content
}
// Source 获取外链URL
func (handler Driver) Source(
ctx context.Context,
path string,
baseURL url.URL,
ttl int64,
isDownload bool,
speed int,
) (string, error) {
func (handler Driver) Source(ctx context.Context, path string, ttl int64, isDownload bool, speed int) (string, error) {
// 尝试从上下文获取文件名
fileName := ""
if file, ok := ctx.Value(fsctx.FileModelCtx).(model.File); ok {

View File

@@ -64,10 +64,6 @@ func (fs *FileSystem) AddFile(ctx context.Context, parent *model.Folder, file fs
UploadSessionID: uploadInfo.UploadSessionID,
}
if fs.Policy.IsThumbExist(uploadInfo.FileName) {
newFile.PicInfo = "1,1"
}
err = newFile.Create()
if err != nil {
@@ -97,9 +93,10 @@ func (fs *FileSystem) GetPhysicalFileContent(ctx context.Context, path string) (
}
// Preview 预览文件
// path - 文件虚拟路径
// isText - 是否为文本文件,文本文件会忽略重定向,直接由
// 服务端拉取中转给用户,故会对文件大小进行限制
//
// path - 文件虚拟路径
// isText - 是否为文本文件,文本文件会忽略重定向,直接由
// 服务端拉取中转给用户,故会对文件大小进行限制
func (fs *FileSystem) Preview(ctx context.Context, id uint, isText bool) (*response.ContentResponse, error) {
err := fs.resetFileIDIfNotExist(ctx, id)
if err != nil {
@@ -174,6 +171,7 @@ func (fs *FileSystem) deleteGroupedFile(ctx context.Context, files map[uint][]*m
// 失败的文件列表
// TODO 并行删除
failed := make(map[uint][]string, len(files))
thumbs := make([]string, 0)
for policyID, toBeDeletedFiles := range files {
// 列举出需要物理删除的文件的物理路径
@@ -188,7 +186,11 @@ func (fs *FileSystem) deleteGroupedFile(ctx context.Context, files map[uint][]*m
uploadSession := session.(serializer.UploadSession)
uploadSessions = append(uploadSessions, &uploadSession)
}
}
// Check if sidecar thumb file exist
if model.IsTrueVal(toBeDeletedFiles[i].MetadataSerialized[model.ThumbSidecarMetadataKey]) {
thumbs = append(thumbs, toBeDeletedFiles[i].ThumbFile())
}
}
@@ -210,8 +212,11 @@ func (fs *FileSystem) deleteGroupedFile(ctx context.Context, files map[uint][]*m
}
// 执行删除
failedFile, _ := fs.Handler.Delete(ctx, sourceNamesAll)
failed[policyID] = failedFile
toBeDeletedSrcs := append(sourceNamesAll, thumbs...)
failedFile, _ := fs.Handler.Delete(ctx, toBeDeletedSrcs)
// Exclude failed results related to thumb file
failed[policyID] = util.SliceDifference(failedFile, thumbs)
}
return failed
@@ -295,8 +300,7 @@ func (fs *FileSystem) SignURL(ctx context.Context, file *model.File, ttl int64,
// 签名最终URL
// 生成外链地址
siteURL := model.GetSiteURL()
source, err := fs.Handler.Source(ctx, fs.FileTarget[0].SourceName, *siteURL, ttl, isDownload, fs.User.Group.SpeedLimit)
source, err := fs.Handler.Source(ctx, fs.FileTarget[0].SourceName, ttl, isDownload, fs.User.Group.SpeedLimit)
if err != nil {
return "", serializer.NewError(serializer.CodeNotSet, "Failed to get source link", err)
}

View File

@@ -58,7 +58,6 @@ func TestFileSystem_AddFile(t *testing.T) {
asserts.NoError(err)
asserts.NoError(mock.ExpectationsWereMet())
asserts.Equal("/Uploads/1_sad.png", f.SourceName)
asserts.NotEmpty(f.PicInfo)
// 前置钩子执行失败
{
@@ -312,6 +311,19 @@ func TestFileSystem_deleteGroupedFile(t *testing.T) {
_, ok := cache.Get(UploadSessionCachePrefix + sessionID)
asserts.False(ok)
}
// 包含缩略图
{
files[0].MetadataSerialized = map[string]string{
model.ThumbSidecarMetadataKey: "1",
}
failed := fs.deleteGroupedFile(ctx, fs.GroupFileByPolicy(ctx, files))
asserts.Equal(map[uint][]string{
1: {},
2: {},
3: {},
}, failed)
}
}
func TestFileSystem_GetSource(t *testing.T) {
@@ -340,8 +352,6 @@ func TestFileSystem_GetSource(t *testing.T) {
sqlmock.NewRows([]string{"id", "type", "is_origin_link_enable"}).
AddRow(35, "local", true),
)
// 查找站点URL
mock.ExpectQuery("SELECT(.+)").WithArgs("siteURL").WillReturnRows(sqlmock.NewRows([]string{"id", "value"}).AddRow(1, "https://cloudreve.org"))
sourceURL, err := fs.GetSource(ctx, 2)
asserts.NoError(mock.ExpectationsWereMet())

View File

@@ -8,6 +8,7 @@ import (
"github.com/cloudreve/Cloudreve/v3/pkg/conf"
"github.com/cloudreve/Cloudreve/v3/pkg/filesystem/driver"
"github.com/cloudreve/Cloudreve/v3/pkg/filesystem/driver/cos"
"github.com/cloudreve/Cloudreve/v3/pkg/filesystem/driver/googledrive"
"github.com/cloudreve/Cloudreve/v3/pkg/filesystem/driver/local"
"github.com/cloudreve/Cloudreve/v3/pkg/filesystem/driver/onedrive"
"github.com/cloudreve/Cloudreve/v3/pkg/filesystem/driver/oss"
@@ -176,6 +177,10 @@ func (fs *FileSystem) DispatchHandler() error {
handler, err := s3.NewDriver(currentPolicy)
fs.Handler = handler
return err
case "googledrive":
handler, err := googledrive.NewDriver(currentPolicy)
fs.Handler = handler
return err
default:
return ErrUnknownPolicyType
}

View File

@@ -35,4 +35,10 @@ const (
CancelFuncCtx
// 文件在从机节点中的路径
SlaveSrcPath
// Webdav目标名称
WebdavDstName
// WebDAVCtx WebDAV
WebDAVCtx
// WebDAV反代Url
WebDAVProxyUrlCtx
)

View File

@@ -2,6 +2,7 @@ package fsctx
import (
"errors"
"github.com/HFO4/aliyun-oss-go-sdk/oss"
"io"
"time"
)
@@ -17,7 +18,7 @@ const (
type UploadTaskInfo struct {
Size uint64
MIMEType string
MimeType string
FileName string
VirtualPath string
Mode WriteMode
@@ -30,6 +31,15 @@ type UploadTaskInfo struct {
Src string
}
// Get mimetype of uploaded file, if it's not defined, detect it from file name
func (u *UploadTaskInfo) DetectMimeType() string {
if u.MimeType != "" {
return u.MimeType
}
return oss.TypeByExtension(u.FileName)
}
// FileHeader 上传来的文件数据处理器
type FileHeader interface {
io.Reader
@@ -51,7 +61,7 @@ type FileStream struct {
Size uint64
VirtualPath string
Name string
MIMEType string
MimeType string
SavePath string
UploadSessionID *string
AppendStart uint64
@@ -90,7 +100,7 @@ func (file *FileStream) Seekable() bool {
func (file *FileStream) Info() *UploadTaskInfo {
return &UploadTaskInfo{
Size: file.Size,
MIMEType: file.MIMEType,
MimeType: file.MimeType,
FileName: file.Name,
VirtualPath: file.VirtualPath,
Mode: file.Mode,

View File

@@ -10,7 +10,10 @@ import (
"github.com/cloudreve/Cloudreve/v3/pkg/serializer"
"github.com/cloudreve/Cloudreve/v3/pkg/util"
"io/ioutil"
"net/http"
"strconv"
"strings"
"time"
)
// Hook 钩子函数
@@ -177,24 +180,12 @@ func GenericAfterUpdate(ctx context.Context, fs *FileSystem, newFile fsctx.FileH
// SlaveAfterUpload Slave模式下上传完成钩子
func SlaveAfterUpload(session *serializer.UploadSession) Hook {
return func(ctx context.Context, fs *FileSystem, fileHeader fsctx.FileHeader) error {
fileInfo := fileHeader.Info()
// 构造一个model.File用于生成缩略图
file := model.File{
Name: fileInfo.FileName,
SourceName: fileInfo.SavePath,
}
fs.GenerateThumbnail(ctx, &file)
if session.Callback == "" {
return nil
}
// 发送回调请求
callbackBody := serializer.UploadCallback{
PicInfo: file.PicInfo,
}
callbackBody := serializer.UploadCallback{}
return cluster.RemoteCallback(session.Callback, callbackBody)
}
}
@@ -231,21 +222,6 @@ func GenericAfterUpload(ctx context.Context, fs *FileSystem, fileHeader fsctx.Fi
return nil
}
// HookGenerateThumb 生成缩略图
func HookGenerateThumb(ctx context.Context, fs *FileSystem, fileHeader fsctx.FileHeader) error {
// 异步尝试生成缩略图
fileMode := fileHeader.Info().Model.(*model.File)
if fs.Policy.IsThumbGenerateNeeded() {
fs.recycleLock.Lock()
go func() {
defer fs.recycleLock.Unlock()
_, _ = fs.Handler.Delete(ctx, []string{fileMode.SourceName + model.GetSettingByNameWithDefault("thumb_file_suffix", "._thumb")})
fs.GenerateThumbnail(ctx, fileMode)
}()
}
return nil
}
// HookClearFileHeaderSize 将FileHeader大小设定为0
func HookClearFileHeaderSize(ctx context.Context, fs *FileSystem, fileHeader fsctx.FileHeader) error {
fileHeader.SetSize(0)
@@ -284,10 +260,6 @@ func HookPopPlaceholderToFile(picInfo string) Hook {
return func(ctx context.Context, fs *FileSystem, fileHeader fsctx.FileHeader) error {
fileInfo := fileHeader.Info()
fileModel := fileInfo.Model.(*model.File)
if picInfo == "" && fs.Policy.IsThumbExist(fileInfo.FileName) {
picInfo = "1,1"
}
return fileModel.PopChunkToFile(fileInfo.LastModified, picInfo)
}
}
@@ -299,3 +271,34 @@ func HookDeleteUploadSession(id string) Hook {
return nil
}
}
// NewWebdavAfterUploadHook 每次创建一个新的钩子函数 rclone 在 PUT 请求里有 OC-Checksum 字符串
// 和 X-OC-Mtime
func NewWebdavAfterUploadHook(request *http.Request) func(ctx context.Context, fs *FileSystem, newFile fsctx.FileHeader) error {
var modtime time.Time
if timeVal := request.Header.Get("X-OC-Mtime"); timeVal != "" {
timeUnix, err := strconv.ParseInt(timeVal, 10, 64)
if err == nil {
modtime = time.Unix(timeUnix, 0)
}
}
checksum := request.Header.Get("OC-Checksum")
return func(ctx context.Context, fs *FileSystem, newFile fsctx.FileHeader) error {
file := newFile.Info().Model.(*model.File)
if !modtime.IsZero() {
err := model.DB.Model(file).UpdateColumn("updated_at", modtime).Error
if err != nil {
return err
}
}
if checksum != "" {
return file.UpdateMetadata(map[string]string{
model.ChecksumMetadataKey: checksum,
})
}
return nil
}
}

View File

@@ -360,7 +360,7 @@ func TestHookClearFileSize(t *testing.T) {
)
mock.ExpectBegin()
mock.ExpectExec("UPDATE(.+)files(.+)").
WithArgs(0, sqlmock.AnyArg(), 1, 10).
WithArgs("", 0, sqlmock.AnyArg(), 1, 10).
WillReturnResult(sqlmock.NewResult(1, 1))
mock.ExpectExec("UPDATE(.+)users(.+)").
WithArgs(10, sqlmock.AnyArg()).
@@ -394,7 +394,7 @@ func TestHookUpdateSourceName(t *testing.T) {
}
ctx := context.WithValue(context.Background(), fsctx.FileModelCtx, originFile)
mock.ExpectBegin()
mock.ExpectExec("UPDATE(.+)").WithArgs("new.txt", sqlmock.AnyArg(), 1).WillReturnResult(sqlmock.NewResult(1, 1))
mock.ExpectExec("UPDATE(.+)").WithArgs("", "new.txt", sqlmock.AnyArg(), 1).WillReturnResult(sqlmock.NewResult(1, 1))
mock.ExpectCommit()
err := HookUpdateSourceName(ctx, fs, nil)
asserts.NoError(mock.ExpectationsWereMet())
@@ -429,7 +429,7 @@ func TestGenericAfterUpdate(t *testing.T) {
fs.Handler = handlerMock
mock.ExpectBegin()
mock.ExpectExec("UPDATE(.+)files(.+)").
WithArgs(10, sqlmock.AnyArg(), 1, 0).
WithArgs("", 10, sqlmock.AnyArg(), 1, 0).
WillReturnResult(sqlmock.NewResult(1, 1))
mock.ExpectExec("UPDATE(.+)users(.+)").
WithArgs(10, sqlmock.AnyArg()).
@@ -462,7 +462,7 @@ func TestGenericAfterUpdate(t *testing.T) {
mock.ExpectBegin()
mock.ExpectExec("UPDATE(.+)").
WithArgs(10, sqlmock.AnyArg(), 1, 0).
WithArgs("", 10, sqlmock.AnyArg(), 1, 0).
WillReturnError(errors.New("error"))
mock.ExpectRollback()
@@ -473,27 +473,6 @@ func TestGenericAfterUpdate(t *testing.T) {
}
}
func TestHookGenerateThumb(t *testing.T) {
a := assert.New(t)
mockHandler := &FileHeaderMock{}
fs := &FileSystem{
User: &model.User{
Model: gorm.Model{ID: 1},
},
Handler: mockHandler,
Policy: &model.Policy{Type: "local"},
}
mockHandler.On("Delete", testMock.Anything, []string{"1.txt._thumb"}).Return([]string{}, nil)
a.NoError(HookGenerateThumb(context.Background(), fs, &fsctx.FileStream{
Model: &model.File{
SourceName: "1.txt",
},
}))
fs.Recycle()
mockHandler.AssertExpectations(t)
}
func TestSlaveAfterUpload(t *testing.T) {
asserts := assert.New(t)
conf.SystemConfig.Mode = "slave"
@@ -625,7 +604,7 @@ func TestHookChunkUploaded(t *testing.T) {
}
mock.ExpectBegin()
mock.ExpectExec("UPDATE(.+)files(.+)").WithArgs(20, sqlmock.AnyArg(), 1, 0).WillReturnResult(sqlmock.NewResult(1, 1))
mock.ExpectExec("UPDATE(.+)files(.+)").WithArgs("", 20, sqlmock.AnyArg(), 1, 0).WillReturnResult(sqlmock.NewResult(1, 1))
mock.ExpectExec("UPDATE(.+)users(.+)").
WithArgs(20, sqlmock.AnyArg()).
WillReturnResult(sqlmock.NewResult(1, 1))
@@ -646,7 +625,7 @@ func TestHookChunkUploadFailed(t *testing.T) {
}
mock.ExpectBegin()
mock.ExpectExec("UPDATE(.+)files(.+)").WithArgs(10, sqlmock.AnyArg(), 1, 0).WillReturnResult(sqlmock.NewResult(1, 1))
mock.ExpectExec("UPDATE(.+)files(.+)").WithArgs("", 10, sqlmock.AnyArg(), 1, 0).WillReturnResult(sqlmock.NewResult(1, 1))
mock.ExpectExec("UPDATE(.+)users(.+)").
WithArgs(10, sqlmock.AnyArg()).
WillReturnResult(sqlmock.NewResult(1, 1))
@@ -704,3 +683,26 @@ func TestHookDeleteUploadSession(t *testing.T) {
_, ok := cache.Get(UploadSessionCachePrefix + "TestHookDeleteUploadSession")
a.False(ok)
}
func TestNewWebdavAfterUploadHook(t *testing.T) {
a := assert.New(t)
fs := &FileSystem{}
file := &fsctx.FileStream{
Model: &model.File{
Model: gorm.Model{ID: 1},
},
}
req, _ := http.NewRequest("get", "http://localhost", nil)
req.Header.Add("X-Oc-Mtime", "1681521402")
req.Header.Add("OC-Checksum", "checksum")
mock.ExpectBegin()
mock.ExpectExec("UPDATE(.+)files(.+)").WillReturnResult(sqlmock.NewResult(1, 1))
mock.ExpectCommit()
mock.ExpectBegin()
mock.ExpectExec("UPDATE(.+)files(.+)").WillReturnResult(sqlmock.NewResult(1, 1))
mock.ExpectCommit()
err := NewWebdavAfterUploadHook(req)(context.Background(), fs, file)
a.NoError(err)
a.NoError(mock.ExpectationsWereMet())
}

View File

@@ -2,13 +2,16 @@ package filesystem
import (
"context"
"errors"
"fmt"
"os"
"sync"
"runtime"
model "github.com/cloudreve/Cloudreve/v3/models"
"github.com/cloudreve/Cloudreve/v3/pkg/conf"
"github.com/cloudreve/Cloudreve/v3/pkg/filesystem/driver"
"github.com/cloudreve/Cloudreve/v3/pkg/filesystem/fsctx"
"github.com/cloudreve/Cloudreve/v3/pkg/filesystem/response"
"github.com/cloudreve/Cloudreve/v3/pkg/thumb"
@@ -20,28 +23,51 @@ import (
================
*/
// HandledExtension 可以生成缩略图的文件扩展名
var HandledExtension = []string{"jpg", "jpeg", "png", "gif"}
// GetThumb 获取文件的缩略图
func (fs *FileSystem) GetThumb(ctx context.Context, id uint) (*response.ContentResponse, error) {
// 根据 ID 查找文件
err := fs.resetFileIDIfNotExist(ctx, id)
if err != nil || fs.FileTarget[0].PicInfo == "" {
return &response.ContentResponse{
Redirect: false,
}, ErrObjectNotExist
if err != nil {
return nil, ErrObjectNotExist
}
file := fs.FileTarget[0]
if !file.ShouldLoadThumb() {
return nil, ErrObjectNotExist
}
w, h := fs.GenerateThumbnailSize(0, 0)
ctx = context.WithValue(ctx, fsctx.ThumbSizeCtx, [2]uint{w, h})
ctx = context.WithValue(ctx, fsctx.FileModelCtx, fs.FileTarget[0])
res, err := fs.Handler.Thumb(ctx, fs.FileTarget[0].SourceName)
// 本地存储策略出错时重新生成缩略图
if err != nil && fs.Policy.Type == "local" {
fs.GenerateThumbnail(ctx, &fs.FileTarget[0])
res, err = fs.Handler.Thumb(ctx, fs.FileTarget[0].SourceName)
ctx = context.WithValue(ctx, fsctx.FileModelCtx, file)
res, err := fs.Handler.Thumb(ctx, &file)
if errors.Is(err, driver.ErrorThumbNotExist) {
// Regenerate thumb if the thumb is not initialized yet
if generateErr := fs.generateThumbnail(ctx, &file); generateErr == nil {
res, err = fs.Handler.Thumb(ctx, &file)
} else {
err = generateErr
}
} else if errors.Is(err, driver.ErrorThumbNotSupported) {
// Policy handler explicitly indicates thumb not available, check if proxy is enabled
if fs.Policy.CouldProxyThumb() {
// if thumb id marked as existed, redirect to "sidecar" thumb file.
if file.MetadataSerialized != nil &&
file.MetadataSerialized[model.ThumbStatusMetadataKey] == model.ThumbStatusExist {
// redirect to sidecar file
res = &response.ContentResponse{
Redirect: true,
}
res.URL, err = fs.Handler.Source(ctx, file.ThumbFile(), int64(model.GetIntSetting("preview_timeout", 60)), false, 0)
} else {
// if not exist, generate and upload the sidecar thumb.
if err = fs.generateThumbnail(ctx, &file); err == nil {
return fs.GetThumb(ctx, id)
}
}
} else {
// thumb not supported and proxy is disabled, mark as not available
_ = updateThumbStatus(&file, model.ThumbStatusNotAvailable)
}
}
if err == nil && conf.SystemConfig.Mode == "master" {
@@ -84,65 +110,109 @@ func (pool *Pool) releaseWorker() {
<-pool.worker
}
// GenerateThumbnail 尝试为本地策略文件生成缩略图并获取图像原始大小
// TODO 失败时,如果之前还有图像信息,则清除
func (fs *FileSystem) GenerateThumbnail(ctx context.Context, file *model.File) {
// 判断是否可以生成缩略图
if !IsInExtensionList(HandledExtension, file.Name) {
return
}
// generateThumbnail generates thumb for given file, upload the thumb file back with given suffix
func (fs *FileSystem) generateThumbnail(ctx context.Context, file *model.File) error {
// 新建上下文
newCtx, cancel := context.WithCancel(context.Background())
defer cancel()
// TODO: check file size
if file.Size > uint64(model.GetIntSetting("thumb_max_src_size", 31457280)) {
_ = updateThumbStatus(file, model.ThumbStatusNotAvailable)
return errors.New("file too large")
}
getThumbWorker().addWorker()
defer getThumbWorker().releaseWorker()
// 获取文件数据
source, err := fs.Handler.Get(newCtx, file.SourceName)
if err != nil {
return
return fmt.Errorf("faield to fetch original file %q: %w", file.SourceName, err)
}
defer source.Close()
getThumbWorker().addWorker()
defer getThumbWorker().releaseWorker()
image, err := thumb.NewThumbFromFile(source, file.Name)
if err != nil {
util.Log().Warning("Cannot generate thumb because of failed to parse image %q: %s", file.SourceName, err)
return
// Provide file source path for local policy files
src := ""
if conf.SystemConfig.Mode == "slave" || file.GetPolicy().Type == "local" {
src = file.SourceName
}
// 获取原始图像尺寸
w, h := image.GetSize()
thumbRes, err := thumb.Generators.Generate(ctx, source, src, file.Name, model.GetSettingByNames(
"thumb_width",
"thumb_height",
"thumb_builtin_enabled",
"thumb_vips_enabled",
"thumb_ffmpeg_enabled",
"thumb_libreoffice_enabled",
))
if err != nil {
_ = updateThumbStatus(file, model.ThumbStatusNotAvailable)
return fmt.Errorf("failed to generate thumb for %q: %w", file.Name, err)
}
defer os.Remove(thumbRes.Path)
thumbFile, err := os.Open(thumbRes.Path)
if err != nil {
return fmt.Errorf("failed to open temp thumb %q: %w", thumbRes.Path, err)
}
defer thumbFile.Close()
fileInfo, err := thumbFile.Stat()
if err != nil {
return fmt.Errorf("failed to stat temp thumb %q: %w", thumbRes.Path, err)
}
if err = fs.Handler.Put(newCtx, &fsctx.FileStream{
Mode: fsctx.Overwrite,
File: thumbFile,
Seeker: thumbFile,
Size: uint64(fileInfo.Size()),
SavePath: file.SourceName + model.GetSettingByNameWithDefault("thumb_file_suffix", "._thumb"),
}); err != nil {
return fmt.Errorf("failed to save thumb for %q: %w", file.Name, err)
}
// 生成缩略图
image.GetThumb(fs.GenerateThumbnailSize(w, h))
// 保存到文件
err = image.Save(util.RelativePath(file.SourceName + model.GetSettingByNameWithDefault("thumb_file_suffix", "._thumb")))
image = nil
if model.IsTrueVal(model.GetSettingByName("thumb_gc_after_gen")) {
util.Log().Debug("GenerateThumbnail runtime.GC")
util.Log().Debug("generateThumbnail runtime.GC")
runtime.GC()
}
if err != nil {
util.Log().Warning("Failed to save thumb: %s", err)
return
}
// 更新文件的图像信息
if file.Model.ID > 0 {
err = file.UpdatePicInfo(fmt.Sprintf("%d,%d", w, h))
} else {
file.PicInfo = fmt.Sprintf("%d,%d", w, h)
}
// Mark this file as thumb available
err = updateThumbStatus(file, model.ThumbStatusExist)
// 失败时删除缩略图文件
if err != nil {
_, _ = fs.Handler.Delete(newCtx, []string{file.SourceName + model.GetSettingByNameWithDefault("thumb_file_suffix", "._thumb")})
}
return nil
}
// GenerateThumbnailSize 获取要生成的缩略图的尺寸
func (fs *FileSystem) GenerateThumbnailSize(w, h int) (uint, uint) {
return uint(model.GetIntSetting("thumb_width", 400)), uint(model.GetIntSetting("thumb_width", 300))
return uint(model.GetIntSetting("thumb_width", 400)), uint(model.GetIntSetting("thumb_height", 300))
}
func updateThumbStatus(file *model.File, status string) error {
if file.Model.ID > 0 {
meta := map[string]string{
model.ThumbStatusMetadataKey: status,
}
if status == model.ThumbStatusExist {
meta[model.ThumbSidecarMetadataKey] = "true"
}
return file.UpdateMetadata(meta)
} else {
if file.MetadataSerialized == nil {
file.MetadataSerialized = map[string]string{}
}
file.MetadataSerialized[model.ThumbStatusMetadataKey] = status
}
return nil
}

View File

@@ -5,8 +5,10 @@ import (
"errors"
model "github.com/cloudreve/Cloudreve/v3/models"
"github.com/cloudreve/Cloudreve/v3/pkg/cache"
"github.com/cloudreve/Cloudreve/v3/pkg/filesystem/driver"
"github.com/cloudreve/Cloudreve/v3/pkg/filesystem/response"
"github.com/cloudreve/Cloudreve/v3/pkg/request"
"github.com/cloudreve/Cloudreve/v3/pkg/mocks/thumbmock"
"github.com/cloudreve/Cloudreve/v3/pkg/thumb"
testMock "github.com/stretchr/testify/mock"
"testing"
@@ -14,30 +16,104 @@ import (
)
func TestFileSystem_GetThumb(t *testing.T) {
asserts := assert.New(t)
a := assert.New(t)
fs := &FileSystem{User: &model.User{}}
// 非图像文件
// file not found
{
fs.SetTargetFile(&[]model.File{{}})
_, err := fs.GetThumb(context.Background(), 1)
asserts.Equal(err, ErrObjectNotExist)
mock.ExpectQuery("SELECT(.+)").WillReturnError(errors.New("error"))
res, err := fs.GetThumb(context.Background(), 1)
a.ErrorIs(err, ErrObjectNotExist)
a.Nil(res)
a.NoError(mock.ExpectationsWereMet())
}
// 成功
// thumb not exist
{
cache.Set("setting_thumb_width", "10", 0)
cache.Set("setting_thumb_height", "10", 0)
cache.Set("setting_preview_timeout", "50", 0)
testHandller2 := new(FileHeaderMock)
testHandller2.On("Thumb", testMock.Anything, "").Return(&response.ContentResponse{}, nil)
fs.CleanTargets()
fs.SetTargetFile(&[]model.File{{PicInfo: "1,1", Policy: model.Policy{Type: "mock"}}})
fs.SetTargetFile(&[]model.File{{
MetadataSerialized: map[string]string{
model.ThumbStatusMetadataKey: model.ThumbStatusNotAvailable,
},
Policy: model.Policy{Type: "mock"},
}})
fs.FileTarget[0].Policy.ID = 1
fs.Handler = testHandller2
res, err := fs.GetThumb(context.Background(), 1)
asserts.NoError(err)
asserts.EqualValues(50, res.MaxAge)
a.ErrorIs(err, ErrObjectNotExist)
a.Nil(res)
}
// thumb not initialized, also failed to generate
{
fs.CleanTargets()
fs.SetTargetFile(&[]model.File{{
Policy: model.Policy{Type: "mock"},
Size: 31457281,
}})
testHandller2 := new(FileHeaderMock)
testHandller2.On("Thumb", testMock.Anything, &fs.FileTarget[0]).Return(&response.ContentResponse{}, driver.ErrorThumbNotExist)
fs.Handler = testHandller2
fs.FileTarget[0].Policy.ID = 1
res, err := fs.GetThumb(context.Background(), 1)
a.Contains(err.Error(), "file too large")
a.Nil(res.Content)
}
// thumb not initialized, failed to get source
{
fs.CleanTargets()
fs.SetTargetFile(&[]model.File{{
Policy: model.Policy{Type: "mock"},
}})
testHandller2 := new(FileHeaderMock)
testHandller2.On("Thumb", testMock.Anything, &fs.FileTarget[0]).Return(&response.ContentResponse{}, driver.ErrorThumbNotExist)
testHandller2.On("Get", testMock.Anything, "").Return(MockRSC{}, errors.New("error"))
fs.Handler = testHandller2
fs.FileTarget[0].Policy.ID = 1
res, err := fs.GetThumb(context.Background(), 1)
a.Contains(err.Error(), "error")
a.Nil(res.Content)
}
// thumb not initialized, no available generators
{
thumb.Generators = []thumb.Generator{}
fs.CleanTargets()
fs.SetTargetFile(&[]model.File{{
Policy: model.Policy{Type: "local"},
}})
testHandller2 := new(FileHeaderMock)
testHandller2.On("Thumb", testMock.Anything, &fs.FileTarget[0]).Return(&response.ContentResponse{}, driver.ErrorThumbNotExist)
testHandller2.On("Get", testMock.Anything, "").Return(MockRSC{}, nil)
fs.Handler = testHandller2
fs.FileTarget[0].Policy.ID = 1
res, err := fs.GetThumb(context.Background(), 1)
a.ErrorIs(err, thumb.ErrNotAvailable)
a.Nil(res)
}
// thumb not initialized, thumb generated but cannot be open
{
mockGenerator := &thumbmock.GeneratorMock{}
thumb.Generators = []thumb.Generator{mockGenerator}
fs.CleanTargets()
fs.SetTargetFile(&[]model.File{{
Policy: model.Policy{Type: "mock"},
}})
cache.Set("setting_thumb_vips_enabled", "1", 0)
testHandller2 := new(FileHeaderMock)
testHandller2.On("Thumb", testMock.Anything, &fs.FileTarget[0]).Return(&response.ContentResponse{}, driver.ErrorThumbNotExist)
testHandller2.On("Get", testMock.Anything, "").Return(MockRSC{}, nil)
mockGenerator.On("Generate", testMock.Anything, testMock.Anything, testMock.Anything, testMock.Anything, testMock.Anything).
Return(&thumb.Result{Path: "not_exit_thumb"}, nil)
fs.Handler = testHandller2
fs.FileTarget[0].Policy.ID = 1
res, err := fs.GetThumb(context.Background(), 1)
a.Contains(err.Error(), "failed to open temp thumb")
a.Nil(res.Content)
testHandller2.AssertExpectations(t)
mockGenerator.AssertExpectations(t)
}
}
@@ -49,22 +125,3 @@ func TestFileSystem_ThumbWorker(t *testing.T) {
getThumbWorker().releaseWorker()
})
}
func TestFileSystem_GenerateThumbnail(t *testing.T) {
fs := &FileSystem{User: &model.User{}}
// 无法生成缩略图
{
fs.SetTargetFile(&[]model.File{{}})
fs.GenerateThumbnail(context.Background(), &model.File{})
}
// 无法获取文件数据
{
testHandller := new(FileHeaderMock)
testHandller.On("Get", testMock.Anything, "").Return(request.NopRSCloser{}, errors.New("error"))
fs.Handler = testHandller
fs.GenerateThumbnail(context.Background(), &model.File{Name: "test.png"})
testHandller.AssertExpectations(t)
}
}

View File

@@ -69,6 +69,11 @@ func (fs *FileSystem) Copy(ctx context.Context, dirs, files []uint, src, dst str
// 记录复制的文件的总容量
var newUsedStorage uint64
// 设置webdav目标名
if dstName, ok := ctx.Value(fsctx.WebdavDstName).(string); ok {
dstFolder.WebdavDstName = dstName
}
// 复制目录
if len(dirs) > 0 {
subFileSizes, err := srcFolder.CopyFolderTo(dirs[0], dstFolder)
@@ -103,6 +108,11 @@ func (fs *FileSystem) Move(ctx context.Context, dirs, files []uint, src, dst str
return ErrPathNotExist
}
// 设置webdav目标名
if dstName, ok := ctx.Value(fsctx.WebdavDstName).(string); ok {
dstFolder.WebdavDstName = dstName
}
// 处理目录及子文件移动
err := srcFolder.MoveFolderTo(dirs, dstFolder)
if err != nil {
@@ -341,7 +351,6 @@ func (fs *FileSystem) listObjects(ctx context.Context, parent string, files []mo
ID: hashid.HashID(subFolder.ID, hashid.FolderID),
Name: subFolder.Name,
Path: processedPath,
Pic: "",
Size: 0,
Type: "dir",
Date: subFolder.UpdatedAt,
@@ -363,7 +372,7 @@ func (fs *FileSystem) listObjects(ctx context.Context, parent string, files []mo
ID: hashid.HashID(file.ID, hashid.FileID),
Name: file.Name,
Path: processedPath,
Pic: file.PicInfo,
Thumb: file.ShouldLoadThumb(),
Size: file.Size,
Type: "file",
Date: file.UpdatedAt,

View File

@@ -682,7 +682,7 @@ func TestFileSystem_Rename(t *testing.T) {
WillReturnRows(sqlmock.NewRows([]string{"id", "name"}).AddRow(10, "old.text"))
mock.ExpectBegin()
mock.ExpectExec("UPDATE(.+)files(.+)SET(.+)").
WithArgs("new.txt", 10).
WithArgs(sqlmock.AnyArg(), "new.txt", sqlmock.AnyArg(), 10).
WillReturnResult(sqlmock.NewResult(1, 1))
mock.ExpectCommit()
err := fs.Rename(ctx, []uint{}, []uint{10}, "new.txt")
@@ -708,7 +708,7 @@ func TestFileSystem_Rename(t *testing.T) {
WillReturnRows(sqlmock.NewRows([]string{"id", "name"}).AddRow(10, "old.text"))
mock.ExpectBegin()
mock.ExpectExec("UPDATE(.+)files(.+)SET(.+)").
WithArgs("new.txt", 10).
WithArgs(sqlmock.AnyArg(), "new.txt", sqlmock.AnyArg(), 10).
WillReturnError(errors.New("error"))
mock.ExpectRollback()
err := fs.Rename(ctx, []uint{}, []uint{10}, "new.txt")

View File

@@ -1,4 +1,4 @@
package onedrive
package oauth
import "sync"

View File

@@ -0,0 +1,8 @@
package oauth
import "context"
type TokenProvider interface {
UpdateCredential(ctx context.Context, isSlave bool) error
AccessToken() string
}

View File

@@ -210,7 +210,6 @@ func (fs *FileSystem) UploadFromStream(ctx context.Context, file *fsctx.FileStre
fs.Use("BeforeUpload", HookValidateCapacity)
fs.Use("AfterUploadCanceled", HookDeleteTempFile)
fs.Use("AfterUpload", GenericAfterUpload)
fs.Use("AfterUpload", HookGenerateThumb)
fs.Use("AfterValidateFailed", HookDeleteTempFile)
}
fs.Lock.Unlock()

View File

@@ -16,7 +16,6 @@ import (
"io/ioutil"
"net/http"
"net/http/httptest"
"net/url"
"strings"
"testing"
)
@@ -55,13 +54,13 @@ func (m FileHeaderMock) Delete(ctx context.Context, files []string) ([]string, e
return args.Get(0).([]string), args.Error(1)
}
func (m FileHeaderMock) Thumb(ctx context.Context, files string) (*response.ContentResponse, error) {
func (m FileHeaderMock) Thumb(ctx context.Context, files *model.File) (*response.ContentResponse, error) {
args := m.Called(ctx, files)
return args.Get(0).(*response.ContentResponse), args.Error(1)
}
func (m FileHeaderMock) Source(ctx context.Context, path string, url url.URL, expires int64, isDownload bool, speed int) (string, error) {
args := m.Called(ctx, path, url, expires, isDownload, speed)
func (m FileHeaderMock) Source(ctx context.Context, path string, expires int64, isDownload bool, speed int) (string, error) {
args := m.Called(ctx, path, expires, isDownload, speed)
return args.Get(0).(string), args.Error(1)
}

View File

@@ -2,7 +2,6 @@ package filesystem
import (
"context"
"path/filepath"
"strings"
"github.com/cloudreve/Cloudreve/v3/pkg/util"
@@ -63,20 +62,5 @@ func (fs *FileSystem) ValidateExtension(ctx context.Context, fileName string) bo
return true
}
return IsInExtensionList(fs.Policy.OptionsSerialized.FileType, fileName)
}
// IsInExtensionList 返回文件的扩展名是否在给定的列表范围内
func IsInExtensionList(extList []string, fileName string) bool {
ext := strings.ToLower(filepath.Ext(fileName))
// 无扩展名时
if len(ext) == 0 {
return false
}
if util.ContainsString(extList, ext[1:]) {
return true
}
return false
return util.IsInExtensionList(fs.Policy.OptionsSerialized.FileType, fileName)
}

View File

@@ -27,3 +27,11 @@ func (c CacheClientMock) Sets(values map[string]interface{}, prefix string) erro
func (c CacheClientMock) Delete(keys []string, prefix string) error {
return c.Called(keys, prefix).Error(0)
}
func (c CacheClientMock) Persist(path string) error {
return c.Called(path).Error(0)
}
func (c CacheClientMock) Restore(path string) error {
return c.Called(path).Error(0)
}

View File

@@ -37,7 +37,7 @@ func (s SlaveControllerMock) GetMasterInfo(s2 string) (*cluster.MasterInfo, erro
return args.Get(0).(*cluster.MasterInfo), args.Error(1)
}
func (s SlaveControllerMock) GetOneDriveToken(s2 string, u uint) (string, error) {
func (s SlaveControllerMock) GetPolicyOauthToken(s2 string, u uint) (string, error) {
args := s.Called(s2, u)
return args.String(0), args.Error(1)
}

View File

@@ -0,0 +1,25 @@
package thumbmock
import (
"context"
"github.com/cloudreve/Cloudreve/v3/pkg/thumb"
"github.com/stretchr/testify/mock"
"io"
)
type GeneratorMock struct {
mock.Mock
}
func (g GeneratorMock) Generate(ctx context.Context, file io.Reader, src string, name string, options map[string]string) (*thumb.Result, error) {
res := g.Called(ctx, file, src, name, options)
return res.Get(0).(*thumb.Result), res.Error(1)
}
func (g GeneratorMock) Priority() int {
return 0
}
func (g GeneratorMock) EnableFlag() string {
return "thumb_vips_enabled"
}

View File

@@ -10,7 +10,7 @@ type WopiClientMock struct {
mock.Mock
}
func (w *WopiClientMock) NewSession(user *model.User, file *model.File, action wopi.ActonType) (*wopi.Session, error) {
func (w *WopiClientMock) NewSession(user uint, file *model.File, action wopi.ActonType) (*wopi.Session, error) {
args := w.Called(user, file, action)
return args.Get(0).(*wopi.Session), args.Error(1)
}

View File

@@ -36,7 +36,7 @@ type Object struct {
ID string `json:"id"`
Name string `json:"name"`
Path string `json:"path"`
Pic string `json:"pic"`
Thumb bool `json:"thumb"`
Size uint64 `json:"size"`
Type string `json:"type"`
Date time.Time `json:"date"`

View File

@@ -42,6 +42,7 @@ type group struct {
WebDAVEnabled bool `json:"webdav"`
SourceBatchSize int `json:"sourceBatch"`
AdvanceDelete bool `json:"advanceDelete"`
AllowWebDAVProxy bool `json:"allowWebDAVProxy"`
}
type tag struct {
@@ -100,6 +101,7 @@ func BuildUser(user model.User) User {
ShareDownload: user.Group.OptionsSerialized.ShareDownload,
CompressEnabled: user.Group.OptionsSerialized.ArchiveTask,
WebDAVEnabled: user.Group.WebDAVEnabled,
AllowWebDAVProxy: user.Group.OptionsSerialized.WebDAVProxy,
SourceBatchSize: user.Group.OptionsSerialized.SourceBatchSize,
AdvanceDelete: user.Group.OptionsSerialized.AdvanceDelete,
},

136
pkg/sessionstore/kv.go Normal file
View File

@@ -0,0 +1,136 @@
package sessionstore
import (
"bytes"
"encoding/base32"
"encoding/gob"
"github.com/cloudreve/Cloudreve/v3/pkg/cache"
"github.com/gorilla/securecookie"
"github.com/gorilla/sessions"
"net/http"
"strings"
)
type kvStore struct {
Codecs []securecookie.Codec
Options *sessions.Options
DefaultMaxAge int
prefix string
serializer SessionSerializer
store cache.Driver
}
func newKvStore(prefix string, store cache.Driver, keyPairs ...[]byte) *kvStore {
return &kvStore{
prefix: prefix,
store: store,
DefaultMaxAge: 60 * 20,
serializer: GobSerializer{},
Codecs: securecookie.CodecsFromPairs(keyPairs...),
Options: &sessions.Options{
Path: "/",
MaxAge: 86400 * 30,
},
}
}
// Get returns a session for the given name after adding it to the registry.
//
// It returns a new session if the sessions doesn't exist. Access IsNew on
// the session to check if it is an existing session or a new one.
//
// It returns a new session and an error if the session exists but could
// not be decoded.
func (s *kvStore) Get(r *http.Request, name string) (*sessions.Session, error) {
return sessions.GetRegistry(r).Get(s, name)
}
// New returns a session for the given name without adding it to the registry.
//
// The difference between New() and Get() is that calling New() twice will
// decode the session data twice, while Get() registers and reuses the same
// decoded session after the first call.
func (s *kvStore) New(r *http.Request, name string) (*sessions.Session, error) {
var (
err error
)
session := sessions.NewSession(s, name)
// make a copy
options := *s.Options
session.Options = &options
session.IsNew = true
if c, errCookie := r.Cookie(name); errCookie == nil {
err = securecookie.DecodeMulti(name, c.Value, &session.ID, s.Codecs...)
if err == nil {
res, ok := s.store.Get(s.prefix + session.ID)
if ok {
err = s.serializer.Deserialize(res.([]byte), session)
}
session.IsNew = !(err == nil && ok) // not new if no error and data available
}
}
return session, err
}
func (s *kvStore) Save(r *http.Request, w http.ResponseWriter, session *sessions.Session) error {
// Marked for deletion.
if session.Options.MaxAge <= 0 {
if err := s.store.Delete([]string{session.ID}, s.prefix); err != nil {
return err
}
http.SetCookie(w, sessions.NewCookie(session.Name(), "", session.Options))
} else {
// Build an alphanumeric key for the redis store.
if session.ID == "" {
session.ID = strings.TrimRight(base32.StdEncoding.EncodeToString(securecookie.GenerateRandomKey(32)), "=")
}
b, err := s.serializer.Serialize(session)
if err != nil {
return err
}
age := session.Options.MaxAge
if age == 0 {
age = s.DefaultMaxAge
}
if err := s.store.Set(s.prefix+session.ID, b, age); err != nil {
return err
}
encoded, err := securecookie.EncodeMulti(session.Name(), session.ID, s.Codecs...)
if err != nil {
return err
}
http.SetCookie(w, sessions.NewCookie(session.Name(), encoded, session.Options))
}
return nil
}
// SessionSerializer provides an interface hook for alternative serializers
type SessionSerializer interface {
Deserialize(d []byte, ss *sessions.Session) error
Serialize(ss *sessions.Session) ([]byte, error)
}
// GobSerializer uses gob package to encode the session map
type GobSerializer struct{}
// Serialize using gob
func (s GobSerializer) Serialize(ss *sessions.Session) ([]byte, error) {
buf := new(bytes.Buffer)
enc := gob.NewEncoder(buf)
err := enc.Encode(ss.Values)
if err == nil {
return buf.Bytes(), nil
}
return nil, err
}
// Deserialize back to map[interface{}]interface{}
func (s GobSerializer) Deserialize(d []byte, ss *sessions.Session) error {
dec := gob.NewDecoder(bytes.NewBuffer(d))
return dec.Decode(&ss.Values)
}

View File

@@ -0,0 +1,22 @@
package sessionstore
import (
"github.com/cloudreve/Cloudreve/v3/pkg/cache"
"github.com/gin-contrib/sessions"
)
type Store interface {
sessions.Store
}
func NewStore(driver cache.Driver, keyPairs ...[]byte) Store {
return &store{newKvStore("cd_session_", driver, keyPairs...)}
}
type store struct {
*kvStore
}
func (c *store) Options(options sessions.Options) {
c.kvStore.Options = options.ToGorillaOptions()
}

View File

@@ -1,7 +1,7 @@
package thumb
import (
"errors"
"context"
"fmt"
"image"
"image/gif"
@@ -13,11 +13,15 @@ import (
model "github.com/cloudreve/Cloudreve/v3/models"
"github.com/cloudreve/Cloudreve/v3/pkg/util"
"github.com/gofrs/uuid"
//"github.com/nfnt/resize"
"golang.org/x/image/draw"
)
func init() {
RegisterGenerator(&Builtin{})
}
// Thumb 缩略图
type Thumb struct {
src image.Image
@@ -30,25 +34,23 @@ func NewThumbFromFile(file io.Reader, name string) (*Thumb, error) {
ext := strings.ToLower(filepath.Ext(name))
// 无扩展名时
if len(ext) == 0 {
return nil, errors.New("未知的图像类型")
return nil, fmt.Errorf("unknown image format: %w", ErrPassThrough)
}
var err error
var img image.Image
switch ext[1:] {
case "jpg":
img, err = jpeg.Decode(file)
case "jpeg":
case "jpg", "jpeg":
img, err = jpeg.Decode(file)
case "gif":
img, err = gif.Decode(file)
case "png":
img, err = png.Decode(file)
default:
return nil, errors.New("unknown image format")
return nil, fmt.Errorf("unknown image format: %w", ErrPassThrough)
}
if err != nil {
return nil, err
return nil, fmt.Errorf("failed to parse image: %w (%w)", err, ErrPassThrough)
}
return &Thumb{
@@ -70,18 +72,12 @@ func (image *Thumb) GetSize() (int, int) {
}
// Save 保存图像到给定路径
func (image *Thumb) Save(path string) (err error) {
out, err := util.CreatNestedFile(path)
if err != nil {
return err
}
defer out.Close()
func (image *Thumb) Save(w io.Writer) (err error) {
switch model.GetSettingByNameWithDefault("thumb_encode_method", "jpg") {
case "png":
err = png.Encode(out, image.src)
err = png.Encode(w, image.src)
default:
err = jpeg.Encode(out, image.src, &jpeg.Options{Quality: model.GetIntSetting("thumb_encode_quality", 85)})
err = jpeg.Encode(w, image.src, &jpeg.Options{Quality: model.GetIntSetting("thumb_encode_quality", 85)})
}
return err
@@ -141,9 +137,15 @@ func (image *Thumb) CreateAvatar(uid uint) error {
// 生成头像缩略图
src := image.src
for k, size := range []int{s, m, l} {
//image.src = resize.Resize(uint(size), uint(size), src, resize.Lanczos3)
out, err := util.CreatNestedFile(filepath.Join(savePath, fmt.Sprintf("avatar_%d_%d.png", uid, k)))
if err != nil {
return err
}
defer out.Close()
image.src = Resize(uint(size), uint(size), src)
err := image.Save(filepath.Join(savePath, fmt.Sprintf("avatar_%d_%d.png", uid, k)))
err = image.Save(out)
if err != nil {
return err
}
@@ -152,3 +154,39 @@ func (image *Thumb) CreateAvatar(uid uint) error {
return nil
}
type Builtin struct{}
func (b Builtin) Generate(ctx context.Context, file io.Reader, src, name string, options map[string]string) (*Result, error) {
img, err := NewThumbFromFile(file, name)
if err != nil {
return nil, err
}
img.GetThumb(thumbSize(options))
tempPath := filepath.Join(
util.RelativePath(model.GetSettingByName("temp_path")),
"thumb",
fmt.Sprintf("thumb_%s", uuid.Must(uuid.NewV4()).String()),
)
thumbFile, err := util.CreatNestedFile(tempPath)
if err != nil {
return nil, fmt.Errorf("failed to create temp file: %w", err)
}
defer thumbFile.Close()
if err := img.Save(thumbFile); err != nil {
return nil, err
}
return &Result{Path: tempPath}, nil
}
func (b Builtin) Priority() int {
return 300
}
func (b Builtin) EnableFlag() string {
return "thumb_builtin_enabled"
}

93
pkg/thumb/ffmpeg.go Normal file
View File

@@ -0,0 +1,93 @@
package thumb
import (
"bytes"
"context"
"fmt"
model "github.com/cloudreve/Cloudreve/v3/models"
"github.com/cloudreve/Cloudreve/v3/pkg/util"
"github.com/gofrs/uuid"
"io"
"os"
"os/exec"
"path/filepath"
"strings"
)
func init() {
RegisterGenerator(&FfmpegGenerator{})
}
type FfmpegGenerator struct {
exts []string
lastRawExts string
}
func (f *FfmpegGenerator) Generate(ctx context.Context, file io.Reader, src, name string, options map[string]string) (*Result, error) {
ffmpegOpts := model.GetSettingByNames("thumb_ffmpeg_path", "thumb_ffmpeg_exts", "thumb_ffmpeg_seek", "thumb_encode_method", "temp_path")
if f.lastRawExts != ffmpegOpts["thumb_ffmpeg_exts"] {
f.exts = strings.Split(ffmpegOpts["thumb_ffmpeg_exts"], ",")
}
if !util.IsInExtensionList(f.exts, name) {
return nil, fmt.Errorf("unsupported video format: %w", ErrPassThrough)
}
tempOutputPath := filepath.Join(
util.RelativePath(ffmpegOpts["temp_path"]),
"thumb",
fmt.Sprintf("thumb_%s.%s", uuid.Must(uuid.NewV4()).String(), ffmpegOpts["thumb_encode_method"]),
)
tempInputPath := src
if tempInputPath == "" {
// If not local policy files, download to temp folder
tempInputPath = filepath.Join(
util.RelativePath(ffmpegOpts["temp_path"]),
"thumb",
fmt.Sprintf("ffmpeg_%s%s", uuid.Must(uuid.NewV4()).String(), filepath.Ext(name)),
)
// Due to limitations of ffmpeg, we need to write the input file to disk first
tempInputFile, err := util.CreatNestedFile(tempInputPath)
if err != nil {
return nil, fmt.Errorf("failed to create temp file: %w", err)
}
defer os.Remove(tempInputPath)
defer tempInputFile.Close()
if _, err = io.Copy(tempInputFile, file); err != nil {
return nil, fmt.Errorf("failed to write input file: %w", err)
}
tempInputFile.Close()
}
// Invoke ffmpeg
scaleOpt := fmt.Sprintf("scale=%s:%s:force_original_aspect_ratio=decrease", options["thumb_width"], options["thumb_height"])
cmd := exec.CommandContext(ctx,
ffmpegOpts["thumb_ffmpeg_path"], "-ss", ffmpegOpts["thumb_ffmpeg_seek"], "-i", tempInputPath,
"-vf", scaleOpt, "-vframes", "1", tempOutputPath)
// Redirect IO
var stdErr bytes.Buffer
cmd.Stdin = file
cmd.Stderr = &stdErr
if err := cmd.Run(); err != nil {
util.Log().Warning("Failed to invoke ffmpeg: %s", stdErr.String())
return nil, fmt.Errorf("failed to invoke ffmpeg: %w", err)
}
return &Result{Path: tempOutputPath}, nil
}
func (f *FfmpegGenerator) Priority() int {
return 200
}
func (f *FfmpegGenerator) EnableFlag() string {
return "thumb_ffmpeg_enabled"
}

View File

@@ -1,146 +0,0 @@
package thumb
import (
"fmt"
"image"
"image/jpeg"
"os"
"testing"
"github.com/cloudreve/Cloudreve/v3/pkg/cache"
"github.com/cloudreve/Cloudreve/v3/pkg/util"
"github.com/stretchr/testify/assert"
)
func CreateTestImage() *os.File {
file, err := os.Create("TestNewThumbFromFile.jpeg")
alpha := image.NewAlpha(image.Rect(0, 0, 500, 200))
jpeg.Encode(file, alpha, nil)
if err != nil {
fmt.Println(err)
}
_, _ = file.Seek(0, 0)
return file
}
func TestNewThumbFromFile(t *testing.T) {
asserts := assert.New(t)
file := CreateTestImage()
defer file.Close()
// 无扩展名时
{
thumb, err := NewThumbFromFile(file, "123")
asserts.Error(err)
asserts.Nil(thumb)
}
{
thumb, err := NewThumbFromFile(file, "123.jpg")
asserts.NoError(err)
asserts.NotNil(thumb)
}
{
thumb, err := NewThumbFromFile(file, "123.jpeg")
asserts.Error(err)
asserts.Nil(thumb)
}
{
thumb, err := NewThumbFromFile(file, "123.png")
asserts.Error(err)
asserts.Nil(thumb)
}
{
thumb, err := NewThumbFromFile(file, "123.gif")
asserts.Error(err)
asserts.Nil(thumb)
}
{
thumb, err := NewThumbFromFile(file, "123.3211")
asserts.Error(err)
asserts.Nil(thumb)
}
}
func TestThumb_GetSize(t *testing.T) {
asserts := assert.New(t)
file := CreateTestImage()
defer file.Close()
thumb, err := NewThumbFromFile(file, "123.jpg")
asserts.NoError(err)
w, h := thumb.GetSize()
asserts.Equal(500, w)
asserts.Equal(200, h)
}
func TestThumb_GetThumb(t *testing.T) {
asserts := assert.New(t)
file := CreateTestImage()
defer file.Close()
thumb, err := NewThumbFromFile(file, "123.jpg")
asserts.NoError(err)
asserts.NotPanics(func() {
thumb.GetThumb(10, 10)
})
}
func TestThumb_Thumbnail(t *testing.T) {
asserts := assert.New(t)
{
img := image.NewRGBA(image.Rect(0, 0, 500, 200))
thumb := Thumbnail(100, 100, img)
asserts.Equal(thumb.Bounds(), image.Rect(0, 0, 100, 40))
}
{
img := image.NewRGBA(image.Rect(0, 0, 200, 200))
thumb := Thumbnail(100, 100, img)
asserts.Equal(thumb.Bounds(), image.Rect(0, 0, 100, 100))
}
{
img := image.NewRGBA(image.Rect(0, 0, 500, 500))
thumb := Thumbnail(100, 100, img)
asserts.Equal(thumb.Bounds(), image.Rect(0, 0, 100, 100))
}
{
img := image.NewRGBA(image.Rect(0, 0, 200, 500))
thumb := Thumbnail(100, 100, img)
asserts.Equal(thumb.Bounds(), image.Rect(0, 0, 40, 100))
}
}
func TestThumb_Save(t *testing.T) {
asserts := assert.New(t)
file := CreateTestImage()
defer file.Close()
thumb, err := NewThumbFromFile(file, "123.jpg")
asserts.NoError(err)
err = thumb.Save("/:noteexist/")
asserts.Error(err)
err = thumb.Save("TestThumb_Save.png")
asserts.NoError(err)
asserts.True(util.Exists("TestThumb_Save.png"))
}
func TestThumb_CreateAvatar(t *testing.T) {
asserts := assert.New(t)
file := CreateTestImage()
defer file.Close()
thumb, err := NewThumbFromFile(file, "123.jpg")
asserts.NoError(err)
cache.Set("setting_avatar_path", "tests", 0)
cache.Set("setting_avatar_size_s", "50", 0)
cache.Set("setting_avatar_size_m", "130", 0)
cache.Set("setting_avatar_size_l", "200", 0)
asserts.NoError(thumb.CreateAvatar(1))
asserts.True(util.Exists(util.RelativePath("tests/avatar_1_1.png")))
asserts.True(util.Exists(util.RelativePath("tests/avatar_1_2.png")))
asserts.True(util.Exists(util.RelativePath("tests/avatar_1_0.png")))
}

99
pkg/thumb/libreoffice.go Normal file
View File

@@ -0,0 +1,99 @@
package thumb
import (
"bytes"
"context"
"fmt"
model "github.com/cloudreve/Cloudreve/v3/models"
"github.com/cloudreve/Cloudreve/v3/pkg/util"
"github.com/gofrs/uuid"
"io"
"os"
"os/exec"
"path/filepath"
"strings"
)
func init() {
RegisterGenerator(&LibreOfficeGenerator{})
}
type LibreOfficeGenerator struct {
exts []string
lastRawExts string
}
func (l *LibreOfficeGenerator) Generate(ctx context.Context, file io.Reader, src string, name string, options map[string]string) (*Result, error) {
sofficeOpts := model.GetSettingByNames("thumb_libreoffice_path", "thumb_libreoffice_exts", "thumb_encode_method", "temp_path")
if l.lastRawExts != sofficeOpts["thumb_libreoffice_exts"] {
l.exts = strings.Split(sofficeOpts["thumb_libreoffice_exts"], ",")
}
if !util.IsInExtensionList(l.exts, name) {
return nil, fmt.Errorf("unsupported document format: %w", ErrPassThrough)
}
tempOutputPath := filepath.Join(
util.RelativePath(sofficeOpts["temp_path"]),
"thumb",
fmt.Sprintf("soffice_%s", uuid.Must(uuid.NewV4()).String()),
)
tempInputPath := src
if tempInputPath == "" {
// If not local policy files, download to temp folder
tempInputPath = filepath.Join(
util.RelativePath(sofficeOpts["temp_path"]),
"thumb",
fmt.Sprintf("soffice_%s%s", uuid.Must(uuid.NewV4()).String(), filepath.Ext(name)),
)
// Due to limitations of ffmpeg, we need to write the input file to disk first
tempInputFile, err := util.CreatNestedFile(tempInputPath)
if err != nil {
return nil, fmt.Errorf("failed to create temp file: %w", err)
}
defer os.Remove(tempInputPath)
defer tempInputFile.Close()
if _, err = io.Copy(tempInputFile, file); err != nil {
return nil, fmt.Errorf("failed to write input file: %w", err)
}
tempInputFile.Close()
}
// Convert the document to an image
cmd := exec.CommandContext(ctx, sofficeOpts["thumb_libreoffice_path"], "--headless",
"-nologo", "--nofirststartwizard", "--invisible", "--norestore", "--convert-to",
sofficeOpts["thumb_encode_method"], "--outdir", tempOutputPath, tempInputPath)
// Redirect IO
var stdErr bytes.Buffer
cmd.Stdin = file
cmd.Stderr = &stdErr
if err := cmd.Run(); err != nil {
util.Log().Warning("Failed to invoke LibreOffice: %s", stdErr.String())
return nil, fmt.Errorf("failed to invoke LibreOffice: %w", err)
}
return &Result{
Path: filepath.Join(
tempOutputPath,
strings.TrimSuffix(filepath.Base(tempInputPath), filepath.Ext(tempInputPath))+"."+sofficeOpts["thumb_encode_method"],
),
Continue: true,
Cleanup: []func(){func() { _ = os.RemoveAll(tempOutputPath) }},
}, nil
}
func (l *LibreOfficeGenerator) Priority() int {
return 50
}
func (l *LibreOfficeGenerator) EnableFlag() string {
return "thumb_libreoffice_enabled"
}

122
pkg/thumb/pipeline.go Normal file
View File

@@ -0,0 +1,122 @@
package thumb
import (
"context"
"errors"
"fmt"
model "github.com/cloudreve/Cloudreve/v3/models"
"github.com/cloudreve/Cloudreve/v3/pkg/util"
"io"
"os"
"path/filepath"
"reflect"
"sort"
"strconv"
)
// Generator generates a thumbnail for a given reader.
type Generator interface {
// Generate generates a thumbnail for a given reader. Src is the original file path, only provided
// for local policy files.
Generate(ctx context.Context, file io.Reader, src string, name string, options map[string]string) (*Result, error)
// Priority of execution order, smaller value means higher priority.
Priority() int
// EnableFlag returns the setting name to enable this generator.
EnableFlag() string
}
type Result struct {
Path string
Continue bool
Cleanup []func()
}
type (
GeneratorType string
GeneratorList []Generator
)
var (
Generators = GeneratorList{}
ErrPassThrough = errors.New("pass through")
ErrNotAvailable = fmt.Errorf("thumbnail not available: %w", ErrPassThrough)
)
func (g GeneratorList) Len() int {
return len(g)
}
func (g GeneratorList) Less(i, j int) bool {
return g[i].Priority() < g[j].Priority()
}
func (g GeneratorList) Swap(i, j int) {
g[i], g[j] = g[j], g[i]
}
// RegisterGenerator registers a thumbnail generator.
func RegisterGenerator(generator Generator) {
Generators = append(Generators, generator)
sort.Sort(Generators)
}
func (p GeneratorList) Generate(ctx context.Context, file io.Reader, src, name string, options map[string]string) (*Result, error) {
inputFile, inputSrc, inputName := file, src, name
for _, generator := range p {
if model.IsTrueVal(options[generator.EnableFlag()]) {
res, err := generator.Generate(ctx, inputFile, inputSrc, inputName, options)
if errors.Is(err, ErrPassThrough) {
util.Log().Debug("Failed to generate thumbnail using %s for %s: %s, passing through to next generator.", reflect.TypeOf(generator).String(), name, err)
continue
}
if res != nil && res.Continue {
util.Log().Debug("Generator %s for %s returned continue, passing through to next generator.", reflect.TypeOf(generator).String(), name)
// defer cleanup funcs
for _, cleanup := range res.Cleanup {
defer cleanup()
}
// prepare file reader for next generator
intermediate, err := os.Open(res.Path)
if err != nil {
return nil, fmt.Errorf("failed to open intermediate thumb file: %w", err)
}
defer intermediate.Close()
inputFile = intermediate
inputSrc = res.Path
inputName = filepath.Base(res.Path)
continue
}
return res, err
}
}
return nil, ErrNotAvailable
}
func (p GeneratorList) Priority() int {
return 0
}
func (p GeneratorList) EnableFlag() string {
return ""
}
func thumbSize(options map[string]string) (uint, uint) {
w, h := uint(400), uint(300)
if wParsed, err := strconv.Atoi(options["thumb_width"]); err == nil {
w = uint(wParsed)
}
if hParsed, err := strconv.Atoi(options["thumb_height"]); err == nil {
h = uint(hParsed)
}
return w, h
}

74
pkg/thumb/tester.go Normal file
View File

@@ -0,0 +1,74 @@
package thumb
import (
"bytes"
"context"
"errors"
"fmt"
"os/exec"
"strings"
)
var (
ErrUnknownGenerator = errors.New("unknown generator type")
ErrUnknownOutput = errors.New("unknown output from generator")
)
// TestGenerator tests thumb generator by getting lib version
func TestGenerator(ctx context.Context, name, executable string) (string, error) {
switch name {
case "vips":
return testVipsGenerator(ctx, executable)
case "ffmpeg":
return testFfmpegGenerator(ctx, executable)
case "libreOffice":
return testLibreOfficeGenerator(ctx, executable)
default:
return "", ErrUnknownGenerator
}
}
func testVipsGenerator(ctx context.Context, executable string) (string, error) {
cmd := exec.CommandContext(ctx, executable, "--version")
var output bytes.Buffer
cmd.Stdout = &output
if err := cmd.Run(); err != nil {
return "", fmt.Errorf("failed to invoke vips executable: %w", err)
}
if !strings.Contains(output.String(), "vips") {
return "", ErrUnknownOutput
}
return output.String(), nil
}
func testFfmpegGenerator(ctx context.Context, executable string) (string, error) {
cmd := exec.CommandContext(ctx, executable, "-version")
var output bytes.Buffer
cmd.Stdout = &output
if err := cmd.Run(); err != nil {
return "", fmt.Errorf("failed to invoke ffmpeg executable: %w", err)
}
if !strings.Contains(output.String(), "ffmpeg") {
return "", ErrUnknownOutput
}
return output.String(), nil
}
func testLibreOfficeGenerator(ctx context.Context, executable string) (string, error) {
cmd := exec.CommandContext(ctx, executable, "--version")
var output bytes.Buffer
cmd.Stdout = &output
if err := cmd.Run(); err != nil {
return "", fmt.Errorf("failed to invoke libreoffice executable: %w", err)
}
if !strings.Contains(output.String(), "LibreOffice") {
return "", ErrUnknownOutput
}
return output.String(), nil
}

78
pkg/thumb/vips.go Normal file
View File

@@ -0,0 +1,78 @@
package thumb
import (
"bytes"
"context"
"fmt"
model "github.com/cloudreve/Cloudreve/v3/models"
"github.com/cloudreve/Cloudreve/v3/pkg/util"
"github.com/gofrs/uuid"
"io"
"os/exec"
"path/filepath"
"strings"
)
func init() {
RegisterGenerator(&VipsGenerator{})
}
type VipsGenerator struct {
exts []string
lastRawExts string
}
func (v *VipsGenerator) Generate(ctx context.Context, file io.Reader, src, name string, options map[string]string) (*Result, error) {
vipsOpts := model.GetSettingByNames("thumb_vips_path", "thumb_vips_exts", "thumb_encode_quality", "thumb_encode_method", "temp_path")
if v.lastRawExts != vipsOpts["thumb_vips_exts"] {
v.exts = strings.Split(vipsOpts["thumb_vips_exts"], ",")
}
if !util.IsInExtensionList(v.exts, name) {
return nil, fmt.Errorf("unsupported image format: %w", ErrPassThrough)
}
outputOpt := ".png"
if vipsOpts["thumb_encode_method"] == "jpg" {
outputOpt = fmt.Sprintf(".jpg[Q=%s]", vipsOpts["thumb_encode_quality"])
}
cmd := exec.CommandContext(ctx,
vipsOpts["thumb_vips_path"], "thumbnail_source", "[descriptor=0]", outputOpt, options["thumb_width"],
"--height", options["thumb_height"])
tempPath := filepath.Join(
util.RelativePath(vipsOpts["temp_path"]),
"thumb",
fmt.Sprintf("thumb_%s", uuid.Must(uuid.NewV4()).String()),
)
thumbFile, err := util.CreatNestedFile(tempPath)
if err != nil {
return nil, fmt.Errorf("failed to create temp file: %w", err)
}
defer thumbFile.Close()
// Redirect IO
var vipsErr bytes.Buffer
cmd.Stdin = file
cmd.Stdout = thumbFile
cmd.Stderr = &vipsErr
if err := cmd.Run(); err != nil {
util.Log().Warning("Failed to invoke vips: %s", vipsErr.String())
return nil, fmt.Errorf("failed to invoke vips: %w", err)
}
return &Result{Path: tempPath}, nil
}
func (v *VipsGenerator) Priority() int {
return 100
}
func (v *VipsGenerator) EnableFlag() string {
return "thumb_vips_enabled"
}

View File

@@ -2,6 +2,7 @@ package util
import (
"math/rand"
"path/filepath"
"regexp"
"strings"
"time"
@@ -32,6 +33,21 @@ func ContainsUint(s []uint, e uint) bool {
return false
}
// IsInExtensionList 返回文件的扩展名是否在给定的列表范围内
func IsInExtensionList(extList []string, fileName string) bool {
ext := strings.ToLower(filepath.Ext(fileName))
// 无扩展名时
if len(ext) == 0 {
return false
}
if ContainsString(extList, ext[1:]) {
return true
}
return false
}
// ContainsString 返回list中是否包含
func ContainsString(s []string, e string) bool {
for _, a := range s {

View File

@@ -9,9 +9,12 @@ import (
"net/http"
"path"
"path/filepath"
"strconv"
"time"
model "github.com/cloudreve/Cloudreve/v3/models"
"github.com/cloudreve/Cloudreve/v3/pkg/filesystem"
"github.com/cloudreve/Cloudreve/v3/pkg/filesystem/fsctx"
)
// slashClean is equivalent to but slightly more efficient than
@@ -23,6 +26,31 @@ func slashClean(name string) string {
return path.Clean(name)
}
// 更新Copy或Move后的修改时间
func updateCopyMoveModtime(req *http.Request, fs *filesystem.FileSystem, dst string) error {
var modtime time.Time
if timeVal := req.Header.Get("X-OC-Mtime"); timeVal != "" {
timeUnix, err := strconv.ParseInt(timeVal, 10, 64)
if err == nil {
modtime = time.Unix(timeUnix, 0)
}
}
if modtime.IsZero() {
return nil
}
ok, fi := isPathExist(req.Context(), fs, dst)
if !ok {
return nil
}
if fi.IsDir() {
return model.DB.Model(fi.(*model.Folder)).UpdateColumn("updated_at", modtime).Error
}
return model.DB.Model(fi.(*model.File)).UpdateColumn("updated_at", modtime).Error
}
// moveFiles moves files and/or directories from src to dst.
//
// See section 9.9.4 for when various HTTP status codes apply.
@@ -38,26 +66,30 @@ func moveFiles(ctx context.Context, fs *filesystem.FileSystem, src FileInfo, dst
fileIDs = []uint{src.(*model.File).ID}
}
// 判断是否需要移动
if src.GetPosition() != path.Dir(dst) {
err = fs.Move(
ctx,
folderIDs,
fileIDs,
src.GetPosition(),
path.Dir(dst),
)
}
if overwrite {
if err := _checkOverwriteFile(ctx, fs, src, dst); err != nil {
return http.StatusInternalServerError, err
}
}
// 判断是否需要重命名
if err == nil && src.GetName() != path.Base(dst) {
err = fs.Rename(
ctx,
folderIDs,
fileIDs,
path.Base(dst),
)
}
// 判断是否需要移动
if src.GetPosition() != path.Dir(dst) {
err = fs.Move(
context.WithValue(ctx, fsctx.WebdavDstName, path.Base(dst)),
folderIDs,
fileIDs,
src.GetPosition(),
path.Dir(dst),
)
} else if src.GetName() != path.Base(dst) {
// 判断是否需要重命名
err = fs.Rename(
ctx,
folderIDs,
fileIDs,
path.Base(dst),
)
}
if err != nil {
return http.StatusInternalServerError, err
@@ -74,26 +106,53 @@ func copyFiles(ctx context.Context, fs *filesystem.FileSystem, src FileInfo, dst
}
recursion++
if src.IsDir() {
err := fs.Copy(
ctx,
[]uint{src.(*model.Folder).ID},
[]uint{}, src.(*model.Folder).Position,
path.Dir(dst),
)
if err != nil {
return http.StatusInternalServerError, err
}
} else {
err := fs.Copy(ctx, []uint{}, []uint{src.(*model.File).ID}, src.(*model.File).Position, path.Dir(dst))
if err != nil {
var (
fileIDs []uint
folderIDs []uint
)
if overwrite {
if err := _checkOverwriteFile(ctx, fs, src, dst); err != nil {
return http.StatusInternalServerError, err
}
}
if src.IsDir() {
folderIDs = []uint{src.(*model.Folder).ID}
} else {
fileIDs = []uint{src.(*model.File).ID}
}
err = fs.Copy(
context.WithValue(ctx, fsctx.WebdavDstName, path.Base(dst)),
folderIDs,
fileIDs,
src.GetPosition(),
path.Dir(dst),
)
if err != nil {
return http.StatusInternalServerError, err
}
return http.StatusNoContent, nil
}
// 判断目标 文件/夹 是否已经存在,存在则先删除目标文件/夹
func _checkOverwriteFile(ctx context.Context, fs *filesystem.FileSystem, src FileInfo, dst string) error {
if src.IsDir() {
ok, folder := fs.IsPathExist(dst)
if ok {
return fs.Delete(ctx, []uint{folder.ID}, []uint{}, false, false)
}
} else {
ok, file := fs.IsFileExist(dst)
if ok {
return fs.Delete(ctx, []uint{}, []uint{file.ID}, false, false)
}
}
return nil
}
// walkFS traverses filesystem fs starting at name up to depth levels.
//
// Allowed values for depth are 0, 1 or infiniteDepth. For each visited node,

View File

@@ -16,9 +16,76 @@ import (
"strconv"
"time"
model "github.com/cloudreve/Cloudreve/v3/models"
"github.com/cloudreve/Cloudreve/v3/pkg/filesystem"
)
type FileDeadProps struct {
*model.File
}
// 实现 webdav.DeadPropsHolder 接口不能在models.file里面定义
func (file *FileDeadProps) DeadProps() (map[xml.Name]Property, error) {
return map[xml.Name]Property{
xml.Name{Space: "http://owncloud.org/ns", Local: "checksums"}: {
XMLName: xml.Name{
Space: "http://owncloud.org/ns", Local: "checksums",
},
InnerXML: []byte("<checksum>" + file.MetadataSerialized[model.ChecksumMetadataKey] + "</checksum>"),
},
}, nil
}
func (file *FileDeadProps) Patch(proppatches []Proppatch) ([]Propstat, error) {
var (
stat Propstat
err error
)
stat.Status = http.StatusOK
for _, patch := range proppatches {
for _, prop := range patch.Props {
stat.Props = append(stat.Props, Property{XMLName: prop.XMLName})
if prop.XMLName.Space == "DAV:" && prop.XMLName.Local == "lastmodified" {
var modtimeUnix int64
modtimeUnix, err = strconv.ParseInt(string(prop.InnerXML), 10, 64)
if err == nil {
err = model.DB.Model(file.File).UpdateColumn("updated_at", time.Unix(modtimeUnix, 0)).Error
}
}
}
}
return []Propstat{stat}, err
}
type FolderDeadProps struct {
*model.Folder
}
func (folder *FolderDeadProps) DeadProps() (map[xml.Name]Property, error) {
return nil, nil
}
func (folder *FolderDeadProps) Patch(proppatches []Proppatch) ([]Propstat, error) {
var (
stat Propstat
err error
)
stat.Status = http.StatusOK
for _, patch := range proppatches {
for _, prop := range patch.Props {
stat.Props = append(stat.Props, Property{XMLName: prop.XMLName})
if prop.XMLName.Space == "DAV:" && prop.XMLName.Local == "lastmodified" {
var modtimeUnix int64
modtimeUnix, err = strconv.ParseInt(string(prop.InnerXML), 10, 64)
if err == nil {
err = model.DB.Model(folder.Folder).UpdateColumn("updated_at", time.Unix(modtimeUnix, 0)).Error
}
}
}
}
return []Propstat{stat}, err
}
type FileInfo interface {
GetSize() uint64
GetName() string
@@ -177,8 +244,18 @@ var liveProps = map[xml.Name]struct {
// of one Propstat element.
func props(ctx context.Context, fs *filesystem.FileSystem, ls LockSystem, fi FileInfo, pnames []xml.Name) ([]Propstat, error) {
isDir := fi.IsDir()
if !isDir {
fi = &FileDeadProps{fi.(*model.File)}
}
var deadProps map[xml.Name]Property
if dph, ok := fi.(DeadPropsHolder); ok {
var err error
deadProps, err = dph.DeadProps()
if err != nil {
return nil, err
}
}
pstatOK := Propstat{Status: http.StatusOK}
pstatNotFound := Propstat{Status: http.StatusNotFound}
@@ -210,8 +287,18 @@ func props(ctx context.Context, fs *filesystem.FileSystem, ls LockSystem, fi Fil
// Propnames returns the property names defined for resource name.
func propnames(ctx context.Context, fs *filesystem.FileSystem, ls LockSystem, fi FileInfo) ([]xml.Name, error) {
isDir := fi.IsDir()
if !isDir {
fi = &FileDeadProps{fi.(*model.File)}
}
var deadProps map[xml.Name]Property
if dph, ok := fi.(DeadPropsHolder); ok {
var err error
deadProps, err = dph.DeadProps()
if err != nil {
return nil, err
}
}
pnames := make([]xml.Name, 0, len(liveProps)+len(deadProps))
for pn, prop := range liveProps {
@@ -219,6 +306,9 @@ func propnames(ctx context.Context, fs *filesystem.FileSystem, ls LockSystem, fi
pnames = append(pnames, pn)
}
}
for pn := range deadProps {
pnames = append(pnames, pn)
}
return pnames, nil
}
@@ -281,6 +371,29 @@ loop:
return makePropstats(pstatForbidden, pstatFailedDep), nil
}
// very unlikely to be false
exist, info := isPathExist(ctx, fs, name)
if exist {
var dph DeadPropsHolder
if info.IsDir() {
dph = &FolderDeadProps{info.(*model.Folder)}
} else {
dph = &FileDeadProps{info.(*model.File)}
}
ret, err := dph.Patch(patches)
if err != nil {
return nil, err
}
// http://www.webdav.org/specs/rfc4918.html#ELEMENT_propstat says that
// "The contents of the prop XML element must only list the names of
// properties to which the result in the status element applies."
for _, pstat := range ret {
for i, p := range pstat.Props {
pstat.Props[i] = Property{XMLName: p.XMLName}
}
}
return ret, nil
}
// The file doesn't implement the optional DeadPropsHolder interface, so
// all patches are forbidden.
pstat := Propstat{Status: http.StatusOK}

View File

@@ -10,6 +10,7 @@ import (
"errors"
"fmt"
"net/http"
"net/http/httputil"
"net/url"
"path"
"strconv"
@@ -241,6 +242,23 @@ func (h *Handler) handleOptions(w http.ResponseWriter, r *http.Request, fs *file
return 0, nil
}
var proxy = &httputil.ReverseProxy{
Director: func(request *http.Request) {
if target, ok := request.Context().Value(fsctx.WebDAVProxyUrlCtx).(*url.URL); ok {
request.URL.Scheme = target.Scheme
request.URL.Host = target.Host
request.URL.Path = target.Path
request.URL.RawPath = target.RawPath
request.URL.RawQuery = target.RawQuery
request.Host = target.Host
request.Header.Del("Authorization")
}
},
ErrorHandler: func(writer http.ResponseWriter, request *http.Request, err error) {
writer.WriteHeader(http.StatusInternalServerError)
},
}
// OK
func (h *Handler) handleGetHeadPost(w http.ResponseWriter, r *http.Request, fs *filesystem.FileSystem) (status int, err error) {
defer fs.Recycle()
@@ -279,7 +297,23 @@ func (h *Handler) handleGetHeadPost(w http.ResponseWriter, r *http.Request, fs *
return 0, nil
}
http.Redirect(w, r, rs.URL, 301)
if application, ok := r.Context().Value(fsctx.WebDAVCtx).(*model.Webdav); ok && application.UseProxy {
target, err := url.Parse(rs.URL)
if err != nil {
return http.StatusInternalServerError, err
}
r = r.Clone(context.WithValue(r.Context(), fsctx.WebDAVProxyUrlCtx, target))
// 忽略反向代理在传输错误时报错
defer func() {
if err := recover(); err != nil && err != http.ErrAbortHandler {
panic(err)
}
}()
proxy.ServeHTTP(w, r)
} else {
http.Redirect(w, r, rs.URL, 301)
}
return 0, nil
}
@@ -345,7 +379,7 @@ func (h *Handler) handlePut(w http.ResponseWriter, r *http.Request, fs *filesyst
fileName := path.Base(reqPath)
filePath := path.Dir(reqPath)
fileData := fsctx.FileStream{
MIMEType: r.Header.Get("Content-Type"),
MimeType: r.Header.Get("Content-Type"),
File: r.Body,
Size: fileSize,
Name: fileName,
@@ -386,10 +420,12 @@ func (h *Handler) handlePut(w http.ResponseWriter, r *http.Request, fs *filesyst
fs.Use("AfterUploadCanceled", filesystem.HookDeleteTempFile)
fs.Use("AfterUploadCanceled", filesystem.HookCancelContext)
fs.Use("AfterUpload", filesystem.GenericAfterUpload)
fs.Use("AfterUpload", filesystem.HookGenerateThumb)
fs.Use("AfterValidateFailed", filesystem.HookDeleteTempFile)
}
// rclone 请求
fs.Use("AfterUpload", filesystem.NewWebdavAfterUploadHook(r))
// 执行上传
err = fs.Upload(ctx, &fileData)
if err != nil {
@@ -494,7 +530,16 @@ func (h *Handler) handleCopyMove(w http.ResponseWriter, r *http.Request, fs *fil
return http.StatusBadRequest, errInvalidDepth
}
}
return copyFiles(ctx, fs, target, dst, r.Header.Get("Overwrite") != "F", depth, 0)
status, err = copyFiles(ctx, fs, target, dst, r.Header.Get("Overwrite") != "F", depth, 0)
if err != nil {
return status, err
}
err = updateCopyMoveModtime(r, fs, dst)
if err != nil {
return http.StatusInternalServerError, err
}
return status, nil
}
// windows下某些情况下网盘根目录下Office保存文件时附带的锁token只包含源文件
@@ -513,7 +558,16 @@ func (h *Handler) handleCopyMove(w http.ResponseWriter, r *http.Request, fs *fil
return http.StatusBadRequest, errInvalidDepth
}
}
return moveFiles(ctx, fs, target, dst, r.Header.Get("Overwrite") == "T")
status, err = moveFiles(ctx, fs, target, dst, r.Header.Get("Overwrite") == "T")
if err != nil {
return status, err
}
err = updateCopyMoveModtime(r, fs, dst)
if err != nil {
return http.StatusInternalServerError, err
}
return status, nil
}
// OK

View File

@@ -18,7 +18,7 @@ import (
type Client interface {
// NewSession creates a new document session with access token.
NewSession(user *model.User, file *model.File, action ActonType) (*Session, error)
NewSession(uid uint, file *model.File, action ActonType) (*Session, error)
// AvailableExts returns a list of file extensions that are supported by WOPI.
AvailableExts() []string
}
@@ -59,6 +59,7 @@ const (
wopiSrcPlaceholder = "WOPI_SOURCE"
wopiSrcParamDefault = "WOPISrc"
languageParamDefault = "lang"
sessionExpiresPadding = 10
wopiHeaderPrefix = "X-WOPI-"
)
@@ -115,7 +116,7 @@ func NewClient(endpoint string, cache cache.Driver, http request.Client) (Client
}, nil
}
func (c *client) NewSession(user *model.User, file *model.File, action ActonType) (*Session, error) {
func (c *client) NewSession(uid uint, file *model.File, action ActonType) (*Session, error) {
if err := c.checkDiscovery(); err != nil {
return nil, err
}
@@ -162,7 +163,7 @@ func (c *client) NewSession(user *model.User, file *model.File, action ActonType
session := &SessionCache{
AccessToken: fmt.Sprintf("%s.%s", sessionID, token),
FileID: file.ID,
UserID: user.ID,
UserID: uid,
Action: action,
}
err = c.cache.Set(SessionCachePrefix+sessionID.String(), *session, ttl)
@@ -214,6 +215,9 @@ func generateActionUrl(src string, fileSrc string) (*url.URL, error) {
queryReplaced.Set(wopiSrcParamDefault, fileSrc)
}
// LibreOffice require this flag to show correct language
queryReplaced.Set(languageParamDefault, "lng")
actionUrl.RawQuery = queryReplaced.Encode()
return actionUrl, nil
}

View File

@@ -55,7 +55,7 @@ func TestNewSession(t *testing.T) {
).Return(&request.Response{
Err: expectedErr,
})
res, err := client.NewSession(&model.User{}, &model.File{}, ActionPreview)
res, err := client.NewSession(0, &model.File{}, ActionPreview)
a.Nil(res)
a.ErrorIs(err, expectedErr)
mockHttp.AssertExpectations(t)
@@ -65,7 +65,7 @@ func TestNewSession(t *testing.T) {
{
client.discovery = &WopiDiscovery{}
client.actions = make(map[string]map[string]Action)
res, err := client.NewSession(&model.User{}, &model.File{}, ActionPreview)
res, err := client.NewSession(0, &model.File{}, ActionPreview)
a.Nil(res)
a.ErrorIs(err, ErrActionNotSupported)
}
@@ -76,7 +76,7 @@ func TestNewSession(t *testing.T) {
client.actions = map[string]map[string]Action{
".doc": {},
}
res, err := client.NewSession(&model.User{}, &model.File{Name: "1.doc"}, ActionPreview)
res, err := client.NewSession(0, &model.File{Name: "1.doc"}, ActionPreview)
a.Nil(res)
a.ErrorIs(err, ErrActionNotSupported)
}
@@ -91,7 +91,7 @@ func TestNewSession(t *testing.T) {
},
},
}
res, err := client.NewSession(&model.User{}, &model.File{Name: "1.doc"}, ActionEdit)
res, err := client.NewSession(0, &model.File{Name: "1.doc"}, ActionEdit)
a.Nil(res)
a.ErrorContains(err, "invalid control character in URL")
}
@@ -106,7 +106,7 @@ func TestNewSession(t *testing.T) {
},
},
}
res, err := client.NewSession(&model.User{}, &model.File{Name: "1.doc"}, ActionEdit)
res, err := client.NewSession(0, &model.File{Name: "1.doc"}, ActionEdit)
a.NotNil(res)
a.NoError(err)
resUrl := res.ActionURL.String()
@@ -123,7 +123,7 @@ func TestNewSession(t *testing.T) {
},
},
}
res, err := client.NewSession(&model.User{}, &model.File{Name: "1.doc"}, ActionEdit)
res, err := client.NewSession(0, &model.File{Name: "1.doc"}, ActionEdit)
a.NotNil(res)
a.NoError(err)
resUrl := res.ActionURL.String()
@@ -147,7 +147,7 @@ func TestNewSession(t *testing.T) {
},
}
mockCache.On("Set", testMock.Anything, testMock.Anything, testMock.Anything).Return(expectedErr)
res, err := client.NewSession(&model.User{}, &model.File{Name: "1.doc"}, ActionEdit)
res, err := client.NewSession(0, &model.File{Name: "1.doc"}, ActionEdit)
a.Nil(res)
a.ErrorIs(err, expectedErr)
}

View File

@@ -98,6 +98,17 @@ func AdminSendTestMail(c *gin.Context) {
}
}
// AdminTestThumbGenerator Tests thumb generator
func AdminTestThumbGenerator(c *gin.Context) {
var service admin.ThumbGeneratorTestService
if err := c.ShouldBindJSON(&service); err == nil {
res := service.Test(c)
c.JSON(200, res)
} else {
c.JSON(200, ErrorResponse(err))
}
}
// AdminTestAria2 测试aria2连接
func AdminTestAria2(c *gin.Context) {
var service admin.Aria2TestService
@@ -181,14 +192,16 @@ func AdminAddSCF(c *gin.Context) {
}
}
// AdminOneDriveOAuth 获取 OneDrive OAuth URL
func AdminOneDriveOAuth(c *gin.Context) {
var service admin.PolicyService
if err := c.ShouldBindUri(&service); err == nil {
res := service.GetOAuth(c)
c.JSON(200, res)
} else {
c.JSON(200, ErrorResponse(err))
// AdminOAuthURL 获取 OneDrive OAuth URL
func AdminOAuthURL(policyType string) gin.HandlerFunc {
return func(c *gin.Context) {
var service admin.PolicyService
if err := c.ShouldBindUri(&service); err == nil {
res := service.GetOAuth(c, policyType)
c.JSON(200, res)
} else {
c.JSON(200, ErrorResponse(err))
}
}
}

View File

@@ -83,9 +83,27 @@ func OneDriveCallback(c *gin.Context) {
// OneDriveOAuth OneDrive 授权回调
func OneDriveOAuth(c *gin.Context) {
var callbackBody callback.OneDriveOauthService
var callbackBody callback.OauthService
if err := c.ShouldBindQuery(&callbackBody); err == nil {
res := callbackBody.Auth(c)
res := callbackBody.OdAuth(c)
redirect := model.GetSiteURL()
redirect.Path = path.Join(redirect.Path, "/admin/policy")
queries := redirect.Query()
queries.Add("code", strconv.Itoa(res.Code))
queries.Add("msg", res.Msg)
queries.Add("err", res.Error)
redirect.RawQuery = queries.Encode()
c.Redirect(303, redirect.String())
} else {
c.JSON(200, ErrorResponse(err))
}
}
// GoogleDriveOAuth Google Drive 授权回调
func GoogleDriveOAuth(c *gin.Context) {
var callbackBody callback.OauthService
if err := c.ShouldBindQuery(&callbackBody); err == nil {
res := callbackBody.GDriveAuth(c)
redirect := model.GetSiteURL()
redirect.Path = path.Join(redirect.Path, "/admin/policy")
queries := redirect.Query()

View File

@@ -223,9 +223,9 @@ func SlaveNotificationPush(c *gin.Context) {
}
}
// SlaveGetOneDriveCredential 从机获取主机的OneDrive存储策略凭证
func SlaveGetOneDriveCredential(c *gin.Context) {
var service node.OneDriveCredentialService
// SlaveGetOauthCredential 从机获取主机的OneDrive存储策略凭证
func SlaveGetOauthCredential(c *gin.Context) {
var service node.OauthCredentialService
if err := c.ShouldBindUri(&service); err == nil {
res := service.Get(c)
c.JSON(200, res)

View File

@@ -1,8 +1,10 @@
package controllers
import (
"context"
model "github.com/cloudreve/Cloudreve/v3/models"
"github.com/cloudreve/Cloudreve/v3/pkg/filesystem"
"github.com/cloudreve/Cloudreve/v3/pkg/filesystem/fsctx"
"github.com/cloudreve/Cloudreve/v3/pkg/util"
"github.com/cloudreve/Cloudreve/v3/pkg/webdav"
"github.com/cloudreve/Cloudreve/v3/service/setting"
@@ -49,6 +51,9 @@ func ServeWebDAV(c *gin.Context) {
return
}
}
// 更新Context
c.Request = c.Request.WithContext(context.WithValue(c.Request.Context(), fsctx.WebDAVCtx, application))
}
handler.ServeHTTP(c.Writer, c.Request, fs)
@@ -76,9 +81,9 @@ func DeleteWebDAVAccounts(c *gin.Context) {
}
}
// UpdateWebDAVAccountsReadonly 更改WebDAV账户只读性
func UpdateWebDAVAccountsReadonly(c *gin.Context) {
var service setting.WebDAVAccountUpdateReadonlyService
// UpdateWebDAVAccounts 更改WebDAV账户只读性和是否使用代理服务
func UpdateWebDAVAccounts(c *gin.Context) {
var service setting.WebDAVAccountUpdateService
if err := c.ShouldBindJSON(&service); err == nil {
res := service.Update(c, CurrentUser(c))
c.JSON(200, res)

View File

@@ -64,7 +64,7 @@ func InitSlaveRouter() *gin.Engine {
// 预览 / 外链
v3.GET("source/:speed/:path/:name", controllers.SlavePreview)
// 缩略图
v3.GET("thumb/:path", controllers.SlaveThumb)
v3.GET("thumb/:path/:ext", controllers.SlaveThumb)
// 删除文件
v3.POST("delete", controllers.SlaveDelete)
// 列出文件
@@ -260,8 +260,8 @@ func InitMasterRouter() *gin.Engine {
// 删除上传会话
upload.DELETE(":sessionId", controllers.SlaveDeleteUploadSession)
}
// OneDrive 存储策略凭证
slave.GET("credential/onedrive/:id", controllers.SlaveGetOneDriveCredential)
// Oauth 存储策略凭证
slave.GET("credential/:id", controllers.SlaveGetOauthCredential)
}
// 回调接口
@@ -310,6 +310,15 @@ func InitMasterRouter() *gin.Engine {
controllers.OneDriveOAuth,
)
}
// Google Drive related
gdrive := callback.Group("googledrive")
{
// OAuth 完成
gdrive.GET(
"auth",
controllers.GoogleDriveOAuth,
)
}
// 腾讯云COS策略上传回调
callback.GET(
"cos/:sessionID",
@@ -422,8 +431,14 @@ func InitMasterRouter() *gin.Engine {
admin.GET("groups", controllers.AdminGetGroups)
// 重新加载子服务
admin.GET("reload/:service", controllers.AdminReloadService)
// 重新加载子服务
admin.POST("mailTest", controllers.AdminSendTestMail)
// 测试设置
test := admin.Group("test")
{
// 测试邮件设置
test.POST("mail", controllers.AdminSendTestMail)
// 测试缩略图生成器调用
test.POST("thumb", controllers.AdminTestThumbGenerator)
}
// 离线下载相关
aria2 := admin.Group("aria2")
@@ -448,7 +463,14 @@ func InitMasterRouter() *gin.Engine {
// 创建COS回调函数
policy.POST("scf", controllers.AdminAddSCF)
// 获取 OneDrive OAuth URL
policy.GET(":id/oauth", controllers.AdminOneDriveOAuth)
oauth := policy.Group(":id/oauth")
{
// 获取 OneDrive OAuth URL
oauth.GET("onedrive", controllers.AdminOAuthURL("onedrive"))
// 获取 Google Drive OAuth URL
oauth.GET("googledrive", controllers.AdminOAuthURL("googledrive"))
}
// 获取 存储策略
policy.GET(":id", controllers.AdminGetPolicy)
// 删除 存储策略
@@ -699,8 +721,8 @@ func InitMasterRouter() *gin.Engine {
webdav.POST("accounts", controllers.CreateWebDAVAccounts)
// 删除账号
webdav.DELETE("accounts/:id", controllers.DeleteWebDAVAccounts)
// 更新账号可读性
webdav.PATCH("accounts", controllers.UpdateWebDAVAccountsReadonly)
// 更新账号可读性和是否使用代理服务
webdav.PATCH("accounts", controllers.UpdateWebDAVAccounts)
}
}

View File

@@ -5,6 +5,7 @@ import (
"context"
"encoding/json"
"fmt"
"github.com/cloudreve/Cloudreve/v3/pkg/filesystem/driver/googledrive"
"net/http"
"net/url"
"os"
@@ -104,27 +105,41 @@ func (service *PolicyService) Get() serializer.Response {
}
// GetOAuth 获取 OneDrive OAuth 地址
func (service *PolicyService) GetOAuth(c *gin.Context) serializer.Response {
func (service *PolicyService) GetOAuth(c *gin.Context, policyType string) serializer.Response {
policy, err := model.GetPolicyByID(service.ID)
if err != nil || policy.Type != "onedrive" {
if err != nil || policy.Type != policyType {
return serializer.Err(serializer.CodePolicyNotExist, "", nil)
}
client, err := onedrive.NewClient(&policy)
if err != nil {
return serializer.Err(serializer.CodeInternalSetting, "Failed to initialize OneDrive client", err)
}
util.SetSession(c, map[string]interface{}{
"onedrive_oauth_policy": policy.ID,
policyType + "_oauth_policy": policy.ID,
})
cache.Deletes([]string{policy.BucketName}, "onedrive_")
var redirect string
switch policy.Type {
case "onedrive":
client, err := onedrive.NewClient(&policy)
if err != nil {
return serializer.Err(serializer.CodeInternalSetting, "Failed to initialize OneDrive client", err)
}
return serializer.Response{Data: client.OAuthURL(context.Background(), []string{
"offline_access",
"files.readwrite.all",
})}
redirect = client.OAuthURL(context.Background(), []string{
"offline_access",
"files.readwrite.all",
})
case "googledrive":
client, err := googledrive.NewClient(&policy)
if err != nil {
return serializer.Err(serializer.CodeInternalSetting, "Failed to initialize Google Drive client", err)
}
redirect = client.OAuthURL(context.Background(), googledrive.RequiredScope)
}
// Delete token cache
cache.Deletes([]string{policy.BucketName}, policyType+"_")
return serializer.Response{Data: redirect}
}
// AddSCF 创建回调云函数

View File

@@ -9,6 +9,8 @@ import (
"github.com/cloudreve/Cloudreve/v3/pkg/conf"
"github.com/cloudreve/Cloudreve/v3/pkg/email"
"github.com/cloudreve/Cloudreve/v3/pkg/serializer"
"github.com/cloudreve/Cloudreve/v3/pkg/thumb"
"github.com/gin-gonic/gin"
)
func init() {
@@ -143,3 +145,21 @@ func (service *NoParamService) Summary() serializer.Response {
Data: resp,
}
}
// ThumbGeneratorTestService 缩略图生成测试服务
type ThumbGeneratorTestService struct {
Name string `json:"name" binding:"required"`
Executable string `json:"executable" binding:"required"`
}
// Test 通过获取生成器版本来测试
func (s *ThumbGeneratorTestService) Test(c *gin.Context) serializer.Response {
version, err := thumb.TestGenerator(c, s.Name, s.Executable)
if err != nil {
return serializer.Err(serializer.CodeParamErr, err.Error(), err)
}
return serializer.Response{
Data: version,
}
}

View File

@@ -6,22 +6,70 @@ import (
model "github.com/cloudreve/Cloudreve/v3/models"
"github.com/cloudreve/Cloudreve/v3/pkg/cache"
"github.com/cloudreve/Cloudreve/v3/pkg/filesystem/driver/googledrive"
"github.com/cloudreve/Cloudreve/v3/pkg/filesystem/driver/onedrive"
"github.com/cloudreve/Cloudreve/v3/pkg/serializer"
"github.com/cloudreve/Cloudreve/v3/pkg/util"
"github.com/gin-gonic/gin"
"github.com/samber/lo"
"strings"
)
// OneDriveOauthService OneDrive 授权回调服务
type OneDriveOauthService struct {
// OauthService OAuth 存储策略授权回调服务
type OauthService struct {
Code string `form:"code"`
Error string `form:"error"`
ErrorMsg string `form:"error_description"`
Scope string `form:"scope"`
}
// Auth 更新认证信息
func (service *OneDriveOauthService) Auth(c *gin.Context) serializer.Response {
// GDriveAuth Google Drive 更新认证信息
func (service *OauthService) GDriveAuth(c *gin.Context) serializer.Response {
if service.Error != "" {
return serializer.ParamErr(service.Error, nil)
}
// validate required scope
if missing, found := lo.Find[string](googledrive.RequiredScope, func(item string) bool {
return !strings.Contains(service.Scope, item)
}); found {
return serializer.ParamErr(fmt.Sprintf("Missing required scope: %s", missing), nil)
}
policyID, ok := util.GetSession(c, "googledrive_oauth_policy").(uint)
if !ok {
return serializer.Err(serializer.CodeNotFound, "", nil)
}
util.DeleteSession(c, "googledrive_oauth_policy")
policy, err := model.GetPolicyByID(policyID)
if err != nil {
return serializer.Err(serializer.CodePolicyNotExist, "", nil)
}
client, err := googledrive.NewClient(&policy)
if err != nil {
return serializer.Err(serializer.CodeInternalSetting, "Failed to initialize Google Drive client", err)
}
credential, err := client.ObtainToken(c, service.Code, "")
if err != nil {
return serializer.Err(serializer.CodeInternalSetting, "Failed to fetch AccessToken", err)
}
// 更新存储策略的 RefreshToken
client.Policy.AccessKey = credential.RefreshToken
if err := client.Policy.SaveAndClearCache(); err != nil {
return serializer.DBErr("Failed to update RefreshToken", err)
}
cache.Deletes([]string{client.Policy.AccessKey}, googledrive.TokenCachePrefix)
return serializer.Response{}
}
// OdAuth OneDrive 更新认证信息
func (service *OauthService) OdAuth(c *gin.Context) serializer.Response {
if service.Error != "" {
return serializer.ParamErr(service.ErrorMsg, nil)
}

View File

@@ -227,6 +227,10 @@ func (service *FileIDService) CreateDocPreviewSession(ctx context.Context, c *gi
return serializer.Err(serializer.CodeNotSet, err.Error(), err)
}
if strings.HasPrefix(downloadURL, "/") {
downloadURL = path.Join(model.GetSiteURL().String(), downloadURL)
}
var resp serializer.DocPreviewSession
// Use WOPI preview if available
@@ -241,7 +245,7 @@ func (service *FileIDService) CreateDocPreviewSession(ctx context.Context, c *gi
action = wopi.ActionEdit
}
session, err := wopi.Default.NewSession(fs.User, &fs.FileTarget[0], action)
session, err := wopi.Default.NewSession(fs.FileTarget[0].UserID, &fs.FileTarget[0], action)
if err != nil {
return serializer.Err(serializer.CodeInternalSetting, "Failed to create WOPI session", err)
}
@@ -408,7 +412,7 @@ func (service *FileIDService) PutContent(ctx context.Context, c *gin.Context) se
}
fileData := fsctx.FileStream{
MIMEType: c.Request.Header.Get("Content-Type"),
MimeType: c.Request.Header.Get("Content-Type"),
File: c.Request.Body,
Size: fileSize,
Mode: fsctx.Overwrite,

View File

@@ -32,6 +32,7 @@ type SlaveDownloadService struct {
// SlaveFileService 从机单文件文件相关服务
type SlaveFileService struct {
PathEncoded string `uri:"path" binding:"required"`
Ext string `uri:"ext"`
}
// SlaveFilesService 从机多文件相关服务
@@ -132,7 +133,7 @@ func (service *SlaveFileService) Thumb(ctx context.Context, c *gin.Context) seri
if err != nil {
return serializer.Err(serializer.CodeFileNotFound, "", err)
}
fs.FileTarget = []model.File{{SourceName: string(fileSource), PicInfo: "1,1"}}
fs.FileTarget = []model.File{{SourceName: string(fileSource), Name: fmt.Sprintf("%s.%s", fileSource, service.Ext), PicInfo: "1,1"}}
// 获取缩略图
resp, err := fs.GetThumb(ctx, 0)

View File

@@ -26,6 +26,7 @@ type CreateUploadSessionService struct {
Name string `json:"name" binding:"required"`
PolicyID string `json:"policy_id" binding:"required"`
LastModified int64 `json:"last_modified"`
MimeType string `json:"mime_type"`
}
// Create 创建新的上传会话
@@ -51,6 +52,7 @@ func (service *CreateUploadSessionService) Create(ctx context.Context, c *gin.Co
Name: service.Name,
VirtualPath: service.Path,
File: ioutil.NopCloser(strings.NewReader("")),
MimeType: service.MimeType,
}
if service.LastModified > 0 {
lastModified := time.UnixMilli(service.LastModified)
@@ -174,7 +176,7 @@ func processChunkUpload(ctx context.Context, c *gin.Context, fs *filesystem.File
}
fileData := fsctx.FileStream{
MIMEType: c.Request.Header.Get("Content-Type"),
MimeType: c.Request.Header.Get("Content-Type"),
File: c.Request.Body,
Size: fileSize,
Name: session.Name,
@@ -196,7 +198,6 @@ func processChunkUpload(ctx context.Context, c *gin.Context, fs *filesystem.File
fs.Use("AfterValidateFailed", filesystem.HookChunkUploadFailed)
if isLastChunk {
fs.Use("AfterUpload", filesystem.HookPopPlaceholderToFile(""))
fs.Use("AfterUpload", filesystem.HookGenerateThumb)
fs.Use("AfterUpload", filesystem.HookDeleteUploadSession(session.Key))
}
} else {

View File

@@ -5,7 +5,9 @@ import (
model "github.com/cloudreve/Cloudreve/v3/models"
"github.com/cloudreve/Cloudreve/v3/pkg/cluster"
"github.com/cloudreve/Cloudreve/v3/pkg/conf"
"github.com/cloudreve/Cloudreve/v3/pkg/filesystem/driver/googledrive"
"github.com/cloudreve/Cloudreve/v3/pkg/filesystem/driver/onedrive"
"github.com/cloudreve/Cloudreve/v3/pkg/filesystem/oauth"
"github.com/cloudreve/Cloudreve/v3/pkg/mq"
"github.com/cloudreve/Cloudreve/v3/pkg/serializer"
"github.com/gin-gonic/gin"
@@ -15,7 +17,7 @@ type SlaveNotificationService struct {
Subject string `uri:"subject" binding:"required"`
}
type OneDriveCredentialService struct {
type OauthCredentialService struct {
PolicyID uint `uri:"id" binding:"required"`
}
@@ -43,21 +45,32 @@ func (s *SlaveNotificationService) HandleSlaveNotificationPush(c *gin.Context) s
return serializer.Response{}
}
// Get 获取主机OneDrive策略的AccessToken
func (s *OneDriveCredentialService) Get(c *gin.Context) serializer.Response {
// Get 获取主机Oauth策略的AccessToken
func (s *OauthCredentialService) Get(c *gin.Context) serializer.Response {
policy, err := model.GetPolicyByID(s.PolicyID)
if err != nil {
return serializer.Err(serializer.CodePolicyNotExist, "", err)
}
client, err := onedrive.NewClient(&policy)
if err != nil {
return serializer.Err(serializer.CodeInternalSetting, "Cannot initialize OneDrive client", err)
var client oauth.TokenProvider
switch policy.Type {
case "onedrive":
client, err = onedrive.NewClient(&policy)
if err != nil {
return serializer.Err(serializer.CodeInternalSetting, "Cannot initialize OneDrive client", err)
}
case "googledrive":
client, err = googledrive.NewClient(&policy)
if err != nil {
return serializer.Err(serializer.CodeInternalSetting, "Cannot initialize Google Drive client", err)
}
default:
return serializer.Err(serializer.CodePolicyNotExist, "", nil)
}
if err := client.UpdateCredential(c, conf.SystemConfig.Mode == "slave"); err != nil {
return serializer.Err(serializer.CodeInternalSetting, "Cannot refresh OneDrive credential", err)
}
return serializer.Response{Data: client.Credential.AccessToken}
return serializer.Response{Data: client.AccessToken()}
}

Some files were not shown because too many files have changed in this diff Show More