Compare commits

..

2 Commits
3.5.0 ... 3.4.3

Author SHA1 Message Date
HFO4
a910a20457 chore: build manually 2022-04-16 14:27:36 +08:00
HFO4
c4fc4a803d update version number 2022-04-16 14:21:33 +08:00
142 changed files with 5002 additions and 5808 deletions

61
.github/stale.yml vendored
View File

@@ -1,61 +0,0 @@
# Configuration for probot-stale - https://github.com/probot/stale
# Number of days of inactivity before an Issue or Pull Request becomes stale
daysUntilStale: 360
# Number of days of inactivity before an Issue or Pull Request with the stale label is closed.
# Set to false to disable. If disabled, issues still need to be closed manually, but will remain marked as stale.
daysUntilClose: 30
# Only issues or pull requests with all of these labels are check if stale. Defaults to `[]` (disabled)
onlyLabels: []
# Issues or Pull Requests with these labels will never be considered stale. Set to `[]` to disable
exemptLabels:
- pinned
- security
- "[Status] Maybe Later"
# Set to true to ignore issues in a project (defaults to false)
exemptProjects: true
# Set to true to ignore issues in a milestone (defaults to false)
exemptMilestones: true
# Set to true to ignore issues with an assignee (defaults to false)
exemptAssignees: true
# Label to use when marking as stale
staleLabel: wontfix
# Comment to post when marking as stale. Set to `false` to disable
markComment: >
This issue has been automatically marked as stale because it has not had
recent activity. It will be closed if no further activity occurs. Thank you
for your contributions.
# Comment to post when removing the stale label.
# unmarkComment: >
# Your comment here.
# Comment to post when closing a stale Issue or Pull Request.
# closeComment: >
# Your comment here.
# Limit the number of actions per hour, from 1-30. Default is 30
limitPerRun: 30
# Limit to only `issues` or `pulls`
# only: issues
# Optionally, specify configuration settings that are specific to just 'issues' or 'pulls':
# pulls:
# daysUntilStale: 30
# markComment: >
# This pull request has been automatically marked as stale because it has not had
# recent activity. It will be closed if no further activity occurs. Thank you
# for your contributions.
# issues:
# exemptLabels:
# - confirmed

View File

@@ -7,49 +7,53 @@ jobs:
name: Build
runs-on: ubuntu-18.04
steps:
- name: Set up Go 1.17
uses: actions/setup-go@v2
with:
go-version: "1.17"
id: go
- name: Check out code into the Go module directory
uses: actions/checkout@v2
with:
clean: false
submodules: "recursive"
- run: |
git fetch --prune --unshallow --tags
- name: Set up Go 1.13
uses: actions/setup-go@v1
with:
go-version: 1.13
id: go
- name: Get dependencies and build
run: |
sudo apt-get update
sudo apt-get -y install gcc-mingw-w64-x86-64
sudo apt-get -y install gcc-arm-linux-gnueabihf libc6-dev-armhf-cross
sudo apt-get -y install gcc-aarch64-linux-gnu libc6-dev-arm64-cross
chmod +x ./build.sh
./build.sh -r b
- name: Check out code into the Go module directory
uses: actions/checkout@v2
with:
clean: false
submodules: 'recursive'
- run: |
git fetch --prune --unshallow --tags
- name: Upload binary files (windows_amd64)
uses: actions/upload-artifact@v2
with:
name: cloudreve_windows_amd64
path: release/cloudreve*windows_amd64.*
- name: Get dependencies and build
run: |
go get github.com/rakyll/statik
export PATH=$PATH:~/go/bin/
statik -src=models -f
sudo apt-get update
sudo apt-get -y install gcc-mingw-w64-x86-64
sudo apt-get -y install gcc-arm-linux-gnueabihf libc6-dev-armhf-cross
sudo apt-get -y install gcc-aarch64-linux-gnu libc6-dev-arm64-cross
chmod +x ./build.sh
./build.sh -r b
- name: Upload binary files (linux_amd64)
uses: actions/upload-artifact@v2
with:
name: cloudreve_linux_amd64
path: release/cloudreve*linux_amd64.*
- name: Upload binary files (windows_amd64)
uses: actions/upload-artifact@v2
with:
name: cloudreve_windows_amd64
path: release/cloudreve*windows_amd64.*
- name: Upload binary files (linux_arm)
uses: actions/upload-artifact@v2
with:
name: cloudreve_linux_arm
path: release/cloudreve*linux_arm.*
- name: Upload binary files (linux_amd64)
uses: actions/upload-artifact@v2
with:
name: cloudreve_linux_amd64
path: release/cloudreve*linux_amd64.*
- name: Upload binary files (linux_arm64)
uses: actions/upload-artifact@v2
with:
name: cloudreve_linux_arm64
path: release/cloudreve*linux_arm64.*
- name: Upload binary files (linux_arm)
uses: actions/upload-artifact@v2
with:
name: cloudreve_linux_arm
path: release/cloudreve*linux_arm.*
- name: Upload binary files (linux_arm64)
uses: actions/upload-artifact@v2
with:
name: cloudreve_linux_arm64
path: release/cloudreve*linux_arm64.*

View File

@@ -1,50 +0,0 @@
name: Build and push docker image
on:
push:
tags:
- 3.* # triggered on every push with tag 3.*
workflow_dispatch: # or just on button clicked
jobs:
docker-build:
runs-on: ubuntu-latest
steps:
- name: Checkout
uses: actions/checkout@v2
- run: git fetch --prune --unshallow
- name: Setup Environments
id: envs
run: |
CLOUDREVE_LATEST_TAG=$(git describe --tags --abbrev=0)
DOCKER_IMAGE="cloudreve/cloudreve"
echo "RELEASE_VERSION=${GITHUB_REF#refs}"
TAGS="${DOCKER_IMAGE}:latest,${DOCKER_IMAGE}:${CLOUDREVE_LATEST_TAG}"
echo "CLOUDREVE_LATEST_TAG:${CLOUDREVE_LATEST_TAG}"
echo ::set-output name=tags::${TAGS}
- name: Setup QEMU Emulator
uses: docker/setup-qemu-action@master
with:
platforms: all
- name: Setup Docker Buildx Command
id: buildx
uses: docker/setup-buildx-action@master
- name: Login to Dockerhub
uses: docker/login-action@v1
with:
username: ${{ secrets.DOCKERHUB_USERNAME }}
password: ${{ secrets.DOCKERHUB_PASSWORD }}
- name: Build Docker Image and Push
id: docker_build
uses: docker/build-push-action@v2
with:
push: true
builder: ${{ steps.buildx.outputs.name }}
context: .
file: ./Dockerfile
platforms: linux/amd64,linux/arm64,linux/arm/v7
tags: ${{ steps.envs.outputs.tags }}
- name: Image Digest
run: echo ${{ steps.docker_build.outputs.digest }}

View File

@@ -2,34 +2,46 @@ name: Test
on:
pull_request:
branches:
branches:
- master
push:
branches: [master]
branches: [ master ]
jobs:
test:
name: Test
runs-on: ubuntu-18.04
steps:
- name: Set up Go 1.17
uses: actions/setup-go@v2
with:
go-version: "1.17"
id: go
- name: Check out code into the Go module directory
uses: actions/checkout@v2
with:
submodules: "recursive"
- name: Set up Go 1.13
uses: actions/setup-go@v1
with:
go-version: 1.13
id: go
- name: Build static files
run: |
mkdir assets/build
touch assets/build/test.html
- name: Check out code into the Go module directory
uses: actions/checkout@v2
with:
submodules: 'recursive'
- name: Test
run: go test -coverprofile=coverage.txt -covermode=atomic ./...
- name: Get dependencies
run: |
go get github.com/rakyll/statik
export PATH=$PATH:~/go/bin/
statik -src=models -f
- name: Upload coverage reports to Codecov
uses: codecov/codecov-action@v2
- name: Test
run: go test -coverprofile=coverage.txt -covermode=atomic ./...
- name: Upload binary files (linux_arm)
uses: actions/upload-artifact@v2
with:
name: cloudreve_linux_arm
path: release/cloudreve*linux_arm.*
- name: Upload binary files (linux_arm64)
uses: actions/upload-artifact@v2
with:
name: cloudreve_linux_arm64
path: release/cloudreve*linux_arm64.*

View File

@@ -1,12 +1,13 @@
language: go
go:
- 1.17.x
- 1.13.x
node_js: "12.16.3"
git:
depth: 1
before_install:
- mkdir assets/build
- touch assets/build/test.html
install:
- go get github.com/rakyll/statik
before_script:
- statik -src=models -f
script:
- go test -coverprofile=coverage.txt -covermode=atomic ./...
after_success:
@@ -26,4 +27,4 @@ deploy:
draft: true
skip_cleanup: true
on:
tags: true
tags: true

View File

@@ -1,41 +1,72 @@
FROM golang:1.17-alpine as cloudreve_builder
# install dependencies and build tools
RUN apk update && apk add --no-cache wget curl git yarn build-base gcc abuild binutils binutils-doc gcc-doc
WORKDIR /cloudreve_builder
RUN git clone --recurse-submodules https://github.com/cloudreve/Cloudreve.git
# build frontend
WORKDIR /cloudreve_builder/Cloudreve/assets
RUN yarn install --network-timeout 1000000
RUN yarn run build && rm -rf build/*.map
FROM node:lts-buster AS fe-builder
COPY ./assets /assets
WORKDIR /assets
# If encountered problems like JavaScript heap out of memory, please uncomment the following options
ENV NODE_OPTIONS --max_old_space_size=4096
# yarn repo connection is unstable, adjust the network timeout to 10 min.
RUN set -ex \
&& yarn install --network-timeout 600000 \
&& yarn run build
# build backend
WORKDIR /cloudreve_builder/Cloudreve
RUN tag_name=$(git describe --tags) \
&& export COMMIT_SHA=$(git rev-parse --short HEAD) \
&& go build -a -o cloudreve -ldflags " -X 'github.com/HFO4/cloudreve/pkg/conf.BackendVersion=$tag_name' -X 'github.com/HFO4/cloudreve/pkg/conf.LastCommit=$COMMIT_SHA'"
FROM golang:1.15.1-alpine3.12 AS be-builder
ENV GO111MODULE on
COPY . /go/src/github.com/cloudreve/Cloudreve/v3
COPY --from=fe-builder /assets/build/ /go/src/github.com/cloudreve/Cloudreve/v3/assets/build/
WORKDIR /go/src/github.com/cloudreve/Cloudreve/v3
RUN set -ex \
&& apk upgrade \
&& apk add gcc libc-dev git \
&& export COMMIT_SHA=$(git rev-parse --short HEAD) \
&& export VERSION=$(git describe --tags) \
&& (cd && go get github.com/rakyll/statik) \
&& statik -src=assets/build/ -include=*.html,*.js,*.json,*.css,*.png,*.svg,*.ico -f \
&& go install -ldflags "-X 'github.com/cloudreve/Cloudreve/v3/pkg/conf.BackendVersion=${VERSION}' \
-X 'github.com/cloudreve/Cloudreve/v3/pkg/conf.LastCommit=${COMMIT_SHA}'\
-w -s"
# build final image
FROM alpine:latest
FROM alpine:3.12 AS dist
WORKDIR /cloudreve
LABEL maintainer="mritd <mritd@linux.com>"
RUN apk update && apk add --no-cache tzdata
# we use the Asia/Shanghai timezone by default, you can be modified
# by `docker build --build-arg=TZ=Other_Timezone ...`
ARG TZ="Asia/Shanghai"
# we using the `Asia/Shanghai` timezone by default, you can do modification at your will
RUN cp /usr/share/zoneinfo/Asia/Shanghai /etc/localtime \
&& echo "Asia/Shanghai" > /etc/timezone
ENV TZ ${TZ}
COPY --from=cloudreve_builder /cloudreve_builder/Cloudreve/cloudreve ./
COPY --from=be-builder /go/bin/Cloudreve /cloudreve/cloudreve
COPY docker-bootstrap.sh /cloudreve/bootstrap.sh
# prepare permissions and aria2 dir
RUN chmod +x ./cloudreve && mkdir -p /data/aria2 && chmod -R 766 /data/aria2
RUN apk upgrade \
&& apk add bash tzdata aria2 \
&& ln -s /cloudreve/cloudreve /usr/bin/cloudreve \
&& ln -sf /usr/share/zoneinfo/${TZ} /etc/localtime \
&& echo ${TZ} > /etc/timezone \
&& rm -rf /var/cache/apk/* \
&& mkdir /etc/cloudreve \
&& ln -s /etc/cloudreve/cloureve.db /cloudreve/cloudreve.db \
&& ln -s /etc/cloudreve/conf.ini /cloudreve/conf.ini
EXPOSE 5212
VOLUME ["/cloudreve/uploads", "/cloudreve/avatar", "/data"]
# cloudreve use tcp 5212 port by default
EXPOSE 5212/tcp
ENTRYPOINT ["./cloudreve"]
# cloudreve stores all files(including executable file) in the `/cloudreve`
# directory by default; users should mount the configfile to the `/etc/cloudreve`
# directory by themselves for persistence considerations, and the data storage
# directory recommends using `/data` directory.
VOLUME /etc/cloudreve
VOLUME /data
ENTRYPOINT ["sh", "/cloudreve/bootstrap.sh"]

View File

@@ -9,19 +9,16 @@
<h4 align="center">支持多家云存储驱动的公有云文件系统.</h4>
<p align="center">
<a href="https://github.com/cloudreve/Cloudreve/actions/workflows/test.yml">
<img src="https://img.shields.io/github/workflow/status/cloudreve/Cloudreve/Test?style=flat-square"
alt="GitHub Test Workflow">
<a href="https://travis-ci.com/github/cloudreve/Cloudreve/">
<img src="https://img.shields.io/travis/com/cloudreve/Cloudreve?style=flat-square"
alt="travis">
</a>
<a href="https://codecov.io/gh/cloudreve/Cloudreve"><img src="https://img.shields.io/codecov/c/github/cloudreve/Cloudreve?style=flat-square"></a>
<a href="https://goreportcard.com/report/github.com/cloudreve/Cloudreve">
<img src="https://goreportcard.com/badge/github.com/cloudreve/Cloudreve?style=flat-square">
</a>
<a href="https://github.com/cloudreve/Cloudreve/releases">
<img src="https://img.shields.io/github/v/release/cloudreve/Cloudreve?include_prereleases&style=flat-square" />
</a>
<a href="https://hub.docker.com/r/cloudreve/cloudreve">
<img src="https://img.shields.io/docker/image-size/cloudreve/cloudreve?style=flat-square"/>
<img src="https://img.shields.io/github/v/release/cloudreve/Cloudreve?include_prereleases&style=flat-square">
</a>
</p>
@@ -71,7 +68,7 @@ chmod +x ./cloudreve
## :gear: 构建
自行构建前需要拥有 `Go >= 1.17``yarn`等必要依赖。
自行构建前需要拥有 `Go >= 1.13``yarn`等必要依赖。
#### 克隆代码
@@ -88,7 +85,19 @@ cd assets
yarn install
# 开始构建
yarn run build
```
#### 嵌入静态资源
```shell
# 回到项目主目录
cd ../
# 安装 statik, 用于嵌入静态资源
go get github.com/rakyll/statik
# 开始嵌入
statik -src=assets/build/ -include=*.html,*.js,*.json,*.css,*.png,*.svg,*.ico -f
```
#### 编译项目
@@ -99,7 +108,7 @@ export COMMIT_SHA=$(git rev-parse --short HEAD)
export VERSION=$(git describe --tags)
# 开始编译
go build -a -o cloudreve -ldflags "-s -w -X 'github.com/cloudreve/Cloudreve/v3/pkg/conf.BackendVersion=$VERSION' -X 'github.com/cloudreve/Cloudreve/v3/pkg/conf.LastCommit=$COMMIT_SHA'"
go build -a -o cloudreve -ldflags " -X 'github.com/cloudreve/Cloudreve/v3/pkg/conf.BackendVersion=$VERSION' -X 'github.com/cloudreve/Cloudreve/v3/pkg/conf.LastCommit=$COMMIT_SHA'"
```
你也可以使用项目根目录下的`build.sh`快速开始构建:

2
assets

Submodule assets updated: ac3af6097f...8e5b5c2d29

View File

@@ -1,7 +1,6 @@
package bootstrap
import (
"embed"
model "github.com/cloudreve/Cloudreve/v3/models"
"github.com/cloudreve/Cloudreve/v3/models/scripts"
"github.com/cloudreve/Cloudreve/v3/pkg/aria2"
@@ -17,7 +16,7 @@ import (
)
// Init 初始化启动
func Init(path string, statics embed.FS) {
func Init(path string) {
InitApplication()
conf.Init(path)
// Debug 关闭时,切换为生产模式
@@ -38,7 +37,7 @@ func Init(path string, statics embed.FS) {
{
"both",
func() {
cache.Init(conf.SystemConfig.Mode == "slave")
cache.Init()
},
},
{
@@ -80,7 +79,7 @@ func Init(path string, statics embed.FS) {
{
"master",
func() {
InitStatic(statics)
InitStatic()
},
},
{

View File

@@ -1,20 +1,17 @@
package bootstrap
import (
"bufio"
"embed"
"encoding/json"
"io"
"io/fs"
"io/ioutil"
"net/http"
"path/filepath"
"github.com/pkg/errors"
"path"
"github.com/cloudreve/Cloudreve/v3/pkg/conf"
"github.com/cloudreve/Cloudreve/v3/pkg/util"
_ "github.com/cloudreve/Cloudreve/v3/statik"
"github.com/gin-contrib/static"
"github.com/rakyll/statik/fs"
)
const StaticFolder = "statics"
@@ -38,100 +35,124 @@ func (b *GinFS) Open(name string) (http.File, error) {
// Exists 文件是否存在
func (b *GinFS) Exists(prefix string, filepath string) bool {
if _, err := b.FS.Open(filepath); err != nil {
return false
}
return true
}
// InitStatic 初始化静态资源文件
func InitStatic(statics embed.FS) {
func InitStatic() {
var err error
if util.Exists(util.RelativePath(StaticFolder)) {
util.Log().Info("检测到 statics 目录存在,将使用此目录下的静态资源文件")
StaticFS = static.LocalFile(util.RelativePath("statics"), false)
// 检查静态资源的版本
f, err := StaticFS.Open("version.json")
if err != nil {
util.Log().Warning("静态资源版本标识文件不存在,请重新构建或删除 statics 目录")
return
}
b, err := ioutil.ReadAll(f)
if err != nil {
util.Log().Warning("无法读取静态资源文件版本,请重新构建或删除 statics 目录")
return
}
var v staticVersion
if err := json.Unmarshal(b, &v); err != nil {
util.Log().Warning("无法解析静态资源文件版本, %s", err)
return
}
staticName := "cloudreve-frontend"
if conf.IsPro == "true" {
staticName += "-pro"
}
if v.Name != staticName {
util.Log().Warning("静态资源版本不匹配,请重新构建或删除 statics 目录")
return
}
if v.Version != conf.RequiredStaticVersion {
util.Log().Warning("静态资源版本不匹配 [当前 %s, 需要: %s],请重新构建或删除 statics 目录", v.Version, conf.RequiredStaticVersion)
return
}
} else {
// 初始化静态资源
embedFS, err := fs.Sub(statics, "assets/build")
StaticFS = &GinFS{}
StaticFS.(*GinFS).FS, err = fs.New()
if err != nil {
util.Log().Panic("无法初始化静态资源, %s", err)
}
StaticFS = &GinFS{
FS: http.FS(embedFS),
}
}
// 检查静态资源的版本
f, err := StaticFS.Open("version.json")
if err != nil {
util.Log().Warning("静态资源版本标识文件不存在,请重新构建或删除 statics 目录")
return
}
b, err := io.ReadAll(f)
if err != nil {
util.Log().Warning("无法读取静态资源文件版本,请重新构建或删除 statics 目录")
return
}
var v staticVersion
if err := json.Unmarshal(b, &v); err != nil {
util.Log().Warning("无法解析静态资源文件版本, %s", err)
return
}
staticName := "cloudreve-frontend"
if conf.IsPro == "true" {
staticName += "-pro"
}
if v.Name != staticName {
util.Log().Warning("静态资源版本不匹配,请重新构建或删除 statics 目录")
return
}
if v.Version != conf.RequiredStaticVersion {
util.Log().Warning("静态资源版本不匹配 [当前 %s, 需要: %s],请重新构建或删除 statics 目录", v.Version, conf.RequiredStaticVersion)
return
}
}
// Eject 抽离内置静态资源
func Eject(statics embed.FS) {
// 初始化静态资源
embedFS, err := fs.Sub(statics, "assets/build")
func Eject() {
staticFS, err := fs.New()
if err != nil {
util.Log().Panic("无法初始化静态资源, %s", err)
}
var walk func(relPath string, d fs.DirEntry, err error) error
walk = func(relPath string, d fs.DirEntry, err error) error {
root, err := staticFS.Open("/")
if err != nil {
util.Log().Panic("根目录不存在, %s", err)
}
var walk func(relPath string, object http.File)
walk = func(relPath string, object http.File) {
stat, err := object.Stat()
if err != nil {
return errors.Errorf("无法获取[%s]的信息, %s, 跳过...", relPath, err)
util.Log().Error("无法获取[%s]的信息, %s, 跳过...", relPath, err)
return
}
if !d.IsDir() {
if !stat.IsDir() {
// 写入文件
out, err := util.CreatNestedFile(filepath.Join(util.RelativePath(""), StaticFolder, relPath))
out, err := util.CreatNestedFile(util.RelativePath(StaticFolder + relPath))
defer out.Close()
if err != nil {
return errors.Errorf("无法创建文件[%s], %s, 跳过...", relPath, err)
util.Log().Error("无法创建文件[%s], %s, 跳过...", relPath, err)
return
}
util.Log().Info("导出 [%s]...", relPath)
obj, _ := embedFS.Open(relPath)
if _, err := io.Copy(out, bufio.NewReader(obj)); err != nil {
return errors.Errorf("无法写入文件[%s], %s, 跳过...", relPath, err)
if _, err := io.Copy(out, object); err != nil {
util.Log().Error("无法写入文件[%s], %s, 跳过...", relPath, err)
return
}
} else {
// 列出目录
objects, err := object.Readdir(0)
if err != nil {
util.Log().Error("无法步入子目录[%s], %s, 跳过...", relPath, err)
return
}
// 递归遍历子目录
for _, newObject := range objects {
newPath := path.Join(relPath, newObject.Name())
newRoot, err := staticFS.Open(newPath)
if err != nil {
util.Log().Error("无法打开对象[%s], %s, 跳过...", newPath, err)
continue
}
walk(newPath, newRoot)
}
}
return nil
}
// util.Log().Info("开始导出内置静态资源...")
err = fs.WalkDir(embedFS, ".", walk)
if err != nil {
util.Log().Error("导出内置静态资源遇到错误:%s", err)
return
}
util.Log().Info("开始导出内置静态资源...")
walk("/", root)
util.Log().Info("内置静态资源导出完成")
}

122
build.sh
View File

@@ -1,16 +1,13 @@
#!/bin/bash
REPO=$(
cd $(dirname $0)
pwd
)
REPO=$(cd $(dirname $0); pwd)
COMMIT_SHA=$(git rev-parse --short HEAD)
VERSION=$(git describe --tags)
ASSETS="false"
BINARY="false"
RELEASE="false"
debugInfo() {
debugInfo () {
echo "Repo: $REPO"
echo "Build assets: $ASSETS"
echo "Build binary: $BINARY"
@@ -19,9 +16,10 @@ debugInfo() {
echo "Commit: $COMMIT_SHA"
}
buildAssets() {
buildAssets () {
cd $REPO
rm -rf assets/build
rm -f statik/statik.go
export CI=false
@@ -29,88 +27,94 @@ buildAssets() {
yarn install
yarn run build
cd build
rm -rf *.map
if ! [ -x "$(command -v statik)" ]; then
export CGO_ENABLED=0
go get github.com/rakyll/statik
fi
cd $REPO
statik -src=assets/build/ -include=*.html,*.js,*.json,*.css,*.png,*.svg,*.ico,*.ttf -f
}
buildBinary() {
buildBinary () {
cd $REPO
go build -a -o cloudreve -ldflags " -X 'github.com/cloudreve/Cloudreve/v3/pkg/conf.BackendVersion=$VERSION' -X 'github.com/cloudreve/Cloudreve/v3/pkg/conf.LastCommit=$COMMIT_SHA'"
}
_build() {
local osarch=$1
IFS=/ read -r -a arr <<<"$osarch"
os="${arr[0]}"
arch="${arr[1]}"
gcc="${arr[2]}"
local osarch=$1
IFS=/ read -r -a arr <<<"$osarch"
os="${arr[0]}"
arch="${arr[1]}"
gcc="${arr[2]}"
# Go build to build the binary.
export GOOS=$os
export GOARCH=$arch
export CC=$gcc
export CGO_ENABLED=1
# Go build to build the binary.
export GOOS=$os
export GOARCH=$arch
export CC=$gcc
export CGO_ENABLED=1
if [ -n "$VERSION" ]; then
out="release/cloudreve_${VERSION}_${os}_${arch}"
else
out="release/cloudreve_${COMMIT_SHA}_${os}_${arch}"
fi
if [ -n "$VERSION" ]; then
out="release/cloudreve_${VERSION}_${os}_${arch}"
else
out="release/cloudreve_${COMMIT_SHA}_${os}_${arch}"
fi
go build -a -o "${out}" -ldflags " -X 'github.com/cloudreve/Cloudreve/v3/pkg/conf.BackendVersion=$VERSION' -X 'github.com/cloudreve/Cloudreve/v3/pkg/conf.LastCommit=$COMMIT_SHA'"
go build -a -o "${out}" -ldflags " -X 'github.com/cloudreve/Cloudreve/v3/pkg/conf.BackendVersion=$VERSION' -X 'github.com/cloudreve/Cloudreve/v3/pkg/conf.LastCommit=$COMMIT_SHA'"
if [ "$os" = "windows" ]; then
mv $out release/cloudreve.exe
zip -j -q "${out}.zip" release/cloudreve.exe
rm -f "release/cloudreve.exe"
else
mv $out release/cloudreve
tar -zcvf "${out}.tar.gz" -C release cloudreve
rm -f "release/cloudreve"
fi
if [ "$os" = "windows" ]; then
mv $out release/cloudreve.exe
zip -j -q "${out}.zip" release/cloudreve.exe
rm -f "release/cloudreve.exe"
else
mv $out release/cloudreve
tar -zcvf "${out}.tar.gz" -C release cloudreve
rm -f "release/cloudreve"
fi
}
release() {
release(){
cd $REPO
## List of architectures and OS to test coss compilation.
SUPPORTED_OSARCH="linux/amd64/gcc linux/arm/arm-linux-gnueabihf-gcc windows/amd64/x86_64-w64-mingw32-gcc linux/arm64/aarch64-linux-gnu-gcc"
echo "Release builds for OS/Arch/CC: ${SUPPORTED_OSARCH}"
for each_osarch in ${SUPPORTED_OSARCH}; do
_build "${each_osarch}"
_build "${each_osarch}"
done
}
usage() {
echo "Usage: $0 [-a] [-c] [-b] [-r]" 1>&2
exit 1
echo "Usage: $0 [-a] [-c] [-b] [-r]" 1>&2;
exit 1;
}
while getopts "bacr:d" o; do
case "${o}" in
b)
ASSETS="true"
BINARY="true"
;;
a)
ASSETS="true"
;;
c)
BINARY="true"
;;
r)
ASSETS="true"
RELEASE="true"
;;
d)
DEBUG="true"
;;
*)
usage
;;
b)
ASSETS="true"
BINARY="true"
;;
a)
ASSETS="true"
;;
c)
BINARY="true"
;;
r)
ASSETS="true"
RELEASE="true"
;;
d)
DEBUG="true"
;;
*)
usage
;;
esac
done
shift $((OPTIND - 1))
shift $((OPTIND-1))
if [ "$DEBUG" = "true" ]; then
debugInfo

View File

@@ -1,33 +0,0 @@
version: "3.8"
services:
cloudreve:
container_name: cloudreve
image: cloudreve/cloudreve:latest
restart: unless-stopped
ports:
- "5212:5212"
volumes:
- temp_data:/data
- ./cloudreve/uploads:/cloudreve/uploads
- ./cloudreve/conf.ini:/cloudreve/conf.ini
- ./cloudreve/cloudreve.db:/cloudreve/cloudreve.db
- ./cloudreve/avatar:/cloudreve/avatar
depends_on:
- aria2
aria2:
container_name: aria2
image: p3terx/aria2-pro # third party image, please keep notice what you are doing
restart: unless-stopped
environment:
- RPC_SECRET=your_aria_rpc_token # aria rpc token, customize your own
- RPC_PORT=6800
volumes:
- ./aria2/config:/config
- temp_data:/data
volumes:
temp_data:
driver: local
driver_opts:
type: none
device: $PWD/data
o: bind

69
go.mod
View File

@@ -1,90 +1,47 @@
module github.com/cloudreve/Cloudreve/v3
go 1.17
go 1.13
require (
github.com/DATA-DOG/go-sqlmock v1.3.3
github.com/HFO4/aliyun-oss-go-sdk v2.2.3+incompatible
github.com/aliyun/aliyun-oss-go-sdk v2.0.5+incompatible
github.com/aws/aws-sdk-go v1.31.5
github.com/baiyubin/aliyun-sts-go-sdk v0.0.0-20180326062324-cfa1a18b161f // indirect
github.com/duo-labs/webauthn v0.0.0-20191119193225-4bf9a0f776d4
github.com/fatih/color v1.7.0
github.com/gin-contrib/cors v1.3.0
github.com/gin-contrib/gzip v0.0.2-0.20200226035851-25bef2ef21e8
github.com/gin-contrib/sessions v0.0.1
github.com/gin-contrib/static v0.0.0-20191128031702-f81c604d8ac2
github.com/gin-gonic/gin v1.7.7
github.com/gin-gonic/gin v1.5.0
github.com/go-ini/ini v1.50.0
github.com/go-mail/mail v2.3.1+incompatible
github.com/gofrs/uuid v4.0.0+incompatible
github.com/gomodule/redigo v2.0.0+incompatible
github.com/google/go-querystring v1.0.0
github.com/gorilla/websocket v1.4.1
github.com/hashicorp/go-version v1.3.0
github.com/hashicorp/go-version v1.2.0
github.com/jinzhu/gorm v1.9.11
github.com/juju/ratelimit v1.0.1
github.com/mattn/go-colorable v0.1.4 // indirect
github.com/mojocn/base64Captcha v0.0.0-20190801020520-752b1cd608b2
github.com/nfnt/resize v0.0.0-20180221191011-83c6a9932646
github.com/pkg/errors v0.9.1
github.com/pquerna/otp v1.2.0
github.com/qiniu/go-sdk/v7 v7.11.1
github.com/qiniu/api.v7/v7 v7.4.0
github.com/rafaeljusto/redigomock v0.0.0-20191117212112-00b2509252a1
github.com/rakyll/statik v0.1.7
github.com/robfig/cron/v3 v3.0.1
github.com/smartystreets/goconvey v1.6.4 // indirect
github.com/speps/go-hashids v2.0.0+incompatible
github.com/stretchr/testify v1.6.1
github.com/stretchr/testify v1.5.1
github.com/tencentcloud/tencentcloud-sdk-go v3.0.125+incompatible
github.com/tencentyun/cos-go-sdk-v5 v0.0.0-20200120023323-87ff3bc489ac
github.com/upyun/go-sdk v2.1.0+incompatible
golang.org/x/image v0.0.0-20211028202545-6944b10bf410
golang.org/x/text v0.3.7
gopkg.in/go-playground/validator.v9 v9.29.1
)
require (
cloud.google.com/go v0.37.4 // indirect
github.com/agl/ed25519 v0.0.0-20170116200512-5312a6153412 // indirect
github.com/baiyubin/aliyun-sts-go-sdk v0.0.0-20180326062324-cfa1a18b161f // indirect
github.com/boj/redistore v0.0.0-20180917114910-cd5dcc76aeff // indirect
github.com/boombuler/barcode v1.0.1-0.20190219062509-6c824513bacc // indirect
github.com/cloudflare/cfssl v0.0.0-20190726000631-633726f6bcb7 // indirect
github.com/davecgh/go-spew v1.1.1 // indirect
github.com/denisenkom/go-mssqldb v0.0.0-20190515213511-eb9f6a1743f3 // indirect
github.com/dgrijalva/jwt-go v3.2.0+incompatible // indirect
github.com/gin-contrib/sse v0.1.0 // indirect
github.com/go-playground/locales v0.14.0 // indirect
github.com/go-playground/universal-translator v0.18.0 // indirect
github.com/go-playground/validator/v10 v10.8.0 // indirect
github.com/go-sql-driver/mysql v1.5.0 // indirect
github.com/golang/freetype v0.0.0-20170609003504-e2365dfdc4a0 // indirect
github.com/golang/protobuf v1.3.3 // indirect
github.com/google/certificate-transparency-go v1.0.21 // indirect
github.com/gorilla/context v1.1.1 // indirect
github.com/gorilla/securecookie v1.1.1 // indirect
github.com/gorilla/sessions v1.1.3 // indirect
github.com/jinzhu/inflection v1.0.0 // indirect
github.com/jmespath/go-jmespath v0.3.0 // indirect
github.com/json-iterator/go v1.1.9 // indirect
github.com/katzenpost/core v0.0.7 // indirect
github.com/leodido/go-urn v1.2.1 // indirect
github.com/lib/pq v1.1.1 // indirect
github.com/mattn/go-colorable v0.1.4 // indirect
github.com/mattn/go-isatty v0.0.12 // indirect
github.com/mattn/go-sqlite3 v1.11.0 // indirect
github.com/mitchellh/mapstructure v1.1.2 // indirect
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
github.com/modern-go/reflect2 v1.0.1 // indirect
github.com/mozillazg/go-httpheader v0.2.1 // indirect
github.com/pmezard/go-difflib v1.0.0 // indirect
github.com/quasoft/memstore v0.0.0-20180925164028-84a050167438 // indirect
github.com/satori/go.uuid v1.2.0 // indirect
github.com/smartystreets/goconvey v1.6.4 // indirect
github.com/stretchr/objx v0.2.0 // indirect
github.com/ugorji/go/codec v1.1.7 // indirect
golang.org/x/crypto v0.0.0-20210921155107-089bfa567519 // indirect
golang.org/x/sync v0.0.0-20201207232520-09787c993a3a // indirect
golang.org/x/sys v0.0.0-20211020174200-9d6173849985 // indirect
golang.org/x/time v0.0.0-20181108054448-85acf8d2951c // indirect
golang.org/x/text v0.3.6
gopkg.in/alexcesaro/quotedprintable.v3 v3.0.0-20150716171945-2caba252f4dc // indirect
gopkg.in/go-playground/validator.v9 v9.29.1
gopkg.in/ini.v1 v1.51.0 // indirect
gopkg.in/mail.v2 v2.3.1 // indirect
gopkg.in/yaml.v2 v2.2.8 // indirect
gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b // indirect
)

97
go.sum
View File

@@ -5,8 +5,6 @@ cloud.google.com/go v0.37.4/go.mod h1:NHPJ89PdicEuT9hdPXMROBD91xc5uRDxsMtSB16k7h
github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
github.com/DATA-DOG/go-sqlmock v1.3.3 h1:CWUqKXe0s8A2z6qCgkP4Kru7wC11YoAnoupUKFDnH08=
github.com/DATA-DOG/go-sqlmock v1.3.3/go.mod h1:f/Ixk793poVmq4qj/V1dPUg2JEAKC73Q5eFN3EC/SaM=
github.com/HFO4/aliyun-oss-go-sdk v2.2.3+incompatible h1:aX/+gJM2dAMDDy3JqWS0DJn3JfOUchf4k37P5TbBKU8=
github.com/HFO4/aliyun-oss-go-sdk v2.2.3+incompatible/go.mod h1:8KDiKVrHK/UbXAhj+iQGp1m40rQa+UAvzBi7m22KywI=
github.com/QcloudApi/qcloud_sign_golang v0.0.0-20141224014652-e4130a326409/go.mod h1:1pk82RBxDY/JZnPQrtqHlUFfCctgdorsd9M06fMynOM=
github.com/Shopify/sarama v1.19.0/go.mod h1:FVkBWblsNy7DGZRfXLU0O9RCGt5g3g3yEuWXgklEdEo=
github.com/Shopify/toxiproxy v2.1.4+incompatible/go.mod h1:OXgGpZ6Cli1/URJOF1DMxUHB2q5Ap20/P/eIdh4G0pI=
@@ -14,6 +12,9 @@ github.com/agl/ed25519 v0.0.0-20170116200512-5312a6153412 h1:w1UutsfOrms1J05zt7I
github.com/agl/ed25519 v0.0.0-20170116200512-5312a6153412/go.mod h1:WPjqKcmVOxf0XSf3YxCJs6N6AOSrOx3obionmG7T0y0=
github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
github.com/aliyun/aliyun-oss-go-sdk v2.0.0/go.mod h1:T/Aws4fEfogEE9v+HPhhw+CntffsBHJ8nXQCwKr0/g8=
github.com/aliyun/aliyun-oss-go-sdk v2.0.5+incompatible h1:A3oZlWPD/Poa19FvNbw+Zu4yKAurDBTjlRDilYGBiS4=
github.com/aliyun/aliyun-oss-go-sdk v2.0.5+incompatible/go.mod h1:T/Aws4fEfogEE9v+HPhhw+CntffsBHJ8nXQCwKr0/g8=
github.com/apache/thrift v0.12.0/go.mod h1:cp2SuWMxlEZw2r+iP2GNCdIi4C1qmUzdZFSVb+bacwQ=
github.com/aws/aws-sdk-go v1.31.5 h1:DFA7BzTydO4etqsTja+x7UfkOKQUv1xzEluLvNk81L0=
github.com/aws/aws-sdk-go v1.31.5/go.mod h1:5zCpMtNQVjRREroY7sYe8lOMRSxkhG6MZveU8YkpAk0=
@@ -30,7 +31,6 @@ github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDk
github.com/cloudflare/cfssl v0.0.0-20190726000631-633726f6bcb7 h1:Puu1hUwfps3+1CUzYdAZXijuvLuRMirgiXdf3zsM2Ig=
github.com/cloudflare/cfssl v0.0.0-20190726000631-633726f6bcb7/go.mod h1:yMWuSON2oQp+43nFtAV/uvKQIFpSPerB57DCt9t8sSA=
github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7DoTY=
github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
@@ -61,9 +61,8 @@ github.com/gin-contrib/sse v0.1.0/go.mod h1:RHrZQHXnP2xjPF+u1gW/2HnVO7nvIa9PG3Gm
github.com/gin-contrib/static v0.0.0-20191128031702-f81c604d8ac2 h1:xLG16iua01X7Gzms9045s2Y2niNpvSY/Zb1oBwgNYZY=
github.com/gin-contrib/static v0.0.0-20191128031702-f81c604d8ac2/go.mod h1:VhW/Ch/3FhimwZb8Oj+qJmdMmoB8r7lmJ5auRjm50oQ=
github.com/gin-gonic/gin v1.4.0/go.mod h1:OW2EZn3DO8Ln9oIKOvM++LBO+5UPHJJDH72/q/3rZdM=
github.com/gin-gonic/gin v1.5.0 h1:fi+bqFAx/oLK54somfCtEZs9HeH1LHVoEPUgARpTqyc=
github.com/gin-gonic/gin v1.5.0/go.mod h1:Nd6IXA8m5kNZdNEHMBd93KT+mdY3+bewLgRvmCsR2Do=
github.com/gin-gonic/gin v1.7.7 h1:3DoBmSbJbZAWqXJC3SLjAPfutPJJRN1U5pALB7EeTTs=
github.com/gin-gonic/gin v1.7.7/go.mod h1:axIBovoeJpVj8S3BwE0uPMTeReE4+AfFtqpqaZ1qq1U=
github.com/globalsign/mgo v0.0.0-20181015135952-eeefdecb41b8/go.mod h1:xkRDCp4j0OGD1HRkm4kmhM+pmpv3AKq5SU7GMg4oO/Q=
github.com/go-ini/ini v1.50.0 h1:ogX6RS8VstVN8MJcwhEP78hHhWaI3klN02+97bByabY=
github.com/go-ini/ini v1.50.0/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3Ies8=
@@ -71,19 +70,10 @@ github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2
github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE=
github.com/go-mail/mail v2.3.1+incompatible h1:UzNOn0k5lpfVtO31cK3hn6I4VEVGhe3lX8AJBAxXExM=
github.com/go-mail/mail v2.3.1+incompatible/go.mod h1:VPWjmmNyRsWXQZHVHT3g0YbIINUkSmuKOiLIDkWbL6M=
github.com/go-playground/assert/v2 v2.0.1 h1:MsBgLAaY856+nPRTKrp3/OZK38U/wa0CcBYNjji3q3A=
github.com/go-playground/assert/v2 v2.0.1/go.mod h1:VDjEfimB/XKnb+ZQfWdccd7VUvScMdVu0Titje2rxJ4=
github.com/go-playground/locales v0.12.1 h1:2FITxuFt/xuCNP1Acdhv62OzaCiviiE4kotfhkmOqEc=
github.com/go-playground/locales v0.12.1/go.mod h1:IUMDtCfWo/w/mtMfIE/IG2K+Ey3ygWanZIBtBW0W2TM=
github.com/go-playground/locales v0.13.0/go.mod h1:taPMhCMXrRLJO55olJkUXHZBHCxTMfnGwq/HNwmWNS8=
github.com/go-playground/locales v0.14.0 h1:u50s323jtVGugKlcYeyzC0etD1HifMjqmJqb8WugfUU=
github.com/go-playground/locales v0.14.0/go.mod h1:sawfccIbzZTqEDETgFXqTho0QybSa7l++s0DH+LDiLs=
github.com/go-playground/universal-translator v0.16.0 h1:X++omBR/4cE2MNg91AoC3rmGrCjJ8eAeUP/K/EKx4DM=
github.com/go-playground/universal-translator v0.16.0/go.mod h1:1AnU7NaIRDWWzGEKwgtJRd2xk99HeFyHw3yid4rvQIY=
github.com/go-playground/universal-translator v0.17.0/go.mod h1:UkSxE5sNxxRwHyU+Scu5vgOQjsIJAF8j9muTVoKLVtA=
github.com/go-playground/universal-translator v0.18.0 h1:82dyy6p4OuJq4/CByFNOn/jYrnRPArHwAcmLoJZxyho=
github.com/go-playground/universal-translator v0.18.0/go.mod h1:UvRDBj+xPUEGrFYl+lu/H90nyDXpg0fqeB/AQUGNTVA=
github.com/go-playground/validator/v10 v10.4.1/go.mod h1:nlOn6nFhuKACm19sB/8EGNn9GlaMV7XkbRSipzJ0Ii4=
github.com/go-playground/validator/v10 v10.8.0 h1:1kAa0fCrnpv+QYdkdcRzrRM7AyYs5o8+jZdJCz9xj6k=
github.com/go-playground/validator/v10 v10.8.0/go.mod h1:9JhgTzTaE31GZDpH/HSvHiRJrJ3iKAgqqH0Bl/Ocjdk=
github.com/go-sql-driver/mysql v1.4.1/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w=
github.com/go-sql-driver/mysql v1.5.0 h1:ozyZYNQW3x3HtqT1jira07DN2PArx2v7/mN66gGcHOs=
github.com/go-sql-driver/mysql v1.5.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg=
@@ -99,9 +89,8 @@ github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfb
github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
github.com/golang/protobuf v1.3.2 h1:6nsPYzhq5kReh6QImI3k5qWzO4PEbvbIW2cwSfR/6xs=
github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
github.com/golang/protobuf v1.3.3 h1:gyjaxf+svBWX08ZjK86iN9geUJF0H6gp2IRKX6Nf6/I=
github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw=
github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
github.com/gomodule/redigo v2.0.0+incompatible h1:K/R+8tc58AaqLkqG2Ol3Qk+DR/TlNuhuh457pBFPtt0=
github.com/gomodule/redigo v2.0.0+incompatible/go.mod h1:B4C85qUVwatsJoIUNIfCRsp7qO0iAmpGFZ4EELWSbC4=
@@ -130,8 +119,8 @@ github.com/gorilla/sessions v1.1.3 h1:uXoZdcdA5XdXF3QzuSlheVRUvjl+1rKY7zBXL68L9R
github.com/gorilla/sessions v1.1.3/go.mod h1:8KCfur6+4Mqcc6S0FEfKuN15Vl5MgXW92AE8ovaJD0w=
github.com/gorilla/websocket v1.4.1 h1:q7AeDBpnBk8AogcD4DSag/Ukw/KV+YhzLj2bP5HvKCM=
github.com/gorilla/websocket v1.4.1/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=
github.com/hashicorp/go-version v1.3.0 h1:McDWVJIU/y+u1BRV06dPaLfLCaT7fUTJLp5r04x7iNw=
github.com/hashicorp/go-version v1.3.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA=
github.com/hashicorp/go-version v1.2.0 h1:3vNe/fWF5CBgRIguda1meWhsZHy3m8gCJ5wx+dIzX/E=
github.com/hashicorp/go-version v1.2.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA=
github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU=
github.com/jinzhu/gorm v1.9.11 h1:gaHGvE+UnWGlbWG4Y3FUwY1EcZ5n6S9WtqBA/uySMLE=
@@ -143,9 +132,8 @@ github.com/jinzhu/now v1.0.1/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/
github.com/jmespath/go-jmespath v0.3.0 h1:OS12ieG61fsCg5+qLJ+SsW9NicxNkg3b25OyT2yCeUc=
github.com/jmespath/go-jmespath v0.3.0/go.mod h1:9QtRXoHjLGCJ5IBSaohpXITPlowMeeYCZ7fLUTSywik=
github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU=
github.com/json-iterator/go v1.1.7 h1:KfgG9LzI+pYjr4xvmz/5H4FXjokeP+rlHLhv3iH62Fo=
github.com/json-iterator/go v1.1.7/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
github.com/json-iterator/go v1.1.9 h1:9yzud/Ht36ygwatGx56VwCZtlI/2AD15T1X2sjSuGns=
github.com/json-iterator/go v1.1.9/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU=
github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo=
github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU=
@@ -158,28 +146,22 @@ github.com/kidstuff/mongostore v0.0.0-20181113001930-e650cd85ee4b/go.mod h1:g2nV
github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc=
github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI=
github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI=
github.com/kr/pretty v0.3.0 h1:WgNl7dwNpEZ6jJ9k1snq4pZsg7DOEN8hP9Xw0Tsjwk0=
github.com/kr/pretty v0.3.0/go.mod h1:640gp4NfQd8pI5XOwp5fnNeVWj67G7CFk/SaSQn7NBk=
github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
github.com/kr/pty v1.1.8/go.mod h1:O1sed60cT9XZ5uDucP5qwvh+TE3NnUj51EiZO/lmSfw=
github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE=
github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
github.com/leodido/go-urn v1.1.0 h1:Sm1gr51B1kKyfD2BlRcLSiEkffoG96g6TPv6eRoEiB8=
github.com/leodido/go-urn v1.1.0/go.mod h1:+cyI34gQWZcE1eQU7NVgKkkzdXDQHr1dBMtdAPozLkw=
github.com/leodido/go-urn v1.2.0/go.mod h1:+8+nEpDfqqsY+g338gtMEUOtuK+4dEMhiQEgxpxOKII=
github.com/leodido/go-urn v1.2.1 h1:BqpAaACuzVSgi/VLzGZIobT2z4v53pjosyNd9Yv6n/w=
github.com/leodido/go-urn v1.2.1/go.mod h1:zt4jvISO2HfUBqxjfIshjdMTYS56ZS/qv49ictyFfxY=
github.com/lib/pq v1.1.1 h1:sJZmqHoEaY7f+NPP8pgLB/WxulyR3fewgCM2qaSlBb4=
github.com/lib/pq v1.1.1/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo=
github.com/mattn/go-colorable v0.1.4 h1:snbPLB8fVfU9iwbbo30TPtbLRzwWu6aJS6Xh4eaaviA=
github.com/mattn/go-colorable v0.1.4/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE=
github.com/mattn/go-isatty v0.0.7/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s=
github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s=
github.com/mattn/go-isatty v0.0.9 h1:d5US/mDsogSGW37IV293h//ZFaeajb69h+EHFsv2xGg=
github.com/mattn/go-isatty v0.0.9/go.mod h1:YNRxwqDuOph6SZLI9vUUz6OYw3QyUt7WiY2yME+cCiQ=
github.com/mattn/go-isatty v0.0.12 h1:wuysRhFDzyxgEmMf5xjvJ2M9dZoWAXNNr5LSBS7uHXY=
github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU=
github.com/mattn/go-sqlite3 v1.11.0 h1:LDdKkqtYlom37fkvqs8rMPFKAMe8+SgjbwZ6ex1/A/Q=
github.com/mattn/go-sqlite3 v1.11.0/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc=
github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0=
@@ -197,12 +179,13 @@ github.com/mojocn/base64Captcha v0.0.0-20190801020520-752b1cd608b2/go.mod h1:wAQ
github.com/mozillazg/go-httpheader v0.2.1 h1:geV7TrjbL8KXSyvghnFm+NyTux/hxwueTSrwhe88TQQ=
github.com/mozillazg/go-httpheader v0.2.1/go.mod h1:jJ8xECTlalr6ValeXYdOF8fFUISeBAdw6E61aqQma60=
github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
github.com/nfnt/resize v0.0.0-20180221191011-83c6a9932646 h1:zYyBkD/k9seD2A7fsi6Oo2LfFZAehjjQMERAvZLEDnQ=
github.com/nfnt/resize v0.0.0-20180221191011-83c6a9932646/go.mod h1:jpp1/29i3P1S/RLdc7JQKbRpFeM1dOBd8T9ki5s+AY8=
github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
github.com/openzipkin/zipkin-go v0.1.6/go.mod h1:QgAqvLzwWbR/WpD4A3cGpPtJrZXNIiJc5AZX7/PBEpw=
github.com/pierrec/lz4 v2.0.5+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY=
github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA=
github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
@@ -217,20 +200,17 @@ github.com/prometheus/client_model v0.0.0-20190115171406-56726106282f/go.mod h1:
github.com/prometheus/common v0.2.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4=
github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk=
github.com/prometheus/procfs v0.0.0-20190117184657-bf6a532e95b1/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk=
github.com/qiniu/dyn v1.3.0/go.mod h1:E8oERcm8TtwJiZvkQPbcAh0RL8jO1G0VXJMW3FAWdkk=
github.com/qiniu/go-sdk/v7 v7.11.1 h1:/LZ9rvFS4p6SnszhGv11FNB1+n4OZvBCwFg7opH5Ovs=
github.com/qiniu/go-sdk/v7 v7.11.1/go.mod h1:btsaOc8CA3hdVloULfFdDgDc+g4f3TDZEFsDY0BLE+w=
github.com/qiniu/x v1.10.5/go.mod h1:03Ni9tj+N2h2aKnAz+6N0Xfl8FwMEDRC2PAlxekASDs=
github.com/qiniu/api.v7/v7 v7.4.0 h1:9dZMVQifh31QGFLVaHls6akCaS2rlj3du8MnEFd7XjQ=
github.com/qiniu/api.v7/v7 v7.4.0/go.mod h1:VE5oC5rkE1xul0u1S2N0b2Uxq9/6hZzhyqjgK25XDcM=
github.com/quasoft/memstore v0.0.0-20180925164028-84a050167438 h1:jnz/4VenymvySjE+Ez511s0pqVzkUOmr1fwCVytNNWk=
github.com/quasoft/memstore v0.0.0-20180925164028-84a050167438/go.mod h1:wTPjTepVu7uJBYgZ0SdWHQlIas582j6cn2jgk4DDdlg=
github.com/rafaeljusto/redigomock v0.0.0-20191117212112-00b2509252a1 h1:leEwA4MD1ew0lNgzz6Q4G76G3AEfeci+TMggN6WuFRs=
github.com/rafaeljusto/redigomock v0.0.0-20191117212112-00b2509252a1/go.mod h1:JaY6n2sDr+z2WTsXkOmNRUfDy6FN0L6Nk7x06ndm4tY=
github.com/rakyll/statik v0.1.7 h1:OF3QCZUuyPxuGEP7B4ypUa7sB/iHtqOTDYZXGM8KOdQ=
github.com/rakyll/statik v0.1.7/go.mod h1:AlZONWzMtEnMs7W4e/1LURLiI49pIMmp6V9Unghqrcc=
github.com/rcrowley/go-metrics v0.0.0-20181016184325-3113b8401b8a/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4=
github.com/robfig/cron/v3 v3.0.1 h1:WdRxkvbJztn8LMz/QEvLN5sBU+xKpSqwwUO1Pjr4qDs=
github.com/robfig/cron/v3 v3.0.1/go.mod h1:eQICP3HwyT7UooqI/z+Ov+PtYAWygg1TEWWzGIFLtro=
github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc=
github.com/rogpeppe/go-internal v1.8.0 h1:FCbCCtXNOY3UtUuHUYaghJg4y7Fd14rXifAYUAtL9R8=
github.com/rogpeppe/go-internal v1.8.0/go.mod h1:WmiCO8CzOY8rg0OYDC4/i/2WRWAB6poM+XZ2dLUbcbE=
github.com/satori/go.uuid v1.2.0 h1:0uYX9dsZ2yD7q2RtLRtPSdGDWzjeM3TbMJP9utgA0ww=
github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0=
github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo=
@@ -247,9 +227,8 @@ github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoH
github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
github.com/stretchr/testify v1.5.1 h1:nOGnQDM7FYENwehXlg/kFVnos3rEvtKTjRvOWSzb6H4=
github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA=
github.com/stretchr/testify v1.6.1 h1:hDPOHmpOpP40lSULcqw7IrRb/u7w6RpDC9399XyoNd0=
github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
github.com/tencentcloud/tencentcloud-sdk-go v3.0.125+incompatible h1:dqpmYaez7VBT7PCRBcBxkzlDOiTk7Td8ATiia1b1GuE=
github.com/tencentcloud/tencentcloud-sdk-go v3.0.125+incompatible/go.mod h1:0PfYow01SHPMhKY31xa+EFz2RStxIqj6JFAJS+IkCi4=
github.com/tencentyun/cos-go-sdk-v5 v0.0.0-20200120023323-87ff3bc489ac h1:PSBhZblOjdwH7SIVgcue+7OlnLHkM45KuScLZ+PiVbQ=
@@ -265,12 +244,10 @@ go.opencensus.io v0.20.1/go.mod h1:6WKK9ahsWS3RSO+PY9ZHZUfv2irvY6gN279GOPZjmmk=
golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
golang.org/x/crypto v0.0.0-20190325154230-a5d413f7728c/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4 h1:HuIa8hRrWRSrqYzx1qI49NNxhdi2PrY7gxVSq1JjLDc=
golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
golang.org/x/crypto v0.0.0-20210711020723-a769d52b0f97/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
golang.org/x/crypto v0.0.0-20210921155107-089bfa567519 h1:7I4JAnoQBe7ZtJcBaYHi5UtiO8tQHbUSXxL+pnGRANg=
golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
golang.org/x/image v0.0.0-20190501045829-6d32002ffd75 h1:TbGuee8sSq15Iguxu4deQ7+Bqq/d2rsQejGcEtADAMQ=
golang.org/x/image v0.0.0-20190501045829-6d32002ffd75/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js=
golang.org/x/image v0.0.0-20211028202545-6944b10bf410 h1:hTftEOvwiOq2+O8k2D5/Q7COC7k5Qcrgc2TFURJYnvQ=
golang.org/x/image v0.0.0-20211028202545-6944b10bf410/go.mod h1:023OzeP/+EPmXeapQh35lcL3II3LrY8Ic+EFFKVhULM=
@@ -289,9 +266,8 @@ golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn
golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/net v0.0.0-20200202094626-16171245cfb2 h1:CCH4IOTTfewWjGOlSp+zGcjutRKlBEZQ6wTn8ozI/nI=
golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/net v0.0.0-20210226172049-e18ecbb05110 h1:qWPm9rbaAMKs8Bq/9LRpbMqxWRVUAQwMI9fVrssnTfw=
golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
@@ -299,8 +275,6 @@ golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJ
golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20201207232520-09787c993a3a h1:DcqTD9SDLc+1P/r1EmRBwnVsrOwW+kk2vWf9n+1sGhs=
golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
@@ -311,21 +285,14 @@ golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5h
golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20190813064441-fde4db37ae7a h1:aYOabOQFp6Vj6W1F80affTUvO9UxmJRx8K0gsfABByQ=
golang.org/x/sys v0.0.0-20190813064441-fde4db37ae7a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20211020174200-9d6173849985 h1:LOlKVhfDyahgmqa97awczplwkjzNaELFg3zRIJ13RYo=
golang.org/x/sys v0.0.0-20211020174200-9d6173849985/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs=
golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
golang.org/x/text v0.3.6 h1:aRYxNxv6iGQlyVaZmk6ZgYEDa+Jg18DxebPSrd6bg1M=
golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
golang.org/x/text v0.3.7 h1:olpwvP2KacW1ZWvsR7uQhoyTYvKAupfQrRGBFM352Gk=
golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
golang.org/x/time v0.0.0-20181108054448-85acf8d2951c h1:fqgJT0MGcGpPgpWU7VRdRjuArfcOvC4AoJmILihzhDg=
golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
golang.org/x/tools v0.0.0-20180828015842-6cd1fcedba52/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
@@ -347,10 +314,8 @@ gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLks
gopkg.in/alexcesaro/quotedprintable.v3 v3.0.0-20150716171945-2caba252f4dc h1:2gGKlE2+asNV9m7xrywl36YYNnBG5ZQ0r/BOOxqPpmk=
gopkg.in/alexcesaro/quotedprintable.v3 v3.0.0-20150716171945-2caba252f4dc/go.mod h1:m7x9LTH6d71AHyAX77c9yqWCCa3UKHcVEj9y7hAtKDk=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY=
gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk=
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q=
gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI=
gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys=
gopkg.in/go-playground/assert.v1 v1.2.1 h1:xoYuJVE7KT85PYWrN730RguIQO0ePzVRfFMXadIrXTM=
gopkg.in/go-playground/assert.v1 v1.2.1/go.mod h1:9RXL0bg/zibRAgZUYszZSwO/z8Y/a8bDuhia5mkpMnE=
@@ -363,12 +328,8 @@ gopkg.in/mail.v2 v2.3.1 h1:WYFn/oANrAGP2C0dcV6/pbkPzv8yGzqTjPmTeO7qoXk=
gopkg.in/mail.v2 v2.3.1/go.mod h1:htwXN1Qh09vZJ1NVKxQqHPBaCBbzKhp5GzuJEA4VJWw=
gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw=
gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw=
gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v2 v2.2.8 h1:obN1ZagJSUGI0Ek/LBmuj4SNLPfIny3KsKFopxRdj10=
gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b h1:h8qDotaEPuJATrMmW04NCwg7v22aHH28wwpauUhK9Oo=
gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
honnef.co/go/tools v0.0.0-20180728063816-88497007e858/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=

View File

@@ -1,7 +1,6 @@
package main
import (
"embed"
"flag"
"github.com/cloudreve/Cloudreve/v3/bootstrap"
@@ -16,21 +15,18 @@ var (
scriptName string
)
//go:embed assets/build
var staticEmbed embed.FS
func init() {
flag.StringVar(&confPath, "c", util.RelativePath("conf.ini"), "配置文件路径")
flag.BoolVar(&isEject, "eject", false, "导出内置静态资源")
flag.StringVar(&scriptName, "database-script", "", "运行内置数据库助手脚本")
flag.Parse()
bootstrap.Init(confPath, staticEmbed)
bootstrap.Init(confPath)
}
func main() {
if isEject {
// 开始导出内置静态资源文件
bootstrap.Eject(staticEmbed)
bootstrap.Eject()
return
}

View File

@@ -5,25 +5,20 @@ import (
"context"
"crypto/md5"
"fmt"
"github.com/cloudreve/Cloudreve/v3/pkg/filesystem"
"github.com/cloudreve/Cloudreve/v3/pkg/filesystem/driver/oss"
"github.com/cloudreve/Cloudreve/v3/pkg/filesystem/driver/upyun"
"github.com/cloudreve/Cloudreve/v3/pkg/mq"
"github.com/cloudreve/Cloudreve/v3/pkg/util"
"github.com/qiniu/go-sdk/v7/auth/qbox"
"io/ioutil"
"net/http"
model "github.com/cloudreve/Cloudreve/v3/models"
"github.com/cloudreve/Cloudreve/v3/pkg/auth"
"github.com/cloudreve/Cloudreve/v3/pkg/cache"
"github.com/cloudreve/Cloudreve/v3/pkg/filesystem/driver/onedrive"
"github.com/cloudreve/Cloudreve/v3/pkg/filesystem/driver/oss"
"github.com/cloudreve/Cloudreve/v3/pkg/filesystem/driver/upyun"
"github.com/cloudreve/Cloudreve/v3/pkg/serializer"
"github.com/cloudreve/Cloudreve/v3/pkg/util"
"github.com/gin-contrib/sessions"
"github.com/gin-gonic/gin"
)
const (
CallbackFailedStatusCode = http.StatusUnauthorized
"github.com/qiniu/api.v7/v7/auth/qbox"
)
// SignRequired 验证请求签名
@@ -122,60 +117,48 @@ func WebDAVAuth() gin.HandlerFunc {
}
}
// 对上传会话进行验证
func UseUploadSession(policyType string) gin.HandlerFunc {
return func(c *gin.Context) {
// 验证key并查找用户
resp := uploadCallbackCheck(c, policyType)
if resp.Code != 0 {
c.JSON(CallbackFailedStatusCode, resp)
c.Abort()
return
}
c.Next()
}
}
// uploadCallbackCheck 对上传回调请求的 callback key 进行验证,如果成功则返回上传用户
func uploadCallbackCheck(c *gin.Context, policyType string) serializer.Response {
func uploadCallbackCheck(c *gin.Context) (serializer.Response, *model.User) {
// 验证 Callback Key
sessionID := c.Param("sessionID")
if sessionID == "" {
return serializer.ParamErr("Session ID 不能为空", nil)
callbackKey := c.Param("key")
if callbackKey == "" {
return serializer.ParamErr("Callback Key 不能为空", nil), nil
}
callbackSessionRaw, exist := cache.Get(filesystem.UploadSessionCachePrefix + sessionID)
callbackSessionRaw, exist := cache.Get("callback_" + callbackKey)
if !exist {
return serializer.ParamErr("上传会话不存在或已过期", nil)
return serializer.ParamErr("回调会话不存在或已过期", nil), nil
}
callbackSession := callbackSessionRaw.(serializer.UploadSession)
c.Set(filesystem.UploadSessionCtx, &callbackSession)
if callbackSession.Policy.Type != policyType {
return serializer.Err(serializer.CodePolicyNotAllowed, "Policy not supported", nil)
}
c.Set("callbackSession", &callbackSession)
// 清理回调会话
_ = cache.Deletes([]string{sessionID}, filesystem.UploadSessionCachePrefix)
_ = cache.Deletes([]string{callbackKey}, "callback_")
// 查找用户
user, err := model.GetActiveUserByID(callbackSession.UID)
if err != nil {
return serializer.Err(serializer.CodeCheckLogin, "找不到用户", err)
return serializer.Err(serializer.CodeCheckLogin, "找不到用户", err), nil
}
c.Set(filesystem.UserCtx, &user)
return serializer.Response{}
c.Set("user", &user)
return serializer.Response{}, &user
}
// RemoteCallbackAuth 远程回调签名验证
func RemoteCallbackAuth() gin.HandlerFunc {
return func(c *gin.Context) {
// 验证key并查找用户
resp, user := uploadCallbackCheck(c)
if resp.Code != 0 {
c.JSON(200, resp)
c.Abort()
return
}
// 验证签名
session := c.MustGet(filesystem.UploadSessionCtx).(*serializer.UploadSession)
authInstance := auth.HMACAuth{SecretKey: []byte(session.Policy.SecretKey)}
authInstance := auth.HMACAuth{SecretKey: []byte(user.Policy.SecretKey)}
if err := auth.CheckRequest(authInstance, c.Request); err != nil {
c.JSON(CallbackFailedStatusCode, serializer.Err(serializer.CodeCredentialInvalid, err.Error(), err))
c.JSON(200, serializer.Err(serializer.CodeCheckLogin, err.Error(), err))
c.Abort()
return
}
@@ -188,10 +171,16 @@ func RemoteCallbackAuth() gin.HandlerFunc {
// QiniuCallbackAuth 七牛回调签名验证
func QiniuCallbackAuth() gin.HandlerFunc {
return func(c *gin.Context) {
session := c.MustGet(filesystem.UploadSessionCtx).(*serializer.UploadSession)
// 验证key并查找用户
resp, user := uploadCallbackCheck(c)
if resp.Code != 0 {
c.JSON(401, serializer.GeneralUploadCallbackFailed{Error: resp.Msg})
c.Abort()
return
}
// 验证回调是否来自qiniu
mac := qbox.NewMac(session.Policy.AccessKey, session.Policy.SecretKey)
mac := qbox.NewMac(user.Policy.AccessKey, user.Policy.SecretKey)
ok, err := mac.VerifyCallback(c.Request)
if err != nil {
util.Log().Debug("无法验证回调请求,%s", err)
@@ -199,7 +188,6 @@ func QiniuCallbackAuth() gin.HandlerFunc {
c.Abort()
return
}
if !ok {
c.JSON(401, serializer.GeneralUploadCallbackFailed{Error: "回调签名无效"})
c.Abort()
@@ -213,6 +201,14 @@ func QiniuCallbackAuth() gin.HandlerFunc {
// OSSCallbackAuth 阿里云OSS回调签名验证
func OSSCallbackAuth() gin.HandlerFunc {
return func(c *gin.Context) {
// 验证key并查找用户
resp, _ := uploadCallbackCheck(c)
if resp.Code != 0 {
c.JSON(401, serializer.GeneralUploadCallbackFailed{Error: resp.Msg})
c.Abort()
return
}
err := oss.VerifyCallbackSignature(c.Request)
if err != nil {
util.Log().Debug("回调签名验证失败,%s", err)
@@ -228,7 +224,13 @@ func OSSCallbackAuth() gin.HandlerFunc {
// UpyunCallbackAuth 又拍云回调签名验证
func UpyunCallbackAuth() gin.HandlerFunc {
return func(c *gin.Context) {
session := c.MustGet(filesystem.UploadSessionCtx).(*serializer.UploadSession)
// 验证key并查找用户
resp, user := uploadCallbackCheck(c)
if resp.Code != 0 {
c.JSON(401, serializer.GeneralUploadCallbackFailed{Error: resp.Msg})
c.Abort()
return
}
// 获取请求正文
body, err := ioutil.ReadAll(c.Request.Body)
@@ -242,7 +244,7 @@ func UpyunCallbackAuth() gin.HandlerFunc {
c.Request.Body = ioutil.NopCloser(bytes.NewReader(body))
// 准备验证Upyun回调签名
handler := upyun.Driver{Policy: &session.Policy}
handler := upyun.Driver{Policy: &user.Policy}
contentMD5 := c.Request.Header.Get("Content-Md5")
date := c.Request.Header.Get("Date")
actualSignature := c.Request.Header.Get("Authorization")
@@ -275,10 +277,50 @@ func UpyunCallbackAuth() gin.HandlerFunc {
}
// OneDriveCallbackAuth OneDrive回调签名验证
// TODO 解耦
func OneDriveCallbackAuth() gin.HandlerFunc {
return func(c *gin.Context) {
// 验证key并查找用户
resp, _ := uploadCallbackCheck(c)
if resp.Code != 0 {
c.JSON(401, serializer.GeneralUploadCallbackFailed{Error: resp.Msg})
c.Abort()
return
}
// 发送回调结束信号
mq.GlobalMQ.Publish(c.Param("sessionID"), mq.Message{})
onedrive.FinishCallback(c.Param("key"))
c.Next()
}
}
// COSCallbackAuth 腾讯云COS回调签名验证
// TODO 解耦 测试
func COSCallbackAuth() gin.HandlerFunc {
return func(c *gin.Context) {
// 验证key并查找用户
resp, _ := uploadCallbackCheck(c)
if resp.Code != 0 {
c.JSON(401, serializer.GeneralUploadCallbackFailed{Error: resp.Msg})
c.Abort()
return
}
c.Next()
}
}
// S3CallbackAuth Amazon S3回调签名验证
func S3CallbackAuth() gin.HandlerFunc {
return func(c *gin.Context) {
// 验证key并查找用户
resp, _ := uploadCallbackCheck(c)
if resp.Code != 0 {
c.JSON(401, serializer.GeneralUploadCallbackFailed{Error: resp.Msg})
c.Abort()
return
}
c.Next()
}

View File

@@ -3,24 +3,21 @@ package middleware
import (
"database/sql"
"errors"
"github.com/cloudreve/Cloudreve/v3/pkg/cache"
"github.com/cloudreve/Cloudreve/v3/pkg/filesystem"
"github.com/cloudreve/Cloudreve/v3/pkg/mq"
"github.com/cloudreve/Cloudreve/v3/pkg/serializer"
"github.com/qiniu/go-sdk/v7/auth/qbox"
"io/ioutil"
"net/http"
"net/http/httptest"
"strings"
"testing"
"time"
"github.com/DATA-DOG/go-sqlmock"
model "github.com/cloudreve/Cloudreve/v3/models"
"github.com/cloudreve/Cloudreve/v3/pkg/auth"
"github.com/cloudreve/Cloudreve/v3/pkg/cache"
"github.com/cloudreve/Cloudreve/v3/pkg/serializer"
"github.com/cloudreve/Cloudreve/v3/pkg/util"
"github.com/gin-gonic/gin"
"github.com/jinzhu/gorm"
"github.com/qiniu/api.v7/v7/auth/qbox"
"github.com/stretchr/testify/assert"
)
@@ -226,31 +223,19 @@ func TestWebDAVAuth(t *testing.T) {
}
func TestUseUploadSession(t *testing.T) {
func TestRemoteCallbackAuth(t *testing.T) {
asserts := assert.New(t)
rec := httptest.NewRecorder()
AuthFunc := UseUploadSession("local")
// sessionID 为空
{
c, _ := gin.CreateTestContext(rec)
c.Params = []gin.Param{}
c.Request, _ = http.NewRequest("POST", "/api/v3/callback/remote/sessionID", nil)
authInstance := auth.HMACAuth{SecretKey: []byte("123")}
auth.SignRequest(authInstance, c.Request, 0)
AuthFunc(c)
asserts.True(c.IsAborted())
}
AuthFunc := RemoteCallbackAuth()
// 成功
{
cache.Set(
filesystem.UploadSessionCachePrefix+"testCallBackRemote",
"callback_testCallBackRemote",
serializer.UploadSession{
UID: 1,
PolicyID: 513,
VirtualPath: "/",
Policy: model.Policy{Type: "local"},
},
0,
)
@@ -263,7 +248,7 @@ func TestUseUploadSession(t *testing.T) {
WillReturnRows(sqlmock.NewRows([]string{"id", "secret_key"}).AddRow(2, "123"))
c, _ := gin.CreateTestContext(rec)
c.Params = []gin.Param{
{"sessionID", "testCallBackRemote"},
{"key", "testCallBackRemote"},
}
c.Request, _ = http.NewRequest("POST", "/api/v3/callback/remote/testCallBackRemote", nil)
authInstance := auth.HMACAuth{SecretKey: []byte("123")}
@@ -272,96 +257,80 @@ func TestUseUploadSession(t *testing.T) {
asserts.NoError(mock.ExpectationsWereMet())
asserts.False(c.IsAborted())
}
}
func TestUploadCallbackCheck(t *testing.T) {
a := assert.New(t)
rec := httptest.NewRecorder()
// 上传会话不存在
// Callback Key 不存在
{
c, _ := gin.CreateTestContext(rec)
c.Params = []gin.Param{
{"sessionID", "testSessionNotExist"},
{"key", "testCallBackRemote"},
}
res := uploadCallbackCheck(c, "local")
a.Contains("上传会话不存在或已过期", res.Msg)
}
// 上传策略不一致
{
c, _ := gin.CreateTestContext(rec)
c.Params = []gin.Param{
{"sessionID", "testPolicyNotMatch"},
}
cache.Set(
filesystem.UploadSessionCachePrefix+"testPolicyNotMatch",
serializer.UploadSession{
UID: 1,
VirtualPath: "/",
Policy: model.Policy{Type: "remote"},
},
0,
)
res := uploadCallbackCheck(c, "local")
a.Contains("Policy not supported", res.Msg)
}
// 用户不存在
{
c, _ := gin.CreateTestContext(rec)
c.Params = []gin.Param{
{"sessionID", "testUserNotExist"},
}
cache.Set(
filesystem.UploadSessionCachePrefix+"testUserNotExist",
serializer.UploadSession{
UID: 313,
VirtualPath: "/",
Policy: model.Policy{Type: "remote"},
},
0,
)
mock.ExpectQuery("SELECT(.+)users(.+)").
WillReturnRows(sqlmock.NewRows([]string{"id", "group_id"}))
res := uploadCallbackCheck(c, "remote")
a.Contains("找不到用户", res.Msg)
a.NoError(mock.ExpectationsWereMet())
_, ok := cache.Get(filesystem.UploadSessionCachePrefix + "testUserNotExist")
a.False(ok)
}
}
func TestRemoteCallbackAuth(t *testing.T) {
asserts := assert.New(t)
rec := httptest.NewRecorder()
AuthFunc := RemoteCallbackAuth()
// 成功
{
c, _ := gin.CreateTestContext(rec)
c.Set(filesystem.UploadSessionCtx, &serializer.UploadSession{
UID: 1,
VirtualPath: "/",
Policy: model.Policy{SecretKey: "123"},
})
c.Request, _ = http.NewRequest("POST", "/api/v3/callback/remote/testCallBackRemote", nil)
authInstance := auth.HMACAuth{SecretKey: []byte("123")}
auth.SignRequest(authInstance, c.Request, 0)
AuthFunc(c)
asserts.False(c.IsAborted())
asserts.True(c.IsAborted())
}
// 用户不存在
{
cache.Set(
"callback_testCallBackRemote",
serializer.UploadSession{
UID: 1,
PolicyID: 550,
VirtualPath: "/",
},
0,
)
cache.Deletes([]string{"1"}, "policy_")
mock.ExpectQuery("SELECT(.+)users(.+)").
WillReturnRows(sqlmock.NewRows([]string{"id", "group_id"}))
c, _ := gin.CreateTestContext(rec)
c.Params = []gin.Param{
{"key", "testCallBackRemote"},
}
c.Request, _ = http.NewRequest("POST", "/api/v3/callback/remote/testCallBackRemote", nil)
authInstance := auth.HMACAuth{SecretKey: []byte("123")}
auth.SignRequest(authInstance, c.Request, 0)
AuthFunc(c)
asserts.NoError(mock.ExpectationsWereMet())
asserts.True(c.IsAborted())
}
// 签名错误
{
cache.Set(
"callback_testCallBackRemote",
serializer.UploadSession{
UID: 1,
PolicyID: 514,
VirtualPath: "/",
},
0,
)
cache.Deletes([]string{"1"}, "policy_")
mock.ExpectQuery("SELECT(.+)users(.+)").
WillReturnRows(sqlmock.NewRows([]string{"id", "group_id"}).AddRow(1, 1))
mock.ExpectQuery("SELECT(.+)groups(.+)").
WillReturnRows(sqlmock.NewRows([]string{"id", "policies"}).AddRow(1, "[514]"))
mock.ExpectQuery("SELECT(.+)policies(.+)").
WillReturnRows(sqlmock.NewRows([]string{"id", "secret_key"}).AddRow(2, "123"))
c, _ := gin.CreateTestContext(rec)
c.Set(filesystem.UploadSessionCtx, &serializer.UploadSession{
UID: 1,
VirtualPath: "/",
Policy: model.Policy{SecretKey: "123"},
})
c.Params = []gin.Param{
{"key", "testCallBackRemote"},
}
c.Request, _ = http.NewRequest("POST", "/api/v3/callback/remote/testCallBackRemote", nil)
AuthFunc(c)
asserts.NoError(mock.ExpectationsWereMet())
asserts.True(c.IsAborted())
}
// Callback Key 为空
{
c, _ := gin.CreateTestContext(rec)
c.Request, _ = http.NewRequest("POST", "/api/v3/callback/remote", nil)
AuthFunc(c)
asserts.True(c.IsAborted())
}
}
@@ -371,17 +340,39 @@ func TestQiniuCallbackAuth(t *testing.T) {
rec := httptest.NewRecorder()
AuthFunc := QiniuCallbackAuth()
// 成功
// Callback Key 相关验证失败
{
c, _ := gin.CreateTestContext(rec)
c.Set(filesystem.UploadSessionCtx, &serializer.UploadSession{
UID: 1,
VirtualPath: "/",
Policy: model.Policy{
SecretKey: "123",
AccessKey: "123",
c.Params = []gin.Param{
{"key", "testQiniuBackRemote"},
}
c.Request, _ = http.NewRequest("POST", "/api/v3/callback/remote/testQiniuBackRemote", nil)
AuthFunc(c)
asserts.True(c.IsAborted())
}
// 成功
{
cache.Set(
"callback_testCallBackQiniu",
serializer.UploadSession{
UID: 1,
PolicyID: 515,
VirtualPath: "/",
},
})
0,
)
cache.Deletes([]string{"1"}, "policy_")
mock.ExpectQuery("SELECT(.+)users(.+)").
WillReturnRows(sqlmock.NewRows([]string{"id", "group_id"}).AddRow(1, 1))
mock.ExpectQuery("SELECT(.+)groups(.+)").
WillReturnRows(sqlmock.NewRows([]string{"id", "policies"}).AddRow(1, "[515]"))
mock.ExpectQuery("SELECT(.+)policies(.+)").
WillReturnRows(sqlmock.NewRows([]string{"id", "access_key", "secret_key"}).AddRow(2, "123", "123"))
c, _ := gin.CreateTestContext(rec)
c.Params = []gin.Param{
{"key", "testCallBackQiniu"},
}
c.Request, _ = http.NewRequest("POST", "/api/v3/callback/qiniu/testCallBackQiniu", nil)
mac := qbox.NewMac("123", "123")
token, err := mac.SignRequest(c.Request)
@@ -394,21 +385,33 @@ func TestQiniuCallbackAuth(t *testing.T) {
// 验证失败
{
c, _ := gin.CreateTestContext(rec)
c.Set(filesystem.UploadSessionCtx, &serializer.UploadSession{
UID: 1,
VirtualPath: "/",
Policy: model.Policy{
SecretKey: "123",
AccessKey: "123",
cache.Set(
"callback_testCallBackQiniu",
serializer.UploadSession{
UID: 1,
PolicyID: 516,
VirtualPath: "/",
},
})
0,
)
cache.Deletes([]string{"1"}, "policy_")
mock.ExpectQuery("SELECT(.+)users(.+)").
WillReturnRows(sqlmock.NewRows([]string{"id", "group_id"}).AddRow(1, 1))
mock.ExpectQuery("SELECT(.+)groups(.+)").
WillReturnRows(sqlmock.NewRows([]string{"id", "policies"}).AddRow(1, "[516]"))
mock.ExpectQuery("SELECT(.+)policies(.+)").
WillReturnRows(sqlmock.NewRows([]string{"id", "access_key", "secret_key"}).AddRow(2, "123", "123"))
c, _ := gin.CreateTestContext(rec)
c.Params = []gin.Param{
{"key", "testCallBackQiniu"},
}
c.Request, _ = http.NewRequest("POST", "/api/v3/callback/qiniu/testCallBackQiniu", nil)
mac := qbox.NewMac("123", "1213")
mac := qbox.NewMac("123", "123")
token, err := mac.SignRequest(c.Request)
asserts.NoError(err)
c.Request.Header["Authorization"] = []string{"QBox " + token}
c.Request.Header["Authorization"] = []string{"QBox " + token + " "}
AuthFunc(c)
asserts.NoError(mock.ExpectationsWereMet())
asserts.True(c.IsAborted())
}
}
@@ -418,41 +421,76 @@ func TestOSSCallbackAuth(t *testing.T) {
rec := httptest.NewRecorder()
AuthFunc := OSSCallbackAuth()
// 签名验证失败
// Callback Key 相关验证失败
{
c, _ := gin.CreateTestContext(rec)
c.Set(filesystem.UploadSessionCtx, &serializer.UploadSession{
UID: 1,
VirtualPath: "/",
Policy: model.Policy{
SecretKey: "123",
AccessKey: "123",
c.Params = []gin.Param{
{"key", "testOSSBackRemote"},
}
c.Request, _ = http.NewRequest("POST", "/api/v3/callback/oss/testQiniuBackRemote", nil)
AuthFunc(c)
asserts.True(c.IsAborted())
}
// 签名验证失败
{
cache.Set(
"callback_testCallBackOSS",
serializer.UploadSession{
UID: 1,
PolicyID: 517,
VirtualPath: "/",
},
})
0,
)
cache.Deletes([]string{"1"}, "policy_")
mock.ExpectQuery("SELECT(.+)users(.+)").
WillReturnRows(sqlmock.NewRows([]string{"id", "group_id"}).AddRow(1, 1))
mock.ExpectQuery("SELECT(.+)groups(.+)").
WillReturnRows(sqlmock.NewRows([]string{"id", "policies"}).AddRow(1, "[517]"))
mock.ExpectQuery("SELECT(.+)policies(.+)").
WillReturnRows(sqlmock.NewRows([]string{"id", "access_key", "secret_key"}).AddRow(2, "123", "123"))
c, _ := gin.CreateTestContext(rec)
c.Params = []gin.Param{
{"key", "testCallBackOSS"},
}
c.Request, _ = http.NewRequest("POST", "/api/v3/callback/oss/testCallBackOSS", nil)
mac := qbox.NewMac("123", "123")
token, err := mac.SignRequest(c.Request)
asserts.NoError(err)
c.Request.Header["Authorization"] = []string{"QBox " + token}
AuthFunc(c)
asserts.NoError(mock.ExpectationsWereMet())
asserts.True(c.IsAborted())
}
// 成功
{
c, _ := gin.CreateTestContext(rec)
c.Set(filesystem.UploadSessionCtx, &serializer.UploadSession{
UID: 1,
VirtualPath: "/",
Policy: model.Policy{
SecretKey: "123",
AccessKey: "123",
cache.Set(
"callback_TnXx5E5VyfJUyM1UdkdDu1rtnJ34EbmH",
serializer.UploadSession{
UID: 1,
PolicyID: 518,
VirtualPath: "/",
},
})
0,
)
cache.Deletes([]string{"1"}, "policy_")
mock.ExpectQuery("SELECT(.+)users(.+)").
WillReturnRows(sqlmock.NewRows([]string{"id", "group_id"}).AddRow(1, 1))
mock.ExpectQuery("SELECT(.+)groups(.+)").
WillReturnRows(sqlmock.NewRows([]string{"id", "policies"}).AddRow(1, "[518]"))
mock.ExpectQuery("SELECT(.+)policies(.+)").
WillReturnRows(sqlmock.NewRows([]string{"id", "access_key", "secret_key"}).AddRow(2, "123", "123"))
c, _ := gin.CreateTestContext(rec)
c.Params = []gin.Param{
{"key", "TnXx5E5VyfJUyM1UdkdDu1rtnJ34EbmH"},
}
c.Request, _ = http.NewRequest("POST", "/api/v3/callback/oss/TnXx5E5VyfJUyM1UdkdDu1rtnJ34EbmH", ioutil.NopCloser(strings.NewReader(`{"name":"2f7b2ccf30e9270ea920f1ab8a4037a546a2f0d5.jpg","source_name":"1/1_hFRtDLgM_2f7b2ccf30e9270ea920f1ab8a4037a546a2f0d5.jpg","size":114020,"pic_info":"810,539"}`)))
c.Request.Header["Authorization"] = []string{"e5LwzwTkP9AFAItT4YzvdJOHd0Y0wqTMWhsV/h5SG90JYGAmMd+8LQyj96R+9qUfJWjMt6suuUh7LaOryR87Dw=="}
c.Request.Header["X-Oss-Pub-Key-Url"] = []string{"aHR0cHM6Ly9nb3NzcHVibGljLmFsaWNkbi5jb20vY2FsbGJhY2tfcHViX2tleV92MS5wZW0="}
AuthFunc(c)
asserts.NoError(mock.ExpectationsWereMet())
asserts.False(c.IsAborted())
}
@@ -469,71 +507,130 @@ func TestUpyunCallbackAuth(t *testing.T) {
rec := httptest.NewRecorder()
AuthFunc := UpyunCallbackAuth()
// 无法获取请求正文
// Callback Key 相关验证失败
{
c, _ := gin.CreateTestContext(rec)
c.Set(filesystem.UploadSessionCtx, &serializer.UploadSession{
UID: 1,
VirtualPath: "/",
Policy: model.Policy{
SecretKey: "123",
AccessKey: "123",
c.Params = []gin.Param{
{"key", "testUpyunBackRemote"},
}
c.Request, _ = http.NewRequest("POST", "/api/v3/callback/upyun/testUpyunBackRemote", nil)
AuthFunc(c)
asserts.True(c.IsAborted())
}
// 无法获取请求正文
{
cache.Set(
"callback_testCallBackUpyun",
serializer.UploadSession{
UID: 1,
PolicyID: 509,
VirtualPath: "/",
},
})
0,
)
cache.Deletes([]string{"1"}, "policy_")
mock.ExpectQuery("SELECT(.+)users(.+)").
WillReturnRows(sqlmock.NewRows([]string{"id", "group_id"}).AddRow(1, 1))
mock.ExpectQuery("SELECT(.+)groups(.+)").
WillReturnRows(sqlmock.NewRows([]string{"id", "policies"}).AddRow(1, "[519]"))
mock.ExpectQuery("SELECT(.+)policies(.+)").
WillReturnRows(sqlmock.NewRows([]string{"id", "access_key", "secret_key"}).AddRow(2, "123", "123"))
c, _ := gin.CreateTestContext(rec)
c.Params = []gin.Param{
{"key", "testCallBackUpyun"},
}
c.Request, _ = http.NewRequest("POST", "/api/v3/callback/upyun/testCallBackUpyun", ioutil.NopCloser(fakeRead("")))
AuthFunc(c)
asserts.NoError(mock.ExpectationsWereMet())
asserts.True(c.IsAborted())
}
// 正文MD5不一致
{
c, _ := gin.CreateTestContext(rec)
c.Set(filesystem.UploadSessionCtx, &serializer.UploadSession{
UID: 1,
VirtualPath: "/",
Policy: model.Policy{
SecretKey: "123",
AccessKey: "123",
cache.Set(
"callback_testCallBackUpyun",
serializer.UploadSession{
UID: 1,
PolicyID: 510,
VirtualPath: "/",
},
})
0,
)
cache.Deletes([]string{"1"}, "policy_")
mock.ExpectQuery("SELECT(.+)users(.+)").
WillReturnRows(sqlmock.NewRows([]string{"id", "group_id"}).AddRow(1, 1))
mock.ExpectQuery("SELECT(.+)groups(.+)").
WillReturnRows(sqlmock.NewRows([]string{"id", "policies"}).AddRow(1, "[520]"))
mock.ExpectQuery("SELECT(.+)policies(.+)").
WillReturnRows(sqlmock.NewRows([]string{"id", "access_key", "secret_key"}).AddRow(2, "123", "123"))
c, _ := gin.CreateTestContext(rec)
c.Params = []gin.Param{
{"key", "testCallBackUpyun"},
}
c.Request, _ = http.NewRequest("POST", "/api/v3/callback/upyun/testCallBackUpyun", ioutil.NopCloser(strings.NewReader("1")))
c.Request.Header["Content-Md5"] = []string{"123"}
AuthFunc(c)
asserts.NoError(mock.ExpectationsWereMet())
asserts.True(c.IsAborted())
}
// 签名不一致
{
c, _ := gin.CreateTestContext(rec)
c.Set(filesystem.UploadSessionCtx, &serializer.UploadSession{
UID: 1,
VirtualPath: "/",
Policy: model.Policy{
SecretKey: "123",
AccessKey: "123",
cache.Set(
"callback_testCallBackUpyun",
serializer.UploadSession{
UID: 1,
PolicyID: 511,
VirtualPath: "/",
},
})
0,
)
cache.Deletes([]string{"1"}, "policy_")
mock.ExpectQuery("SELECT(.+)users(.+)").
WillReturnRows(sqlmock.NewRows([]string{"id", "group_id"}).AddRow(1, 1))
mock.ExpectQuery("SELECT(.+)groups(.+)").
WillReturnRows(sqlmock.NewRows([]string{"id", "policies"}).AddRow(1, "[521]"))
mock.ExpectQuery("SELECT(.+)policies(.+)").
WillReturnRows(sqlmock.NewRows([]string{"id", "access_key", "secret_key"}).AddRow(2, "123", "123"))
c, _ := gin.CreateTestContext(rec)
c.Params = []gin.Param{
{"key", "testCallBackUpyun"},
}
c.Request, _ = http.NewRequest("POST", "/api/v3/callback/upyun/testCallBackUpyun", ioutil.NopCloser(strings.NewReader("1")))
c.Request.Header["Content-Md5"] = []string{"c4ca4238a0b923820dcc509a6f75849b"}
AuthFunc(c)
asserts.NoError(mock.ExpectationsWereMet())
asserts.True(c.IsAborted())
}
// 成功
{
c, _ := gin.CreateTestContext(rec)
c.Set(filesystem.UploadSessionCtx, &serializer.UploadSession{
UID: 1,
VirtualPath: "/",
Policy: model.Policy{
SecretKey: "123",
AccessKey: "123",
cache.Set(
"callback_testCallBackUpyun",
serializer.UploadSession{
UID: 1,
PolicyID: 512,
VirtualPath: "/",
},
})
0,
)
cache.Deletes([]string{"1"}, "policy_")
mock.ExpectQuery("SELECT(.+)users(.+)").
WillReturnRows(sqlmock.NewRows([]string{"id", "group_id"}).AddRow(1, 1))
mock.ExpectQuery("SELECT(.+)groups(.+)").
WillReturnRows(sqlmock.NewRows([]string{"id", "policies"}).AddRow(1, "[522]"))
mock.ExpectQuery("SELECT(.+)policies(.+)").
WillReturnRows(sqlmock.NewRows([]string{"id", "access_key", "secret_key"}).AddRow(2, "123", "123"))
c, _ := gin.CreateTestContext(rec)
c.Params = []gin.Param{
{"key", "testCallBackUpyun"},
}
c.Request, _ = http.NewRequest("POST", "/api/v3/callback/upyun/testCallBackUpyun", ioutil.NopCloser(strings.NewReader("1")))
c.Request.Header["Content-Md5"] = []string{"c4ca4238a0b923820dcc509a6f75849b"}
c.Request.Header["Authorization"] = []string{"UPYUN 123:GWueK9x493BKFFk5gmfdO2Mn6EM="}
AuthFunc(c)
asserts.NoError(mock.ExpectationsWereMet())
asserts.False(c.IsAborted())
}
}
@@ -543,28 +640,87 @@ func TestOneDriveCallbackAuth(t *testing.T) {
rec := httptest.NewRecorder()
AuthFunc := OneDriveCallbackAuth()
// 成功
// Callback Key 相关验证失败
{
c, _ := gin.CreateTestContext(rec)
c.Params = []gin.Param{
{"sessionID", "TestOneDriveCallbackAuth"},
{"key", "testUpyunBackRemote"},
}
c.Set(filesystem.UploadSessionCtx, &serializer.UploadSession{
UID: 1,
VirtualPath: "/",
Policy: model.Policy{
SecretKey: "123",
AccessKey: "123",
},
})
c.Request, _ = http.NewRequest("POST", "/api/v3/callback/upyun/TestOneDriveCallbackAuth", ioutil.NopCloser(strings.NewReader("1")))
res := mq.GlobalMQ.Subscribe("TestOneDriveCallbackAuth", 1)
c.Request, _ = http.NewRequest("POST", "/api/v3/callback/upyun/testUpyunBackRemote", nil)
AuthFunc(c)
select {
case <-res:
case <-time.After(time.Millisecond * 500):
asserts.Fail("mq message should be published")
asserts.True(c.IsAborted())
}
// 成功
{
cache.Set(
"callback_testCallBackUpyun",
serializer.UploadSession{
UID: 1,
PolicyID: 512,
VirtualPath: "/",
},
0,
)
cache.Deletes([]string{"1"}, "policy_")
mock.ExpectQuery("SELECT(.+)users(.+)").
WillReturnRows(sqlmock.NewRows([]string{"id", "group_id"}).AddRow(1, 1))
mock.ExpectQuery("SELECT(.+)groups(.+)").
WillReturnRows(sqlmock.NewRows([]string{"id", "policies"}).AddRow(1, "[657]"))
mock.ExpectQuery("SELECT(.+)policies(.+)").
WillReturnRows(sqlmock.NewRows([]string{"id", "access_key", "secret_key"}).AddRow(2, "123", "123"))
c, _ := gin.CreateTestContext(rec)
c.Params = []gin.Param{
{"key", "testCallBackUpyun"},
}
c.Request, _ = http.NewRequest("POST", "/api/v3/callback/upyun/testCallBackUpyun", ioutil.NopCloser(strings.NewReader("1")))
AuthFunc(c)
asserts.NoError(mock.ExpectationsWereMet())
asserts.False(c.IsAborted())
}
}
func TestCOSCallbackAuth(t *testing.T) {
asserts := assert.New(t)
rec := httptest.NewRecorder()
AuthFunc := COSCallbackAuth()
// Callback Key 相关验证失败
{
c, _ := gin.CreateTestContext(rec)
c.Params = []gin.Param{
{"key", "testUpyunBackRemote"},
}
c.Request, _ = http.NewRequest("POST", "/api/v3/callback/upyun/testUpyunBackRemote", nil)
AuthFunc(c)
asserts.True(c.IsAborted())
}
// 成功
{
cache.Set(
"callback_testCallBackUpyun",
serializer.UploadSession{
UID: 1,
PolicyID: 512,
VirtualPath: "/",
},
0,
)
cache.Deletes([]string{"1"}, "policy_")
mock.ExpectQuery("SELECT(.+)users(.+)").
WillReturnRows(sqlmock.NewRows([]string{"id", "group_id"}).AddRow(1, 1))
mock.ExpectQuery("SELECT(.+)groups(.+)").
WillReturnRows(sqlmock.NewRows([]string{"id", "policies"}).AddRow(1, "[702]"))
mock.ExpectQuery("SELECT(.+)policies(.+)").
WillReturnRows(sqlmock.NewRows([]string{"id", "access_key", "secret_key"}).AddRow(2, "123", "123"))
c, _ := gin.CreateTestContext(rec)
c.Params = []gin.Param{
{"key", "testCallBackUpyun"},
}
c.Request, _ = http.NewRequest("POST", "/api/v3/callback/upyun/testCallBackUpyun", ioutil.NopCloser(strings.NewReader("1")))
AuthFunc(c)
asserts.NoError(mock.ExpectationsWereMet())
asserts.False(c.IsAborted())
}
}
@@ -603,3 +759,46 @@ func TestIsAdmin(t *testing.T) {
asserts.False(c.IsAborted())
}
}
func TestS3CallbackAuth(t *testing.T) {
asserts := assert.New(t)
rec := httptest.NewRecorder()
AuthFunc := S3CallbackAuth()
// Callback Key 相关验证失败
{
c, _ := gin.CreateTestContext(rec)
c.Params = []gin.Param{
{"key", "testUpyunBackRemote"},
}
c.Request, _ = http.NewRequest("POST", "/api/v3/callback/upyun/testUpyunBackRemote", nil)
AuthFunc(c)
asserts.True(c.IsAborted())
}
// 成功
{
cache.Set(
"callback_testCallBackUpyun",
serializer.UploadSession{
UID: 1,
PolicyID: 512,
VirtualPath: "/",
},
0,
)
cache.Deletes([]string{"1"}, "policy_")
mock.ExpectQuery("SELECT(.+)users(.+)").
WillReturnRows(sqlmock.NewRows([]string{"id", "group_id"}).AddRow(1, 1))
mock.ExpectQuery("SELECT(.+)groups(.+)").
WillReturnRows(sqlmock.NewRows([]string{"id", "policies"}).AddRow(1, "[702]"))
c, _ := gin.CreateTestContext(rec)
c.Params = []gin.Param{
{"key", "testCallBackUpyun"},
}
c.Request, _ = http.NewRequest("POST", "/api/v3/callback/upyun/testCallBackUpyun", ioutil.NopCloser(strings.NewReader("1")))
AuthFunc(c)
asserts.False(c.IsAborted())
asserts.NoError(mock.ExpectationsWereMet())
}
}

View File

@@ -1,7 +1,6 @@
package middleware
import (
"github.com/cloudreve/Cloudreve/v3/pkg/auth"
"github.com/cloudreve/Cloudreve/v3/pkg/cluster"
"github.com/cloudreve/Cloudreve/v3/pkg/serializer"
"github.com/gin-gonic/gin"
@@ -11,9 +10,9 @@ import (
// MasterMetadata 解析主机节点发来请求的包含主机节点信息的元数据
func MasterMetadata() gin.HandlerFunc {
return func(c *gin.Context) {
c.Set("MasterSiteID", c.GetHeader(auth.CrHeaderPrefix+"Site-Id"))
c.Set("MasterSiteURL", c.GetHeader(auth.CrHeaderPrefix+"Site-Url"))
c.Set("MasterVersion", c.GetHeader(auth.CrHeaderPrefix+"Cloudreve-Version"))
c.Set("MasterSiteID", c.GetHeader("X-Cr-Site-Id"))
c.Set("MasterSiteURL", c.GetHeader("X-Cr-Site-Url"))
c.Set("MasterVersion", c.GetHeader("X-Cr-Cloudreve-Version"))
c.Next()
}
}
@@ -42,7 +41,7 @@ func UseSlaveAria2Instance(clusterController cluster.Controller) gin.HandlerFunc
func SlaveRPCSignRequired(nodePool cluster.Pool) gin.HandlerFunc {
return func(c *gin.Context) {
nodeID, err := strconv.ParseUint(c.GetHeader(auth.CrHeaderPrefix+"Node-Id"), 10, 64)
nodeID, err := strconv.ParseUint(c.GetHeader("X-Cr-Node-Id"), 10, 64)
if err != nil {
c.JSON(200, serializer.ParamErr("未知的主机节点ID", err))
c.Abort()

View File

@@ -62,10 +62,6 @@ func FrontendFileHandler() gin.HandlerFunc {
return
}
if path == "/service-worker.js" {
c.Header("Cache-Control", "public, no-cache")
}
// 存在的静态文件
fileServer.ServeHTTP(c.Writer, c.Request)
c.Abort()

View File

@@ -38,10 +38,3 @@ func IsFunctionEnabled(key string) gin.HandlerFunc {
c.Next()
}
}
// CacheControl 屏蔽客户端缓存
func CacheControl() gin.HandlerFunc {
return func(c *gin.Context) {
c.Header("Cache-Control", "private, no-cache")
}
}

View File

@@ -76,12 +76,3 @@ func TestIsFunctionEnabled(t *testing.T) {
}
}
func TestCacheControl(t *testing.T) {
a := assert.New(t)
TestFunc := CacheControl()
rec := httptest.NewRecorder()
c, _ := gin.CreateTestContext(rec)
TestFunc(c)
a.Contains(c.Writer.Header().Get("Cache-Control"), "no-cache")
}

View File

@@ -1,117 +0,0 @@
package model
import (
"github.com/cloudreve/Cloudreve/v3/pkg/conf"
"github.com/cloudreve/Cloudreve/v3/pkg/util"
"github.com/gofrs/uuid"
)
var defaultSettings = []Setting{
{Name: "siteURL", Value: `http://localhost`, Type: "basic"},
{Name: "siteName", Value: `Cloudreve`, Type: "basic"},
{Name: "siteICPId", Value: ``, Type: "basic"},
{Name: "register_enabled", Value: `1`, Type: "register"},
{Name: "default_group", Value: `2`, Type: "register"},
{Name: "siteKeywords", Value: `网盘,网盘`, Type: "basic"},
{Name: "siteDes", Value: `Cloudreve`, Type: "basic"},
{Name: "siteTitle", Value: `平步云端`, Type: "basic"},
{Name: "siteScript", Value: ``, Type: "basic"},
{Name: "siteID", Value: uuid.Must(uuid.NewV4()).String(), Type: "basic"},
{Name: "fromName", Value: `Cloudreve`, Type: "mail"},
{Name: "mail_keepalive", Value: `30`, Type: "mail"},
{Name: "fromAdress", Value: `no-reply@acg.blue`, Type: "mail"},
{Name: "smtpHost", Value: `smtp.mxhichina.com`, Type: "mail"},
{Name: "smtpPort", Value: `25`, Type: "mail"},
{Name: "replyTo", Value: `abslant@126.com`, Type: "mail"},
{Name: "smtpUser", Value: `no-reply@acg.blue`, Type: "mail"},
{Name: "smtpPass", Value: ``, Type: "mail"},
{Name: "smtpEncryption", Value: `0`, Type: "mail"},
{Name: "maxEditSize", Value: `4194304`, Type: "file_edit"},
{Name: "archive_timeout", Value: `600`, Type: "timeout"},
{Name: "download_timeout", Value: `600`, Type: "timeout"},
{Name: "preview_timeout", Value: `600`, Type: "timeout"},
{Name: "doc_preview_timeout", Value: `600`, Type: "timeout"},
{Name: "upload_session_timeout", Value: `86400`, Type: "timeout"},
{Name: "slave_api_timeout", Value: `60`, Type: "timeout"},
{Name: "slave_node_retry", Value: `3`, Type: "slave"},
{Name: "slave_ping_interval", Value: `60`, Type: "slave"},
{Name: "slave_recover_interval", Value: `120`, Type: "slave"},
{Name: "slave_transfer_timeout", Value: `172800`, Type: "timeout"},
{Name: "onedrive_monitor_timeout", Value: `600`, Type: "timeout"},
{Name: "share_download_session_timeout", Value: `2073600`, Type: "timeout"},
{Name: "onedrive_callback_check", Value: `20`, Type: "timeout"},
{Name: "folder_props_timeout", Value: `300`, Type: "timeout"},
{Name: "chunk_retries", Value: `5`, Type: "retry"},
{Name: "onedrive_source_timeout", Value: `1800`, Type: "timeout"},
{Name: "reset_after_upload_failed", Value: `0`, Type: "upload"},
{Name: "use_temp_chunk_buffer", Value: `1`, Type: "upload"},
{Name: "login_captcha", Value: `0`, Type: "login"},
{Name: "reg_captcha", Value: `0`, Type: "login"},
{Name: "email_active", Value: `0`, Type: "register"},
{Name: "mail_activation_template", Value: `<!DOCTYPE html PUBLIC"-//W3C//DTD XHTML 1.0 Transitional//EN""http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd"><html xmlns="http://www.w3.org/1999/xhtml"style="font-family: 'Helvetica Neue', Helvetica, Arial, sans-serif; box-sizing: border-box;
font-size: 14px; margin: 0;"><head><meta name="viewport"content="width=device-width"/><meta http-equiv="Content-Type"content="text/html; charset=UTF-8"/><title>激活您的账户</title><style type="text/css">img{max-width:100%}body{-webkit-font-smoothing:antialiased;-webkit-text-size-adjust:none;width:100%!important;height:100%;line-height:1.6em}body{background-color:#f6f6f6}@media only screen and(max-width:640px){body{padding:0!important}h1{font-weight:800!important;margin:20px 0 5px!important}h2{font-weight:800!important;margin:20px 0 5px!important}h3{font-weight:800!important;margin:20px 0 5px!important}h4{font-weight:800!important;margin:20px 0 5px!important}h1{font-size:22px!important}h2{font-size:18px!important}h3{font-size:16px!important}.container{padding:0!important;width:100%!important}.content{padding:0!important}.content-wrap{padding:10px!important}.invoice{width:100%!important}}</style></head><body itemscope itemtype="http://schema.org/EmailMessage"style="font-family: 'Helvetica Neue',Helvetica,Arial,sans-serif; box-sizing:
border-box; font-size: 14px; -webkit-font-smoothing: antialiased; -webkit-text-size-adjust: none; width: 100% !important; height: 100%; line-height: 1.6em; background-color: #f6f6f6; margin: 0;"bgcolor="#f6f6f6"><table class="body-wrap"style="font-family: 'Helvetica Neue',Helvetica,Arial,sans-serif; box-sizing: border-box; font-size: 14px; width: 100%; background-color: #f6f6f6; margin: 0;"bgcolor="#f6f6f6"><tr style="font-family: 'Helvetica Neue',Helvetica,Arial,sans-serif;
box-sizing: border-box; font-size: 14px; margin: 0;"><td style="font-family: 'Helvetica Neue',Helvetica,Arial,sans-serif; box-sizing: border-box; font-size: 14px; vertical-align: top; margin: 0;"valign="top"></td><td class="container"width="600"style="font-family: 'Helvetica Neue',Helvetica,Arial,sans-serif; box-sizing: border-box; font-size: 14px; vertical-align: top; display: block !important; max-width: 600px !important; clear: both !important; margin: 0 auto;"valign="top"><div class="content"style="font-family: 'Helvetica Neue',Helvetica,Arial,sans-serif; box-sizing: border-box; font-size: 14px; max-width: 600px; display: block; margin: 0 auto; padding: 20px;"><table class="main"width="100%"cellpadding="0"cellspacing="0"style="font-family: 'Helvetica Neue',Helvetica,Arial,sans-serif; box-sizing: border-box; font-size: 14px; border-radius: 3px; background-color: #fff; margin: 0; border: 1px
solid #e9e9e9;"bgcolor="#fff"><tr style="font-family: 'Helvetica Neue',Helvetica,Arial,sans-serif; box-sizing: border-box; font-size:
14px; margin: 0;"><td class="alert alert-warning"style="font-family: 'Helvetica Neue',Helvetica,Arial,sans-serif; box-sizing: border-box; font-size: 16px; vertical-align: top; color: #fff; font-weight: 500; text-align: center; border-radius: 3px 3px 0 0; background-color: #009688; margin: 0; padding: 20px;"align="center"bgcolor="#FF9F00"valign="top">激活{siteTitle}账户</td></tr><tr style="font-family: 'Helvetica Neue',Helvetica,Arial,sans-serif; box-sizing: border-box; font-size: 14px; margin: 0;"><td class="content-wrap"style="font-family: 'Helvetica Neue',Helvetica,Arial,sans-serif; box-sizing: border-box; font-size: 14px; vertical-align: top; margin: 0; padding: 20px;"valign="top"><table width="100%"cellpadding="0"cellspacing="0"style="font-family: 'Helvetica Neue',Helvetica,Arial,sans-serif; box-sizing: border-box; font-size: 14px; margin: 0;"><tr style="font-family: 'Helvetica Neue',Helvetica,Arial,sans-serif; box-sizing: border-box; font-size: 14px; margin: 0;"><td class="content-block"style="font-family: 'Helvetica
Neue',Helvetica,Arial,sans-serif; box-sizing: border-box; font-size: 14px; vertical-align: top; margin: 0; padding: 0 0 20px;"valign="top">亲爱的<strong style="font-family: 'Helvetica Neue',Helvetica,Arial,sans-serif; box-sizing: border-box; font-size: 14px; margin: 0;">{userName}</strong></td></tr><tr style="font-family: 'Helvetica Neue',Helvetica,Arial,sans-serif; box-sizing: border-box; font-size: 14px; margin: 0;"><td class="content-block"style="font-family: 'Helvetica Neue',Helvetica,Arial,sans-serif; box-sizing: border-box; font-size: 14px; vertical-align: top; margin: 0; padding: 0 0 20px;"valign="top">感谢您注册{siteTitle},请点击下方按钮完成账户激活。</td></tr><tr style="font-family: 'Helvetica Neue',Helvetica,Arial,sans-serif; box-sizing: border-box; font-size: 14px; margin: 0;"><td class="content-block"style="font-family: 'Helvetica Neue',Helvetica,Arial,sans-serif; box-sizing: border-box; font-size: 14px; vertical-align: top; margin: 0; padding: 0 0 20px;"valign="top"><a href="{activationUrl}"class="btn-primary"style="font-family: 'Helvetica Neue',Helvetica,Arial,sans-serif; box-sizing: border-box; font-size: 14px; color: #FFF; text-decoration: none; line-height: 2em; font-weight: bold; text-align: center; cursor: pointer; display: inline-block; border-radius: 5px; text-transform: capitalize; background-color: #009688; margin: 0; border-color: #009688; border-style: solid; border-width: 10px 20px;">激活账户</a></td></tr><tr style="font-family: 'Helvetica Neue',Helvetica,Arial,sans-serif; box-sizing: border-box; font-size: 14px; margin: 0;"><td class="content-block"style="font-family: 'Helvetica Neue',Helvetica,Arial,sans-serif; box-sizing: border-box; font-size: 14px; vertical-align: top; margin: 0; padding: 0 0 20px;"valign="top">感谢您选择{siteTitle}。</td></tr></table></td></tr></table><div class="footer"style="font-family: 'Helvetica Neue',Helvetica,Arial,sans-serif; box-sizing: border-box; font-size: 14px; width: 100%; clear: both; color: #999; margin: 0; padding: 20px;"><table width="100%"style="font-family: 'Helvetica Neue',Helvetica,Arial,sans-serif; box-sizing: border-box; font-size: 14px; margin: 0;"><tr style="font-family: 'Helvetica Neue',Helvetica,Arial,sans-serif; box-sizing: border-box; font-size: 14px; margin: 0;"><td class="aligncenter content-block"style="font-family: 'Helvetica Neue',Helvetica,Arial,sans-serif; box-sizing: border-box; font-size: 12px; vertical-align: top; color: #999; text-align: center; margin: 0; padding: 0 0 20px;"align="center"valign="top">此邮件由系统自动发送,请不要直接回复。</td></tr></table></div></div></td><td style="font-family: 'Helvetica Neue',Helvetica,Arial,sans-serif; box-sizing: border-box; font-size: 14px; vertical-align: top; margin: 0;"valign="top"></td></tr></table></body></html>`, Type: "mail_template"},
{Name: "forget_captcha", Value: `0`, Type: "login"},
{Name: "mail_reset_pwd_template", Value: `<!DOCTYPE html PUBLIC"-//W3C//DTD XHTML 1.0 Transitional//EN""http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd"><html xmlns="http://www.w3.org/1999/xhtml"style="font-family: 'Helvetica Neue', Helvetica, Arial, sans-serif; box-sizing: border-box;
font-size: 14px; margin: 0;"><head><meta name="viewport"content="width=device-width"/><meta http-equiv="Content-Type"content="text/html; charset=UTF-8"/><title>重设密码</title><style type="text/css">img{max-width:100%}body{-webkit-font-smoothing:antialiased;-webkit-text-size-adjust:none;width:100%!important;height:100%;line-height:1.6em}body{background-color:#f6f6f6}@media only screen and(max-width:640px){body{padding:0!important}h1{font-weight:800!important;margin:20px 0 5px!important}h2{font-weight:800!important;margin:20px 0 5px!important}h3{font-weight:800!important;margin:20px 0 5px!important}h4{font-weight:800!important;margin:20px 0 5px!important}h1{font-size:22px!important}h2{font-size:18px!important}h3{font-size:16px!important}.container{padding:0!important;width:100%!important}.content{padding:0!important}.content-wrap{padding:10px!important}.invoice{width:100%!important}}</style></head><body itemscope itemtype="http://schema.org/EmailMessage"style="font-family: 'Helvetica Neue',Helvetica,Arial,sans-serif; box-sizing:
border-box; font-size: 14px; -webkit-font-smoothing: antialiased; -webkit-text-size-adjust: none; width: 100% !important; height: 100%; line-height: 1.6em; background-color: #f6f6f6; margin: 0;"bgcolor="#f6f6f6"><table class="body-wrap"style="font-family: 'Helvetica Neue',Helvetica,Arial,sans-serif; box-sizing: border-box; font-size: 14px; width: 100%; background-color: #f6f6f6; margin: 0;"bgcolor="#f6f6f6"><tr style="font-family: 'Helvetica Neue',Helvetica,Arial,sans-serif;
box-sizing: border-box; font-size: 14px; margin: 0;"><td style="font-family: 'Helvetica Neue',Helvetica,Arial,sans-serif; box-sizing: border-box; font-size: 14px; vertical-align: top; margin: 0;"valign="top"></td><td class="container"width="600"style="font-family: 'Helvetica Neue',Helvetica,Arial,sans-serif; box-sizing: border-box; font-size: 14px; vertical-align: top; display: block !important; max-width: 600px !important; clear: both !important; margin: 0 auto;"valign="top"><div class="content"style="font-family: 'Helvetica Neue',Helvetica,Arial,sans-serif; box-sizing: border-box; font-size: 14px; max-width: 600px; display: block; margin: 0 auto; padding: 20px;"><table class="main"width="100%"cellpadding="0"cellspacing="0"style="font-family: 'Helvetica Neue',Helvetica,Arial,sans-serif; box-sizing: border-box; font-size: 14px; border-radius: 3px; background-color: #fff; margin: 0; border: 1px
solid #e9e9e9;"bgcolor="#fff"><tr style="font-family: 'Helvetica Neue',Helvetica,Arial,sans-serif; box-sizing: border-box; font-size:
14px; margin: 0;"><td class="alert alert-warning"style="font-family: 'Helvetica Neue',Helvetica,Arial,sans-serif; box-sizing: border-box; font-size: 16px; vertical-align: top; color: #fff; font-weight: 500; text-align: center; border-radius: 3px 3px 0 0; background-color: #2196F3; margin: 0; padding: 20px;"align="center"bgcolor="#FF9F00"valign="top">重设{siteTitle}密码</td></tr><tr style="font-family: 'Helvetica Neue',Helvetica,Arial,sans-serif; box-sizing: border-box; font-size: 14px; margin: 0;"><td class="content-wrap"style="font-family: 'Helvetica Neue',Helvetica,Arial,sans-serif; box-sizing: border-box; font-size: 14px; vertical-align: top; margin: 0; padding: 20px;"valign="top"><table width="100%"cellpadding="0"cellspacing="0"style="font-family: 'Helvetica Neue',Helvetica,Arial,sans-serif; box-sizing: border-box; font-size: 14px; margin: 0;"><tr style="font-family: 'Helvetica Neue',Helvetica,Arial,sans-serif; box-sizing: border-box; font-size: 14px; margin: 0;"><td class="content-block"style="font-family: 'Helvetica
Neue',Helvetica,Arial,sans-serif; box-sizing: border-box; font-size: 14px; vertical-align: top; margin: 0; padding: 0 0 20px;"valign="top">亲爱的<strong style="font-family: 'Helvetica Neue',Helvetica,Arial,sans-serif; box-sizing: border-box; font-size: 14px; margin: 0;">{userName}</strong></td></tr><tr style="font-family: 'Helvetica Neue',Helvetica,Arial,sans-serif; box-sizing: border-box; font-size: 14px; margin: 0;"><td class="content-block"style="font-family: 'Helvetica Neue',Helvetica,Arial,sans-serif; box-sizing: border-box; font-size: 14px; vertical-align: top; margin: 0; padding: 0 0 20px;"valign="top">请点击下方按钮完成密码重设。如果非你本人操作,请忽略此邮件。</td></tr><tr style="font-family: 'Helvetica Neue',Helvetica,Arial,sans-serif; box-sizing: border-box; font-size: 14px; margin: 0;"><td class="content-block"style="font-family: 'Helvetica Neue',Helvetica,Arial,sans-serif; box-sizing: border-box; font-size: 14px; vertical-align: top; margin: 0; padding: 0 0 20px;"valign="top"><a href="{resetUrl}"class="btn-primary"style="font-family: 'Helvetica Neue',Helvetica,Arial,sans-serif; box-sizing: border-box; font-size: 14px; color: #FFF; text-decoration: none; line-height: 2em; font-weight: bold; text-align: center; cursor: pointer; display: inline-block; border-radius: 5px; text-transform: capitalize; background-color: #2196F3; margin: 0; border-color: #2196F3; border-style: solid; border-width: 10px 20px;">重设密码</a></td></tr><tr style="font-family: 'Helvetica Neue',Helvetica,Arial,sans-serif; box-sizing: border-box; font-size: 14px; margin: 0;"><td class="content-block"style="font-family: 'Helvetica Neue',Helvetica,Arial,sans-serif; box-sizing: border-box; font-size: 14px; vertical-align: top; margin: 0; padding: 0 0 20px;"valign="top">感谢您选择{siteTitle}。</td></tr></table></td></tr></table><div class="footer"style="font-family: 'Helvetica Neue',Helvetica,Arial,sans-serif; box-sizing: border-box; font-size: 14px; width: 100%; clear: both; color: #999; margin: 0; padding: 20px;"><table width="100%"style="font-family: 'Helvetica Neue',Helvetica,Arial,sans-serif; box-sizing: border-box; font-size: 14px; margin: 0;"><tr style="font-family: 'Helvetica Neue',Helvetica,Arial,sans-serif; box-sizing: border-box; font-size: 14px; margin: 0;"><td class="aligncenter content-block"style="font-family: 'Helvetica Neue',Helvetica,Arial,sans-serif; box-sizing: border-box; font-size: 12px; vertical-align: top; color: #999; text-align: center; margin: 0; padding: 0 0 20px;"align="center"valign="top">此邮件由系统自动发送,请不要直接回复。</td></tr></table></div></div></td><td style="font-family: 'Helvetica Neue',Helvetica,Arial,sans-serif; box-sizing: border-box; font-size: 14px; vertical-align: top; margin: 0;"valign="top"></td></tr></table></body></html>`, Type: "mail_template"},
{Name: "db_version_" + conf.RequiredDBVersion, Value: `installed`, Type: "version"},
{Name: "hot_share_num", Value: `10`, Type: "share"},
{Name: "gravatar_server", Value: `https://www.gravatar.com/`, Type: "avatar"},
{Name: "defaultTheme", Value: `#3f51b5`, Type: "basic"},
{Name: "themes", Value: `{"#3f51b5":{"palette":{"primary":{"main":"#3f51b5"},"secondary":{"main":"#f50057"}}},"#2196f3":{"palette":{"primary":{"main":"#2196f3"},"secondary":{"main":"#FFC107"}}},"#673AB7":{"palette":{"primary":{"main":"#673AB7"},"secondary":{"main":"#2196F3"}}},"#E91E63":{"palette":{"primary":{"main":"#E91E63"},"secondary":{"main":"#42A5F5","contrastText":"#fff"}}},"#FF5722":{"palette":{"primary":{"main":"#FF5722"},"secondary":{"main":"#3F51B5"}}},"#FFC107":{"palette":{"primary":{"main":"#FFC107"},"secondary":{"main":"#26C6DA"}}},"#8BC34A":{"palette":{"primary":{"main":"#8BC34A","contrastText":"#fff"},"secondary":{"main":"#FF8A65","contrastText":"#fff"}}},"#009688":{"palette":{"primary":{"main":"#009688"},"secondary":{"main":"#4DD0E1","contrastText":"#fff"}}},"#607D8B":{"palette":{"primary":{"main":"#607D8B"},"secondary":{"main":"#F06292"}}},"#795548":{"palette":{"primary":{"main":"#795548"},"secondary":{"main":"#4CAF50","contrastText":"#fff"}}}}`, Type: "basic"},
{Name: "max_worker_num", Value: `10`, Type: "task"},
{Name: "max_parallel_transfer", Value: `4`, Type: "task"},
{Name: "secret_key", Value: util.RandStringRunes(256), Type: "auth"},
{Name: "temp_path", Value: "temp", Type: "path"},
{Name: "avatar_path", Value: "avatar", Type: "path"},
{Name: "avatar_size", Value: "2097152", Type: "avatar"},
{Name: "avatar_size_l", Value: "200", Type: "avatar"},
{Name: "avatar_size_m", Value: "130", Type: "avatar"},
{Name: "avatar_size_s", Value: "50", Type: "avatar"},
{Name: "home_view_method", Value: "icon", Type: "view"},
{Name: "share_view_method", Value: "list", Type: "view"},
{Name: "cron_garbage_collect", Value: "@hourly", Type: "cron"},
{Name: "cron_recycle_upload_session", Value: "@every 1h30m", Type: "cron"},
{Name: "authn_enabled", Value: "0", Type: "authn"},
{Name: "captcha_type", Value: "normal", Type: "captcha"},
{Name: "captcha_height", Value: "60", Type: "captcha"},
{Name: "captcha_width", Value: "240", Type: "captcha"},
{Name: "captcha_mode", Value: "3", Type: "captcha"},
{Name: "captcha_ComplexOfNoiseText", Value: "0", Type: "captcha"},
{Name: "captcha_ComplexOfNoiseDot", Value: "0", Type: "captcha"},
{Name: "captcha_IsShowHollowLine", Value: "0", Type: "captcha"},
{Name: "captcha_IsShowNoiseDot", Value: "1", Type: "captcha"},
{Name: "captcha_IsShowNoiseText", Value: "0", Type: "captcha"},
{Name: "captcha_IsShowSlimeLine", Value: "1", Type: "captcha"},
{Name: "captcha_IsShowSineLine", Value: "0", Type: "captcha"},
{Name: "captcha_CaptchaLen", Value: "6", Type: "captcha"},
{Name: "captcha_ReCaptchaKey", Value: "defaultKey", Type: "captcha"},
{Name: "captcha_ReCaptchaSecret", Value: "defaultSecret", Type: "captcha"},
{Name: "captcha_TCaptcha_CaptchaAppId", Value: "", Type: "captcha"},
{Name: "captcha_TCaptcha_AppSecretKey", Value: "", Type: "captcha"},
{Name: "captcha_TCaptcha_SecretId", Value: "", Type: "captcha"},
{Name: "captcha_TCaptcha_SecretKey", Value: "", Type: "captcha"},
{Name: "thumb_width", Value: "400", Type: "thumb"},
{Name: "thumb_height", Value: "300", Type: "thumb"},
{Name: "thumb_file_suffix", Value: "._thumb", Type: "thumb"},
{Name: "thumb_max_task_count", Value: "-1", Type: "thumb"},
{Name: "thumb_encode_method", Value: "jpg", Type: "thumb"},
{Name: "thumb_gc_after_gen", Value: "0", Type: "thumb"},
{Name: "thumb_encode_quality", Value: "85", Type: "thumb"},
{Name: "pwa_small_icon", Value: "/static/img/favicon.ico", Type: "pwa"},
{Name: "pwa_medium_icon", Value: "/static/img/logo192.png", Type: "pwa"},
{Name: "pwa_large_icon", Value: "/static/img/logo512.png", Type: "pwa"},
{Name: "pwa_display", Value: "standalone", Type: "pwa"},
{Name: "pwa_theme_color", Value: "#000000", Type: "pwa"},
{Name: "pwa_background_color", Value: "#ffffff", Type: "pwa"},
{Name: "office_preview_service", Value: "https://view.officeapps.live.com/op/view.aspx?src={$src}", Type: "preview"},
}

View File

@@ -2,8 +2,6 @@ package model
import (
"encoding/gob"
"encoding/json"
"errors"
"path"
"time"
@@ -15,22 +13,19 @@ import (
type File struct {
// 表字段
gorm.Model
Name string `gorm:"unique_index:idx_only_one"`
SourceName string `gorm:"type:text"`
UserID uint `gorm:"index:user_id;unique_index:idx_only_one"`
Size uint64
PicInfo string
FolderID uint `gorm:"index:folder_id;unique_index:idx_only_one"`
PolicyID uint
UploadSessionID *string `gorm:"index:session_id;unique_index:session_only_one"`
Metadata string `gorm:"type:text"`
Name string `gorm:"unique_index:idx_only_one"`
SourceName string `gorm:"type:text"`
UserID uint `gorm:"index:user_id;unique_index:idx_only_one"`
Size uint64
PicInfo string
FolderID uint `gorm:"index:folder_id;unique_index:idx_only_one"`
PolicyID uint
// 关联模型
Policy Policy `gorm:"PRELOAD:false,association_autoupdate:false"`
// 数据库忽略字段
Position string `gorm:"-"`
MetadataSerialized map[string]string `gorm:"-"`
Position string `gorm:"-"`
}
func init() {
@@ -39,40 +34,12 @@ func init() {
}
// Create 创建文件记录
func (file *File) Create() error {
tx := DB.Begin()
if err := tx.Create(file).Error; err != nil {
func (file *File) Create() (uint, error) {
if err := DB.Create(file).Error; err != nil {
util.Log().Warning("无法插入文件记录, %s", err)
tx.Rollback()
return err
return 0, err
}
user := &User{}
user.ID = file.UserID
if err := user.ChangeStorage(tx, "+", file.Size); err != nil {
tx.Rollback()
return err
}
return tx.Commit().Error
}
// AfterFind 找到文件后的钩子
func (file *File) AfterFind() (err error) {
// 反序列化文件元数据
if file.Metadata != "" {
err = json.Unmarshal([]byte(file.Metadata), &file.MetadataSerialized)
}
return
}
// BeforeSave Save策略前的钩子
func (file *File) BeforeSave() (err error) {
metaValue, err := json.Marshal(&file.MetadataSerialized)
file.Metadata = string(metaValue)
return err
return file.ID, nil
}
// GetChildFile 查找目录下名为name的子文件
@@ -151,19 +118,6 @@ func GetChildFilesOfFolders(folders *[]Folder) ([]File, error) {
return files, result.Error
}
// GetUploadPlaceholderFiles 获取所有上传占位文件
// UID为0表示忽略用户
func GetUploadPlaceholderFiles(uid uint) []*File {
query := DB
if uid != 0 {
query = query.Where("user_id = ?", uid)
}
var files []*File
query.Where("upload_session_id is not NULL").Find(&files)
return files
}
// GetPolicy 获取文件所属策略
func (file *File) GetPolicy() *Policy {
if file.Policy.Model.ID == 0 {
@@ -212,38 +166,10 @@ func RemoveFilesWithSoftLinks(files []File) ([]File, error) {
}
// DeleteFiles 批量删除文件记录并归还容量
func DeleteFiles(files []*File, uid uint) error {
tx := DB.Begin()
user := &User{}
user.ID = uid
var size uint64
for _, file := range files {
if file.UserID != uid {
tx.Rollback()
return errors.New("user id not consistent")
}
result := tx.Unscoped().Where("size = ?", file.Size).Delete(file)
if result.Error != nil {
tx.Rollback()
return result.Error
}
if result.RowsAffected == 0 {
tx.Rollback()
return errors.New("file size is dirty")
}
size += file.Size
}
if err := user.ChangeStorage(tx, "-", size); err != nil {
tx.Rollback()
return err
}
return tx.Commit().Error
// DeleteFileByIDs 根据给定ID批量删除文件记录
func DeleteFileByIDs(ids []uint) error {
result := DB.Where("id in (?)", ids).Unscoped().Delete(&File{})
return result.Error
}
// GetFilesByParentIDs 根据父目录ID查找文件
@@ -253,13 +179,6 @@ func GetFilesByParentIDs(ids []uint, uid uint) ([]File, error) {
return files, result.Error
}
// GetFilesByUploadSession 查找上传会话对应的文件
func GetFilesByUploadSession(sessionID string, uid uint) (*File, error) {
file := File{}
result := DB.Where("user_id = ? and upload_session_id = ?", uid, sessionID).Find(&file)
return &file, result.Error
}
// Rename 重命名文件
func (file *File) Rename(new string) error {
return DB.Model(&file).Update("name", new).Error
@@ -267,39 +186,12 @@ func (file *File) Rename(new string) error {
// UpdatePicInfo 更新文件的图像信息
func (file *File) UpdatePicInfo(value string) error {
return DB.Model(&file).Set("gorm:association_autoupdate", false).UpdateColumns(File{PicInfo: value}).Error
return DB.Model(&file).Set("gorm:association_autoupdate", false).Update("pic_info", value).Error
}
// UpdateSize 更新文件的大小信息
// TODO: 全局锁
func (file *File) UpdateSize(value uint64) error {
tx := DB.Begin()
var sizeDelta uint64
operator := "+"
user := User{}
user.ID = file.UserID
if value > file.Size {
sizeDelta = value - file.Size
} else {
operator = "-"
sizeDelta = file.Size - value
}
if res := tx.Model(&file).
Where("size = ?", file.Size).
Set("gorm:association_autoupdate", false).
Update("size", value); res.Error != nil {
tx.Rollback()
return res.Error
}
if err := user.ChangeStorage(tx, operator, sizeDelta); err != nil {
tx.Rollback()
return err
}
file.Size = value
return tx.Commit().Error
return DB.Model(&file).Set("gorm:association_autoupdate", false).Update("size", value).Error
}
// UpdateSourceName 更新文件的源文件名
@@ -307,24 +199,6 @@ func (file *File) UpdateSourceName(value string) error {
return DB.Model(&file).Set("gorm:association_autoupdate", false).Update("source_name", value).Error
}
func (file *File) PopChunkToFile(lastModified *time.Time, picInfo string) error {
file.UploadSessionID = nil
if lastModified != nil {
file.UpdatedAt = *lastModified
}
return DB.Model(file).UpdateColumns(map[string]interface{}{
"upload_session_id": file.UploadSessionID,
"updated_at": file.UpdatedAt,
"pic_info": picInfo,
}).Error
}
// CanCopy 返回文件是否可被复制
func (file *File) CanCopy() bool {
return file.UploadSessionID == nil
}
/*
实现 webdav.FileInfo 接口
*/

View File

@@ -15,62 +15,22 @@ func TestFile_Create(t *testing.T) {
Name: "123",
}
// 无法插入文件记录
{
mock.ExpectBegin()
mock.ExpectExec("INSERT(.+)").WillReturnError(errors.New("error"))
mock.ExpectRollback()
err := file.Create()
asserts.Error(err)
asserts.NoError(mock.ExpectationsWereMet())
}
mock.ExpectBegin()
mock.ExpectExec("INSERT(.+)").WillReturnResult(sqlmock.NewResult(5, 1))
mock.ExpectCommit()
fileID, err := file.Create()
asserts.NoError(err)
asserts.Equal(uint(5), fileID)
asserts.Equal(uint(5), file.ID)
asserts.NoError(mock.ExpectationsWereMet())
// 无法更新用户容量
{
mock.ExpectBegin()
mock.ExpectExec("INSERT(.+)").WillReturnResult(sqlmock.NewResult(5, 1))
mock.ExpectExec("UPDATE(.+)").WillReturnError(errors.New("error"))
mock.ExpectRollback()
err := file.Create()
asserts.Error(err)
asserts.NoError(mock.ExpectationsWereMet())
}
mock.ExpectBegin()
mock.ExpectExec("INSERT(.+)").WillReturnError(errors.New("error"))
mock.ExpectRollback()
fileID, err = file.Create()
asserts.Error(err)
asserts.NoError(mock.ExpectationsWereMet())
// 成功
{
mock.ExpectBegin()
mock.ExpectExec("INSERT(.+)").WillReturnResult(sqlmock.NewResult(5, 1))
mock.ExpectExec("UPDATE(.+)storage(.+)").WillReturnResult(sqlmock.NewResult(0, 1))
mock.ExpectCommit()
err := file.Create()
asserts.NoError(err)
asserts.Equal(uint(5), file.ID)
asserts.NoError(mock.ExpectationsWereMet())
}
}
func TestFile_AfterFind(t *testing.T) {
a := assert.New(t)
file := File{
Name: "123",
Metadata: "{\"name\":\"123\"}",
}
a.NoError(file.AfterFind())
a.Equal("123", file.MetadataSerialized["name"])
}
func TestFile_BeforeSave(t *testing.T) {
a := assert.New(t)
file := File{
Name: "123",
MetadataSerialized: map[string]string{
"name": "123",
},
}
a.NoError(file.BeforeSave())
a.Equal("{\"name\":\"123\"}", file.Metadata)
}
func TestFolder_GetChildFile(t *testing.T) {
@@ -215,17 +175,6 @@ func TestGetChildFilesOfFolders(t *testing.T) {
}
}
func TestGetUploadPlaceholderFiles(t *testing.T) {
a := assert.New(t)
mock.ExpectQuery("SELECT(.+)upload_session_id(.+)").
WithArgs(1).
WillReturnRows(sqlmock.NewRows([]string{"id", "name"}).AddRow(1, "1"))
files := GetUploadPlaceholderFiles(1)
a.NoError(mock.ExpectationsWereMet())
a.Len(files, 1)
}
func TestFile_GetPolicy(t *testing.T) {
asserts := assert.New(t)
@@ -333,62 +282,28 @@ func TestRemoveFilesWithSoftLinks(t *testing.T) {
}
}
func TestDeleteFiles(t *testing.T) {
a := assert.New(t)
func TestDeleteFileByIDs(t *testing.T) {
asserts := assert.New(t)
// uid 不一致
{
err := DeleteFiles([]*File{{}}, 1)
a.Contains("user id not consistent", err.Error())
}
// 删除失败
// 出错
{
mock.ExpectBegin()
mock.ExpectExec("DELETE(.+)").
WillReturnError(errors.New("error"))
mock.ExpectRollback()
err := DeleteFiles([]*File{{}}, 0)
a.NoError(mock.ExpectationsWereMet())
a.Error(err)
err := DeleteFileByIDs([]uint{1, 2, 3})
asserts.NoError(mock.ExpectationsWereMet())
asserts.Error(err)
}
// 无法变更用户容量
{
mock.ExpectBegin()
mock.ExpectExec("DELETE(.+)").
WillReturnResult(sqlmock.NewResult(1, 1))
mock.ExpectExec("UPDATE(.+)storage(.+)").WillReturnError(errors.New("error"))
mock.ExpectRollback()
err := DeleteFiles([]*File{{}}, 0)
a.NoError(mock.ExpectationsWereMet())
a.Error(err)
}
// 文件脏读
{
mock.ExpectBegin()
mock.ExpectExec("DELETE(.+)").
WillReturnResult(sqlmock.NewResult(1, 0))
mock.ExpectRollback()
err := DeleteFiles([]*File{{Size: 1}, {Size: 2}}, 0)
a.NoError(mock.ExpectationsWereMet())
a.Error(err)
a.Contains("file size is dirty", err.Error())
}
// 成功
{
mock.ExpectBegin()
mock.ExpectExec("DELETE(.+)").
WillReturnResult(sqlmock.NewResult(2, 1))
mock.ExpectExec("DELETE(.+)").
WillReturnResult(sqlmock.NewResult(2, 1))
mock.ExpectExec("UPDATE(.+)storage(.+)").WithArgs(uint64(3), sqlmock.AnyArg()).WillReturnResult(sqlmock.NewResult(1, 1))
WillReturnResult(sqlmock.NewResult(0, 3))
mock.ExpectCommit()
err := DeleteFiles([]*File{{Size: 1}, {Size: 2}}, 0)
a.NoError(mock.ExpectationsWereMet())
a.NoError(err)
err := DeleteFileByIDs([]uint{1, 2, 3})
asserts.NoError(mock.ExpectationsWereMet())
asserts.NoError(err)
}
}
@@ -409,19 +324,6 @@ func TestGetFilesByParentIDs(t *testing.T) {
asserts.Len(files, 3)
}
func TestGetFilesByUploadSession(t *testing.T) {
a := assert.New(t)
mock.ExpectQuery("SELECT(.+)").
WithArgs(1, "sessionID").
WillReturnRows(
sqlmock.NewRows([]string{"id", "name"}).AddRow(4, "4.txt"))
files, err := GetFilesByUploadSession("sessionID", 1)
a.NoError(err)
a.NoError(mock.ExpectationsWereMet())
a.Equal("4.txt", files.Name)
}
func TestFile_Updates(t *testing.T) {
asserts := assert.New(t)
file := File{Model: gorm.Model{ID: 1}}
@@ -438,91 +340,22 @@ func TestFile_Updates(t *testing.T) {
// UpdatePicInfo
{
mock.ExpectBegin()
mock.ExpectExec("UPDATE(.+)").WithArgs("1,1", 1).WillReturnResult(sqlmock.NewResult(1, 1))
mock.ExpectExec("UPDATE(.+)").WithArgs(10, sqlmock.AnyArg(), 1).WillReturnResult(sqlmock.NewResult(1, 1))
mock.ExpectCommit()
err := file.UpdateSize(10)
asserts.NoError(mock.ExpectationsWereMet())
asserts.NoError(err)
}
// UpdatePicInfo
{
mock.ExpectBegin()
mock.ExpectExec("UPDATE(.+)").WithArgs("1,1", sqlmock.AnyArg(), 1).WillReturnResult(sqlmock.NewResult(1, 1))
mock.ExpectCommit()
err := file.UpdatePicInfo("1,1")
asserts.NoError(mock.ExpectationsWereMet())
asserts.NoError(err)
}
// UpdateSourceName
{
mock.ExpectBegin()
mock.ExpectExec("UPDATE(.+)").WithArgs("newName", sqlmock.AnyArg(), 1).WillReturnResult(sqlmock.NewResult(1, 1))
mock.ExpectCommit()
err := file.UpdateSourceName("newName")
asserts.NoError(mock.ExpectationsWereMet())
asserts.NoError(err)
}
}
func TestFile_UpdateSize(t *testing.T) {
a := assert.New(t)
// 增加成功
{
file := File{Size: 10}
mock.ExpectBegin()
mock.ExpectExec("UPDATE(.+)files(.+)").WithArgs(11, sqlmock.AnyArg(), 10).WillReturnResult(sqlmock.NewResult(1, 1))
mock.ExpectExec("UPDATE(.+)storage(.+)+(.+)").WithArgs(uint64(1), sqlmock.AnyArg()).WillReturnResult(sqlmock.NewResult(1, 1))
mock.ExpectCommit()
a.NoError(file.UpdateSize(11))
a.NoError(mock.ExpectationsWereMet())
}
// 减少成功
{
file := File{Size: 10}
mock.ExpectBegin()
mock.ExpectExec("UPDATE(.+)files(.+)").WithArgs(8, sqlmock.AnyArg(), 10).WillReturnResult(sqlmock.NewResult(1, 1))
mock.ExpectExec("UPDATE(.+)storage(.+)-(.+)").WithArgs(uint64(2), sqlmock.AnyArg()).WillReturnResult(sqlmock.NewResult(1, 1))
mock.ExpectCommit()
a.NoError(file.UpdateSize(8))
a.NoError(mock.ExpectationsWereMet())
}
// 文件更新失败
{
file := File{Size: 10}
mock.ExpectBegin()
mock.ExpectExec("UPDATE(.+)files(.+)").WithArgs(8, sqlmock.AnyArg(), 10).WillReturnError(errors.New("error"))
mock.ExpectRollback()
a.Error(file.UpdateSize(8))
a.NoError(mock.ExpectationsWereMet())
}
// 用户容量更新失败
{
file := File{Size: 10}
mock.ExpectBegin()
mock.ExpectExec("UPDATE(.+)files(.+)").WithArgs(8, sqlmock.AnyArg(), 10).WillReturnResult(sqlmock.NewResult(1, 1))
mock.ExpectExec("UPDATE(.+)storage(.+)-(.+)").WithArgs(uint64(2), sqlmock.AnyArg()).WillReturnError(errors.New("error"))
mock.ExpectRollback()
a.Error(file.UpdateSize(8))
a.NoError(mock.ExpectationsWereMet())
}
}
func TestFile_PopChunkToFile(t *testing.T) {
a := assert.New(t)
timeNow := time.Now()
file := File{}
mock.ExpectBegin()
mock.ExpectExec("UPDATE(.+)files(.+)").WillReturnResult(sqlmock.NewResult(1, 1))
mock.ExpectCommit()
a.NoError(file.PopChunkToFile(&timeNow, "1,1"))
}
func TestFile_CanCopy(t *testing.T) {
a := assert.New(t)
file := File{}
a.True(file.CanCopy())
file.UploadSessionID = &file.Name
a.False(file.CanCopy())
}
func TestFile_FileInfoInterface(t *testing.T) {

View File

@@ -23,15 +23,11 @@ type Folder struct {
// Create 创建目录
func (folder *Folder) Create() (uint, error) {
tx := DB.Begin()
if err := tx.FirstOrCreate(folder, *folder).Error; err != nil {
tx.Rollback()
if err := DB.Create(folder).Error; err != nil {
util.Log().Warning("无法插入目录记录, %s", err)
return 0, err
}
err := tx.Commit().Error
return folder.ID, err
return folder.ID, nil
}
// GetChild 返回folder下名为name的子目录不存在则返回错误
@@ -162,11 +158,6 @@ func (folder *Folder) MoveOrCopyFileTo(files []uint, dstFolder *Folder, isCopy b
// 复制文件记录
for _, oldFile := range originFiles {
if !oldFile.CanCopy() {
util.Log().Warning("无法复制正在上传中的文件 [%s] 跳过...", oldFile.Name)
continue
}
oldFile.Model = gorm.Model{}
oldFile.FolderID = dstFolder.ID
oldFile.UserID = dstFolder.OwnerID
@@ -255,11 +246,6 @@ func (folder *Folder) CopyFolderTo(folderID uint, dstFolder *Folder) (size uint6
// 复制文件记录
for _, oldFile := range originFiles {
if !oldFile.CanCopy() {
util.Log().Warning("无法复制正在上传中的文件 [%s] 跳过...", oldFile.Name)
continue
}
oldFile.Model = gorm.Model{}
oldFile.FolderID = newIDCache[oldFile.FolderID]
oldFile.UserID = dstFolder.OwnerID
@@ -277,13 +263,6 @@ func (folder *Folder) CopyFolderTo(folderID uint, dstFolder *Folder) (size uint6
// MoveFolderTo 将folder目录下的dirs子目录复制或移动到dstFolder
// 返回此过程中增加的容量
func (folder *Folder) MoveFolderTo(dirs []uint, dstFolder *Folder) error {
// 如果目标位置为待移动的目录,会导致 parent 为自己
// 造成死循环且无法被除搜索以外的组件展示
if folder.OwnerID == dstFolder.OwnerID && util.ContainsUint(dirs, dstFolder.ID) {
return errors.New("cannot move a folder into itself")
}
// 更改顶级要移动目录的父目录指向
err := DB.Model(Folder{}).Where(
"id in (?) and owner_id = ? and parent_id = ?",

View File

@@ -17,9 +17,8 @@ func TestFolder_Create(t *testing.T) {
Name: "new folder",
}
// 不存在,插入成功
// 插入成功
mock.ExpectBegin()
mock.ExpectQuery("SELECT(.+)folders(.+)").WillReturnRows(sqlmock.NewRows([]string{"id"}))
mock.ExpectExec("INSERT(.+)").WillReturnResult(sqlmock.NewResult(5, 1))
mock.ExpectCommit()
fid, err := folder.Create()
@@ -29,22 +28,12 @@ func TestFolder_Create(t *testing.T) {
// 插入失败
mock.ExpectBegin()
mock.ExpectQuery("SELECT(.+)folders(.+)").WillReturnRows(sqlmock.NewRows([]string{"id"}))
mock.ExpectExec("INSERT(.+)").WillReturnError(errors.New("error"))
mock.ExpectRollback()
fid, err = folder.Create()
asserts.Error(err)
asserts.Equal(uint(0), fid)
asserts.NoError(mock.ExpectationsWereMet())
// 存在,直接返回
mock.ExpectBegin()
mock.ExpectQuery("SELECT(.+)folders(.+)").WillReturnRows(sqlmock.NewRows([]string{"id"}).AddRow(5))
mock.ExpectCommit()
fid, err = folder.Create()
asserts.NoError(err)
asserts.Equal(uint(5), fid)
asserts.NoError(mock.ExpectationsWereMet())
}
func TestFolder_GetChild(t *testing.T) {
@@ -223,14 +212,12 @@ func TestFolder_MoveOrCopyFileTo(t *testing.T) {
WithArgs(
1,
2,
3,
1,
1,
).WillReturnRows(
sqlmock.NewRows([]string{"id", "size", "upload_session_id"}).
AddRow(1, 10, nil).
AddRow(2, 20, nil).
AddRow(2, 20, &folder.Name),
sqlmock.NewRows([]string{"id", "size"}).
AddRow(1, 10).
AddRow(2, 20),
)
mock.ExpectBegin()
mock.ExpectExec("INSERT(.+)").WillReturnResult(sqlmock.NewResult(1, 1))
@@ -239,7 +226,7 @@ func TestFolder_MoveOrCopyFileTo(t *testing.T) {
mock.ExpectExec("INSERT(.+)").WillReturnResult(sqlmock.NewResult(1, 1))
mock.ExpectCommit()
storage, err := folder.MoveOrCopyFileTo(
[]uint{1, 2, 3},
[]uint{1, 2},
&dstFolder,
true,
)
@@ -348,7 +335,7 @@ func TestFolder_CopyFolderTo(t *testing.T) {
// 测试复制目录结构
// test(2)(5)
// 1(3)(6) 2.txt
// 3(4)(7) 4.txt 5.txt(上传中)
// 3(4)(7) 4.txt
// 正常情况 成功
{
@@ -373,10 +360,9 @@ func TestFolder_CopyFolderTo(t *testing.T) {
mock.ExpectQuery("SELECT(.+)").
WithArgs(1, 2, 3, 4).
WillReturnRows(
sqlmock.NewRows([]string{"id", "name", "folder_id", "size", "upload_session_id"}).
AddRow(1, "2.txt", 2, 10, nil).
AddRow(2, "3.txt", 3, 20, nil).
AddRow(3, "5.txt", 3, 20, &dstFolder.Name),
sqlmock.NewRows([]string{"id", "name", "folder_id", "size"}).
AddRow(1, "2.txt", 2, 10).
AddRow(2, "3.txt", 3, 20),
)
// 复制子文件
@@ -507,8 +493,7 @@ func TestFolder_MoveOrCopyFolderTo_Move(t *testing.T) {
}
// 目标目录
dstFolder := Folder{
Model: gorm.Model{ID: 10},
OwnerID: 1,
Model: gorm.Model{ID: 10},
}
// 成功
@@ -522,12 +507,6 @@ func TestFolder_MoveOrCopyFolderTo_Move(t *testing.T) {
asserts.NoError(mock.ExpectationsWereMet())
asserts.NoError(err)
}
// 移动自己到自己内部,失败
{
err := parFolder.MoveFolderTo([]uint{10, 2}, &dstFolder)
asserts.Error(err)
}
}
func TestFolder_FileInfoInterface(t *testing.T) {

View File

@@ -73,13 +73,10 @@ func Init() {
}
//设置连接池
//空闲
db.DB().SetMaxIdleConns(50)
if conf.DatabaseConfig.Type == "sqlite" || conf.DatabaseConfig.Type == "sqlite3" || conf.DatabaseConfig.Type == "UNSET" {
db.DB().SetMaxOpenConns(1)
} else {
db.DB().SetMaxOpenConns(100)
}
//打开
db.DB().SetMaxOpenConns(100)
//超时
db.DB().SetConnMaxLifetime(time.Second * 30)

View File

@@ -7,6 +7,7 @@ import (
"github.com/cloudreve/Cloudreve/v3/pkg/conf"
"github.com/cloudreve/Cloudreve/v3/pkg/util"
"github.com/fatih/color"
"github.com/gofrs/uuid"
"github.com/hashicorp/go-version"
"github.com/jinzhu/gorm"
"sort"
@@ -77,9 +78,6 @@ func addDefaultPolicy() {
DirNameRule: "uploads/{uid}/{path}",
FileNameRule: "{uid}_{randomkey8}_{originname}",
IsOriginLinkEnable: false,
OptionsSerialized: PolicyOption{
ChunkSize: 25 << 20, // 25MB
},
}
if err := DB.Create(&defaultPolicy).Error; err != nil {
util.Log().Panic("无法创建初始存储策略, %s", err)
@@ -88,6 +86,112 @@ func addDefaultPolicy() {
}
func addDefaultSettings() {
siteID, _ := uuid.NewV4()
defaultSettings := []Setting{
{Name: "siteURL", Value: `http://localhost`, Type: "basic"},
{Name: "siteName", Value: `Cloudreve`, Type: "basic"},
{Name: "siteICPId", Value: ``, Type: "basic"},
{Name: "register_enabled", Value: `1`, Type: "register"},
{Name: "default_group", Value: `2`, Type: "register"},
{Name: "siteKeywords", Value: `网盘,网盘`, Type: "basic"},
{Name: "siteDes", Value: `Cloudreve`, Type: "basic"},
{Name: "siteTitle", Value: `平步云端`, Type: "basic"},
{Name: "siteScript", Value: ``, Type: "basic"},
{Name: "siteID", Value: siteID.String(), Type: "basic"},
{Name: "fromName", Value: `Cloudreve`, Type: "mail"},
{Name: "mail_keepalive", Value: `30`, Type: "mail"},
{Name: "fromAdress", Value: `no-reply@acg.blue`, Type: "mail"},
{Name: "smtpHost", Value: `smtp.mxhichina.com`, Type: "mail"},
{Name: "smtpPort", Value: `25`, Type: "mail"},
{Name: "replyTo", Value: `abslant@126.com`, Type: "mail"},
{Name: "smtpUser", Value: `no-reply@acg.blue`, Type: "mail"},
{Name: "smtpPass", Value: ``, Type: "mail"},
{Name: "smtpEncryption", Value: `0`, Type: "mail"},
{Name: "maxEditSize", Value: `4194304`, Type: "file_edit"},
{Name: "archive_timeout", Value: `60`, Type: "timeout"},
{Name: "download_timeout", Value: `60`, Type: "timeout"},
{Name: "preview_timeout", Value: `60`, Type: "timeout"},
{Name: "doc_preview_timeout", Value: `60`, Type: "timeout"},
{Name: "upload_credential_timeout", Value: `1800`, Type: "timeout"},
{Name: "upload_session_timeout", Value: `86400`, Type: "timeout"},
{Name: "slave_api_timeout", Value: `60`, Type: "timeout"},
{Name: "slave_node_retry", Value: `3`, Type: "slave"},
{Name: "slave_ping_interval", Value: `60`, Type: "slave"},
{Name: "slave_recover_interval", Value: `120`, Type: "slave"},
{Name: "slave_transfer_timeout", Value: `172800`, Type: "timeout"},
{Name: "onedrive_monitor_timeout", Value: `600`, Type: "timeout"},
{Name: "share_download_session_timeout", Value: `2073600`, Type: "timeout"},
{Name: "onedrive_callback_check", Value: `20`, Type: "timeout"},
{Name: "folder_props_timeout", Value: `300`, Type: "timeout"},
{Name: "onedrive_chunk_retries", Value: `1`, Type: "retry"},
{Name: "onedrive_source_timeout", Value: `1800`, Type: "timeout"},
{Name: "reset_after_upload_failed", Value: `0`, Type: "upload"},
{Name: "login_captcha", Value: `0`, Type: "login"},
{Name: "reg_captcha", Value: `0`, Type: "login"},
{Name: "email_active", Value: `0`, Type: "register"},
{Name: "mail_activation_template", Value: `<!DOCTYPE html PUBLIC"-//W3C//DTD XHTML 1.0 Transitional//EN""http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd"><html xmlns="http://www.w3.org/1999/xhtml"style="font-family: 'Helvetica Neue', Helvetica, Arial, sans-serif; box-sizing: border-box;
font-size: 14px; margin: 0;"><head><meta name="viewport"content="width=device-width"/><meta http-equiv="Content-Type"content="text/html; charset=UTF-8"/><title>激活您的账户</title><style type="text/css">img{max-width:100%}body{-webkit-font-smoothing:antialiased;-webkit-text-size-adjust:none;width:100%!important;height:100%;line-height:1.6em}body{background-color:#f6f6f6}@media only screen and(max-width:640px){body{padding:0!important}h1{font-weight:800!important;margin:20px 0 5px!important}h2{font-weight:800!important;margin:20px 0 5px!important}h3{font-weight:800!important;margin:20px 0 5px!important}h4{font-weight:800!important;margin:20px 0 5px!important}h1{font-size:22px!important}h2{font-size:18px!important}h3{font-size:16px!important}.container{padding:0!important;width:100%!important}.content{padding:0!important}.content-wrap{padding:10px!important}.invoice{width:100%!important}}</style></head><body itemscope itemtype="http://schema.org/EmailMessage"style="font-family: 'Helvetica Neue',Helvetica,Arial,sans-serif; box-sizing:
border-box; font-size: 14px; -webkit-font-smoothing: antialiased; -webkit-text-size-adjust: none; width: 100% !important; height: 100%; line-height: 1.6em; background-color: #f6f6f6; margin: 0;"bgcolor="#f6f6f6"><table class="body-wrap"style="font-family: 'Helvetica Neue',Helvetica,Arial,sans-serif; box-sizing: border-box; font-size: 14px; width: 100%; background-color: #f6f6f6; margin: 0;"bgcolor="#f6f6f6"><tr style="font-family: 'Helvetica Neue',Helvetica,Arial,sans-serif;
box-sizing: border-box; font-size: 14px; margin: 0;"><td style="font-family: 'Helvetica Neue',Helvetica,Arial,sans-serif; box-sizing: border-box; font-size: 14px; vertical-align: top; margin: 0;"valign="top"></td><td class="container"width="600"style="font-family: 'Helvetica Neue',Helvetica,Arial,sans-serif; box-sizing: border-box; font-size: 14px; vertical-align: top; display: block !important; max-width: 600px !important; clear: both !important; margin: 0 auto;"valign="top"><div class="content"style="font-family: 'Helvetica Neue',Helvetica,Arial,sans-serif; box-sizing: border-box; font-size: 14px; max-width: 600px; display: block; margin: 0 auto; padding: 20px;"><table class="main"width="100%"cellpadding="0"cellspacing="0"style="font-family: 'Helvetica Neue',Helvetica,Arial,sans-serif; box-sizing: border-box; font-size: 14px; border-radius: 3px; background-color: #fff; margin: 0; border: 1px
solid #e9e9e9;"bgcolor="#fff"><tr style="font-family: 'Helvetica Neue',Helvetica,Arial,sans-serif; box-sizing: border-box; font-size:
14px; margin: 0;"><td class="alert alert-warning"style="font-family: 'Helvetica Neue',Helvetica,Arial,sans-serif; box-sizing: border-box; font-size: 16px; vertical-align: top; color: #fff; font-weight: 500; text-align: center; border-radius: 3px 3px 0 0; background-color: #009688; margin: 0; padding: 20px;"align="center"bgcolor="#FF9F00"valign="top">激活{siteTitle}账户</td></tr><tr style="font-family: 'Helvetica Neue',Helvetica,Arial,sans-serif; box-sizing: border-box; font-size: 14px; margin: 0;"><td class="content-wrap"style="font-family: 'Helvetica Neue',Helvetica,Arial,sans-serif; box-sizing: border-box; font-size: 14px; vertical-align: top; margin: 0; padding: 20px;"valign="top"><table width="100%"cellpadding="0"cellspacing="0"style="font-family: 'Helvetica Neue',Helvetica,Arial,sans-serif; box-sizing: border-box; font-size: 14px; margin: 0;"><tr style="font-family: 'Helvetica Neue',Helvetica,Arial,sans-serif; box-sizing: border-box; font-size: 14px; margin: 0;"><td class="content-block"style="font-family: 'Helvetica
Neue',Helvetica,Arial,sans-serif; box-sizing: border-box; font-size: 14px; vertical-align: top; margin: 0; padding: 0 0 20px;"valign="top">亲爱的<strong style="font-family: 'Helvetica Neue',Helvetica,Arial,sans-serif; box-sizing: border-box; font-size: 14px; margin: 0;">{userName}</strong></td></tr><tr style="font-family: 'Helvetica Neue',Helvetica,Arial,sans-serif; box-sizing: border-box; font-size: 14px; margin: 0;"><td class="content-block"style="font-family: 'Helvetica Neue',Helvetica,Arial,sans-serif; box-sizing: border-box; font-size: 14px; vertical-align: top; margin: 0; padding: 0 0 20px;"valign="top">感谢您注册{siteTitle},请点击下方按钮完成账户激活。</td></tr><tr style="font-family: 'Helvetica Neue',Helvetica,Arial,sans-serif; box-sizing: border-box; font-size: 14px; margin: 0;"><td class="content-block"style="font-family: 'Helvetica Neue',Helvetica,Arial,sans-serif; box-sizing: border-box; font-size: 14px; vertical-align: top; margin: 0; padding: 0 0 20px;"valign="top"><a href="{activationUrl}"class="btn-primary"style="font-family: 'Helvetica Neue',Helvetica,Arial,sans-serif; box-sizing: border-box; font-size: 14px; color: #FFF; text-decoration: none; line-height: 2em; font-weight: bold; text-align: center; cursor: pointer; display: inline-block; border-radius: 5px; text-transform: capitalize; background-color: #009688; margin: 0; border-color: #009688; border-style: solid; border-width: 10px 20px;">激活账户</a></td></tr><tr style="font-family: 'Helvetica Neue',Helvetica,Arial,sans-serif; box-sizing: border-box; font-size: 14px; margin: 0;"><td class="content-block"style="font-family: 'Helvetica Neue',Helvetica,Arial,sans-serif; box-sizing: border-box; font-size: 14px; vertical-align: top; margin: 0; padding: 0 0 20px;"valign="top">感谢您选择{siteTitle}。</td></tr></table></td></tr></table><div class="footer"style="font-family: 'Helvetica Neue',Helvetica,Arial,sans-serif; box-sizing: border-box; font-size: 14px; width: 100%; clear: both; color: #999; margin: 0; padding: 20px;"><table width="100%"style="font-family: 'Helvetica Neue',Helvetica,Arial,sans-serif; box-sizing: border-box; font-size: 14px; margin: 0;"><tr style="font-family: 'Helvetica Neue',Helvetica,Arial,sans-serif; box-sizing: border-box; font-size: 14px; margin: 0;"><td class="aligncenter content-block"style="font-family: 'Helvetica Neue',Helvetica,Arial,sans-serif; box-sizing: border-box; font-size: 12px; vertical-align: top; color: #999; text-align: center; margin: 0; padding: 0 0 20px;"align="center"valign="top">此邮件由系统自动发送,请不要直接回复。</td></tr></table></div></div></td><td style="font-family: 'Helvetica Neue',Helvetica,Arial,sans-serif; box-sizing: border-box; font-size: 14px; vertical-align: top; margin: 0;"valign="top"></td></tr></table></body></html>`, Type: "mail_template"},
{Name: "forget_captcha", Value: `0`, Type: "login"},
{Name: "mail_reset_pwd_template", Value: `<!DOCTYPE html PUBLIC"-//W3C//DTD XHTML 1.0 Transitional//EN""http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd"><html xmlns="http://www.w3.org/1999/xhtml"style="font-family: 'Helvetica Neue', Helvetica, Arial, sans-serif; box-sizing: border-box;
font-size: 14px; margin: 0;"><head><meta name="viewport"content="width=device-width"/><meta http-equiv="Content-Type"content="text/html; charset=UTF-8"/><title>重设密码</title><style type="text/css">img{max-width:100%}body{-webkit-font-smoothing:antialiased;-webkit-text-size-adjust:none;width:100%!important;height:100%;line-height:1.6em}body{background-color:#f6f6f6}@media only screen and(max-width:640px){body{padding:0!important}h1{font-weight:800!important;margin:20px 0 5px!important}h2{font-weight:800!important;margin:20px 0 5px!important}h3{font-weight:800!important;margin:20px 0 5px!important}h4{font-weight:800!important;margin:20px 0 5px!important}h1{font-size:22px!important}h2{font-size:18px!important}h3{font-size:16px!important}.container{padding:0!important;width:100%!important}.content{padding:0!important}.content-wrap{padding:10px!important}.invoice{width:100%!important}}</style></head><body itemscope itemtype="http://schema.org/EmailMessage"style="font-family: 'Helvetica Neue',Helvetica,Arial,sans-serif; box-sizing:
border-box; font-size: 14px; -webkit-font-smoothing: antialiased; -webkit-text-size-adjust: none; width: 100% !important; height: 100%; line-height: 1.6em; background-color: #f6f6f6; margin: 0;"bgcolor="#f6f6f6"><table class="body-wrap"style="font-family: 'Helvetica Neue',Helvetica,Arial,sans-serif; box-sizing: border-box; font-size: 14px; width: 100%; background-color: #f6f6f6; margin: 0;"bgcolor="#f6f6f6"><tr style="font-family: 'Helvetica Neue',Helvetica,Arial,sans-serif;
box-sizing: border-box; font-size: 14px; margin: 0;"><td style="font-family: 'Helvetica Neue',Helvetica,Arial,sans-serif; box-sizing: border-box; font-size: 14px; vertical-align: top; margin: 0;"valign="top"></td><td class="container"width="600"style="font-family: 'Helvetica Neue',Helvetica,Arial,sans-serif; box-sizing: border-box; font-size: 14px; vertical-align: top; display: block !important; max-width: 600px !important; clear: both !important; margin: 0 auto;"valign="top"><div class="content"style="font-family: 'Helvetica Neue',Helvetica,Arial,sans-serif; box-sizing: border-box; font-size: 14px; max-width: 600px; display: block; margin: 0 auto; padding: 20px;"><table class="main"width="100%"cellpadding="0"cellspacing="0"style="font-family: 'Helvetica Neue',Helvetica,Arial,sans-serif; box-sizing: border-box; font-size: 14px; border-radius: 3px; background-color: #fff; margin: 0; border: 1px
solid #e9e9e9;"bgcolor="#fff"><tr style="font-family: 'Helvetica Neue',Helvetica,Arial,sans-serif; box-sizing: border-box; font-size:
14px; margin: 0;"><td class="alert alert-warning"style="font-family: 'Helvetica Neue',Helvetica,Arial,sans-serif; box-sizing: border-box; font-size: 16px; vertical-align: top; color: #fff; font-weight: 500; text-align: center; border-radius: 3px 3px 0 0; background-color: #2196F3; margin: 0; padding: 20px;"align="center"bgcolor="#FF9F00"valign="top">重设{siteTitle}密码</td></tr><tr style="font-family: 'Helvetica Neue',Helvetica,Arial,sans-serif; box-sizing: border-box; font-size: 14px; margin: 0;"><td class="content-wrap"style="font-family: 'Helvetica Neue',Helvetica,Arial,sans-serif; box-sizing: border-box; font-size: 14px; vertical-align: top; margin: 0; padding: 20px;"valign="top"><table width="100%"cellpadding="0"cellspacing="0"style="font-family: 'Helvetica Neue',Helvetica,Arial,sans-serif; box-sizing: border-box; font-size: 14px; margin: 0;"><tr style="font-family: 'Helvetica Neue',Helvetica,Arial,sans-serif; box-sizing: border-box; font-size: 14px; margin: 0;"><td class="content-block"style="font-family: 'Helvetica
Neue',Helvetica,Arial,sans-serif; box-sizing: border-box; font-size: 14px; vertical-align: top; margin: 0; padding: 0 0 20px;"valign="top">亲爱的<strong style="font-family: 'Helvetica Neue',Helvetica,Arial,sans-serif; box-sizing: border-box; font-size: 14px; margin: 0;">{userName}</strong></td></tr><tr style="font-family: 'Helvetica Neue',Helvetica,Arial,sans-serif; box-sizing: border-box; font-size: 14px; margin: 0;"><td class="content-block"style="font-family: 'Helvetica Neue',Helvetica,Arial,sans-serif; box-sizing: border-box; font-size: 14px; vertical-align: top; margin: 0; padding: 0 0 20px;"valign="top">请点击下方按钮完成密码重设。如果非你本人操作,请忽略此邮件。</td></tr><tr style="font-family: 'Helvetica Neue',Helvetica,Arial,sans-serif; box-sizing: border-box; font-size: 14px; margin: 0;"><td class="content-block"style="font-family: 'Helvetica Neue',Helvetica,Arial,sans-serif; box-sizing: border-box; font-size: 14px; vertical-align: top; margin: 0; padding: 0 0 20px;"valign="top"><a href="{resetUrl}"class="btn-primary"style="font-family: 'Helvetica Neue',Helvetica,Arial,sans-serif; box-sizing: border-box; font-size: 14px; color: #FFF; text-decoration: none; line-height: 2em; font-weight: bold; text-align: center; cursor: pointer; display: inline-block; border-radius: 5px; text-transform: capitalize; background-color: #2196F3; margin: 0; border-color: #2196F3; border-style: solid; border-width: 10px 20px;">重设密码</a></td></tr><tr style="font-family: 'Helvetica Neue',Helvetica,Arial,sans-serif; box-sizing: border-box; font-size: 14px; margin: 0;"><td class="content-block"style="font-family: 'Helvetica Neue',Helvetica,Arial,sans-serif; box-sizing: border-box; font-size: 14px; vertical-align: top; margin: 0; padding: 0 0 20px;"valign="top">感谢您选择{siteTitle}。</td></tr></table></td></tr></table><div class="footer"style="font-family: 'Helvetica Neue',Helvetica,Arial,sans-serif; box-sizing: border-box; font-size: 14px; width: 100%; clear: both; color: #999; margin: 0; padding: 20px;"><table width="100%"style="font-family: 'Helvetica Neue',Helvetica,Arial,sans-serif; box-sizing: border-box; font-size: 14px; margin: 0;"><tr style="font-family: 'Helvetica Neue',Helvetica,Arial,sans-serif; box-sizing: border-box; font-size: 14px; margin: 0;"><td class="aligncenter content-block"style="font-family: 'Helvetica Neue',Helvetica,Arial,sans-serif; box-sizing: border-box; font-size: 12px; vertical-align: top; color: #999; text-align: center; margin: 0; padding: 0 0 20px;"align="center"valign="top">此邮件由系统自动发送,请不要直接回复。</td></tr></table></div></div></td><td style="font-family: 'Helvetica Neue',Helvetica,Arial,sans-serif; box-sizing: border-box; font-size: 14px; vertical-align: top; margin: 0;"valign="top"></td></tr></table></body></html>`, Type: "mail_template"},
{Name: "db_version_" + conf.RequiredDBVersion, Value: `installed`, Type: "version"},
{Name: "hot_share_num", Value: `10`, Type: "share"},
{Name: "gravatar_server", Value: `https://www.gravatar.com/`, Type: "avatar"},
{Name: "defaultTheme", Value: `#3f51b5`, Type: "basic"},
{Name: "themes", Value: `{"#3f51b5":{"palette":{"primary":{"main":"#3f51b5"},"secondary":{"main":"#f50057"}}},"#2196f3":{"palette":{"primary":{"main":"#2196f3"},"secondary":{"main":"#FFC107"}}},"#673AB7":{"palette":{"primary":{"main":"#673AB7"},"secondary":{"main":"#2196F3"}}},"#E91E63":{"palette":{"primary":{"main":"#E91E63"},"secondary":{"main":"#42A5F5","contrastText":"#fff"}}},"#FF5722":{"palette":{"primary":{"main":"#FF5722"},"secondary":{"main":"#3F51B5"}}},"#FFC107":{"palette":{"primary":{"main":"#FFC107"},"secondary":{"main":"#26C6DA"}}},"#8BC34A":{"palette":{"primary":{"main":"#8BC34A","contrastText":"#fff"},"secondary":{"main":"#FF8A65","contrastText":"#fff"}}},"#009688":{"palette":{"primary":{"main":"#009688"},"secondary":{"main":"#4DD0E1","contrastText":"#fff"}}},"#607D8B":{"palette":{"primary":{"main":"#607D8B"},"secondary":{"main":"#F06292"}}},"#795548":{"palette":{"primary":{"main":"#795548"},"secondary":{"main":"#4CAF50","contrastText":"#fff"}}}}`, Type: "basic"},
{Name: "max_worker_num", Value: `10`, Type: "task"},
{Name: "max_parallel_transfer", Value: `4`, Type: "task"},
{Name: "secret_key", Value: util.RandStringRunes(256), Type: "auth"},
{Name: "temp_path", Value: "temp", Type: "path"},
{Name: "avatar_path", Value: "avatar", Type: "path"},
{Name: "avatar_size", Value: "2097152", Type: "avatar"},
{Name: "avatar_size_l", Value: "200", Type: "avatar"},
{Name: "avatar_size_m", Value: "130", Type: "avatar"},
{Name: "avatar_size_s", Value: "50", Type: "avatar"},
{Name: "home_view_method", Value: "icon", Type: "view"},
{Name: "share_view_method", Value: "list", Type: "view"},
{Name: "cron_garbage_collect", Value: "@hourly", Type: "cron"},
{Name: "authn_enabled", Value: "0", Type: "authn"},
{Name: "captcha_type", Value: "normal", Type: "captcha"},
{Name: "captcha_height", Value: "60", Type: "captcha"},
{Name: "captcha_width", Value: "240", Type: "captcha"},
{Name: "captcha_mode", Value: "3", Type: "captcha"},
{Name: "captcha_ComplexOfNoiseText", Value: "0", Type: "captcha"},
{Name: "captcha_ComplexOfNoiseDot", Value: "0", Type: "captcha"},
{Name: "captcha_IsShowHollowLine", Value: "0", Type: "captcha"},
{Name: "captcha_IsShowNoiseDot", Value: "1", Type: "captcha"},
{Name: "captcha_IsShowNoiseText", Value: "0", Type: "captcha"},
{Name: "captcha_IsShowSlimeLine", Value: "1", Type: "captcha"},
{Name: "captcha_IsShowSineLine", Value: "0", Type: "captcha"},
{Name: "captcha_CaptchaLen", Value: "6", Type: "captcha"},
{Name: "captcha_ReCaptchaKey", Value: "defaultKey", Type: "captcha"},
{Name: "captcha_ReCaptchaSecret", Value: "defaultSecret", Type: "captcha"},
{Name: "captcha_TCaptcha_CaptchaAppId", Value: "", Type: "captcha"},
{Name: "captcha_TCaptcha_AppSecretKey", Value: "", Type: "captcha"},
{Name: "captcha_TCaptcha_SecretId", Value: "", Type: "captcha"},
{Name: "captcha_TCaptcha_SecretKey", Value: "", Type: "captcha"},
{Name: "thumb_width", Value: "400", Type: "thumb"},
{Name: "thumb_height", Value: "300", Type: "thumb"},
{Name: "pwa_small_icon", Value: "/static/img/favicon.ico", Type: "pwa"},
{Name: "pwa_medium_icon", Value: "/static/img/logo192.png", Type: "pwa"},
{Name: "pwa_large_icon", Value: "/static/img/logo512.png", Type: "pwa"},
{Name: "pwa_display", Value: "standalone", Type: "pwa"},
{Name: "pwa_theme_color", Value: "#000000", Type: "pwa"},
{Name: "pwa_background_color", Value: "#ffffff", Type: "pwa"},
{Name: "office_preview_service", Value: "https://view.officeapps.live.com/op/view.aspx?src={$src}", Type: "preview"},
}
for _, value := range defaultSettings {
DB.Where(Setting{Name: value.Name}).Create(&value)
}

View File

@@ -3,7 +3,8 @@ package model
import (
"encoding/gob"
"encoding/json"
"github.com/gofrs/uuid"
"fmt"
"net/url"
"path"
"path/filepath"
"strconv"
@@ -57,10 +58,6 @@ type PolicyOption struct {
Region string `json:"region,omitempty"`
// ServerSideEndpoint 服务端请求使用的 Endpoint为空时使用 Policy.Server 字段
ServerSideEndpoint string `json:"server_side_endpoint,omitempty"`
// 分片上传的分片大小
ChunkSize uint64 `json:"chunk_size,omitempty"`
// 分片上传时是否需要预留空间
PlaceholderWithSize bool `json:"placeholder_with_size,omitempty"`
}
var thumbSuffix = map[string][]string{
@@ -151,7 +148,7 @@ func (policy *Policy) GeneratePath(uid uint, origin string) string {
func (policy *Policy) GenerateFileName(uid uint, origin string) string {
// 未开启自动重命名时,直接返回原始文件名
if !policy.AutoRename {
return origin
return policy.getOriginNameRule(origin)
}
fileRule := policy.FileNameRule
@@ -170,15 +167,35 @@ func (policy *Policy) GenerateFileName(uid uint, origin string) string {
"{hour}": time.Now().Format("15"),
"{minute}": time.Now().Format("04"),
"{second}": time.Now().Format("05"),
"{originname}": origin,
"{ext}": filepath.Ext(origin),
"{uuid}": uuid.Must(uuid.NewV4()).String(),
}
replaceTable["{originname}"] = policy.getOriginNameRule(origin)
fileRule = util.Replace(replaceTable, fileRule)
return fileRule
}
func (policy Policy) getOriginNameRule(origin string) string {
// 部分存储策略可以使用{origin}代表原始文件名
if origin == "" {
// 如果上游未传回原始文件名,则使用占位符,让云存储端替换
switch policy.Type {
case "qiniu":
// 七牛会将$(fname)自动替换为原始文件名
return "$(fname)"
case "local", "remote":
return origin
case "oss", "cos":
// OSS会将${filename}自动替换为原始文件名
return "${filename}"
case "upyun":
// Upyun会将{filename}{.suffix}自动替换为原始文件名
return "{filename}{.suffix}"
}
}
return origin
}
// IsDirectlyPreview 返回此策略下文件是否可以直接预览(不需要重定向)
func (policy *Policy) IsDirectlyPreview() bool {
return policy.Type == "local"
@@ -197,7 +214,18 @@ func (policy *Policy) IsThumbExist(name string) bool {
// IsTransitUpload 返回此策略上传给定size文件时是否需要服务端中转
func (policy *Policy) IsTransitUpload(size uint64) bool {
return policy.Type == "local"
if policy.Type == "local" {
return true
}
if policy.Type == "onedrive" && size < 4*1024*1024 {
return true
}
return false
}
// IsPathGenerateNeeded 返回此策略是否需要在生成上传凭证时生成存储路径
func (policy *Policy) IsPathGenerateNeeded() bool {
return policy.Type != "remote"
}
// IsThumbGenerateNeeded 返回此策略是否需要在上传后生成缩略图
@@ -205,24 +233,44 @@ func (policy *Policy) IsThumbGenerateNeeded() bool {
return policy.Type == "local"
}
// IsUploadPlaceholderWithSize 返回此策略创建上传会话时是否需要预留空间
func (policy *Policy) IsUploadPlaceholderWithSize() bool {
if policy.Type == "remote" {
return true
}
if util.ContainsString([]string{"onedrive", "oss", "qiniu", "cos", "s3"}, policy.Type) {
return policy.OptionsSerialized.PlaceholderWithSize
}
return false
}
// CanStructureBeListed 返回存储策略是否能被前台列物理目录
func (policy *Policy) CanStructureBeListed() bool {
return policy.Type != "local" && policy.Type != "remote"
}
// GetUploadURL 获取文件上传服务API地址
func (policy *Policy) GetUploadURL() string {
server, err := url.Parse(policy.Server)
if err != nil {
return policy.Server
}
controller, _ := url.Parse("")
switch policy.Type {
case "local", "onedrive":
return "/api/v3/file/upload"
case "remote":
controller, _ = url.Parse("/api/v3/slave/upload")
case "oss":
return "https://" + policy.BucketName + "." + policy.Server
case "cos":
return policy.Server
case "upyun":
return "https://v0.api.upyun.com/" + policy.BucketName
case "s3":
if policy.Server == "" {
return fmt.Sprintf("https://%s.s3.%s.amazonaws.com/", policy.BucketName,
policy.OptionsSerialized.Region)
}
if !strings.Contains(policy.Server, policy.BucketName) {
controller, _ = url.Parse("/" + policy.BucketName)
}
}
return server.ResolveReference(controller).String()
}
// SaveAndClearCache 更新并清理缓存
func (policy *Policy) SaveAndClearCache() error {
err := DB.Save(policy).Error

View File

@@ -104,7 +104,7 @@ func TestPolicy_GenerateFileName(t *testing.T) {
asserts.Equal("123.txt", testPolicy.GenerateFileName(1, "123.txt"))
testPolicy.Type = "oss"
asserts.Equal("origin", testPolicy.GenerateFileName(1, "origin"))
asserts.Equal("${filename}", testPolicy.GenerateFileName(1, ""))
}
// 重命名开启
@@ -145,23 +145,19 @@ func TestPolicy_GenerateFileName(t *testing.T) {
testPolicy.Type = "oss"
testPolicy.FileNameRule = "{uid}123{originname}"
asserts.Equal("1123123321", testPolicy.GenerateFileName(1, "123321"))
asserts.Equal("1123${filename}", testPolicy.GenerateFileName(1, ""))
testPolicy.Type = "upyun"
testPolicy.FileNameRule = "{uid}123{originname}"
asserts.Equal("1123123321", testPolicy.GenerateFileName(1, "123321"))
asserts.Equal("1123{filename}{.suffix}", testPolicy.GenerateFileName(1, ""))
testPolicy.Type = "qiniu"
testPolicy.FileNameRule = "{uid}123{originname}"
asserts.Equal("1123123321", testPolicy.GenerateFileName(1, "123321"))
asserts.Equal("1123$(fname)", testPolicy.GenerateFileName(1, ""))
testPolicy.Type = "local"
testPolicy.FileNameRule = "{uid}123{originname}"
asserts.Equal("1123", testPolicy.GenerateFileName(1, ""))
testPolicy.Type = "local"
testPolicy.FileNameRule = "{ext}123{uuid}"
asserts.Contains(testPolicy.GenerateFileName(1, "123.txt"), ".txt123")
}
}
@@ -174,6 +170,78 @@ func TestPolicy_IsDirectlyPreview(t *testing.T) {
asserts.False(policy.IsDirectlyPreview())
}
func TestPolicy_GetUploadURL(t *testing.T) {
asserts := assert.New(t)
// 本地
{
cache.Set("setting_siteURL", "http://127.0.0.1", 0)
policy := Policy{Type: "local", Server: "http://127.0.0.1"}
asserts.Equal("/api/v3/file/upload", policy.GetUploadURL())
}
// 远程
{
policy := Policy{Type: "remote", Server: "http://127.0.0.1"}
asserts.Equal("http://127.0.0.1/api/v3/slave/upload", policy.GetUploadURL())
}
// OSS
{
policy := Policy{Type: "oss", BucketName: "base", Server: "127.0.0.1"}
asserts.Equal("https://base.127.0.0.1", policy.GetUploadURL())
}
// cos
{
policy := Policy{Type: "cos", BaseURL: "base", Server: "http://127.0.0.1"}
asserts.Equal("http://127.0.0.1", policy.GetUploadURL())
}
// upyun
{
policy := Policy{Type: "upyun", BucketName: "base", Server: "http://127.0.0.1"}
asserts.Equal("https://v0.api.upyun.com/base", policy.GetUploadURL())
}
// 未知
{
policy := Policy{Type: "unknown", Server: "http://127.0.0.1"}
asserts.Equal("http://127.0.0.1", policy.GetUploadURL())
}
// S3 未填写自动生成
{
policy := Policy{
Type: "s3",
Server: "",
BucketName: "bucket",
OptionsSerialized: PolicyOption{Region: "us-east"},
}
asserts.Equal("https://bucket.s3.us-east.amazonaws.com/", policy.GetUploadURL())
}
// s3 自己指定
{
policy := Policy{
Type: "s3",
Server: "https://s3.us-east.amazonaws.com/",
BucketName: "bucket",
OptionsSerialized: PolicyOption{Region: "us-east"},
}
asserts.Equal("https://s3.us-east.amazonaws.com/bucket", policy.GetUploadURL())
}
}
func TestPolicy_IsPathGenerateNeeded(t *testing.T) {
asserts := assert.New(t)
policy := Policy{Type: "qiniu"}
asserts.True(policy.IsPathGenerateNeeded())
policy.Type = "remote"
asserts.False(policy.IsPathGenerateNeeded())
}
func TestPolicy_ClearCache(t *testing.T) {
asserts := assert.New(t)
cache.Set("policy_202", 1, 0)
@@ -198,18 +266,15 @@ func TestPolicy_UpdateAccessKey(t *testing.T) {
func TestPolicy_Props(t *testing.T) {
asserts := assert.New(t)
policy := Policy{Type: "onedrive"}
policy.OptionsSerialized.PlaceholderWithSize = true
asserts.False(policy.IsThumbGenerateNeeded())
asserts.False(policy.IsTransitUpload(4))
asserts.True(policy.IsPathGenerateNeeded())
asserts.True(policy.IsTransitUpload(4))
asserts.False(policy.IsTransitUpload(5 * 1024 * 1024))
asserts.True(policy.CanStructureBeListed())
asserts.True(policy.IsUploadPlaceholderWithSize())
policy.Type = "local"
asserts.True(policy.IsThumbGenerateNeeded())
asserts.True(policy.IsPathGenerateNeeded())
asserts.False(policy.CanStructureBeListed())
asserts.False(policy.IsUploadPlaceholderWithSize())
policy.Type = "remote"
asserts.True(policy.IsUploadPlaceholderWithSize())
}
func TestPolicy_IsThumbExist(t *testing.T) {

View File

@@ -43,15 +43,6 @@ func GetSettingByName(name string) string {
return ""
}
// GetSettingByNameWithDefault 用 Name 获取设置值, 取不到时使用缺省值
func GetSettingByNameWithDefault(name, fallback string) string {
res := GetSettingByName(name)
if res == "" {
return fallback
}
return res
}
// GetSettingByNames 用多个 Name 获取设置值
func GetSettingByNames(names ...string) map[string]string {
var queryRes []Setting

View File

@@ -59,15 +59,6 @@ func TestGetSettingByType(t *testing.T) {
asserts.Equal(map[string]string{}, settings)
}
func TestGetSettingByNameWithDefault(t *testing.T) {
a := assert.New(t)
rows := sqlmock.NewRows([]string{"name", "value", "type"})
mock.ExpectQuery("^SELECT \\* FROM `(.+)` WHERE `(.+)`\\.`deleted_at` IS NULL AND(.+)$").WillReturnRows(rows)
settings := GetSettingByNameWithDefault("123", "123321")
a.Equal("123321", settings)
}
func TestGetSettingByNames(t *testing.T) {
cache.Store = cache.NewMemoStore()
asserts := assert.New(t)

View File

@@ -56,8 +56,8 @@ func TestGetTagsByUID(t *testing.T) {
func TestGetTagsByID(t *testing.T) {
asserts := assert.New(t)
mock.ExpectQuery("SELECT(.+)").WillReturnRows(sqlmock.NewRows([]string{"name"}).AddRow("tag"))
res, err := GetTagsByID(1, 1)
res, err := GetTagsByUID(1)
asserts.NoError(mock.ExpectationsWereMet())
asserts.NoError(err)
asserts.EqualValues("tag", res.Name)
asserts.EqualValues("tag", res[0].Name)
}

View File

@@ -64,7 +64,7 @@ func ListTasks(uid uint, page, pageSize int, order string) ([]Task, int) {
dbChain = dbChain.Where("user_id = ?", uid)
// 计算总数用于分页
dbChain.Model(&Task{}).Count(&total)
dbChain.Model(&Share{}).Count(&total)
// 查询记录
dbChain.Limit(pageSize).Offset((page - 1) * pageSize).Order(order).Find(&tasks)

View File

@@ -3,7 +3,6 @@ package model
import (
"crypto/md5"
"crypto/sha1"
"encoding/gob"
"encoding/hex"
"encoding/json"
"strings"
@@ -47,10 +46,6 @@ type User struct {
OptionsSerialized UserOption `gorm:"-"`
}
func init() {
gob.Register(User{})
}
// UserOption 用户个性化配置字段
type UserOption struct {
ProfileOff bool `json:"profile_off,omitempty"`
@@ -94,11 +89,6 @@ func (user *User) IncreaseStorage(size uint64) bool {
return false
}
// ChangeStorage 更新用户容量
func (user *User) ChangeStorage(tx *gorm.DB, operator string, size uint64) error {
return tx.Model(user).Update("storage", gorm.Expr("storage "+operator+" ?", size)).Error
}
// IncreaseStorageWithoutCheck 忽略可用容量,增加用户已用容量
func (user *User) IncreaseStorageWithoutCheck(size uint64) {
if size == 0 {

View File

@@ -13,6 +13,7 @@ import (
"github.com/cloudreve/Cloudreve/v3/pkg/aria2/rpc"
"github.com/cloudreve/Cloudreve/v3/pkg/cluster"
"github.com/cloudreve/Cloudreve/v3/pkg/filesystem"
"github.com/cloudreve/Cloudreve/v3/pkg/filesystem/driver/local"
"github.com/cloudreve/Cloudreve/v3/pkg/filesystem/fsctx"
"github.com/cloudreve/Cloudreve/v3/pkg/mq"
"github.com/cloudreve/Cloudreve/v3/pkg/task"
@@ -190,12 +191,12 @@ func (monitor *Monitor) ValidateFile() error {
defer fs.Recycle()
// 创建上下文环境
file := &fsctx.FileStream{
ctx := context.WithValue(context.Background(), fsctx.FileHeaderCtx, local.FileStream{
Size: monitor.Task.TotalSize,
}
})
// 验证用户容量
if err := filesystem.HookValidateCapacity(context.Background(), fs, file); err != nil {
if err := filesystem.HookValidateCapacityWithoutIncrease(ctx, fs); err != nil {
return err
}
@@ -204,11 +205,11 @@ func (monitor *Monitor) ValidateFile() error {
if fileInfo.Selected == "true" {
// 创建上下文环境
fileSize, _ := strconv.ParseUint(fileInfo.Length, 10, 64)
file := &fsctx.FileStream{
ctx := context.WithValue(context.Background(), fsctx.FileHeaderCtx, local.FileStream{
Size: fileSize,
Name: filepath.Base(fileInfo.Path),
}
if err := filesystem.HookValidateFile(context.Background(), fs, file); err != nil {
})
if err := filesystem.HookValidateFile(ctx, fs); err != nil {
return err
}
}

View File

@@ -11,7 +11,7 @@ type StatusInfo struct {
UploadLength string `json:"uploadLength"` // Uploaded length of the download in bytes.
BitField string `json:"bitfield"` // Hexadecimal representation of the download progress. The highest bit corresponds to the piece at index 0. Any set bits indicate loaded pieces, while unset bits indicate not yet loaded and/or missing pieces. Any overflow bits at the end are set to zero. When the download was not started yet, this key will not be included in the response.
DownloadSpeed string `json:"downloadSpeed"` // Download speed of this download measured in bytes/sec.
UploadSpeed string `json:"uploadSpeed"` // LocalUpload speed of this download measured in bytes/sec.
UploadSpeed string `json:"uploadSpeed"` // Upload speed of this download measured in bytes/sec.
InfoHash string `json:"infoHash"` // InfoHash. BitTorrent only.
NumSeeders string `json:"numSeeders"` // The number of seeders aria2 has connected to. BitTorrent only.
Seeder string `json:"seeder"` // true if the local endpoint is a seeder. Otherwise false. BitTorrent only.
@@ -60,7 +60,7 @@ type PeerInfo struct {
AmChoking string `json:"amChoking"` // true if aria2 is choking the peer. Otherwise false.
PeerChoking string `json:"peerChoking"` // true if the peer is choking aria2. Otherwise false.
DownloadSpeed string `json:"downloadSpeed"` // Download speed (byte/sec) that this client obtains from the peer.
UploadSpeed string `json:"uploadSpeed"` // LocalUpload speed(byte/sec) that this client uploads to the peer.
UploadSpeed string `json:"uploadSpeed"` // Upload speed(byte/sec) that this client uploads to the peer.
Seeder string `json:"seeder"` // true if this peer is a seeder. Otherwise false.
}

View File

@@ -23,8 +23,6 @@ var (
ErrExpired = serializer.NewError(serializer.CodeSignExpired, "签名已过期", nil)
)
const CrHeaderPrefix = "X-Cr-"
// General 通用的认证接口
var General Auth
@@ -66,12 +64,12 @@ func CheckRequest(instance Auth, r *http.Request) error {
return instance.Check(getSignContent(r), sign[0])
}
// getSignContent 签名请求 path、正文、以`X-`开头的 Header. 如果请求 path 为从机上传 API
// getSignContent 签名请求 path、正文、以`X-`开头的 Header. 如果 Header 中包含 `X-Policy`
// 则不对正文签名。返回待签名/验证的字符串
func getSignContent(r *http.Request) (rawSignString string) {
// 读取所有body正文
var body = []byte{}
if !strings.Contains(r.URL.Path, "/api/v3/slave/upload/") {
if _, ok := r.Header["X-Cr-Policy"]; !ok {
if r.Body != nil {
body, _ = ioutil.ReadAll(r.Body)
_ = r.Body.Close()
@@ -82,7 +80,7 @@ func getSignContent(r *http.Request) (rawSignString string) {
// 决定要签名的header
var signedHeader []string
for k, _ := range r.Header {
if strings.HasPrefix(k, CrHeaderPrefix) && k != CrHeaderPrefix+"Filename" {
if strings.HasPrefix(k, "X-Cr-") && k != "X-Cr-Filename" {
signedHeader = append(signedHeader, fmt.Sprintf("%s=%s", k, r.Header.Get(k)))
}
}

12
pkg/cache/driver.go vendored
View File

@@ -2,7 +2,6 @@ package cache
import (
"github.com/cloudreve/Cloudreve/v3/pkg/conf"
"github.com/cloudreve/Cloudreve/v3/pkg/util"
"github.com/gin-gonic/gin"
)
@@ -10,7 +9,9 @@ import (
var Store Driver = NewMemoStore()
// Init 初始化缓存
func Init(isSlave bool) {
func Init() {
//Store = NewRedisStore(10, "tcp", "127.0.0.1:6379", "", "0")
//return
if conf.RedisConfig.Server != "" && gin.Mode() != gin.TestMode {
Store = NewRedisStore(
10,
@@ -20,13 +21,6 @@ func Init(isSlave bool) {
conf.RedisConfig.DB,
)
}
if isSlave {
err := Store.Sets(conf.OptionOverwrite, "setting_")
if err != nil {
util.Log().Warning("无法覆盖数据库设置: %s", err)
}
}
}
// Driver 键值缓存存储容器

View File

@@ -56,10 +56,6 @@ func TestInit(t *testing.T) {
asserts := assert.New(t)
asserts.NotPanics(func() {
Init(false)
})
asserts.NotPanics(func() {
Init(true)
Init()
})
}

View File

@@ -1,15 +1,11 @@
package cluster
import (
"bytes"
"encoding/json"
"errors"
"fmt"
model "github.com/cloudreve/Cloudreve/v3/models"
"github.com/cloudreve/Cloudreve/v3/pkg/aria2/common"
"github.com/cloudreve/Cloudreve/v3/pkg/aria2/rpc"
"github.com/cloudreve/Cloudreve/v3/pkg/auth"
"github.com/cloudreve/Cloudreve/v3/pkg/conf"
"github.com/cloudreve/Cloudreve/v3/pkg/request"
"github.com/cloudreve/Cloudreve/v3/pkg/serializer"
"github.com/cloudreve/Cloudreve/v3/pkg/util"
@@ -44,7 +40,7 @@ func (node *SlaveNode) Init(nodeModel *model.Node) {
var endpoint *url.URL
if serverURL, err := url.Parse(node.Model.Server); err == nil {
var controller *url.URL
controller, _ = url.Parse("/api/v3/slave/")
controller, _ = url.Parse("/api/v3/slave")
endpoint = serverURL.ResolveReference(controller)
}
@@ -412,40 +408,3 @@ func getAria2RequestBody(body *serializer.SlaveAria2Call) (io.Reader, error) {
return strings.NewReader(string(reqBodyEncoded)), nil
}
// RemoteCallback 发送远程存储策略上传回调请求
func RemoteCallback(url string, body serializer.UploadCallback) error {
callbackBody, err := json.Marshal(struct {
Data serializer.UploadCallback `json:"data"`
}{
Data: body,
})
if err != nil {
return serializer.NewError(serializer.CodeCallbackError, "无法编码回调正文", err)
}
resp := request.GeneralClient.Request(
"POST",
url,
bytes.NewReader(callbackBody),
request.WithTimeout(time.Duration(conf.SlaveConfig.CallbackTimeout)*time.Second),
request.WithCredential(auth.General, int64(conf.SlaveConfig.SignatureTTL)),
)
if resp.Err != nil {
return serializer.NewError(serializer.CodeCallbackError, "从机无法发起回调请求", resp.Err)
}
// 解析回调服务端响应
response, err := resp.DecodeResponse()
if err != nil {
msg := fmt.Sprintf("从机无法解析主机返回的响应 (StatusCode=%d)", resp.Response.StatusCode)
return serializer.NewError(serializer.CodeCallbackError, msg, err)
}
if response.Code != 0 {
return serializer.NewError(response.Code, response.Msg, errors.New(response.Error))
}
return nil
}

View File

@@ -1,12 +1,8 @@
package cluster
import (
"bytes"
"encoding/json"
"errors"
model "github.com/cloudreve/Cloudreve/v3/models"
"github.com/cloudreve/Cloudreve/v3/pkg/cache"
"github.com/cloudreve/Cloudreve/v3/pkg/mocks/requestmock"
"github.com/cloudreve/Cloudreve/v3/pkg/request"
"github.com/cloudreve/Cloudreve/v3/pkg/serializer"
"github.com/stretchr/testify/assert"
@@ -445,115 +441,3 @@ func TestSlaveCaller_DeleteTempFile(t *testing.T) {
a.NoError(err)
}
}
func TestRemoteCallback(t *testing.T) {
asserts := assert.New(t)
// 回调成功
{
clientMock := requestmock.RequestMock{}
mockResp, _ := json.Marshal(serializer.Response{Code: 0})
clientMock.On(
"Request",
"POST",
"http://test/test/url",
testMock.Anything,
testMock.Anything,
).Return(&request.Response{
Err: nil,
Response: &http.Response{
StatusCode: 200,
Body: ioutil.NopCloser(bytes.NewReader(mockResp)),
},
})
request.GeneralClient = clientMock
resp := RemoteCallback("http://test/test/url", serializer.UploadCallback{})
asserts.NoError(resp)
clientMock.AssertExpectations(t)
}
// 服务端返回业务错误
{
clientMock := requestmock.RequestMock{}
mockResp, _ := json.Marshal(serializer.Response{Code: 401})
clientMock.On(
"Request",
"POST",
"http://test/test/url",
testMock.Anything,
testMock.Anything,
).Return(&request.Response{
Err: nil,
Response: &http.Response{
StatusCode: 200,
Body: ioutil.NopCloser(bytes.NewReader(mockResp)),
},
})
request.GeneralClient = clientMock
resp := RemoteCallback("http://test/test/url", serializer.UploadCallback{})
asserts.EqualValues(401, resp.(serializer.AppError).Code)
clientMock.AssertExpectations(t)
}
// 无法解析回调响应
{
clientMock := requestmock.RequestMock{}
clientMock.On(
"Request",
"POST",
"http://test/test/url",
testMock.Anything,
testMock.Anything,
).Return(&request.Response{
Err: nil,
Response: &http.Response{
StatusCode: 200,
Body: ioutil.NopCloser(strings.NewReader("mockResp")),
},
})
request.GeneralClient = clientMock
resp := RemoteCallback("http://test/test/url", serializer.UploadCallback{})
asserts.Error(resp)
clientMock.AssertExpectations(t)
}
// HTTP状态码非200
{
clientMock := requestmock.RequestMock{}
clientMock.On(
"Request",
"POST",
"http://test/test/url",
testMock.Anything,
testMock.Anything,
).Return(&request.Response{
Err: nil,
Response: &http.Response{
StatusCode: 404,
Body: ioutil.NopCloser(strings.NewReader("mockResp")),
},
})
request.GeneralClient = clientMock
resp := RemoteCallback("http://test/test/url", serializer.UploadCallback{})
asserts.Error(resp)
clientMock.AssertExpectations(t)
}
// 无法发起回调
{
clientMock := requestmock.RequestMock{}
clientMock.On(
"Request",
"POST",
"http://test/test/url",
testMock.Anything,
testMock.Anything,
).Return(&request.Response{
Err: errors.New("error"),
})
request.GeneralClient = clientMock
resp := RemoteCallback("http://test/test/url", serializer.UploadCallback{})
asserts.Error(resp)
clientMock.AssertExpectations(t)
}
}

View File

@@ -45,6 +45,21 @@ type slave struct {
SignatureTTL int `validate:"omitempty,gte=1"`
}
// captcha 验证码配置
type captcha struct {
Height int `validate:"gte=0"`
Width int `validate:"gte=0"`
Mode int `validate:"gte=0,lte=3"`
ComplexOfNoiseText int `validate:"gte=0,lte=2"`
ComplexOfNoiseDot int `validate:"gte=0,lte=2"`
IsShowHollowLine bool
IsShowNoiseDot bool
IsShowNoiseText bool
IsShowSlimeLine bool
IsShowSineLine bool
CaptchaLen int `validate:"gt=0"`
}
// redis 配置
type redis struct {
Network string
@@ -53,6 +68,17 @@ type redis struct {
DB string
}
// 缩略图 配置
type thumb struct {
MaxWidth uint
MaxHeight uint
FileSuffix string `validate:"min=1"`
MaxTaskCount int
EncodeMethod string `validate:"eq=jpg|eq=png"`
EncodeQuality int `validate:"gte=1,lte=100"`
GCAfterGen bool
}
// 跨域配置
type cors struct {
AllowOrigins []string
@@ -65,7 +91,6 @@ type cors struct {
var cfg *ini.File
const defaultConf = `[System]
Debug = false
Mode = master
Listen = :5212
SessionSecret = {SessionSecret}
@@ -106,7 +131,9 @@ func Init(path string) {
"System": SystemConfig,
"SSL": SSLConfig,
"UnixSocket": UnixConfig,
"Captcha": CaptchaConfig,
"Redis": RedisConfig,
"Thumbnail": ThumbConfig,
"CORS": CORSConfig,
"Slave": SlaveConfig,
}
@@ -117,11 +144,6 @@ func Init(path string) {
}
}
// 映射数据库配置覆盖
for _, key := range cfg.Section("OptionOverwrite").Keys() {
OptionOverwrite[key.Name()] = key.Value()
}
// 重设log等级
if !SystemConfig.Debug {
util.Level = util.LevelInformational

View File

@@ -56,11 +56,7 @@ User = root
Password = root
Host = 127.0.0.1:3306
Name = v3
TablePrefix = v3_
[OptionOverwrite]
key=value
`
TablePrefix = v3_`
err := ioutil.WriteFile("testConf.ini", []byte(testCase), 0644)
defer func() { err = os.Remove("testConf.ini") }()
if err != nil {
@@ -69,7 +65,6 @@ key=value
asserts.NotPanics(func() {
Init("testConf.ini")
})
asserts.Equal(OptionOverwrite["key"], "value")
}
func TestMapSection(t *testing.T) {

View File

@@ -1,5 +1,7 @@
package conf
import "github.com/mojocn/base64Captcha"
// RedisConfig Redis服务器配置
var RedisConfig = &redis{
Network: "tcp",
@@ -23,6 +25,21 @@ var SystemConfig = &system{
Listen: ":5212",
}
// CaptchaConfig 验证码配置
var CaptchaConfig = &captcha{
Height: 60,
Width: 240,
Mode: 3,
ComplexOfNoiseText: base64Captcha.CaptchaComplexLower,
ComplexOfNoiseDot: base64Captcha.CaptchaComplexLower,
IsShowHollowLine: false,
IsShowNoiseDot: false,
IsShowNoiseText: false,
IsShowSlimeLine: false,
IsShowSineLine: false,
CaptchaLen: 6,
}
// CORSConfig 跨域配置
var CORSConfig = &cors{
AllowOrigins: []string{"UNSET"},
@@ -32,6 +49,17 @@ var CORSConfig = &cors{
ExposeHeaders: nil,
}
// ThumbConfig 缩略图配置
var ThumbConfig = &thumb{
MaxWidth: 400,
MaxHeight: 300,
FileSuffix: "._thumb",
MaxTaskCount: -1,
EncodeMethod: "jpg",
GCAfterGen: false,
EncodeQuality: 85,
}
// SlaveConfig 从机配置
var SlaveConfig = &slave{
CallbackTimeout: 20,
@@ -47,5 +75,3 @@ var SSLConfig = &ssl{
var UnixConfig = &unix{
Listen: "",
}
var OptionOverwrite = map[string]interface{}{}

View File

@@ -1,13 +1,13 @@
package conf
// BackendVersion 当前后端版本号
var BackendVersion = "3.5.0"
var BackendVersion = "3.4.3"
// RequiredDBVersion 与当前版本匹配的数据库版本
var RequiredDBVersion = "3.5.0"
var RequiredDBVersion = "3.4.0"
// RequiredStaticVersion 与当前版本匹配的静态资源版本
var RequiredStaticVersion = "3.5.0"
var RequiredStaticVersion = "3.4.3"
// IsPro 是否为Pro版本
var IsPro = "false"

View File

@@ -1,7 +1,6 @@
package crontab
import (
"context"
"os"
"path/filepath"
"strings"
@@ -9,7 +8,6 @@ import (
model "github.com/cloudreve/Cloudreve/v3/models"
"github.com/cloudreve/Cloudreve/v3/pkg/cache"
"github.com/cloudreve/Cloudreve/v3/pkg/filesystem"
"github.com/cloudreve/Cloudreve/v3/pkg/util"
)
@@ -55,45 +53,3 @@ func collectCache(store *cache.MemoStore) {
util.Log().Debug("清理内存缓存")
store.GarbageCollect()
}
func uploadSessionCollect() {
placeholders := model.GetUploadPlaceholderFiles(0)
// 将过期的上传会话按照用户分组
userToFiles := make(map[uint][]uint)
for _, file := range placeholders {
_, sessionExist := cache.Get(filesystem.UploadSessionCachePrefix + *file.UploadSessionID)
if sessionExist {
continue
}
if _, ok := userToFiles[file.UserID]; !ok {
userToFiles[file.UserID] = make([]uint, 0)
}
userToFiles[file.UserID] = append(userToFiles[file.UserID], file.ID)
}
// 删除过期的会话
for uid, filesIDs := range userToFiles {
user, err := model.GetUserByID(uid)
if err != nil {
util.Log().Warning("上传会话所属用户不存在, %s", err)
continue
}
fs, err := filesystem.NewFileSystem(&user)
if err != nil {
util.Log().Warning("无法初始化文件系统, %s", err)
continue
}
if err = fs.Delete(context.Background(), []uint{}, filesIDs, false); err != nil {
util.Log().Warning("无法删除上传会话, %s", err)
}
fs.Recycle()
}
util.Log().Info("定时任务 [cron_recycle_upload_session] 执行完毕")
}

View File

@@ -21,18 +21,13 @@ func Reload() {
func Init() {
util.Log().Info("初始化定时任务...")
// 读取cron日程设置
options := model.GetSettingByNames(
"cron_garbage_collect",
"cron_recycle_upload_session",
)
options := model.GetSettingByNames("cron_garbage_collect")
Cron := cron.New()
for k, v := range options {
var handler func()
switch k {
case "cron_garbage_collect":
handler = garbageCollect
case "cron_recycle_upload_session":
handler = uploadSessionCollect
default:
util.Log().Warning("未知定时任务类型 [%s],跳过", k)
continue

View File

@@ -28,17 +28,17 @@ import (
*/
// Compress 创建给定目录和文件的压缩文件
func (fs *FileSystem) Compress(ctx context.Context, writer io.Writer, folderIDs, fileIDs []uint, isArchive bool) error {
func (fs *FileSystem) Compress(ctx context.Context, folderIDs, fileIDs []uint, isArchive bool) (string, error) {
// 查找待压缩目录
folders, err := model.GetFoldersByIDs(folderIDs, fs.User.ID)
if err != nil && len(folderIDs) != 0 {
return ErrDBListObjects
return "", ErrDBListObjects
}
// 查找待压缩文件
files, err := model.GetFilesByIDs(fileIDs, fs.User.ID)
if err != nil && len(fileIDs) != 0 {
return ErrDBListObjects
return "", ErrDBListObjects
}
// 如果上下文限制了父目录,则进行检查
@@ -46,14 +46,14 @@ func (fs *FileSystem) Compress(ctx context.Context, writer io.Writer, folderIDs,
// 检查目录
for _, folder := range folders {
if *folder.ParentID != parent.ID {
return ErrObjectNotExist
return "", ErrObjectNotExist
}
}
// 检查文件
for _, file := range files {
if file.FolderID != parent.ID {
return ErrObjectNotExist
return "", ErrObjectNotExist
}
}
}
@@ -73,8 +73,25 @@ func (fs *FileSystem) Compress(ctx context.Context, writer io.Writer, folderIDs,
files[i].Position = ""
}
// 创建临时压缩文件
saveFolder := "archive"
if !isArchive {
saveFolder = "compress"
}
zipFilePath := filepath.Join(
util.RelativePath(model.GetSettingByName("temp_path")),
saveFolder,
fmt.Sprintf("archive_%d.zip", time.Now().UnixNano()),
)
zipFile, err := util.CreatNestedFile(zipFilePath)
if err != nil {
util.Log().Warning("%s", err)
return "", err
}
defer zipFile.Close()
// 创建压缩文件Writer
zipWriter := zip.NewWriter(writer)
zipWriter := zip.NewWriter(zipFile)
defer zipWriter.Close()
ctx = reqContext
@@ -84,9 +101,10 @@ func (fs *FileSystem) Compress(ctx context.Context, writer io.Writer, folderIDs,
select {
case <-reqContext.Done():
// 取消压缩请求
return ErrClientCanceled
fs.cancelCompress(ctx, zipWriter, zipFile, zipFilePath)
return "", ErrClientCanceled
default:
fs.doCompress(reqContext, nil, &folders[i], zipWriter, isArchive)
fs.doCompress(ctx, nil, &folders[i], zipWriter, isArchive)
}
}
@@ -94,13 +112,22 @@ func (fs *FileSystem) Compress(ctx context.Context, writer io.Writer, folderIDs,
select {
case <-reqContext.Done():
// 取消压缩请求
return ErrClientCanceled
fs.cancelCompress(ctx, zipWriter, zipFile, zipFilePath)
return "", ErrClientCanceled
default:
fs.doCompress(reqContext, &files[i], nil, zipWriter, isArchive)
fs.doCompress(ctx, &files[i], nil, zipWriter, isArchive)
}
}
return nil
return zipFilePath, nil
}
// cancelCompress 取消压缩进程
func (fs *FileSystem) cancelCompress(ctx context.Context, zipWriter *zip.Writer, file *os.File, path string) {
util.Log().Debug("客户端取消压缩请求")
zipWriter.Close()
file.Close()
_ = os.Remove(path)
}
func (fs *FileSystem) doCompress(ctx context.Context, file *model.File, folder *model.Folder, zipWriter *zip.Writer, isArchive bool) {
@@ -190,8 +217,6 @@ func (fs *FileSystem) Decompress(ctx context.Context, src, dst string) error {
return err
}
defer fileStream.Close()
tempZipFilePath = filepath.Join(
util.RelativePath(model.GetSettingByName("temp_path")),
"decompress",
@@ -213,7 +238,6 @@ func (fs *FileSystem) Decompress(ctx context.Context, src, dst string) error {
}
zipFile.Close()
fileStream.Close()
// 解压缩文件
r, err := zip.OpenReader(tempZipFilePath)
@@ -279,12 +303,7 @@ func (fs *FileSystem) Decompress(ctx context.Context, src, dst string) error {
}
}()
err = fs.UploadFromStream(ctx, &fsctx.FileStream{
File: fileStream,
Size: uint64(size),
Name: path.Base(savePath),
VirtualPath: path.Dir(savePath),
}, true)
err = fs.UploadFromStream(ctx, fileStream, savePath, uint64(size))
fileStream.Close()
if err != nil {
util.Log().Debug("无法上传压缩包内的文件%s , %s , 跳过", rawPath, err)

View File

@@ -1,11 +1,8 @@
package filesystem
import (
"bytes"
"context"
"errors"
"github.com/cloudreve/Cloudreve/v3/pkg/util"
testMock "github.com/stretchr/testify/mock"
"io"
"os"
"strings"
@@ -15,8 +12,11 @@ import (
model "github.com/cloudreve/Cloudreve/v3/models"
"github.com/cloudreve/Cloudreve/v3/pkg/cache"
"github.com/cloudreve/Cloudreve/v3/pkg/filesystem/fsctx"
"github.com/cloudreve/Cloudreve/v3/pkg/request"
"github.com/cloudreve/Cloudreve/v3/pkg/util"
"github.com/jinzhu/gorm"
"github.com/stretchr/testify/assert"
testMock "github.com/stretchr/testify/mock"
)
func TestFileSystem_Compress(t *testing.T) {
@@ -58,11 +58,12 @@ func TestFileSystem_Compress(t *testing.T) {
)
// 查找上传策略
asserts.NoError(cache.Set("policy_1", model.Policy{Type: "local"}, -1))
w := &bytes.Buffer{}
err := fs.Compress(ctx, w, []uint{1}, []uint{1}, true)
zipFile, err := fs.Compress(ctx, []uint{1}, []uint{1}, true)
asserts.NoError(err)
asserts.NotEmpty(w.Len())
asserts.NotEmpty(zipFile)
asserts.Contains(zipFile, "archive_")
asserts.Contains(zipFile, "tests")
}
// 上下文取消
@@ -83,10 +84,9 @@ func TestFileSystem_Compress(t *testing.T) {
)
asserts.NoError(cache.Set("setting_temp_path", "tests", -1))
w := &bytes.Buffer{}
err := fs.Compress(ctx, w, []uint{1}, []uint{1}, true)
zipFile, err := fs.Compress(ctx, []uint{1}, []uint{1}, true)
asserts.Error(err)
asserts.NotEmpty(w.Len())
asserts.Empty(zipFile)
}
// 限制父目录
@@ -108,11 +108,10 @@ func TestFileSystem_Compress(t *testing.T) {
)
asserts.NoError(cache.Set("setting_temp_path", "tests", -1))
w := &bytes.Buffer{}
err := fs.Compress(ctx, w, []uint{1}, []uint{1}, true)
zipFile, err := fs.Compress(ctx, []uint{1}, []uint{1}, true)
asserts.Error(err)
asserts.Equal(ErrObjectNotExist, err)
asserts.Empty(w.Len())
asserts.Empty(zipFile)
}
}
@@ -172,7 +171,7 @@ func TestFileSystem_Decompress(t *testing.T) {
fs.FileTarget = []model.File{{SourceName: "1.zip", Policy: model.Policy{Type: "mock"}}}
fs.FileTarget[0].Policy.ID = 1
testHandler := new(FileHeaderMock)
testHandler.On("Get", testMock.Anything, "1.zip").Return(MockRSC{}, errors.New("error"))
testHandler.On("Get", testMock.Anything, "1.zip").Return(request.NopRSCloser{}, errors.New("error"))
fs.Handler = testHandler
err := fs.Decompress(ctx, "/1.zip", "/")
asserts.NoError(mock.ExpectationsWereMet())
@@ -186,7 +185,7 @@ func TestFileSystem_Decompress(t *testing.T) {
fs.FileTarget = []model.File{{SourceName: "1.zip", Policy: model.Policy{Type: "mock"}}}
fs.FileTarget[0].Policy.ID = 1
testHandler := new(FileHeaderMock)
testHandler.On("Get", testMock.Anything, "1.zip").Return(MockRSC{}, nil)
testHandler.On("Get", testMock.Anything, "1.zip").Return(request.NopRSCloser{}, nil)
fs.Handler = testHandler
err := fs.Decompress(ctx, "/1.zip", "/")
asserts.NoError(mock.ExpectationsWereMet())

View File

@@ -1,31 +0,0 @@
package backoff
import "time"
// Backoff used for retry sleep backoff
type Backoff interface {
Next() bool
Reset()
}
// ConstantBackoff implements Backoff interface with constant sleep time
type ConstantBackoff struct {
Sleep time.Duration
Max int
tried int
}
func (c *ConstantBackoff) Next() bool {
c.tried++
if c.tried > c.Max {
return false
}
time.Sleep(c.Sleep)
return true
}
func (c *ConstantBackoff) Reset() {
c.tried = 0
}

View File

@@ -1,22 +0,0 @@
package backoff
import (
"github.com/stretchr/testify/assert"
"testing"
"time"
)
func TestConstantBackoff_Next(t *testing.T) {
a := assert.New(t)
b := &ConstantBackoff{Sleep: time.Duration(0), Max: 3}
a.True(b.Next())
a.True(b.Next())
a.True(b.Next())
a.False(b.Next())
b.Reset()
a.True(b.Next())
a.True(b.Next())
a.True(b.Next())
a.False(b.Next())
}

View File

@@ -1,162 +0,0 @@
package chunk
import (
"context"
"fmt"
"github.com/cloudreve/Cloudreve/v3/pkg/filesystem/chunk/backoff"
"github.com/cloudreve/Cloudreve/v3/pkg/filesystem/fsctx"
"github.com/cloudreve/Cloudreve/v3/pkg/util"
"io"
"os"
)
const bufferTempPattern = "cdChunk.*.tmp"
// ChunkProcessFunc callback function for processing a chunk
type ChunkProcessFunc func(c *ChunkGroup, chunk io.Reader) error
// ChunkGroup manage groups of chunks
type ChunkGroup struct {
file fsctx.FileHeader
chunkSize uint64
backoff backoff.Backoff
enableRetryBuffer bool
fileInfo *fsctx.UploadTaskInfo
currentIndex int
chunkNum uint64
bufferTemp *os.File
}
func NewChunkGroup(file fsctx.FileHeader, chunkSize uint64, backoff backoff.Backoff, useBuffer bool) *ChunkGroup {
c := &ChunkGroup{
file: file,
chunkSize: chunkSize,
backoff: backoff,
fileInfo: file.Info(),
currentIndex: -1,
enableRetryBuffer: useBuffer,
}
if c.chunkSize == 0 {
c.chunkSize = c.fileInfo.Size
}
if c.fileInfo.Size == 0 {
c.chunkNum = 1
} else {
c.chunkNum = c.fileInfo.Size / c.chunkSize
if c.fileInfo.Size%c.chunkSize != 0 {
c.chunkNum++
}
}
return c
}
// TempAvailable returns if current chunk temp file is available to be read
func (c *ChunkGroup) TempAvailable() bool {
if c.bufferTemp != nil {
state, _ := c.bufferTemp.Stat()
return state != nil && state.Size() == c.Length()
}
return false
}
// Process a chunk with retry logic
func (c *ChunkGroup) Process(processor ChunkProcessFunc) error {
reader := io.LimitReader(c.file, int64(c.chunkSize))
// If useBuffer is enabled, tee the reader to a temp file
if c.enableRetryBuffer && c.bufferTemp == nil && !c.file.Seekable() {
c.bufferTemp, _ = os.CreateTemp("", bufferTempPattern)
reader = io.TeeReader(reader, c.bufferTemp)
}
if c.bufferTemp != nil {
defer func() {
if c.bufferTemp != nil {
c.bufferTemp.Close()
os.Remove(c.bufferTemp.Name())
c.bufferTemp = nil
}
}()
// if temp buffer file is available, use it
if c.TempAvailable() {
if _, err := c.bufferTemp.Seek(0, io.SeekStart); err != nil {
return fmt.Errorf("failed to seek temp file back to chunk start: %w", err)
}
util.Log().Debug("Chunk %d will be read from temp file %q.", c.Index(), c.bufferTemp.Name())
reader = c.bufferTemp
}
}
err := processor(c, reader)
if err != nil {
if err != context.Canceled && (c.file.Seekable() || c.TempAvailable()) && c.backoff.Next() {
if c.file.Seekable() {
if _, seekErr := c.file.Seek(c.Start(), io.SeekStart); seekErr != nil {
return fmt.Errorf("failed to seek back to chunk start: %w, last error: %s", seekErr, err)
}
}
util.Log().Debug("Retrying chunk %d, last error: %s", c.currentIndex, err)
return c.Process(processor)
}
return err
}
util.Log().Debug("Chunk %d processed", c.currentIndex)
return nil
}
// Start returns the byte index of current chunk
func (c *ChunkGroup) Start() int64 {
return int64(uint64(c.Index()) * c.chunkSize)
}
// Total returns the total length
func (c *ChunkGroup) Total() int64 {
return int64(c.fileInfo.Size)
}
// Num returns the total chunk number
func (c *ChunkGroup) Num() int {
return int(c.chunkNum)
}
// RangeHeader returns header value of Content-Range
func (c *ChunkGroup) RangeHeader() string {
return fmt.Sprintf("bytes %d-%d/%d", c.Start(), c.Start()+c.Length()-1, c.Total())
}
// Index returns current chunk index, starts from 0
func (c *ChunkGroup) Index() int {
return c.currentIndex
}
// Next switch to next chunk, returns whether all chunks are processed
func (c *ChunkGroup) Next() bool {
c.currentIndex++
c.backoff.Reset()
return c.currentIndex < int(c.chunkNum)
}
// Length returns the length of current chunk
func (c *ChunkGroup) Length() int64 {
contentLength := c.chunkSize
if c.Index() == int(c.chunkNum-1) {
contentLength = c.fileInfo.Size - c.chunkSize*(c.chunkNum-1)
}
return int64(contentLength)
}
// IsLast returns if current chunk is the last one
func (c *ChunkGroup) IsLast() bool {
return c.Index() == int(c.chunkNum-1)
}

View File

@@ -1,250 +0,0 @@
package chunk
import (
"errors"
"github.com/cloudreve/Cloudreve/v3/pkg/filesystem/chunk/backoff"
"github.com/cloudreve/Cloudreve/v3/pkg/filesystem/fsctx"
"github.com/stretchr/testify/assert"
"io"
"os"
"strings"
"testing"
)
func TestNewChunkGroup(t *testing.T) {
a := assert.New(t)
testCases := []struct {
fileSize uint64
chunkSize uint64
expectedInnerChunkSize uint64
expectedChunkNum uint64
expectedInfo [][2]int //Start, Index,Length
}{
{10, 0, 10, 1, [][2]int{{0, 10}}},
{0, 0, 0, 1, [][2]int{{0, 0}}},
{0, 10, 10, 1, [][2]int{{0, 0}}},
{50, 10, 10, 5, [][2]int{
{0, 10},
{10, 10},
{20, 10},
{30, 10},
{40, 10},
}},
{50, 50, 50, 1, [][2]int{
{0, 50},
}},
{50, 15, 15, 4, [][2]int{
{0, 15},
{15, 15},
{30, 15},
{45, 5},
}},
}
for index, testCase := range testCases {
file := &fsctx.FileStream{Size: testCase.fileSize}
chunkGroup := NewChunkGroup(file, testCase.chunkSize, &backoff.ConstantBackoff{}, true)
a.EqualValues(testCase.expectedChunkNum, chunkGroup.Num(),
"TestCase:%d,ChunkNum()", index)
a.EqualValues(testCase.expectedInnerChunkSize, chunkGroup.chunkSize,
"TestCase:%d,InnerChunkSize()", index)
a.EqualValues(testCase.expectedChunkNum, chunkGroup.Num(),
"TestCase:%d,len(Chunks)", index)
a.EqualValues(testCase.fileSize, chunkGroup.Total())
for cIndex, info := range testCase.expectedInfo {
a.True(chunkGroup.Next())
a.EqualValues(info[1], chunkGroup.Length(),
"TestCase:%d,Chunks[%d].Length()", index, cIndex)
a.EqualValues(info[0], chunkGroup.Start(),
"TestCase:%d,Chunks[%d].Start()", index, cIndex)
a.Equal(cIndex == len(testCase.expectedInfo)-1, chunkGroup.IsLast(),
"TestCase:%d,Chunks[%d].IsLast()", index, cIndex)
a.NotEmpty(chunkGroup.RangeHeader())
}
a.False(chunkGroup.Next())
}
}
func TestChunkGroup_TempAvailablet(t *testing.T) {
a := assert.New(t)
file := &fsctx.FileStream{Size: 1}
c := NewChunkGroup(file, 0, &backoff.ConstantBackoff{}, true)
a.False(c.TempAvailable())
f, err := os.CreateTemp("", "TestChunkGroup_TempAvailablet.*")
defer func() {
f.Close()
os.Remove(f.Name())
}()
a.NoError(err)
c.bufferTemp = f
a.False(c.TempAvailable())
f.Write([]byte("1"))
a.True(c.TempAvailable())
}
func TestChunkGroup_Process(t *testing.T) {
a := assert.New(t)
file := &fsctx.FileStream{Size: 10}
// success
{
file.File = io.NopCloser(strings.NewReader("1234567890"))
c := NewChunkGroup(file, 5, &backoff.ConstantBackoff{}, true)
count := 0
a.True(c.Next())
a.NoError(c.Process(func(c *ChunkGroup, chunk io.Reader) error {
count++
res, err := io.ReadAll(chunk)
a.NoError(err)
a.EqualValues("12345", string(res))
return nil
}))
a.True(c.Next())
a.NoError(c.Process(func(c *ChunkGroup, chunk io.Reader) error {
count++
res, err := io.ReadAll(chunk)
a.NoError(err)
a.EqualValues("67890", string(res))
return nil
}))
a.False(c.Next())
a.Equal(2, count)
}
// retry, read from buffer file
{
file.File = io.NopCloser(strings.NewReader("1234567890"))
c := NewChunkGroup(file, 5, &backoff.ConstantBackoff{Max: 2}, true)
count := 0
a.True(c.Next())
a.NoError(c.Process(func(c *ChunkGroup, chunk io.Reader) error {
count++
res, err := io.ReadAll(chunk)
a.NoError(err)
a.EqualValues("12345", string(res))
return nil
}))
a.True(c.Next())
a.NoError(c.Process(func(c *ChunkGroup, chunk io.Reader) error {
count++
res, err := io.ReadAll(chunk)
a.NoError(err)
a.EqualValues("67890", string(res))
if count == 2 {
return errors.New("error")
}
return nil
}))
a.False(c.Next())
a.Equal(3, count)
}
// retry, read from seeker
{
f, _ := os.CreateTemp("", "TestChunkGroup_Process.*")
f.Write([]byte("1234567890"))
f.Seek(0, 0)
defer func() {
f.Close()
os.Remove(f.Name())
}()
file.File = f
file.Seeker = f
c := NewChunkGroup(file, 5, &backoff.ConstantBackoff{Max: 2}, false)
count := 0
a.True(c.Next())
a.NoError(c.Process(func(c *ChunkGroup, chunk io.Reader) error {
count++
res, err := io.ReadAll(chunk)
a.NoError(err)
a.EqualValues("12345", string(res))
return nil
}))
a.True(c.Next())
a.NoError(c.Process(func(c *ChunkGroup, chunk io.Reader) error {
count++
res, err := io.ReadAll(chunk)
a.NoError(err)
a.EqualValues("67890", string(res))
if count == 2 {
return errors.New("error")
}
return nil
}))
a.False(c.Next())
a.Equal(3, count)
}
// retry, seek error
{
f, _ := os.CreateTemp("", "TestChunkGroup_Process.*")
f.Write([]byte("1234567890"))
f.Seek(0, 0)
defer func() {
f.Close()
os.Remove(f.Name())
}()
file.File = f
file.Seeker = f
c := NewChunkGroup(file, 5, &backoff.ConstantBackoff{Max: 2}, false)
count := 0
a.True(c.Next())
a.NoError(c.Process(func(c *ChunkGroup, chunk io.Reader) error {
count++
res, err := io.ReadAll(chunk)
a.NoError(err)
a.EqualValues("12345", string(res))
return nil
}))
a.True(c.Next())
f.Close()
a.Error(c.Process(func(c *ChunkGroup, chunk io.Reader) error {
count++
if count == 2 {
return errors.New("error")
}
return nil
}))
a.False(c.Next())
a.Equal(2, count)
}
// retry, finally error
{
f, _ := os.CreateTemp("", "TestChunkGroup_Process.*")
f.Write([]byte("1234567890"))
f.Seek(0, 0)
defer func() {
f.Close()
os.Remove(f.Name())
}()
file.File = f
file.Seeker = f
c := NewChunkGroup(file, 5, &backoff.ConstantBackoff{Max: 2}, false)
count := 0
a.True(c.Next())
a.NoError(c.Process(func(c *ChunkGroup, chunk io.Reader) error {
count++
res, err := io.ReadAll(chunk)
a.NoError(err)
a.EqualValues("12345", string(res))
return nil
}))
a.True(c.Next())
a.Error(c.Process(func(c *ChunkGroup, chunk io.Reader) error {
count++
return errors.New("error")
}))
a.False(c.Next())
a.Equal(4, count)
}
}

View File

@@ -183,11 +183,9 @@ func (handler Driver) Get(ctx context.Context, path string) (response.RSCloser,
}
// Put 将文件流保存到指定目录
func (handler Driver) Put(ctx context.Context, file fsctx.FileHeader) error {
defer file.Close()
func (handler Driver) Put(ctx context.Context, file io.ReadCloser, dst string, size uint64) error {
opt := &cossdk.ObjectPutOptions{}
_, err := handler.Client.Object.Put(ctx, file.Info().SavePath, file, opt)
_, err := handler.Client.Object.Put(ctx, dst, file, opt)
return err
}
@@ -326,16 +324,21 @@ func (handler Driver) signSourceURL(ctx context.Context, path string, ttl int64,
}
// Token 获取上传策略和认证Token
func (handler Driver) Token(ctx context.Context, ttl int64, uploadSession *serializer.UploadSession, file fsctx.FileHeader) (*serializer.UploadCredential, error) {
func (handler Driver) Token(ctx context.Context, TTL int64, key string) (serializer.UploadCredential, error) {
// 读取上下文中生成的存储路径
savePath, ok := ctx.Value(fsctx.SavePathCtx).(string)
if !ok {
return serializer.UploadCredential{}, errors.New("无法获取存储路径")
}
// 生成回调地址
siteURL := model.GetSiteURL()
apiBaseURI, _ := url.Parse("/api/v3/callback/cos/" + uploadSession.Key)
apiBaseURI, _ := url.Parse("/api/v3/callback/cos/" + key)
apiURL := siteURL.ResolveReference(apiBaseURI).String()
// 上传策略
savePath := file.Info().SavePath
startTime := time.Now()
endTime := startTime.Add(time.Duration(ttl) * time.Second)
endTime := startTime.Add(time.Duration(TTL) * time.Second)
keyTime := fmt.Sprintf("%d;%d", startTime.Unix(), endTime.Unix())
postPolicy := UploadPolicy{
Expiration: endTime.UTC().Format(time.RFC3339),
@@ -343,7 +346,7 @@ func (handler Driver) Token(ctx context.Context, ttl int64, uploadSession *seria
map[string]string{"bucket": handler.Policy.BucketName},
map[string]string{"$key": savePath},
map[string]string{"x-cos-meta-callback": apiURL},
map[string]string{"x-cos-meta-key": uploadSession.Key},
map[string]string{"x-cos-meta-key": key},
map[string]string{"q-sign-algorithm": "sha1"},
map[string]string{"q-ak": handler.Policy.AccessKey},
map[string]string{"q-sign-time": keyTime},
@@ -355,22 +358,16 @@ func (handler Driver) Token(ctx context.Context, ttl int64, uploadSession *seria
[]interface{}{"content-length-range", 0, handler.Policy.MaxSize})
}
res, err := handler.getUploadCredential(ctx, postPolicy, keyTime, savePath)
res, err := handler.getUploadCredential(ctx, postPolicy, keyTime)
if err == nil {
res.SessionID = uploadSession.Key
res.Callback = apiURL
res.UploadURLs = []string{handler.Policy.Server}
res.Key = key
}
return res, err
}
// 取消上传凭证
func (handler Driver) CancelToken(ctx context.Context, uploadSession *serializer.UploadSession) error {
return nil
}
// Meta 获取文件信息
func (handler Driver) Meta(ctx context.Context, path string) (*MetaData, error) {
res, err := handler.Client.Object.Head(ctx, path, &cossdk.ObjectHeadOptions{})
@@ -384,11 +381,17 @@ func (handler Driver) Meta(ctx context.Context, path string) (*MetaData, error)
}, nil
}
func (handler Driver) getUploadCredential(ctx context.Context, policy UploadPolicy, keyTime string, savePath string) (*serializer.UploadCredential, error) {
func (handler Driver) getUploadCredential(ctx context.Context, policy UploadPolicy, keyTime string) (serializer.UploadCredential, error) {
// 读取上下文中生成的存储路径
savePath, ok := ctx.Value(fsctx.SavePathCtx).(string)
if !ok {
return serializer.UploadCredential{}, errors.New("无法获取存储路径")
}
// 编码上传策略
policyJSON, err := json.Marshal(policy)
if err != nil {
return nil, err
return serializer.UploadCredential{}, err
}
policyEncoded := base64.StdEncoding.EncodeToString(policyJSON)
@@ -396,14 +399,14 @@ func (handler Driver) getUploadCredential(ctx context.Context, policy UploadPoli
hmacSign := hmac.New(sha1.New, []byte(handler.Policy.SecretKey))
_, err = io.WriteString(hmacSign, keyTime)
if err != nil {
return nil, err
return serializer.UploadCredential{}, err
}
signKey := fmt.Sprintf("%x", hmacSign.Sum(nil))
sha1Sign := sha1.New()
_, err = sha1Sign.Write(policyJSON)
if err != nil {
return nil, err
return serializer.UploadCredential{}, err
}
stringToSign := fmt.Sprintf("%x", sha1Sign.Sum(nil))
@@ -411,15 +414,15 @@ func (handler Driver) getUploadCredential(ctx context.Context, policy UploadPoli
hmacFinalSign := hmac.New(sha1.New, []byte(signKey))
_, err = hmacFinalSign.Write([]byte(stringToSign))
if err != nil {
return nil, err
return serializer.UploadCredential{}, err
}
signature := hmacFinalSign.Sum(nil)
return &serializer.UploadCredential{
Policy: policyEncoded,
Path: savePath,
AccessKey: handler.Policy.AccessKey,
Credential: fmt.Sprintf("%x", signature),
KeyTime: keyTime,
return serializer.UploadCredential{
Policy: policyEncoded,
Path: savePath,
AccessKey: handler.Policy.AccessKey,
Token: fmt.Sprintf("%x", signature),
KeyTime: keyTime,
}, nil
}

View File

@@ -2,9 +2,9 @@ package driver
import (
"context"
"github.com/cloudreve/Cloudreve/v3/pkg/filesystem/fsctx"
"github.com/cloudreve/Cloudreve/v3/pkg/filesystem/response"
"github.com/cloudreve/Cloudreve/v3/pkg/serializer"
"io"
"net/url"
)
@@ -12,7 +12,7 @@ import (
type Handler interface {
// 上传文件, dst为文件存储路径size 为文件大小。上下文关闭
// 时,应取消上传并清理临时文件
Put(ctx context.Context, file fsctx.FileHeader) error
Put(ctx context.Context, file io.ReadCloser, dst string, size uint64) error
// 删除一个或多个给定路径的文件,返回删除失败的文件路径列表及错误
Delete(ctx context.Context, files []string) ([]string, error)
@@ -29,11 +29,8 @@ type Handler interface {
// isDownload - 是否直接下载
Source(ctx context.Context, path string, url url.URL, ttl int64, isDownload bool, speed int) (string, error)
// Token 获取有效期为ttl的上传凭证和签名
Token(ctx context.Context, ttl int64, uploadSession *serializer.UploadSession, file fsctx.FileHeader) (*serializer.UploadCredential, error)
// CancelToken 取消已经创建的有状态上传凭证
CancelToken(ctx context.Context, uploadSession *serializer.UploadSession) error
// Token 获取有效期为ttl的上传凭证和签名同时回调会话有效期为sessionTTL
Token(ctx context.Context, ttl int64, callbackKey string) (serializer.UploadCredential, error)
// List 递归列取远程端path路径下文件、目录不包含path本身
// 返回的对象路径以path作为起始根目录.

View File

@@ -0,0 +1,38 @@
package local
import (
"io"
)
// FileStream 用户传来的文件
type FileStream struct {
File io.ReadCloser
Size uint64
VirtualPath string
Name string
MIMEType string
}
func (file FileStream) Read(p []byte) (n int, err error) {
return file.File.Read(p)
}
func (file FileStream) GetMIMEType() string {
return file.MIMEType
}
func (file FileStream) GetSize() uint64 {
return file.Size
}
func (file FileStream) Close() error {
return file.File.Close()
}
func (file FileStream) GetFileName() string {
return file.Name
}
func (file FileStream) GetVirtualPath() string {
return file.VirtualPath
}

View File

@@ -0,0 +1,48 @@
package local
import (
"github.com/stretchr/testify/assert"
"io/ioutil"
"strings"
"testing"
)
func TestFileStream_GetFileName(t *testing.T) {
asserts := assert.New(t)
file := FileStream{Name: "123"}
asserts.Equal("123", file.GetFileName())
}
func TestFileStream_GetMIMEType(t *testing.T) {
asserts := assert.New(t)
file := FileStream{MIMEType: "123"}
asserts.Equal("123", file.GetMIMEType())
}
func TestFileStream_GetSize(t *testing.T) {
asserts := assert.New(t)
file := FileStream{Size: 123}
asserts.Equal(uint64(123), file.GetSize())
}
func TestFileStream_Read(t *testing.T) {
asserts := assert.New(t)
file := FileStream{
File: ioutil.NopCloser(strings.NewReader("123")),
}
var p = make([]byte, 3)
{
n, err := file.Read(p)
asserts.Equal(3, n)
asserts.NoError(err)
}
}
func TestFileStream_Close(t *testing.T) {
asserts := assert.New(t)
file := FileStream{
File: ioutil.NopCloser(strings.NewReader("123")),
}
err := file.Close()
asserts.NoError(err)
}

View File

@@ -12,16 +12,13 @@ import (
model "github.com/cloudreve/Cloudreve/v3/models"
"github.com/cloudreve/Cloudreve/v3/pkg/auth"
"github.com/cloudreve/Cloudreve/v3/pkg/cache"
"github.com/cloudreve/Cloudreve/v3/pkg/conf"
"github.com/cloudreve/Cloudreve/v3/pkg/filesystem/fsctx"
"github.com/cloudreve/Cloudreve/v3/pkg/filesystem/response"
"github.com/cloudreve/Cloudreve/v3/pkg/serializer"
"github.com/cloudreve/Cloudreve/v3/pkg/util"
)
const (
Perm = 0744
)
// Driver 本地策略适配器
type Driver struct {
Policy *model.Policy
@@ -82,17 +79,29 @@ func (handler Driver) Get(ctx context.Context, path string) (response.RSCloser,
return nil, err
}
// 开启一个协程用于请求结束后关闭reader
// go closeReader(ctx, file)
return file, nil
}
// Put 将文件流保存到指定目录
func (handler Driver) Put(ctx context.Context, file fsctx.FileHeader) error {
defer file.Close()
fileInfo := file.Info()
dst := util.RelativePath(filepath.FromSlash(fileInfo.SavePath))
// closeReader 用于在请求结束后关闭reader
// TODO 让业务代码自己关闭
func closeReader(ctx context.Context, closer io.Closer) {
select {
case <-ctx.Done():
_ = closer.Close()
// 如果非 Overwrite则检查是否有重名冲突
if fileInfo.Mode&fsctx.Overwrite != fsctx.Overwrite {
}
}
// Put 将文件流保存到指定目录
func (handler Driver) Put(ctx context.Context, file io.ReadCloser, dst string, size uint64) error {
defer file.Close()
dst = util.RelativePath(filepath.FromSlash(dst))
// 如果禁止了 Overwrite则检查是否有重名冲突
if ctx.Value(fsctx.DisableOverwrite) != nil {
if util.Exists(dst) {
util.Log().Warning("物理同名文件已存在或不可用: %s", dst)
return errors.New("物理同名文件已存在或不可用")
@@ -102,73 +111,26 @@ func (handler Driver) Put(ctx context.Context, file fsctx.FileHeader) error {
// 如果目标目录不存在,创建
basePath := filepath.Dir(dst)
if !util.Exists(basePath) {
err := os.MkdirAll(basePath, Perm)
err := os.MkdirAll(basePath, 0744)
if err != nil {
util.Log().Warning("无法创建目录,%s", err)
return err
}
}
var (
out *os.File
err error
)
openMode := os.O_CREATE | os.O_RDWR
if fileInfo.Mode&fsctx.Append == fsctx.Append {
openMode |= os.O_APPEND
} else {
openMode |= os.O_TRUNC
}
out, err = os.OpenFile(dst, openMode, Perm)
// 创建目标文件
out, err := os.Create(dst)
if err != nil {
util.Log().Warning("无法打开或创建文件,%s", err)
util.Log().Warning("无法创建文件,%s", err)
return err
}
defer out.Close()
if fileInfo.Mode&fsctx.Append == fsctx.Append {
stat, err := out.Stat()
if err != nil {
util.Log().Warning("无法读取文件信息,%s", err)
return err
}
if uint64(stat.Size()) < fileInfo.AppendStart {
return errors.New("未上传完成的文件分片与预期大小不一致")
} else if uint64(stat.Size()) > fileInfo.AppendStart {
out.Close()
if err := handler.Truncate(ctx, dst, fileInfo.AppendStart); err != nil {
return fmt.Errorf("覆盖分片时发生错误: %w", err)
}
out, err = os.OpenFile(dst, openMode, Perm)
defer out.Close()
if err != nil {
util.Log().Warning("无法打开或创建文件,%s", err)
return err
}
}
}
// 写入文件内容
_, err = io.Copy(out, file)
return err
}
func (handler Driver) Truncate(ctx context.Context, src string, size uint64) error {
util.Log().Warning("截断文件 [%s] 至 [%d]", src, size)
out, err := os.OpenFile(src, os.O_WRONLY, Perm)
if err != nil {
util.Log().Warning("无法打开文件,%s", err)
return err
}
defer out.Close()
return out.Truncate(int64(size))
}
// Delete 删除一个或多个文件,
// 返回未删除的文件,及遇到的最后一个错误
func (handler Driver) Delete(ctx context.Context, files []string) ([]string, error) {
@@ -187,7 +149,7 @@ func (handler Driver) Delete(ctx context.Context, files []string) ([]string, err
}
// 尝试删除文件的缩略图(如果有)
_ = os.Remove(util.RelativePath(value + model.GetSettingByNameWithDefault("thumb_file_suffix", "._thumb")))
_ = os.Remove(util.RelativePath(value + conf.ThumbConfig.FileSuffix))
}
return deleteFailed, retErr
@@ -195,7 +157,7 @@ func (handler Driver) Delete(ctx context.Context, files []string) ([]string, err
// Thumb 获取文件缩略图
func (handler Driver) Thumb(ctx context.Context, path string) (*response.ContentResponse, error) {
file, err := handler.Get(ctx, path+model.GetSettingByNameWithDefault("thumb_file_suffix", "._thumb"))
file, err := handler.Get(ctx, path+conf.ThumbConfig.FileSuffix)
if err != nil {
return nil, err
}
@@ -265,18 +227,6 @@ func (handler Driver) Source(
}
// Token 获取上传策略和认证Token本地策略直接返回空值
func (handler Driver) Token(ctx context.Context, ttl int64, uploadSession *serializer.UploadSession, file fsctx.FileHeader) (*serializer.UploadCredential, error) {
if util.Exists(uploadSession.SavePath) {
return nil, errors.New("placeholder file already exist")
}
return &serializer.UploadCredential{
SessionID: uploadSession.Key,
ChunkSize: handler.Policy.OptionsSerialized.ChunkSize,
}, nil
}
// 取消上传凭证
func (handler Driver) CancelToken(ctx context.Context, uploadSession *serializer.UploadSession) error {
return nil
func (handler Driver) Token(ctx context.Context, ttl int64, key string) (serializer.UploadCredential, error) {
return serializer.UploadCredential{}, nil
}

View File

@@ -4,12 +4,13 @@ import (
"context"
model "github.com/cloudreve/Cloudreve/v3/models"
"github.com/cloudreve/Cloudreve/v3/pkg/auth"
"github.com/cloudreve/Cloudreve/v3/pkg/conf"
"github.com/cloudreve/Cloudreve/v3/pkg/filesystem/fsctx"
"github.com/cloudreve/Cloudreve/v3/pkg/serializer"
"github.com/cloudreve/Cloudreve/v3/pkg/util"
"github.com/jinzhu/gorm"
"github.com/stretchr/testify/assert"
"io"
"io/ioutil"
"net/url"
"os"
"strings"
@@ -19,74 +20,52 @@ import (
func TestHandler_Put(t *testing.T) {
asserts := assert.New(t)
handler := Driver{}
defer func() {
os.Remove(util.RelativePath("TestHandler_Put.txt"))
os.Remove(util.RelativePath("inner/TestHandler_Put.txt"))
}()
ctx := context.WithValue(context.Background(), fsctx.DisableOverwrite, true)
os.Remove(util.RelativePath("test/test/txt"))
testCases := []struct {
file fsctx.FileHeader
errContains string
file io.ReadCloser
dst string
err bool
}{
{&fsctx.FileStream{
SavePath: "TestHandler_Put.txt",
File: io.NopCloser(strings.NewReader("")),
}, ""},
{&fsctx.FileStream{
SavePath: "TestHandler_Put.txt",
File: io.NopCloser(strings.NewReader("")),
}, "物理同名文件已存在或不可用"},
{&fsctx.FileStream{
SavePath: "inner/TestHandler_Put.txt",
File: io.NopCloser(strings.NewReader("")),
}, ""},
{&fsctx.FileStream{
Mode: fsctx.Append | fsctx.Overwrite,
SavePath: "inner/TestHandler_Put.txt",
File: io.NopCloser(strings.NewReader("123")),
}, ""},
{&fsctx.FileStream{
AppendStart: 10,
Mode: fsctx.Append | fsctx.Overwrite,
SavePath: "inner/TestHandler_Put.txt",
File: io.NopCloser(strings.NewReader("123")),
}, "未上传完成的文件分片与预期大小不一致"},
{&fsctx.FileStream{
Mode: fsctx.Append | fsctx.Overwrite,
SavePath: "inner/TestHandler_Put.txt",
File: io.NopCloser(strings.NewReader("123")),
}, ""},
{
file: ioutil.NopCloser(strings.NewReader("test input file")),
dst: "test/test/txt",
err: false,
},
{
file: ioutil.NopCloser(strings.NewReader("test input file")),
dst: "test/test/txt",
err: true,
},
{
file: ioutil.NopCloser(strings.NewReader("test input file")),
dst: "/notexist:/S.TXT",
err: true,
},
}
for _, testCase := range testCases {
err := handler.Put(context.Background(), testCase.file)
if testCase.errContains != "" {
err := handler.Put(ctx, testCase.file, testCase.dst, 15)
if testCase.err {
asserts.Error(err)
asserts.Contains(err.Error(), testCase.errContains)
} else {
asserts.NoError(err)
asserts.True(util.Exists(util.RelativePath(testCase.file.Info().SavePath)))
asserts.True(util.Exists(util.RelativePath(testCase.dst)))
}
}
}
func TestDriver_TruncateFailed(t *testing.T) {
a := assert.New(t)
h := Driver{}
a.Error(h.Truncate(context.Background(), "TestDriver_TruncateFailed", 0))
}
func TestHandler_Delete(t *testing.T) {
asserts := assert.New(t)
handler := Driver{}
ctx := context.Background()
filePath := util.RelativePath("TestHandler_Delete.file")
filePath := util.RelativePath("test.file")
file, err := os.Create(filePath)
asserts.NoError(err)
_ = file.Close()
list, err := handler.Delete(ctx, []string{"TestHandler_Delete.file"})
list, err := handler.Delete(ctx, []string{"test.file"})
asserts.Equal([]string{}, list)
asserts.NoError(err)
@@ -94,7 +73,7 @@ func TestHandler_Delete(t *testing.T) {
_ = file.Close()
file, _ = os.OpenFile(filePath, os.O_RDWR, os.FileMode(0))
asserts.NoError(err)
list, err = handler.Delete(ctx, []string{"TestHandler_Delete.file", "test.notexist"})
list, err = handler.Delete(ctx, []string{"test.file", "test.notexist"})
file.Close()
asserts.Equal([]string{}, list)
asserts.NoError(err)
@@ -105,7 +84,7 @@ func TestHandler_Delete(t *testing.T) {
file, err = os.Create(filePath)
asserts.NoError(err)
list, err = handler.Delete(ctx, []string{"TestHandler_Delete.file"})
list, err = handler.Delete(ctx, []string{"test.file"})
_ = file.Close()
asserts.Equal([]string{}, list)
asserts.NoError(err)
@@ -137,7 +116,7 @@ func TestHandler_Thumb(t *testing.T) {
asserts := assert.New(t)
handler := Driver{}
ctx := context.Background()
file, err := os.Create(util.RelativePath("TestHandler_Thumb._thumb"))
file, err := os.Create(util.RelativePath("TestHandler_Thumb" + conf.ThumbConfig.FileSuffix))
asserts.NoError(err)
file.Close()
@@ -181,25 +160,6 @@ func TestHandler_Source(t *testing.T) {
asserts.Contains(sourceURL, "https://cloudreve.org")
}
// 下载
{
file := model.File{
Model: gorm.Model{
ID: 1,
},
Name: "test.jpg",
}
ctx := context.WithValue(ctx, fsctx.FileModelCtx, file)
baseURL, err := url.Parse("https://cloudreve.org")
asserts.NoError(err)
sourceURL, err := handler.Source(ctx, "", *baseURL, 0, true, 0)
asserts.NoError(err)
asserts.NotEmpty(sourceURL)
asserts.Contains(sourceURL, "sign=")
asserts.Contains(sourceURL, "download")
asserts.Contains(sourceURL, "https://cloudreve.org")
}
// 无法获取上下文
{
baseURL, err := url.Parse("https://cloudreve.org")
@@ -281,29 +241,10 @@ func TestHandler_GetDownloadURL(t *testing.T) {
func TestHandler_Token(t *testing.T) {
asserts := assert.New(t)
handler := Driver{
Policy: &model.Policy{},
}
ctx := context.Background()
upSession := &serializer.UploadSession{SavePath: "TestHandler_Token"}
_, err := handler.Token(ctx, 10, upSession, &fsctx.FileStream{})
asserts.NoError(err)
file, _ := os.Create("TestHandler_Token")
defer func() {
file.Close()
os.Remove("TestHandler_Token")
}()
_, err = handler.Token(ctx, 10, upSession, &fsctx.FileStream{})
asserts.Error(err)
asserts.Contains(err.Error(), "already exist")
}
func TestDriver_CancelToken(t *testing.T) {
a := assert.New(t)
handler := Driver{}
a.NoError(handler.CancelToken(context.Background(), &serializer.UploadSession{}))
ctx := context.Background()
_, err := handler.Token(ctx, 10, "123")
asserts.NoError(err)
}
func TestDriver_List(t *testing.T) {

View File

@@ -1,6 +1,7 @@
package onedrive
import (
"bytes"
"context"
"encoding/json"
"errors"
@@ -17,10 +18,7 @@ import (
model "github.com/cloudreve/Cloudreve/v3/models"
"github.com/cloudreve/Cloudreve/v3/pkg/cache"
"github.com/cloudreve/Cloudreve/v3/pkg/filesystem/chunk"
"github.com/cloudreve/Cloudreve/v3/pkg/filesystem/chunk/backoff"
"github.com/cloudreve/Cloudreve/v3/pkg/filesystem/fsctx"
"github.com/cloudreve/Cloudreve/v3/pkg/mq"
"github.com/cloudreve/Cloudreve/v3/pkg/request"
"github.com/cloudreve/Cloudreve/v3/pkg/util"
)
@@ -31,8 +29,7 @@ const (
// ChunkSize 服务端中转分片上传分片大小
ChunkSize uint64 = 10 * 1024 * 1024
// ListRetry 列取请求重试次数
ListRetry = 1
chunkRetrySleep = time.Second * 5
ListRetry = 1
)
// GetSourcePath 获取文件的绝对路径
@@ -222,21 +219,28 @@ func (client *Client) GetUploadSessionStatus(ctx context.Context, uploadURL stri
}
// UploadChunk 上传分片
func (client *Client) UploadChunk(ctx context.Context, uploadURL string, content io.Reader, current *chunk.ChunkGroup) (*UploadSessionResponse, error) {
func (client *Client) UploadChunk(ctx context.Context, uploadURL string, chunk *Chunk) (*UploadSessionResponse, error) {
res, err := client.request(
ctx, "PUT", uploadURL, content,
request.WithContentLength(current.Length()),
ctx, "PUT", uploadURL, bytes.NewReader(chunk.Data[0:chunk.ChunkSize]),
request.WithContentLength(int64(chunk.ChunkSize)),
request.WithHeader(http.Header{
"Content-Range": {current.RangeHeader()},
"Content-Range": {fmt.Sprintf("bytes %d-%d/%d", chunk.Offset, chunk.Offset+chunk.ChunkSize-1, chunk.Total)},
}),
request.WithoutHeader([]string{"Authorization", "Content-Type"}),
request.WithTimeout(time.Duration(300)*time.Second),
)
if err != nil {
return nil, fmt.Errorf("failed to upload OneDrive chunk #%d: %w", current.Index(), err)
// 如果重试次数小于限制5秒后重试
if chunk.Retried < model.GetIntSetting("onedrive_chunk_retries", 1) {
chunk.Retried++
util.Log().Debug("分片偏移%d上传失败[%s]5秒钟后重试", chunk.Offset, err)
time.Sleep(time.Duration(5) * time.Second)
return client.UploadChunk(ctx, uploadURL, chunk)
}
return nil, err
}
if current.IsLast() {
if chunk.IsLast() {
return nil, nil
}
@@ -253,17 +257,13 @@ func (client *Client) UploadChunk(ctx context.Context, uploadURL string, content
}
// Upload 上传文件
func (client *Client) Upload(ctx context.Context, file fsctx.FileHeader) error {
fileInfo := file.Info()
func (client *Client) Upload(ctx context.Context, dst string, size int, file io.Reader) error {
// 决定是否覆盖文件
overwrite := "fail"
if fileInfo.Mode&fsctx.Overwrite == fsctx.Overwrite {
overwrite = "replace"
overwrite := "replace"
if ctx.Value(fsctx.DisableOverwrite) != nil {
overwrite = "fail"
}
size := int(fileInfo.Size)
dst := fileInfo.SavePath
// 小文件,使用简单上传接口上传
if size <= int(SmallFileSize) {
_, err := client.SimpleUpload(ctx, dst, file, int64(size), WithConflictBehavior(overwrite))
@@ -277,24 +277,46 @@ func (client *Client) Upload(ctx context.Context, file fsctx.FileHeader) error {
return err
}
// Initial chunk groups
chunks := chunk.NewChunkGroup(file, client.Policy.OptionsSerialized.ChunkSize, &backoff.ConstantBackoff{
Max: model.GetIntSetting("chunk_retries", 5),
Sleep: chunkRetrySleep,
}, model.IsTrueVal(model.GetSettingByName("use_temp_chunk_buffer")))
uploadFunc := func(current *chunk.ChunkGroup, content io.Reader) error {
_, err := client.UploadChunk(ctx, uploadURL, content, current)
return err
offset := 0
chunkNum := size / int(ChunkSize)
if size%int(ChunkSize) != 0 {
chunkNum++
}
// upload chunks
for chunks.Next() {
if err := chunks.Process(uploadFunc); err != nil {
return fmt.Errorf("failed to upload chunk #%d: %w", chunks.Index(), err)
chunkData := make([]byte, ChunkSize)
for i := 0; i < chunkNum; i++ {
select {
case <-ctx.Done():
util.Log().Debug("OneDrive 客户端取消")
return ErrClientCanceled
default:
// 分块
chunkSize := int(ChunkSize)
if size-offset < chunkSize {
chunkSize = size - offset
}
// 因为后面需要错误重试,这里要把分片内容读到内存中
chunkContent := chunkData[:chunkSize]
_, err := io.ReadFull(file, chunkContent)
chunk := Chunk{
Offset: offset,
ChunkSize: chunkSize,
Total: size,
Data: chunkContent,
}
// 上传
_, err = client.UploadChunk(ctx, uploadURL, &chunk)
if err != nil {
return err
}
offset += chunkSize
}
}
}
return nil
}
@@ -323,6 +345,16 @@ func (client *Client) SimpleUpload(ctx context.Context, dst string, body io.Read
request.WithTimeout(time.Duration(150)*time.Second),
)
if err != nil {
retried := 0
if v, ok := ctx.Value(fsctx.RetryCtx).(int); ok {
retried = v
}
if retried < model.GetIntSetting("onedrive_chunk_retries", 1) {
retried++
util.Log().Debug("文件[%s]上传失败[%s]5秒钟后重试", dst, err)
time.Sleep(time.Duration(5) * time.Second)
return client.SimpleUpload(context.WithValue(ctx, fsctx.RetryCtx, retried), dst, body, size, opts...)
}
return nil, err
}
@@ -451,9 +483,9 @@ func (client *Client) GetThumbURL(ctx context.Context, dst string, w, h uint) (s
// MonitorUpload 监控客户端分片上传进度
func (client *Client) MonitorUpload(uploadURL, callbackKey, path string, size uint64, ttl int64) {
// 回调完成通知chan
callbackChan := mq.GlobalMQ.Subscribe(callbackKey, 1)
defer mq.GlobalMQ.Unsubscribe(callbackKey, callbackChan)
callbackChan := make(chan bool)
callbackSignal.Store(callbackKey, callbackChan)
defer callbackSignal.Delete(callbackKey)
timeout := model.GetIntSetting("onedrive_monitor_timeout", 600)
interval := model.GetIntSetting("onedrive_callback_check", 20)
@@ -478,16 +510,16 @@ func (client *Client) MonitorUpload(uploadURL, callbackKey, path string, size ui
if resErr, ok := err.(*RespError); ok {
if resErr.APIError.Code == "itemNotFound" {
util.Log().Debug("上传会话已完成,稍后检查回调")
select {
case <-time.After(time.Duration(interval) * time.Second):
time.Sleep(time.Duration(interval) * time.Second)
util.Log().Debug("开始检查回调")
_, ok := cache.Get("callback_" + callbackKey)
if ok {
util.Log().Warning("未发送回调,删除文件")
cache.Deletes([]string{callbackKey}, "callback_")
_, err = client.Delete(context.Background(), []string{path})
if err != nil {
util.Log().Warning("无法删除未回调的文件,%s", err)
}
case <-callbackChan:
util.Log().Debug("客户端完成回调")
}
return
}
@@ -524,6 +556,15 @@ func (client *Client) MonitorUpload(uploadURL, callbackKey, path string, size ui
}
}
// FinishCallback 向Monitor发送回调结束信号
func FinishCallback(key string) {
if signal, ok := callbackSignal.Load(key); ok {
if signalChan, ok := signal.(chan bool); ok {
close(signalChan)
}
}
}
func sysError(err error) *RespError {
return &RespError{APIError: APIError{
Code: "system",

View File

@@ -4,11 +4,6 @@ import (
"context"
"errors"
"fmt"
"github.com/cloudreve/Cloudreve/v3/pkg/filesystem/chunk"
"github.com/cloudreve/Cloudreve/v3/pkg/filesystem/chunk/backoff"
"github.com/cloudreve/Cloudreve/v3/pkg/filesystem/fsctx"
"github.com/cloudreve/Cloudreve/v3/pkg/mq"
"io"
"io/ioutil"
"net/http"
"strings"
@@ -17,6 +12,7 @@ import (
model "github.com/cloudreve/Cloudreve/v3/models"
"github.com/cloudreve/Cloudreve/v3/pkg/cache"
"github.com/cloudreve/Cloudreve/v3/pkg/filesystem/fsctx"
"github.com/cloudreve/Cloudreve/v3/pkg/request"
"github.com/stretchr/testify/assert"
testMock "github.com/stretchr/testify/mock"
@@ -311,31 +307,6 @@ func TestClient_Meta(t *testing.T) {
asserts.NotNil(res)
asserts.Equal("123321", res.Name)
}
// 返回正常, 使用资源id
{
client.Credential.ExpiresIn = time.Now().Add(time.Duration(100) * time.Hour).Unix()
clientMock := ClientMock{}
clientMock.On(
"Request",
"GET",
testMock.Anything,
testMock.Anything,
testMock.Anything,
).Return(&request.Response{
Err: nil,
Response: &http.Response{
StatusCode: 200,
Body: ioutil.NopCloser(strings.NewReader(`{"name":"123321"}`)),
},
})
client.Request = clientMock
res, err := client.Meta(context.Background(), "123321", "123")
clientMock.AssertExpectations(t)
asserts.NoError(err)
asserts.NotNil(res)
asserts.Equal("123321", res.Name)
}
}
func TestClient_CreateUploadSession(t *testing.T) {
@@ -471,11 +442,9 @@ func TestClient_UploadChunk(t *testing.T) {
client, _ := NewClient(&model.Policy{})
client.Credential.AccessToken = "AccessToken"
client.Credential.ExpiresIn = time.Now().Add(time.Duration(100) * time.Hour).Unix()
cg := chunk.NewChunkGroup(&fsctx.FileStream{Size: 15}, 10, &backoff.ConstantBackoff{}, false)
// 非最后分片,正常
{
cg.Next()
client.Credential.ExpiresIn = time.Now().Add(time.Duration(100) * time.Hour).Unix()
clientMock := ClientMock{}
clientMock.On(
@@ -484,10 +453,6 @@ func TestClient_UploadChunk(t *testing.T) {
"http://dev.com",
testMock.Anything,
testMock.Anything,
testMock.Anything,
testMock.Anything,
testMock.Anything,
testMock.Anything,
).Return(&request.Response{
Err: nil,
Response: &http.Response{
@@ -496,7 +461,13 @@ func TestClient_UploadChunk(t *testing.T) {
},
})
client.Request = clientMock
res, err := client.UploadChunk(context.Background(), "http://dev.com", strings.NewReader("1234567890"), cg)
res, err := client.UploadChunk(context.Background(), "http://dev.com", &Chunk{
Offset: 0,
ChunkSize: 10,
Total: 100,
Retried: 0,
Data: []byte("12313121231312"),
})
clientMock.AssertExpectations(t)
asserts.NoError(err)
asserts.Equal("http://dev.com/2", res.UploadURL)
@@ -520,7 +491,13 @@ func TestClient_UploadChunk(t *testing.T) {
},
})
client.Request = clientMock
res, err := client.UploadChunk(context.Background(), "http://dev.com", strings.NewReader("1234567890"), cg)
res, err := client.UploadChunk(context.Background(), "http://dev.com", &Chunk{
Offset: 0,
ChunkSize: 10,
Total: 100,
Retried: 0,
Data: []byte("12313112313122"),
})
clientMock.AssertExpectations(t)
asserts.Error(err)
asserts.Nil(res)
@@ -528,7 +505,6 @@ func TestClient_UploadChunk(t *testing.T) {
// 最后分片,正常
{
cg.Next()
client.Credential.ExpiresIn = time.Now().Add(time.Duration(100) * time.Hour).Unix()
clientMock := ClientMock{}
clientMock.On(
@@ -545,26 +521,53 @@ func TestClient_UploadChunk(t *testing.T) {
},
})
client.Request = clientMock
res, err := client.UploadChunk(context.Background(), "http://dev.com", strings.NewReader("12345"), cg)
res, err := client.UploadChunk(context.Background(), "http://dev.com", &Chunk{
Offset: 95,
ChunkSize: 5,
Total: 100,
Retried: 0,
Data: []byte("1231312"),
})
clientMock.AssertExpectations(t)
asserts.NoError(err)
asserts.Nil(res)
}
// 最后分片,失败
// 最后分片,第一次失败,重试后成功
{
cache.Set("setting_chunk_retries", "1", 0)
cache.Set("setting_onedrive_chunk_retries", "1", 0)
client.Credential.ExpiresIn = 0
go func() {
time.Sleep(time.Duration(2) * time.Second)
client.Credential.ExpiresIn = time.Now().Add(time.Duration(100) * time.Hour).Unix()
}()
clientMock := ClientMock{}
clientMock.On(
"Request",
"PUT",
"http://dev.com",
testMock.Anything,
testMock.Anything,
).Return(&request.Response{
Err: nil,
Response: &http.Response{
StatusCode: 200,
Body: ioutil.NopCloser(strings.NewReader(`???`)),
},
})
client.Request = clientMock
res, err := client.UploadChunk(context.Background(), "http://dev.com", strings.NewReader("12345"), cg)
chunk := &Chunk{
Offset: 95,
ChunkSize: 5,
Total: 100,
Retried: 0,
Data: []byte("1231312"),
}
res, err := client.UploadChunk(context.Background(), "http://dev.com", chunk)
clientMock.AssertExpectations(t)
asserts.Error(err)
asserts.NoError(err)
asserts.Nil(res)
asserts.EqualValues(1, chunk.Retried)
}
}
@@ -573,20 +576,41 @@ func TestClient_Upload(t *testing.T) {
client, _ := NewClient(&model.Policy{})
client.Credential.AccessToken = "AccessToken"
client.Credential.ExpiresIn = time.Now().Add(time.Duration(100) * time.Hour).Unix()
ctx := context.Background()
cache.Set("setting_chunk_retries", "1", 0)
cache.Set("setting_use_temp_chunk_buffer", "false", 0)
ctx := context.WithValue(context.Background(), fsctx.DisableOverwrite, true)
// 小文件,简单上传,失败
{
client.Credential.ExpiresIn = 0
err := client.Upload(ctx, &fsctx.FileStream{
Size: 5,
File: io.NopCloser(strings.NewReader("12345")),
})
err := client.Upload(ctx, "123.jpg", 3, strings.NewReader("123"))
asserts.Error(err)
}
// 上下文取消
{
client.Credential.ExpiresIn = time.Now().Add(time.Duration(100) * time.Hour).Unix()
clientMock := ClientMock{}
clientMock.On(
"Request",
"POST",
testMock.Anything,
testMock.Anything,
testMock.Anything,
).Return(&request.Response{
Err: nil,
Response: &http.Response{
StatusCode: 200,
Body: ioutil.NopCloser(strings.NewReader(`{"uploadUrl":"123321"}`)),
},
})
client.Request = clientMock
ctx, cancel := context.WithCancel(context.Background())
cancel()
err := client.Upload(ctx, "123.jpg", 15*1024*1024, strings.NewReader("123"))
clientMock.AssertExpectations(t)
asserts.Error(err)
asserts.Equal(ErrClientCanceled, err)
}
// 无法创建分片会话
{
client.Credential.ExpiresIn = time.Now().Add(time.Duration(100) * time.Hour).Unix()
@@ -605,54 +629,11 @@ func TestClient_Upload(t *testing.T) {
},
})
client.Request = clientMock
err := client.Upload(context.Background(), &fsctx.FileStream{
Size: SmallFileSize + 1,
File: io.NopCloser(strings.NewReader("12345")),
})
err := client.Upload(context.Background(), "123.jpg", 15*1024*1024, strings.NewReader("123"))
clientMock.AssertExpectations(t)
asserts.Error(err)
}
// 分片上传失败
{
client.Credential.ExpiresIn = time.Now().Add(time.Duration(100) * time.Hour).Unix()
clientMock := ClientMock{}
clientMock.On(
"Request",
"POST",
testMock.Anything,
testMock.Anything,
testMock.Anything,
).Return(&request.Response{
Err: nil,
Response: &http.Response{
StatusCode: 200,
Body: ioutil.NopCloser(strings.NewReader(`{"uploadUrl":"123321"}`)),
},
})
clientMock.On(
"Request",
"PUT",
testMock.Anything,
testMock.Anything,
testMock.Anything,
).Return(&request.Response{
Err: nil,
Response: &http.Response{
StatusCode: 400,
Body: ioutil.NopCloser(strings.NewReader(`{"uploadUrl":"123321"}`)),
},
})
client.Request = clientMock
err := client.Upload(context.Background(), &fsctx.FileStream{
Size: SmallFileSize + 1,
File: io.NopCloser(strings.NewReader("12345")),
})
clientMock.AssertExpectations(t)
asserts.Error(err)
asserts.Contains(err.Error(), "failed to upload chunk")
}
}
func TestClient_SimpleUpload(t *testing.T) {
@@ -660,9 +641,9 @@ func TestClient_SimpleUpload(t *testing.T) {
client, _ := NewClient(&model.Policy{})
client.Credential.AccessToken = "AccessToken"
client.Credential.ExpiresIn = time.Now().Add(time.Duration(100) * time.Hour).Unix()
cache.Set("setting_chunk_retries", "1", 0)
cache.Set("setting_onedrive_chunk_retries", "1", 0)
// 请求失败
// 请求失败,并重试
{
client.Credential.ExpiresIn = 0
res, err := client.SimpleUpload(context.Background(), "123.jpg", strings.NewReader("123"), 3)
@@ -670,6 +651,7 @@ func TestClient_SimpleUpload(t *testing.T) {
asserts.Nil(res)
}
cache.Set("setting_onedrive_chunk_retries", "0", 0)
// 返回未知响应
{
client.Credential.ExpiresIn = time.Now().Add(time.Duration(100) * time.Hour).Unix()
@@ -1006,7 +988,7 @@ func TestClient_MonitorUpload(t *testing.T) {
asserts.NotPanics(func() {
go func() {
time.Sleep(time.Duration(1) * time.Second)
mq.GlobalMQ.Publish("key", mq.Message{})
FinishCallback("key")
}()
client.MonitorUpload("url", "key", "path", 10, 10)
})

View File

@@ -4,6 +4,7 @@ import (
"context"
"errors"
"fmt"
"io"
"net/url"
"path"
"path/filepath"
@@ -30,10 +31,6 @@ type Driver struct {
// NewDriver 从存储策略初始化新的Driver实例
func NewDriver(policy *model.Policy) (driver.Handler, error) {
client, err := NewClient(policy)
if policy.OptionsSerialized.ChunkSize == 0 {
policy.OptionsSerialized.ChunkSize = 50 << 20 // 50MB
}
return Driver{
Policy: policy,
Client: client,
@@ -124,10 +121,9 @@ func (handler Driver) Get(ctx context.Context, path string) (response.RSCloser,
}
// Put 将文件流保存到指定目录
func (handler Driver) Put(ctx context.Context, file fsctx.FileHeader) error {
func (handler Driver) Put(ctx context.Context, file io.ReadCloser, dst string, size uint64) error {
defer file.Close()
return handler.Client.Upload(ctx, file)
return handler.Client.Upload(ctx, dst, int(size), file)
}
// Delete 删除一个或多个文件,
@@ -227,26 +223,38 @@ func (handler Driver) replaceSourceHost(origin string) (string, error) {
}
// Token 获取上传会话URL
func (handler Driver) Token(ctx context.Context, ttl int64, uploadSession *serializer.UploadSession, file fsctx.FileHeader) (*serializer.UploadCredential, error) {
fileInfo := file.Info()
func (handler Driver) Token(ctx context.Context, TTL int64, key string) (serializer.UploadCredential, error) {
uploadURL, err := handler.Client.CreateUploadSession(ctx, fileInfo.SavePath, WithConflictBehavior("fail"))
// 读取上下文中生成的存储路径和文件大小
savePath, ok := ctx.Value(fsctx.SavePathCtx).(string)
if !ok {
return serializer.UploadCredential{}, errors.New("无法获取存储路径")
}
fileSize, ok := ctx.Value(fsctx.FileSizeCtx).(uint64)
if !ok {
return serializer.UploadCredential{}, errors.New("无法获取文件大小")
}
// 如果小于4MB则由服务端中转
if fileSize <= SmallFileSize {
return serializer.UploadCredential{}, nil
}
// 生成回调地址
siteURL := model.GetSiteURL()
apiBaseURI, _ := url.Parse("/api/v3/callback/onedrive/finish/" + key)
apiURL := siteURL.ResolveReference(apiBaseURI)
uploadURL, err := handler.Client.CreateUploadSession(ctx, savePath, WithConflictBehavior("fail"))
if err != nil {
return nil, err
return serializer.UploadCredential{}, err
}
// 监控回调及上传
go handler.Client.MonitorUpload(uploadURL, uploadSession.Key, fileInfo.SavePath, fileInfo.Size, ttl)
go handler.Client.MonitorUpload(uploadURL, key, savePath, fileSize, TTL)
uploadSession.UploadURL = uploadURL
return &serializer.UploadCredential{
SessionID: uploadSession.Key,
ChunkSize: handler.Policy.OptionsSerialized.ChunkSize,
UploadURLs: []string{uploadURL},
return serializer.UploadCredential{
Policy: uploadURL,
Token: apiURL.String(),
}, nil
}
// 取消上传凭证
func (handler Driver) CancelToken(ctx context.Context, uploadSession *serializer.UploadSession) error {
return handler.Client.DeleteUploadSession(ctx, uploadSession.UploadURL)
}

View File

@@ -4,9 +4,6 @@ import (
"context"
"fmt"
"github.com/cloudreve/Cloudreve/v3/pkg/auth"
"github.com/cloudreve/Cloudreve/v3/pkg/mq"
"github.com/cloudreve/Cloudreve/v3/pkg/serializer"
"github.com/jinzhu/gorm"
"io"
"io/ioutil"
"net/http"
@@ -15,23 +12,51 @@ import (
"testing"
"time"
"github.com/DATA-DOG/go-sqlmock"
model "github.com/cloudreve/Cloudreve/v3/models"
"github.com/cloudreve/Cloudreve/v3/pkg/cache"
"github.com/cloudreve/Cloudreve/v3/pkg/filesystem/fsctx"
"github.com/cloudreve/Cloudreve/v3/pkg/request"
"github.com/cloudreve/Cloudreve/v3/pkg/serializer"
"github.com/stretchr/testify/assert"
testMock "github.com/stretchr/testify/mock"
)
func TestDriver_Token(t *testing.T) {
asserts := assert.New(t)
h, _ := NewDriver(&model.Policy{
AccessKey: "ak",
SecretKey: "sk",
BucketName: "test",
Server: "test.com",
})
handler := h.(Driver)
handler := Driver{
Policy: &model.Policy{
AccessKey: "ak",
SecretKey: "sk",
BucketName: "test",
Server: "test.com",
},
}
// 无法获取文件路径
{
ctx := context.WithValue(context.Background(), fsctx.FileSizeCtx, uint64(10))
res, err := handler.Token(ctx, 10, "key")
asserts.Error(err)
asserts.Equal(serializer.UploadCredential{}, res)
}
// 无法获取文件大小
{
ctx := context.WithValue(context.Background(), fsctx.SavePathCtx, "/123")
res, err := handler.Token(ctx, 10, "key")
asserts.Error(err)
asserts.Equal(serializer.UploadCredential{}, res)
}
// 小文件成功
{
ctx := context.WithValue(context.Background(), fsctx.SavePathCtx, "/123")
ctx = context.WithValue(ctx, fsctx.FileSizeCtx, uint64(10))
res, err := handler.Token(ctx, 10, "key")
asserts.NoError(err)
asserts.Equal(serializer.UploadCredential{}, res)
}
// 分片上传 失败
{
@@ -53,9 +78,11 @@ func TestDriver_Token(t *testing.T) {
},
})
handler.Client.Request = clientMock
res, err := handler.Token(context.Background(), 10, &serializer.UploadSession{}, &fsctx.FileStream{})
ctx := context.WithValue(context.Background(), fsctx.SavePathCtx, "/123")
ctx = context.WithValue(ctx, fsctx.FileSizeCtx, uint64(20*1024*1024))
res, err := handler.Token(ctx, 10, "key")
asserts.Error(err)
asserts.Nil(res)
asserts.Equal(serializer.UploadCredential{}, res)
}
// 分片上传 成功
@@ -81,13 +108,15 @@ func TestDriver_Token(t *testing.T) {
},
})
handler.Client.Request = clientMock
ctx := context.WithValue(context.Background(), fsctx.SavePathCtx, "/123")
ctx = context.WithValue(ctx, fsctx.FileSizeCtx, uint64(20*1024*1024))
go func() {
time.Sleep(time.Duration(1) * time.Second)
mq.GlobalMQ.Publish("TestDriver_Token", mq.Message{})
FinishCallback("key")
}()
res, err := handler.Token(context.Background(), 10, &serializer.UploadSession{Key: "TestDriver_Token"}, &fsctx.FileStream{})
res, err := handler.Token(ctx, 10, "key")
asserts.NoError(err)
asserts.Equal("123321", res.UploadURLs[0])
asserts.Equal("123321", res.Policy)
}
}
@@ -266,8 +295,12 @@ func TestDriver_Thumb(t *testing.T) {
// 失败
{
ctx := context.WithValue(context.Background(), fsctx.ThumbSizeCtx, [2]uint{10, 20})
ctx = context.WithValue(ctx, fsctx.FileModelCtx, model.File{PicInfo: "1,1", Model: gorm.Model{ID: 1}})
ctx = context.WithValue(ctx, fsctx.FileModelCtx, model.File{})
mock.ExpectBegin()
mock.ExpectExec("UPDATE(.+)").WillReturnResult(sqlmock.NewResult(1, 1))
mock.ExpectCommit()
res, err := handler.Thumb(ctx, "123.jpg")
asserts.NoError(mock.ExpectationsWereMet())
asserts.Error(err)
asserts.Empty(res.URL)
}
@@ -275,6 +308,7 @@ func TestDriver_Thumb(t *testing.T) {
// 上下文错误
{
_, err := handler.Thumb(context.Background(), "123.jpg")
asserts.NoError(mock.ExpectationsWereMet())
asserts.Error(err)
}
}
@@ -295,6 +329,7 @@ func TestDriver_Delete(t *testing.T) {
// 失败
{
_, err := handler.Delete(context.Background(), []string{"1"})
asserts.NoError(mock.ExpectationsWereMet())
asserts.Error(err)
}
@@ -315,7 +350,7 @@ func TestDriver_Put(t *testing.T) {
// 失败
{
err := handler.Put(context.Background(), &fsctx.FileStream{})
err := handler.Put(context.Background(), ioutil.NopCloser(strings.NewReader("")), "dst", 0)
asserts.Error(err)
}
}
@@ -383,55 +418,3 @@ func TestDriver_Get(t *testing.T) {
asserts.NoError(err)
asserts.Equal("123", string(content))
}
func TestDriver_replaceSourceHost(t *testing.T) {
tests := []struct {
name string
origin string
cdn string
want string
wantErr bool
}{
{"TestNoReplace", "http://1dr.ms/download.aspx?123456", "", "http://1dr.ms/download.aspx?123456", false},
{"TestReplaceCorrect", "http://1dr.ms/download.aspx?123456", "https://test.com:8080", "https://test.com:8080/download.aspx?123456", false},
{"TestCdnFormatError", "http://1dr.ms/download.aspx?123456", string([]byte{0x7f}), "", true},
{"TestSrcFormatError", string([]byte{0x7f}), "https://test.com:8080", "", true},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
policy := &model.Policy{}
policy.OptionsSerialized.OdProxy = tt.cdn
handler := Driver{
Policy: policy,
}
got, err := handler.replaceSourceHost(tt.origin)
if (err != nil) != tt.wantErr {
t.Errorf("replaceSourceHost() error = %v, wantErr %v", err, tt.wantErr)
return
}
if got != tt.want {
t.Errorf("replaceSourceHost() got = %v, want %v", got, tt.want)
}
})
}
}
func TestDriver_CancelToken(t *testing.T) {
asserts := assert.New(t)
handler := Driver{
Policy: &model.Policy{
AccessKey: "ak",
SecretKey: "sk",
BucketName: "test",
Server: "test.com",
},
}
handler.Client, _ = NewClient(&model.Policy{})
handler.Client.Credential.ExpiresIn = time.Now().Add(time.Duration(100) * time.Hour).Unix()
// 失败
{
err := handler.CancelToken(context.Background(), &serializer.UploadSession{})
asserts.Error(err)
}
}

View File

@@ -0,0 +1,38 @@
package onedrive
import (
model "github.com/cloudreve/Cloudreve/v3/models"
"testing"
)
func TestDriver_replaceSourceHost(t *testing.T) {
tests := []struct {
name string
origin string
cdn string
want string
wantErr bool
}{
{"TestNoReplace", "http://1dr.ms/download.aspx?123456", "", "http://1dr.ms/download.aspx?123456", false},
{"TestReplaceCorrect", "http://1dr.ms/download.aspx?123456", "https://test.com:8080", "https://test.com:8080/download.aspx?123456", false},
{"TestCdnFormatError", "http://1dr.ms/download.aspx?123456", string([]byte{0x7f}), "", true},
{"TestSrcFormatError", string([]byte{0x7f}), "https://test.com:8080", "", true},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
policy := &model.Policy{}
policy.OptionsSerialized.OdProxy = tt.cdn
handler := Driver{
Policy: policy,
}
got, err := handler.replaceSourceHost(tt.origin)
if (err != nil) != tt.wantErr {
t.Errorf("replaceSourceHost() error = %v, wantErr %v", err, tt.wantErr)
return
}
if got != tt.want {
t.Errorf("replaceSourceHost() got = %v, want %v", got, tt.want)
}
})
}
}

View File

@@ -3,6 +3,7 @@ package onedrive
import (
"encoding/gob"
"net/url"
"sync"
)
// RespError 接口返回错误
@@ -98,6 +99,15 @@ type ListResponse struct {
Context string `json:"@odata.context"`
}
// Chunk 文件分片
type Chunk struct {
Offset int
ChunkSize int
Total int
Retried int
Data []byte
}
// oauthEndpoint OAuth接口地址
type oauthEndpoint struct {
token url.URL
@@ -133,3 +143,10 @@ type Site struct {
func init() {
gob.Register(Credential{})
}
// IsLast 返回是否为最后一个分片
func (chunk *Chunk) IsLast() bool {
return chunk.Total-chunk.Offset == chunk.ChunkSize
}
var callbackSignal sync.Map

View File

@@ -0,0 +1,188 @@
package oss
import (
"io/ioutil"
"net/http"
"net/url"
"strings"
"testing"
"github.com/cloudreve/Cloudreve/v3/pkg/cache"
"github.com/stretchr/testify/assert"
)
func TestGetPublicKey(t *testing.T) {
asserts := assert.New(t)
testCases := []struct {
Request http.Request
ResNil bool
Error bool
}{
// Header解码失败
{
Request: http.Request{
Header: http.Header{
"X-Oss-Pub-Key-Url": {"中文"},
},
},
ResNil: true,
Error: true,
},
// 公钥URL无效
{
Request: http.Request{
Header: http.Header{
"X-Oss-Pub-Key-Url": {"aHR0cHM6Ly9wb3JuaHViLmNvbQ=="},
},
},
ResNil: true,
Error: true,
},
// 请求失败
{
Request: http.Request{
Header: http.Header{
"X-Oss-Pub-Key-Url": {"aHR0cDovL2dvc3NwdWJsaWMuYWxpY2RuLmNvbS8yMzQyMzQ="},
},
},
ResNil: true,
Error: true,
},
// 成功
{
Request: http.Request{
Header: http.Header{
"X-Oss-Pub-Key-Url": {"aHR0cDovL2dvc3NwdWJsaWMuYWxpY2RuLmNvbS9jYWxsYmFja19wdWJfa2V5X3YxLnBlbQ=="},
},
},
ResNil: false,
Error: false,
},
}
for i, testCase := range testCases {
asserts.NoError(cache.Deletes([]string{"oss_public_key"}, ""))
res, err := GetPublicKey(&testCase.Request)
if testCase.Error {
asserts.Error(err, "Test Case #%d", i)
} else {
asserts.NoError(err, "Test Case #%d", i)
}
if testCase.ResNil {
asserts.Empty(res, "Test Case #%d", i)
} else {
asserts.NotEmpty(res, "Test Case #%d", i)
}
}
// 测试缓存
asserts.NoError(cache.Set("oss_public_key", []byte("123"), 0))
res, err := GetPublicKey(nil)
asserts.NoError(err)
asserts.Equal([]byte("123"), res)
}
func TestVerifyCallbackSignature(t *testing.T) {
asserts := assert.New(t)
testPubKey := `-----BEGIN PUBLIC KEY-----
MFwwDQYJKoZIhvcNAQEBBQADSwAwSAJBAKs/JBGzwUB2aVht4crBx3oIPBLNsjGs
C0fTXv+nvlmklvkcolvpvXLTjaxUHR3W9LXxQ2EHXAJfCB+6H2YF1k8CAwEAAQ==
-----END PUBLIC KEY-----
`
// 成功
{
asserts.NoError(cache.Set("oss_public_key", []byte(testPubKey), 0))
r := http.Request{
URL: &url.URL{Path: "/api/v3/callback/oss/TnXx5E5VyfJUyM1UdkdDu1rtnJ34EbmH"},
Header: map[string][]string{
"Authorization": {"e5LwzwTkP9AFAItT4YzvdJOHd0Y0wqTMWhsV/h5SG90JYGAmMd+8LQyj96R+9qUfJWjMt6suuUh7LaOryR87Dw=="},
"X-Oss-Pub-Key-Url": {"aHR0cHM6Ly9nb3NzcHVibGljLmFsaWNkbi5jb20vY2FsbGJhY2tfcHViX2tleV92MS5wZW0="},
},
Body: ioutil.NopCloser(strings.NewReader(`{"name":"2f7b2ccf30e9270ea920f1ab8a4037a546a2f0d5.jpg","source_name":"1/1_hFRtDLgM_2f7b2ccf30e9270ea920f1ab8a4037a546a2f0d5.jpg","size":114020,"pic_info":"810,539"}`)),
}
asserts.NoError(VerifyCallbackSignature(&r))
}
// 签名错误
{
asserts.NoError(cache.Set("oss_public_key", []byte(testPubKey), 0))
r := http.Request{
URL: &url.URL{Path: "/api/v3/callback/oss/TnXx5E5VyfJUyM1UdkdDu1rtnJ34EbmH"},
Header: map[string][]string{
"Authorization": {"e3LwzwTkP9AFAItT4YzvdJOHd0Y0wqTMWhsV/h5SG90JYGAmMd+8LQyj96R+9qUfJWjMt6suuUh7LaOryR87Dw=="},
"X-Oss-Pub-Key-Url": {"aHR0cHM6Ly9nb3NzcHVibGljLmFsaWNkbi5jb20vY2FsbGJhY2tfcHViX2tleV92MS5wZW0="},
},
Body: ioutil.NopCloser(strings.NewReader(`{"name":"2f7b2ccf30e9270ea920f1ab8a4037a546a2f0d5.jpg","source_name":"1/1_hFRtDLgM_2f7b2ccf30e9270ea920f1ab8a4037a546a2f0d5.jpg","size":114020,"pic_info":"810,539"}`)),
}
asserts.Error(VerifyCallbackSignature(&r))
}
// GetPubKey 失败
{
asserts.NoError(cache.Deletes([]string{"oss_public_key"}, ""))
r := http.Request{
URL: &url.URL{Path: "/api/v3/callback/oss/TnXx5E5VyfJUyM1UdkdDu1rtnJ34EbmH"},
Header: map[string][]string{
"Authorization": {"e5LwzwTkP9AFAItT4YzvdJOHd0Y0wqTMWhsV/h5SG90JYGAmMd+8LQyj96R+9qUfJWjMt6suuUh7LaOryR87Dw=="},
},
Body: ioutil.NopCloser(strings.NewReader(`{"name":"2f7b2ccf30e9270ea920f1ab8a4037a546a2f0d5.jpg","source_name":"1/1_hFRtDLgM_2f7b2ccf30e9270ea920f1ab8a4037a546a2f0d5.jpg","size":114020,"pic_info":"810,539"}`)),
}
asserts.Error(VerifyCallbackSignature(&r))
}
// getRequestMD5 失败
{
asserts.NoError(cache.Set("oss_public_key", []byte(testPubKey), 0))
r := http.Request{
URL: &url.URL{Path: "%测试"},
Header: map[string][]string{
"Authorization": {"e5LwzwTkP9AFAItT4YzvdJOHd0Y0wqTMWhsV/h5SG90JYGAmMd+8LQyj96R+9qUfJWjMt6suuUh7LaOryR87Dw=="},
"X-Oss-Pub-Key-Url": {"aHR0cHM6Ly9nb3NzcHVibGljLmFsaWNkbi5jb20vY2FsbGJhY2tfcHViX2tleV92MS5wZW0="},
},
Body: ioutil.NopCloser(strings.NewReader(`{"name":"2f7b2ccf30e9270ea920f1ab8a4037a546a2f0d5.jpg","source_name":"1/1_hFRtDLgM_2f7b2ccf30e9270ea920f1ab8a4037a546a2f0d5.jpg","size":114020,"pic_info":"810,539"}`)),
}
asserts.Error(VerifyCallbackSignature(&r))
}
// 无 Authorization 头
{
asserts.NoError(cache.Set("oss_public_key", []byte(testPubKey), 0))
r := http.Request{
URL: &url.URL{Path: "/api/v3/callback/oss/TnXx5E5VyfJUyM1UdkdDu1rtnJ34EbmH"},
Header: map[string][]string{
"X-Oss-Pub-Key-Url": {"aHR0cHM6Ly9nb3NzcHVibGljLmFsaWNkbi5jb20vY2FsbGJhY2tfcHViX2tleV92MS5wZW0="},
},
Body: ioutil.NopCloser(strings.NewReader(`{"name":"2f7b2ccf30e9270ea920f1ab8a4037a546a2f0d5.jpg","source_name":"1/1_hFRtDLgM_2f7b2ccf30e9270ea920f1ab8a4037a546a2f0d5.jpg","size":114020,"pic_info":"810,539"}`)),
}
asserts.Error(VerifyCallbackSignature(&r))
}
// pub block 不存在
{
asserts.NoError(cache.Set("oss_public_key", []byte(""), 0))
r := http.Request{
URL: &url.URL{Path: "/api/v3/callback/oss/TnXx5E5VyfJUyM1UdkdDu1rtnJ34EbmH"},
Header: map[string][]string{
"Authorization": {"e5LwzwTkP9AFAItT4YzvdJOHd0Y0wqTMWhsV/h5SG90JYGAmMd+8LQyj96R+9qUfJWjMt6suuUh7LaOryR87Dw=="},
"X-Oss-Pub-Key-Url": {"aHR0cHM6Ly9nb3NzcHVibGljLmFsaWNkbi5jb20vY2FsbGJhY2tfcHViX2tleV92MS5wZW0="},
},
Body: ioutil.NopCloser(strings.NewReader(`{"name":"2f7b2ccf30e9270ea920f1ab8a4037a546a2f0d5.jpg","source_name":"1/1_hFRtDLgM_2f7b2ccf30e9270ea920f1ab8a4037a546a2f0d5.jpg","size":114020,"pic_info":"810,539"}`)),
}
asserts.Error(VerifyCallbackSignature(&r))
}
// ParsePKIXPublicKey出错
{
asserts.NoError(cache.Set("oss_public_key", []byte("-----BEGIN PUBLIC KEY-----\n-----END PUBLIC KEY-----"), 0))
r := http.Request{
URL: &url.URL{Path: "/api/v3/callback/oss/TnXx5E5VyfJUyM1UdkdDu1rtnJ34EbmH"},
Header: map[string][]string{
"Authorization": {"e5LwzwTkP9AFAItT4YzvdJOHd0Y0wqTMWhsV/h5SG90JYGAmMd+8LQyj96R+9qUfJWjMt6suuUh7LaOryR87Dw=="},
"X-Oss-Pub-Key-Url": {"aHR0cHM6Ly9nb3NzcHVibGljLmFsaWNkbi5jb20vY2FsbGJhY2tfcHViX2tleV92MS5wZW0="},
},
Body: ioutil.NopCloser(strings.NewReader(`{"name":"2f7b2ccf30e9270ea920f1ab8a4037a546a2f0d5.jpg","source_name":"1/1_hFRtDLgM_2f7b2ccf30e9270ea920f1ab8a4037a546a2f0d5.jpg","size":114020,"pic_info":"810,539"}`)),
}
asserts.Error(VerifyCallbackSignature(&r))
}
}

View File

@@ -2,6 +2,8 @@ package oss
import (
"context"
"crypto/hmac"
"crypto/sha1"
"encoding/base64"
"encoding/json"
"errors"
@@ -13,10 +15,8 @@ import (
"strings"
"time"
"github.com/HFO4/aliyun-oss-go-sdk/oss"
"github.com/aliyun/aliyun-oss-go-sdk/oss"
model "github.com/cloudreve/Cloudreve/v3/models"
"github.com/cloudreve/Cloudreve/v3/pkg/filesystem/chunk"
"github.com/cloudreve/Cloudreve/v3/pkg/filesystem/chunk/backoff"
"github.com/cloudreve/Cloudreve/v3/pkg/filesystem/fsctx"
"github.com/cloudreve/Cloudreve/v3/pkg/filesystem/response"
"github.com/cloudreve/Cloudreve/v3/pkg/request"
@@ -48,29 +48,17 @@ type Driver struct {
type key int
const (
chunkRetrySleep = time.Duration(5) * time.Second
// MultiPartUploadThreshold 服务端使用分片上传的阈值
MultiPartUploadThreshold uint64 = 5 * (1 << 30) // 5GB
// VersionID 文件版本标识
VersionID key = iota
)
func NewDriver(policy *model.Policy) (*Driver, error) {
if policy.OptionsSerialized.ChunkSize == 0 {
policy.OptionsSerialized.ChunkSize = 25 << 20 // 25 MB
}
driver := &Driver{
Policy: policy,
HTTPClient: request.NewClient(),
}
return driver, driver.InitOSSClient(false)
}
// CORS 创建跨域策略
func (handler *Driver) CORS() error {
// 初始化客户端
if err := handler.InitOSSClient(false); err != nil {
return err
}
return handler.client.SetBucketCORS(handler.Policy.BucketName, []oss.CORSRule{
{
AllowedOrigin: []string{"*"},
@@ -94,31 +82,39 @@ func (handler *Driver) InitOSSClient(forceUsePublicEndpoint bool) error {
return errors.New("存储策略为空")
}
// 决定是否使用内网 Endpoint
endpoint := handler.Policy.Server
if handler.Policy.OptionsSerialized.ServerSideEndpoint != "" && !forceUsePublicEndpoint {
endpoint = handler.Policy.OptionsSerialized.ServerSideEndpoint
}
if handler.client == nil {
// 决定是否使用内网 Endpoint
endpoint := handler.Policy.Server
if handler.Policy.OptionsSerialized.ServerSideEndpoint != "" && !forceUsePublicEndpoint {
endpoint = handler.Policy.OptionsSerialized.ServerSideEndpoint
}
// 初始化客户端
client, err := oss.New(endpoint, handler.Policy.AccessKey, handler.Policy.SecretKey)
if err != nil {
return err
}
handler.client = client
// 初始化客户端
client, err := oss.New(endpoint, handler.Policy.AccessKey, handler.Policy.SecretKey)
if err != nil {
return err
}
handler.client = client
// 初始化存储桶
bucket, err := client.Bucket(handler.Policy.BucketName)
if err != nil {
return err
}
handler.bucket = bucket
// 初始化存储桶
bucket, err := client.Bucket(handler.Policy.BucketName)
if err != nil {
return err
}
handler.bucket = bucket
return nil
}
// List 列出OSS上的文件
func (handler *Driver) List(ctx context.Context, base string, recursive bool) ([]response.Object, error) {
func (handler Driver) List(ctx context.Context, base string, recursive bool) ([]response.Object, error) {
// 初始化客户端
if err := handler.InitOSSClient(false); err != nil {
return nil, err
}
// 列取文件
base = strings.TrimPrefix(base, "/")
if base != "" {
@@ -185,7 +181,7 @@ func (handler *Driver) List(ctx context.Context, base string, recursive bool) ([
}
// Get 获取文件
func (handler *Driver) Get(ctx context.Context, path string) (response.RSCloser, error) {
func (handler Driver) Get(ctx context.Context, path string) (response.RSCloser, error) {
// 通过VersionID禁止缓存
ctx = context.WithValue(ctx, VersionID, time.Now().UnixNano())
@@ -228,54 +224,45 @@ func (handler *Driver) Get(ctx context.Context, path string) (response.RSCloser,
}
// Put 将文件流保存到指定目录
func (handler *Driver) Put(ctx context.Context, file fsctx.FileHeader) error {
func (handler Driver) Put(ctx context.Context, file io.ReadCloser, dst string, size uint64) error {
defer file.Close()
fileInfo := file.Info()
// 初始化客户端
if err := handler.InitOSSClient(false); err != nil {
return err
}
// 凭证有效期
credentialTTL := model.GetIntSetting("upload_session_timeout", 3600)
credentialTTL := model.GetIntSetting("upload_credential_timeout", 3600)
// 是否允许覆盖
overwrite := fileInfo.Mode&fsctx.Overwrite == fsctx.Overwrite
overwrite := true
if ctx.Value(fsctx.DisableOverwrite) != nil {
overwrite = false
}
options := []oss.Option{
oss.Expires(time.Now().Add(time.Duration(credentialTTL) * time.Second)),
oss.ForbidOverWrite(!overwrite),
}
// 小文件直接上传
if fileInfo.Size < MultiPartUploadThreshold {
return handler.bucket.PutObject(fileInfo.SavePath, file, options...)
}
// 超过阈值时使用分片上传
imur, err := handler.bucket.InitiateMultipartUpload(fileInfo.SavePath, options...)
// 上传文件
err := handler.bucket.PutObject(dst, file, options...)
if err != nil {
return fmt.Errorf("failed to initiate multipart upload: %w", err)
}
chunks := chunk.NewChunkGroup(file, handler.Policy.OptionsSerialized.ChunkSize, &backoff.ConstantBackoff{
Max: model.GetIntSetting("chunk_retries", 5),
Sleep: chunkRetrySleep,
}, model.IsTrueVal(model.GetSettingByName("use_temp_chunk_buffer")))
uploadFunc := func(current *chunk.ChunkGroup, content io.Reader) error {
_, err := handler.bucket.UploadPart(imur, content, current.Length(), current.Index()+1)
return err
}
for chunks.Next() {
if err := chunks.Process(uploadFunc); err != nil {
return fmt.Errorf("failed to upload chunk #%d: %w", chunks.Index(), err)
}
}
_, err = handler.bucket.CompleteMultipartUpload(imur, oss.CompleteAll("yes"), oss.ForbidOverWrite(!overwrite))
return err
return nil
}
// Delete 删除一个或多个文件,
// 返回未删除的文件
func (handler *Driver) Delete(ctx context.Context, files []string) ([]string, error) {
func (handler Driver) Delete(ctx context.Context, files []string) ([]string, error) {
// 初始化客户端
if err := handler.InitOSSClient(false); err != nil {
return files, err
}
// 删除文件
delRes, err := handler.bucket.DeleteObjects(files)
@@ -293,7 +280,7 @@ func (handler *Driver) Delete(ctx context.Context, files []string) ([]string, er
}
// Thumb 获取文件缩略图
func (handler *Driver) Thumb(ctx context.Context, path string) (*response.ContentResponse, error) {
func (handler Driver) Thumb(ctx context.Context, path string) (*response.ContentResponse, error) {
// 初始化客户端
if err := handler.InitOSSClient(true); err != nil {
return nil, err
@@ -327,7 +314,7 @@ func (handler *Driver) Thumb(ctx context.Context, path string) (*response.Conten
}
// Source 获取外链URL
func (handler *Driver) Source(
func (handler Driver) Source(
ctx context.Context,
path string,
baseURL url.URL,
@@ -372,7 +359,7 @@ func (handler *Driver) Source(
return handler.signSourceURL(ctx, path, ttl, signOptions)
}
func (handler *Driver) signSourceURL(ctx context.Context, path string, ttl int64, options []oss.Option) (string, error) {
func (handler Driver) signSourceURL(ctx context.Context, path string, ttl int64, options []oss.Option) (string, error) {
signedURL, err := handler.bucket.SignURL(path, oss.HTTPGet, ttl, options...)
if err != nil {
return "", err
@@ -384,6 +371,9 @@ func (handler *Driver) signSourceURL(ctx context.Context, path string, ttl int64
return "", err
}
// 优先使用https
finalURL.Scheme = "https"
// 公有空间替换掉Key及不支持的头
if !handler.Policy.IsPrivate {
query := finalURL.Query()
@@ -407,11 +397,16 @@ func (handler *Driver) signSourceURL(ctx context.Context, path string, ttl int64
}
// Token 获取上传策略和认证Token
func (handler *Driver) Token(ctx context.Context, ttl int64, uploadSession *serializer.UploadSession, file fsctx.FileHeader) (*serializer.UploadCredential, error) {
func (handler Driver) Token(ctx context.Context, TTL int64, key string) (serializer.UploadCredential, error) {
// 读取上下文中生成的存储路径
savePath, ok := ctx.Value(fsctx.SavePathCtx).(string)
if !ok {
return serializer.UploadCredential{}, errors.New("无法获取存储路径")
}
// 生成回调地址
siteURL := model.GetSiteURL()
apiBaseURI, _ := url.Parse("/api/v3/callback/oss/" + uploadSession.Key)
apiBaseURI, _ := url.Parse("/api/v3/callback/oss/" + key)
apiURL := siteURL.ResolveReference(apiBaseURI)
// 回调策略
@@ -420,66 +415,61 @@ func (handler *Driver) Token(ctx context.Context, ttl int64, uploadSession *seri
CallbackBody: `{"name":${x:fname},"source_name":${object},"size":${size},"pic_info":"${imageInfo.width},${imageInfo.height}"}`,
CallbackBodyType: "application/json",
}
callbackPolicyJSON, err := json.Marshal(callbackPolicy)
if err != nil {
return nil, fmt.Errorf("failed to encode callback policy: %w", err)
}
callbackPolicyEncoded := base64.StdEncoding.EncodeToString(callbackPolicyJSON)
// 初始化分片上传
fileInfo := file.Info()
options := []oss.Option{
oss.Expires(time.Now().Add(time.Duration(ttl) * time.Second)),
oss.ForbidOverWrite(true),
// 上传策略
postPolicy := UploadPolicy{
Expiration: time.Now().UTC().Add(time.Duration(TTL) * time.Second).Format(time.RFC3339),
Conditions: []interface{}{
map[string]string{"bucket": handler.Policy.BucketName},
[]string{"starts-with", "$key", path.Dir(savePath)},
},
}
imur, err := handler.bucket.InitiateMultipartUpload(fileInfo.SavePath, options...)
if err != nil {
return nil, fmt.Errorf("failed to initialize multipart upload: %w", err)
if handler.Policy.MaxSize > 0 {
postPolicy.Conditions = append(postPolicy.Conditions,
[]interface{}{"content-length-range", 0, handler.Policy.MaxSize})
}
uploadSession.UploadID = imur.UploadID
// 为每个分片签名上传 URL
chunks := chunk.NewChunkGroup(file, handler.Policy.OptionsSerialized.ChunkSize, &backoff.ConstantBackoff{}, false)
urls := make([]string, chunks.Num())
for chunks.Next() {
err := chunks.Process(func(c *chunk.ChunkGroup, chunk io.Reader) error {
signedURL, err := handler.bucket.SignURL(fileInfo.SavePath, oss.HTTPPut, ttl,
oss.PartNumber(c.Index()+1),
oss.UploadID(imur.UploadID),
oss.ContentType("application/octet-stream"))
if err != nil {
return err
}
return handler.getUploadCredential(ctx, postPolicy, callbackPolicy, TTL)
}
urls[c.Index()] = signedURL
return nil
})
func (handler Driver) getUploadCredential(ctx context.Context, policy UploadPolicy, callback CallbackPolicy, TTL int64) (serializer.UploadCredential, error) {
// 读取上下文中生成的存储路径
savePath, ok := ctx.Value(fsctx.SavePathCtx).(string)
if !ok {
return serializer.UploadCredential{}, errors.New("无法获取存储路径")
}
// 处理回调策略
callbackPolicyEncoded := ""
if callback.CallbackURL != "" {
callbackPolicyJSON, err := json.Marshal(callback)
if err != nil {
return nil, err
return serializer.UploadCredential{}, err
}
callbackPolicyEncoded = base64.StdEncoding.EncodeToString(callbackPolicyJSON)
policy.Conditions = append(policy.Conditions, map[string]string{"callback": callbackPolicyEncoded})
}
// 签名完成分片上传的URL
completeURL, err := handler.bucket.SignURL(fileInfo.SavePath, oss.HTTPPost, ttl,
oss.UploadID(imur.UploadID),
oss.Expires(time.Now().Add(time.Duration(ttl)*time.Second)),
oss.CompleteAll("yes"),
oss.ForbidOverWrite(true),
oss.CallbackParam(callbackPolicyEncoded))
// 编码上传策略
policyJSON, err := json.Marshal(policy)
if err != nil {
return nil, err
return serializer.UploadCredential{}, err
}
policyEncoded := base64.StdEncoding.EncodeToString(policyJSON)
return &serializer.UploadCredential{
SessionID: uploadSession.Key,
ChunkSize: handler.Policy.OptionsSerialized.ChunkSize,
UploadID: imur.UploadID,
UploadURLs: urls,
CompleteURL: completeURL,
// 签名上传策略
hmacSign := hmac.New(sha1.New, []byte(handler.Policy.SecretKey))
_, err = io.WriteString(hmacSign, policyEncoded)
if err != nil {
return serializer.UploadCredential{}, err
}
signature := base64.StdEncoding.EncodeToString(hmacSign.Sum(nil))
return serializer.UploadCredential{
Policy: fmt.Sprintf("%s:%s", callbackPolicyEncoded, policyEncoded),
Path: savePath,
AccessKey: handler.Policy.AccessKey,
Token: signature,
}, nil
}
// 取消上传凭证
func (handler *Driver) CancelToken(ctx context.Context, uploadSession *serializer.UploadSession) error {
return handler.bucket.AbortMultipartUpload(oss.InitiateMultipartUploadResult{UploadID: uploadSession.UploadID, Key: uploadSession.SavePath}, nil)
}

View File

@@ -0,0 +1,354 @@
package oss
import (
"context"
"io"
"io/ioutil"
"net/http"
"net/url"
"strings"
"testing"
model "github.com/cloudreve/Cloudreve/v3/models"
"github.com/cloudreve/Cloudreve/v3/pkg/cache"
"github.com/cloudreve/Cloudreve/v3/pkg/filesystem/fsctx"
"github.com/cloudreve/Cloudreve/v3/pkg/request"
"github.com/stretchr/testify/assert"
testMock "github.com/stretchr/testify/mock"
)
func TestDriver_InitOSSClient(t *testing.T) {
asserts := assert.New(t)
handler := Driver{
Policy: &model.Policy{
AccessKey: "ak",
SecretKey: "sk",
BucketName: "test",
Server: "test.com",
},
}
// 成功
{
asserts.NoError(handler.InitOSSClient(false))
}
// 使用内网Endpoint
{
handler.Policy.OptionsSerialized.ServerSideEndpoint = "endpoint2"
asserts.NoError(handler.InitOSSClient(false))
}
// 未指定存储策略
{
handler := Driver{}
asserts.Error(handler.InitOSSClient(false))
}
}
func TestDriver_CORS(t *testing.T) {
asserts := assert.New(t)
handler := Driver{
Policy: &model.Policy{
AccessKey: "ak",
SecretKey: "sk",
BucketName: "test",
Server: "test.com",
},
}
// 失败
{
asserts.NotPanics(func() {
handler.CORS()
})
}
}
func TestDriver_Token(t *testing.T) {
asserts := assert.New(t)
handler := Driver{
Policy: &model.Policy{
AccessKey: "ak",
SecretKey: "sk",
BucketName: "test",
Server: "test.com",
},
}
// 成功
{
ctx := context.WithValue(context.Background(), fsctx.SavePathCtx, "/123")
cache.Set("setting_siteURL", "http://test.cloudreve.org", 0)
res, err := handler.Token(ctx, 10, "key")
asserts.NoError(err)
asserts.NotEmpty(res.Policy)
asserts.NotEmpty(res.Token)
asserts.Equal(handler.Policy.AccessKey, res.AccessKey)
asserts.Equal("/123", res.Path)
}
// 上下文错误
{
ctx := context.Background()
_, err := handler.Token(ctx, 10, "key")
asserts.Error(err)
}
}
func TestDriver_Source(t *testing.T) {
asserts := assert.New(t)
handler := Driver{
Policy: &model.Policy{
AccessKey: "ak",
SecretKey: "sk",
BucketName: "test",
Server: "test.com",
IsPrivate: true,
},
}
// 正常 非下载 无限速
{
res, err := handler.Source(context.Background(), "/123", url.URL{}, 10, false, 0)
asserts.NoError(err)
resURL, err := url.Parse(res)
asserts.NoError(err)
query := resURL.Query()
asserts.NotEmpty(query.Get("Signature"))
asserts.NotEmpty(query.Get("Expires"))
asserts.Equal("ak", query.Get("OSSAccessKeyId"))
}
// 限速 + 下载
{
ctx := context.WithValue(context.Background(), fsctx.FileModelCtx, model.File{Name: "123.txt"})
res, err := handler.Source(ctx, "/123", url.URL{}, 10, true, 102401)
asserts.NoError(err)
resURL, err := url.Parse(res)
asserts.NoError(err)
query := resURL.Query()
asserts.NotEmpty(query.Get("Signature"))
asserts.NotEmpty(query.Get("Expires"))
asserts.Equal("ak", query.Get("OSSAccessKeyId"))
asserts.EqualValues("819208", query.Get("x-oss-traffic-limit"))
asserts.NotEmpty(query.Get("response-content-disposition"))
}
// 限速超出范围 + 下载
{
ctx := context.WithValue(context.Background(), fsctx.FileModelCtx, model.File{Name: "123.txt"})
res, err := handler.Source(ctx, "/123", url.URL{}, 10, true, 10)
asserts.NoError(err)
resURL, err := url.Parse(res)
asserts.NoError(err)
query := resURL.Query()
asserts.NotEmpty(query.Get("Signature"))
asserts.NotEmpty(query.Get("Expires"))
asserts.Equal("ak", query.Get("OSSAccessKeyId"))
asserts.EqualValues("819200", query.Get("x-oss-traffic-limit"))
asserts.NotEmpty(query.Get("response-content-disposition"))
}
// 限速超出范围 + 下载
{
ctx := context.WithValue(context.Background(), fsctx.FileModelCtx, model.File{Name: "123.txt"})
res, err := handler.Source(ctx, "/123", url.URL{}, 10, true, 838860801)
asserts.NoError(err)
resURL, err := url.Parse(res)
asserts.NoError(err)
query := resURL.Query()
asserts.NotEmpty(query.Get("Signature"))
asserts.NotEmpty(query.Get("Expires"))
asserts.Equal("ak", query.Get("OSSAccessKeyId"))
asserts.EqualValues("838860800", query.Get("x-oss-traffic-limit"))
asserts.NotEmpty(query.Get("response-content-disposition"))
}
// 公共空间
{
handler.Policy.IsPrivate = false
res, err := handler.Source(context.Background(), "/123", url.URL{}, 10, false, 0)
asserts.NoError(err)
resURL, err := url.Parse(res)
asserts.NoError(err)
query := resURL.Query()
asserts.Empty(query.Get("Signature"))
}
// 正常 指定了CDN域名
{
handler.Policy.BaseURL = "https://cqu.edu.cn"
res, err := handler.Source(context.Background(), "/123", url.URL{}, 10, false, 0)
asserts.NoError(err)
resURL, err := url.Parse(res)
asserts.NoError(err)
query := resURL.Query()
asserts.Empty(query.Get("Signature"))
asserts.Contains(resURL.String(), handler.Policy.BaseURL)
}
// 强制使用公网 Endpoint
{
handler.Policy.BaseURL = ""
handler.Policy.OptionsSerialized.ServerSideEndpoint = "endpoint.com"
res, err := handler.Source(context.WithValue(context.Background(), fsctx.ForceUsePublicEndpointCtx, false), "/123", url.URL{}, 10, false, 0)
asserts.NoError(err)
resURL, err := url.Parse(res)
asserts.NoError(err)
query := resURL.Query()
asserts.Empty(query.Get("Signature"))
asserts.Contains(resURL.String(), "endpoint.com")
}
}
func TestDriver_Thumb(t *testing.T) {
asserts := assert.New(t)
handler := Driver{
Policy: &model.Policy{
AccessKey: "ak",
SecretKey: "sk",
BucketName: "test",
Server: "test.com",
},
}
// 上下文不存在
{
ctx := context.Background()
res, err := handler.Thumb(ctx, "/123.txt")
asserts.Error(err)
asserts.Nil(res)
}
// 成功
{
cache.Set("setting_preview_timeout", "60", 0)
ctx := context.WithValue(context.Background(), fsctx.ThumbSizeCtx, [2]uint{10, 20})
res, err := handler.Thumb(ctx, "/123.jpg")
asserts.NoError(err)
resURL, err := url.Parse(res.URL)
asserts.NoError(err)
urlQuery := resURL.Query()
asserts.Equal("image/resize,m_lfit,h_20,w_10", urlQuery.Get("x-oss-process"))
}
}
func TestDriver_Delete(t *testing.T) {
asserts := assert.New(t)
handler := Driver{
Policy: &model.Policy{
AccessKey: "ak",
SecretKey: "sk",
BucketName: "test",
Server: "oss-cn-shanghai.aliyuncs.com",
},
}
// 失败
{
res, err := handler.Delete(context.Background(), []string{"1", "2", "3"})
asserts.Error(err)
asserts.Equal([]string{"1", "2", "3"}, res)
}
}
func TestDriver_Put(t *testing.T) {
asserts := assert.New(t)
handler := Driver{
Policy: &model.Policy{
AccessKey: "ak",
SecretKey: "sk",
BucketName: "test",
Server: "oss-cn-shanghai.aliyuncs.com",
},
}
cache.Set("setting_upload_credential_timeout", "3600", 0)
ctx := context.WithValue(context.Background(), fsctx.DisableOverwrite, true)
// 失败
{
err := handler.Put(ctx, ioutil.NopCloser(strings.NewReader("123")), "/123.txt", 3)
asserts.Error(err)
}
}
type ClientMock struct {
testMock.Mock
}
func (m ClientMock) Request(method, target string, body io.Reader, opts ...request.Option) *request.Response {
args := m.Called(method, target, body, opts)
return args.Get(0).(*request.Response)
}
func TestDriver_Get(t *testing.T) {
asserts := assert.New(t)
handler := Driver{
Policy: &model.Policy{
AccessKey: "ak",
SecretKey: "sk",
BucketName: "test",
Server: "oss-cn-shanghai.aliyuncs.com",
},
HTTPClient: request.NewClient(),
}
cache.Set("setting_preview_timeout", "3600", 0)
// 响应失败
{
res, err := handler.Get(context.Background(), "123.txt")
asserts.Error(err)
asserts.Nil(res)
}
// 响应成功
{
ctx := context.WithValue(context.Background(), fsctx.FileModelCtx, model.File{Size: 3})
clientMock := ClientMock{}
clientMock.On(
"Request",
"GET",
testMock.Anything,
testMock.Anything,
testMock.Anything,
).Return(&request.Response{
Err: nil,
Response: &http.Response{
StatusCode: 200,
Body: ioutil.NopCloser(strings.NewReader(`123`)),
},
})
handler.HTTPClient = clientMock
res, err := handler.Get(ctx, "123.txt")
clientMock.AssertExpectations(t)
asserts.NoError(err)
n, err := res.Seek(0, io.SeekEnd)
asserts.NoError(err)
asserts.EqualValues(3, n)
content, err := ioutil.ReadAll(res)
asserts.NoError(err)
asserts.Equal("123", string(content))
}
}
func TestDriver_List(t *testing.T) {
asserts := assert.New(t)
handler := Driver{
Policy: &model.Policy{
AccessKey: "ak",
SecretKey: "sk",
BucketName: "test",
Server: "test.com",
IsPrivate: true,
},
}
// 连接失败
{
res, err := handler.List(context.Background(), "/", true)
asserts.Error(err)
asserts.Empty(res)
}
}

View File

@@ -2,9 +2,9 @@ package qiniu
import (
"context"
"encoding/base64"
"errors"
"fmt"
"io"
"net/http"
"net/url"
"path"
@@ -17,35 +17,17 @@ import (
"github.com/cloudreve/Cloudreve/v3/pkg/filesystem/response"
"github.com/cloudreve/Cloudreve/v3/pkg/request"
"github.com/cloudreve/Cloudreve/v3/pkg/serializer"
"github.com/qiniu/go-sdk/v7/auth/qbox"
"github.com/qiniu/go-sdk/v7/storage"
"github.com/qiniu/api.v7/v7/auth/qbox"
"github.com/qiniu/api.v7/v7/storage"
)
// Driver 本地策略适配器
type Driver struct {
Policy *model.Policy
mac *qbox.Mac
cfg *storage.Config
bucket *storage.BucketManager
}
func NewDriver(policy *model.Policy) *Driver {
if policy.OptionsSerialized.ChunkSize == 0 {
policy.OptionsSerialized.ChunkSize = 25 << 20 // 25 MB
}
mac := qbox.NewMac(policy.AccessKey, policy.SecretKey)
cfg := &storage.Config{UseHTTPS: true}
return &Driver{
Policy: policy,
mac: mac,
cfg: cfg,
bucket: storage.NewBucketManager(mac, cfg),
}
}
// List 列出给定路径下的文件
func (handler *Driver) List(ctx context.Context, base string, recursive bool) ([]response.Object, error) {
func (handler Driver) List(ctx context.Context, base string, recursive bool) ([]response.Object, error) {
base = strings.TrimPrefix(base, "/")
if base != "" {
base += "/"
@@ -61,8 +43,14 @@ func (handler *Driver) List(ctx context.Context, base string, recursive bool) ([
delimiter = "/"
}
mac := qbox.NewMac(handler.Policy.AccessKey, handler.Policy.SecretKey)
cfg := storage.Config{
UseHTTPS: true,
}
bucketManager := storage.NewBucketManager(mac, &cfg)
for {
entries, folders, nextMarker, hashNext, err := handler.bucket.ListFiles(
entries, folders, nextMarker, hashNext, err := bucketManager.ListFiles(
handler.Policy.BucketName,
base, delimiter, marker, 1000)
if err != nil {
@@ -112,7 +100,7 @@ func (handler *Driver) List(ctx context.Context, base string, recursive bool) ([
}
// Get 获取文件
func (handler *Driver) Get(ctx context.Context, path string) (response.RSCloser, error) {
func (handler Driver) Get(ctx context.Context, path string) (response.RSCloser, error) {
// 给文件名加上随机参数以强制拉取
path = fmt.Sprintf("%s?v=%d", path, time.Now().UnixNano())
@@ -130,7 +118,7 @@ func (handler *Driver) Get(ctx context.Context, path string) (response.RSCloser,
}
// 获取文件数据流
client := request.NewClient()
client := request.HTTPClient{}
resp, err := client.Request(
"GET",
downloadURL,
@@ -156,25 +144,19 @@ func (handler *Driver) Get(ctx context.Context, path string) (response.RSCloser,
}
// Put 将文件流保存到指定目录
func (handler *Driver) Put(ctx context.Context, file fsctx.FileHeader) error {
func (handler Driver) Put(ctx context.Context, file io.ReadCloser, dst string, size uint64) error {
defer file.Close()
// 凭证有效期
credentialTTL := model.GetIntSetting("upload_session_timeout", 3600)
credentialTTL := model.GetIntSetting("upload_credential_timeout", 3600)
// 生成上传策略
fileInfo := file.Info()
scope := handler.Policy.BucketName
if fileInfo.Mode&fsctx.Overwrite == fsctx.Overwrite {
scope = fmt.Sprintf("%s:%s", handler.Policy.BucketName, fileInfo.SavePath)
}
putPolicy := storage.PutPolicy{
// 指定为覆盖策略
Scope: scope,
SaveKey: fileInfo.SavePath,
Scope: fmt.Sprintf("%s:%s", handler.Policy.BucketName, dst),
SaveKey: dst,
ForceSaveKey: true,
FsizeLimit: int64(fileInfo.Size),
FsizeLimit: int64(size),
}
// 是否开启了MIMEType限制
if handler.Policy.OptionsSerialized.MimeType != "" {
@@ -182,7 +164,7 @@ func (handler *Driver) Put(ctx context.Context, file fsctx.FileHeader) error {
}
// 生成上传凭证
token, err := handler.getUploadCredential(ctx, putPolicy, fileInfo, int64(credentialTTL), false)
token, err := handler.getUploadCredential(ctx, putPolicy, int64(credentialTTL))
if err != nil {
return err
}
@@ -196,7 +178,7 @@ func (handler *Driver) Put(ctx context.Context, file fsctx.FileHeader) error {
}
// 开始上传
err = formUploader.Put(ctx, &ret, token.Credential, fileInfo.SavePath, file, int64(fileInfo.Size), &putExtra)
err = formUploader.Put(ctx, &ret, token.Token, dst, file, int64(size), &putExtra)
if err != nil {
return err
}
@@ -206,14 +188,19 @@ func (handler *Driver) Put(ctx context.Context, file fsctx.FileHeader) error {
// Delete 删除一个或多个文件,
// 返回未删除的文件
func (handler *Driver) Delete(ctx context.Context, files []string) ([]string, error) {
func (handler Driver) Delete(ctx context.Context, files []string) ([]string, error) {
// TODO 大于一千个文件需要分批发送
deleteOps := make([]string, 0, len(files))
for _, key := range files {
deleteOps = append(deleteOps, storage.URIDelete(handler.Policy.BucketName, key))
}
rets, err := handler.bucket.Batch(deleteOps)
mac := qbox.NewMac(handler.Policy.AccessKey, handler.Policy.SecretKey)
cfg := storage.Config{
UseHTTPS: true,
}
bucketManager := storage.NewBucketManager(mac, &cfg)
rets, err := bucketManager.Batch(deleteOps)
// 处理删除结果
if err != nil {
@@ -230,7 +217,7 @@ func (handler *Driver) Delete(ctx context.Context, files []string) ([]string, er
}
// Thumb 获取文件缩略图
func (handler *Driver) Thumb(ctx context.Context, path string) (*response.ContentResponse, error) {
func (handler Driver) Thumb(ctx context.Context, path string) (*response.ContentResponse, error) {
var (
thumbSize = [2]uint{400, 300}
ok = false
@@ -251,7 +238,7 @@ func (handler *Driver) Thumb(ctx context.Context, path string) (*response.Conten
}
// Source 获取外链URL
func (handler *Driver) Source(
func (handler Driver) Source(
ctx context.Context,
path string,
baseURL url.URL,
@@ -274,11 +261,12 @@ func (handler *Driver) Source(
return handler.signSourceURL(ctx, path, ttl), nil
}
func (handler *Driver) signSourceURL(ctx context.Context, path string, ttl int64) string {
func (handler Driver) signSourceURL(ctx context.Context, path string, ttl int64) string {
var sourceURL string
if handler.Policy.IsPrivate {
mac := qbox.NewMac(handler.Policy.AccessKey, handler.Policy.SecretKey)
deadline := time.Now().Add(time.Second * time.Duration(ttl)).Unix()
sourceURL = storage.MakePrivateURL(handler.mac, handler.Policy.BaseURL, path, deadline)
sourceURL = storage.MakePrivateURL(mac, handler.Policy.BaseURL, path, deadline)
} else {
sourceURL = storage.MakePublicURL(handler.Policy.BaseURL, path)
}
@@ -286,20 +274,25 @@ func (handler *Driver) signSourceURL(ctx context.Context, path string, ttl int64
}
// Token 获取上传策略和认证Token
func (handler *Driver) Token(ctx context.Context, ttl int64, uploadSession *serializer.UploadSession, file fsctx.FileHeader) (*serializer.UploadCredential, error) {
func (handler Driver) Token(ctx context.Context, TTL int64, key string) (serializer.UploadCredential, error) {
// 生成回调地址
siteURL := model.GetSiteURL()
apiBaseURI, _ := url.Parse("/api/v3/callback/qiniu/" + uploadSession.Key)
apiBaseURI, _ := url.Parse("/api/v3/callback/qiniu/" + key)
apiURL := siteURL.ResolveReference(apiBaseURI)
// 读取上下文中生成的存储路径
savePath, ok := ctx.Value(fsctx.SavePathCtx).(string)
if !ok {
return serializer.UploadCredential{}, errors.New("无法获取存储路径")
}
// 创建上传策略
fileInfo := file.Info()
putPolicy := storage.PutPolicy{
Scope: handler.Policy.BucketName,
CallbackURL: apiURL.String(),
CallbackBody: `{"size":$(fsize),"pic_info":"$(imageInfo.width),$(imageInfo.height)"}`,
CallbackBody: `{"name":"$(fname)","source_name":"$(key)","size":$(fsize),"pic_info":"$(imageInfo.width),$(imageInfo.height)"}`,
CallbackBodyType: "application/json",
SaveKey: fileInfo.SavePath,
SaveKey: savePath,
ForceSaveKey: true,
FsizeLimit: int64(handler.Policy.MaxSize),
}
@@ -308,46 +301,16 @@ func (handler *Driver) Token(ctx context.Context, ttl int64, uploadSession *seri
putPolicy.MimeLimit = handler.Policy.OptionsSerialized.MimeType
}
credential, err := handler.getUploadCredential(ctx, putPolicy, fileInfo, ttl, true)
if err != nil {
return nil, fmt.Errorf("failed to init parts: %w", err)
}
credential.SessionID = uploadSession.Key
credential.ChunkSize = handler.Policy.OptionsSerialized.ChunkSize
uploadSession.UploadURL = credential.UploadURLs[0]
uploadSession.Credential = credential.Credential
return credential, nil
return handler.getUploadCredential(ctx, putPolicy, TTL)
}
// getUploadCredential 签名上传策略并创建上传会话
func (handler *Driver) getUploadCredential(ctx context.Context, policy storage.PutPolicy, file *fsctx.UploadTaskInfo, TTL int64, resume bool) (*serializer.UploadCredential, error) {
// 上传凭证
// getUploadCredential 签名上传策略
func (handler Driver) getUploadCredential(ctx context.Context, policy storage.PutPolicy, TTL int64) (serializer.UploadCredential, error) {
policy.Expires = uint64(TTL)
upToken := policy.UploadToken(handler.mac)
mac := qbox.NewMac(handler.Policy.AccessKey, handler.Policy.SecretKey)
upToken := policy.UploadToken(mac)
// 初始化分片上传
resumeUploader := storage.NewResumeUploaderV2(handler.cfg)
upHost, err := resumeUploader.UpHost(handler.Policy.AccessKey, handler.Policy.BucketName)
if err != nil {
return nil, err
}
ret := &storage.InitPartsRet{}
if resume {
err = resumeUploader.InitParts(ctx, upToken, upHost, handler.Policy.BucketName, file.SavePath, true, ret)
}
return &serializer.UploadCredential{
UploadURLs: []string{upHost + "/buckets/" + handler.Policy.BucketName + "/objects/" + base64.URLEncoding.EncodeToString([]byte(file.SavePath)) + "/uploads/" + ret.UploadID},
Credential: upToken,
}, err
}
// 取消上传凭证
func (handler Driver) CancelToken(ctx context.Context, uploadSession *serializer.UploadSession) error {
resumeUploader := storage.NewResumeUploaderV2(handler.cfg)
return resumeUploader.Client.CallWith(ctx, nil, "DELETE", uploadSession.UploadURL, http.Header{"Authorization": {"UpToken " + uploadSession.Credential}}, nil, 0)
return serializer.UploadCredential{
Token: upToken,
}, nil
}

View File

@@ -1,195 +0,0 @@
package remote
import (
"context"
"encoding/json"
"fmt"
model "github.com/cloudreve/Cloudreve/v3/models"
"github.com/cloudreve/Cloudreve/v3/pkg/auth"
"github.com/cloudreve/Cloudreve/v3/pkg/filesystem/chunk"
"github.com/cloudreve/Cloudreve/v3/pkg/filesystem/chunk/backoff"
"github.com/cloudreve/Cloudreve/v3/pkg/filesystem/fsctx"
"github.com/cloudreve/Cloudreve/v3/pkg/request"
"github.com/cloudreve/Cloudreve/v3/pkg/serializer"
"github.com/cloudreve/Cloudreve/v3/pkg/util"
"github.com/gofrs/uuid"
"io"
"net/http"
"net/url"
"path"
"strings"
"time"
)
const (
basePath = "/api/v3/slave/"
OverwriteHeader = auth.CrHeaderPrefix + "Overwrite"
chunkRetrySleep = time.Duration(5) * time.Second
)
// Client to operate uploading to remote slave server
type Client interface {
// CreateUploadSession creates remote upload session
CreateUploadSession(ctx context.Context, session *serializer.UploadSession, ttl int64, overwrite bool) error
// GetUploadURL signs an url for uploading file
GetUploadURL(ttl int64, sessionID string) (string, string, error)
// Upload uploads file to remote server
Upload(ctx context.Context, file fsctx.FileHeader) error
// DeleteUploadSession deletes remote upload session
DeleteUploadSession(ctx context.Context, sessionID string) error
}
// NewClient creates new Client from given policy
func NewClient(policy *model.Policy) (Client, error) {
authInstance := auth.HMACAuth{[]byte(policy.SecretKey)}
serverURL, err := url.Parse(policy.Server)
if err != nil {
return nil, err
}
base, _ := url.Parse(basePath)
signTTL := model.GetIntSetting("slave_api_timeout", 60)
return &remoteClient{
policy: policy,
authInstance: authInstance,
httpClient: request.NewClient(
request.WithEndpoint(serverURL.ResolveReference(base).String()),
request.WithCredential(authInstance, int64(signTTL)),
request.WithMasterMeta(),
request.WithSlaveMeta(policy.AccessKey),
),
}, nil
}
type remoteClient struct {
policy *model.Policy
authInstance auth.Auth
httpClient request.Client
}
func (c *remoteClient) Upload(ctx context.Context, file fsctx.FileHeader) error {
ttl := model.GetIntSetting("upload_session_timeout", 86400)
fileInfo := file.Info()
session := &serializer.UploadSession{
Key: uuid.Must(uuid.NewV4()).String(),
VirtualPath: fileInfo.VirtualPath,
Name: fileInfo.FileName,
Size: fileInfo.Size,
SavePath: fileInfo.SavePath,
LastModified: fileInfo.LastModified,
Policy: *c.policy,
}
// Create upload session
overwrite := fileInfo.Mode&fsctx.Overwrite == fsctx.Overwrite
if err := c.CreateUploadSession(ctx, session, int64(ttl), overwrite); err != nil {
return fmt.Errorf("failed to create upload session: %w", err)
}
// Initial chunk groups
chunks := chunk.NewChunkGroup(file, c.policy.OptionsSerialized.ChunkSize, &backoff.ConstantBackoff{
Max: model.GetIntSetting("chunk_retries", 5),
Sleep: chunkRetrySleep,
}, model.IsTrueVal(model.GetSettingByName("use_temp_chunk_buffer")))
uploadFunc := func(current *chunk.ChunkGroup, content io.Reader) error {
return c.uploadChunk(ctx, session.Key, current.Index(), content, overwrite, current.Length())
}
// upload chunks
for chunks.Next() {
if err := chunks.Process(uploadFunc); err != nil {
if err := c.DeleteUploadSession(ctx, session.Key); err != nil {
util.Log().Warning("failed to delete upload session: %s", err)
}
return fmt.Errorf("failed to upload chunk #%d: %w", chunks.Index(), err)
}
}
return nil
}
func (c *remoteClient) DeleteUploadSession(ctx context.Context, sessionID string) error {
resp, err := c.httpClient.Request(
"DELETE",
"upload/"+sessionID,
nil,
request.WithContext(ctx),
).CheckHTTPResponse(200).DecodeResponse()
if err != nil {
return err
}
if resp.Code != 0 {
return serializer.NewErrorFromResponse(resp)
}
return nil
}
func (c *remoteClient) CreateUploadSession(ctx context.Context, session *serializer.UploadSession, ttl int64, overwrite bool) error {
reqBodyEncoded, err := json.Marshal(map[string]interface{}{
"session": session,
"ttl": ttl,
"overwrite": overwrite,
})
if err != nil {
return err
}
bodyReader := strings.NewReader(string(reqBodyEncoded))
resp, err := c.httpClient.Request(
"PUT",
"upload",
bodyReader,
request.WithContext(ctx),
).CheckHTTPResponse(200).DecodeResponse()
if err != nil {
return err
}
if resp.Code != 0 {
return serializer.NewErrorFromResponse(resp)
}
return nil
}
func (c *remoteClient) GetUploadURL(ttl int64, sessionID string) (string, string, error) {
base, err := url.Parse(c.policy.Server)
if err != nil {
return "", "", err
}
base.Path = path.Join(base.Path, basePath, "upload", sessionID)
req, err := http.NewRequest("POST", base.String(), nil)
if err != nil {
return "", "", err
}
req = auth.SignRequest(c.authInstance, req, ttl)
return req.URL.String(), req.Header["Authorization"][0], nil
}
func (c *remoteClient) uploadChunk(ctx context.Context, sessionID string, index int, chunk io.Reader, overwrite bool, size int64) error {
resp, err := c.httpClient.Request(
"POST",
fmt.Sprintf("upload/%s?chunk=%d", sessionID, index),
chunk,
request.WithContext(ctx),
request.WithTimeout(time.Duration(0)),
request.WithContentLength(size),
request.WithHeader(map[string][]string{OverwriteHeader: {fmt.Sprintf("%t", overwrite)}}),
).CheckHTTPResponse(200).DecodeResponse()
if err != nil {
return err
}
if resp.Code != 0 {
return serializer.NewErrorFromResponse(resp)
}
return nil
}

View File

@@ -1,262 +0,0 @@
package remote
import (
"context"
"errors"
model "github.com/cloudreve/Cloudreve/v3/models"
"github.com/cloudreve/Cloudreve/v3/pkg/cache"
"github.com/cloudreve/Cloudreve/v3/pkg/filesystem/fsctx"
"github.com/cloudreve/Cloudreve/v3/pkg/mocks/requestmock"
"github.com/cloudreve/Cloudreve/v3/pkg/request"
"github.com/stretchr/testify/assert"
testMock "github.com/stretchr/testify/mock"
"io/ioutil"
"net/http"
"strings"
"testing"
)
func TestNewClient(t *testing.T) {
a := assert.New(t)
policy := &model.Policy{}
// 无法解析服务端url
{
policy.Server = string([]byte{0x7f})
c, err := NewClient(policy)
a.Error(err)
a.Nil(c)
}
// 成功
{
policy.Server = ""
c, err := NewClient(policy)
a.NoError(err)
a.NotNil(c)
}
}
func TestRemoteClient_Upload(t *testing.T) {
a := assert.New(t)
c, _ := NewClient(&model.Policy{})
// 无法创建上传会话
{
clientMock := requestmock.RequestMock{}
c.(*remoteClient).httpClient = &clientMock
clientMock.On(
"Request",
"PUT",
"upload",
testMock.Anything,
testMock.Anything,
).Return(&request.Response{
Err: errors.New("error"),
})
err := c.Upload(context.Background(), &fsctx.FileStream{})
a.Error(err)
a.Contains(err.Error(), "error")
clientMock.AssertExpectations(t)
}
// 分片上传失败,成功删除上传会话
{
cache.Set("setting_chunk_retries", "1", 0)
clientMock := requestmock.RequestMock{}
c.(*remoteClient).httpClient = &clientMock
clientMock.On(
"Request",
"PUT",
"upload",
testMock.Anything,
testMock.Anything,
).Return(&request.Response{
Err: nil,
Response: &http.Response{
StatusCode: 200,
Body: ioutil.NopCloser(strings.NewReader(`{"code":0}`)),
},
})
clientMock.On(
"Request",
"POST",
testMock.Anything,
testMock.Anything,
testMock.Anything,
).Return(&request.Response{
Err: errors.New("error"),
})
clientMock.On(
"Request",
"DELETE",
testMock.Anything,
testMock.Anything,
testMock.Anything,
).Return(&request.Response{
Err: nil,
Response: &http.Response{
StatusCode: 200,
Body: ioutil.NopCloser(strings.NewReader(`{"code":0}`)),
},
})
err := c.Upload(context.Background(), &fsctx.FileStream{})
a.Error(err)
a.Contains(err.Error(), "error")
clientMock.AssertExpectations(t)
}
// 分片上传失败,无法删除上传会话
{
cache.Set("setting_chunk_retries", "1", 0)
clientMock := requestmock.RequestMock{}
c.(*remoteClient).httpClient = &clientMock
clientMock.On(
"Request",
"PUT",
"upload",
testMock.Anything,
testMock.Anything,
).Return(&request.Response{
Err: nil,
Response: &http.Response{
StatusCode: 200,
Body: ioutil.NopCloser(strings.NewReader(`{"code":0}`)),
},
})
clientMock.On(
"Request",
"POST",
testMock.Anything,
testMock.Anything,
testMock.Anything,
).Return(&request.Response{
Err: errors.New("error"),
})
clientMock.On(
"Request",
"DELETE",
testMock.Anything,
testMock.Anything,
testMock.Anything,
).Return(&request.Response{
Err: errors.New("error2"),
Response: &http.Response{
StatusCode: 200,
Body: ioutil.NopCloser(strings.NewReader(`{"code":0}`)),
},
})
err := c.Upload(context.Background(), &fsctx.FileStream{})
a.Error(err)
a.Contains(err.Error(), "error")
clientMock.AssertExpectations(t)
}
// 成功
{
cache.Set("setting_chunk_retries", "1", 0)
clientMock := requestmock.RequestMock{}
c.(*remoteClient).httpClient = &clientMock
clientMock.On(
"Request",
"PUT",
"upload",
testMock.Anything,
testMock.Anything,
).Return(&request.Response{
Err: nil,
Response: &http.Response{
StatusCode: 200,
Body: ioutil.NopCloser(strings.NewReader(`{"code":0}`)),
},
})
clientMock.On(
"Request",
"POST",
testMock.Anything,
testMock.Anything,
testMock.Anything,
).Return(&request.Response{
Response: &http.Response{
StatusCode: 200,
Body: ioutil.NopCloser(strings.NewReader(`{"code":0}`)),
},
})
err := c.Upload(context.Background(), &fsctx.FileStream{})
a.NoError(err)
clientMock.AssertExpectations(t)
}
}
func TestRemoteClient_CreateUploadSessionFailed(t *testing.T) {
a := assert.New(t)
c, _ := NewClient(&model.Policy{})
clientMock := requestmock.RequestMock{}
c.(*remoteClient).httpClient = &clientMock
clientMock.On(
"Request",
"PUT",
"upload",
testMock.Anything,
testMock.Anything,
).Return(&request.Response{
Err: nil,
Response: &http.Response{
StatusCode: 200,
Body: ioutil.NopCloser(strings.NewReader(`{"code":500,"msg":"error"}`)),
},
})
err := c.Upload(context.Background(), &fsctx.FileStream{})
a.Error(err)
a.Contains(err.Error(), "error")
clientMock.AssertExpectations(t)
}
func TestRemoteClient_UploadChunkFailed(t *testing.T) {
a := assert.New(t)
c, _ := NewClient(&model.Policy{})
clientMock := requestmock.RequestMock{}
c.(*remoteClient).httpClient = &clientMock
clientMock.On(
"Request",
"POST",
testMock.Anything,
testMock.Anything,
testMock.Anything,
).Return(&request.Response{
Err: nil,
Response: &http.Response{
StatusCode: 200,
Body: ioutil.NopCloser(strings.NewReader(`{"code":500,"msg":"error"}`)),
},
})
err := c.(*remoteClient).uploadChunk(context.Background(), "", 0, strings.NewReader(""), false, 0)
a.Error(err)
a.Contains(err.Error(), "error")
clientMock.AssertExpectations(t)
}
func TestRemoteClient_GetUploadURL(t *testing.T) {
a := assert.New(t)
c, _ := NewClient(&model.Policy{})
// url 解析失败
{
c.(*remoteClient).policy.Server = string([]byte{0x7f})
res, sign, err := c.GetUploadURL(0, "")
a.Error(err)
a.Empty(res)
a.Empty(sign)
}
// 成功
{
c.(*remoteClient).policy.Server = ""
res, sign, err := c.GetUploadURL(0, "")
a.NoError(err)
a.NotEmpty(res)
a.NotEmpty(sign)
}
}

View File

@@ -6,6 +6,8 @@ import (
"encoding/json"
"errors"
"fmt"
"io"
"net/http"
"net/url"
"path"
"strings"
@@ -24,28 +26,10 @@ type Driver struct {
Client request.Client
Policy *model.Policy
AuthInstance auth.Auth
uploadClient Client
}
// NewDriver initializes a new Driver from policy
// TODO: refactor all method into upload client
func NewDriver(policy *model.Policy) (*Driver, error) {
client, err := NewClient(policy)
if err != nil {
return nil, err
}
return &Driver{
Policy: policy,
Client: request.NewClient(),
AuthInstance: auth.HMACAuth{[]byte(policy.SecretKey)},
uploadClient: client,
}, nil
}
// List 列取文件
func (handler *Driver) List(ctx context.Context, path string, recursive bool) ([]response.Object, error) {
func (handler Driver) List(ctx context.Context, path string, recursive bool) ([]response.Object, error) {
var res []response.Object
reqBody := serializer.ListRequest{
@@ -87,7 +71,7 @@ func (handler *Driver) List(ctx context.Context, path string, recursive bool) ([
}
// getAPIUrl 获取接口请求地址
func (handler *Driver) getAPIUrl(scope string, routes ...string) string {
func (handler Driver) getAPIUrl(scope string, routes ...string) string {
serverURL, err := url.Parse(handler.Policy.Server)
if err != nil {
return ""
@@ -113,7 +97,7 @@ func (handler *Driver) getAPIUrl(scope string, routes ...string) string {
}
// Get 获取文件内容
func (handler *Driver) Get(ctx context.Context, path string) (response.RSCloser, error) {
func (handler Driver) Get(ctx context.Context, path string) (response.RSCloser, error) {
// 尝试获取速度限制
speedLimit := 0
if user, ok := ctx.Value(fsctx.UserCtx).(model.User); ok {
@@ -150,15 +134,62 @@ func (handler *Driver) Get(ctx context.Context, path string) (response.RSCloser,
}
// Put 将文件流保存到指定目录
func (handler *Driver) Put(ctx context.Context, file fsctx.FileHeader) error {
func (handler Driver) Put(ctx context.Context, file io.ReadCloser, dst string, size uint64) error {
defer file.Close()
return handler.uploadClient.Upload(ctx, file)
// 凭证有效期
credentialTTL := model.GetIntSetting("upload_credential_timeout", 3600)
// 生成上传策略
policy := serializer.UploadPolicy{
SavePath: path.Dir(dst),
FileName: path.Base(dst),
AutoRename: false,
MaxSize: size,
}
credential, err := handler.getUploadCredential(ctx, policy, int64(credentialTTL))
if err != nil {
return err
}
// 对文件名进行URLEncode
fileName := url.QueryEscape(path.Base(dst))
// 决定是否要禁用文件覆盖
overwrite := "true"
if ctx.Value(fsctx.DisableOverwrite) != nil {
overwrite = "false"
}
// 上传文件
resp, err := handler.Client.Request(
"POST",
handler.Policy.GetUploadURL(),
file,
request.WithHeader(map[string][]string{
"X-Cr-Policy": {credential.Policy},
"X-Cr-FileName": {fileName},
"X-Cr-Overwrite": {overwrite},
}),
request.WithContentLength(int64(size)),
request.WithTimeout(time.Duration(0)),
request.WithMasterMeta(),
request.WithSlaveMeta(handler.Policy.AccessKey),
request.WithCredential(handler.AuthInstance, int64(credentialTTL)),
).CheckHTTPResponse(200).DecodeResponse()
if err != nil {
return err
}
if resp.Code != 0 {
return errors.New(resp.Msg)
}
return nil
}
// Delete 删除一个或多个文件,
// 返回未删除的文件,及遇到的最后一个错误
func (handler *Driver) Delete(ctx context.Context, files []string) ([]string, error) {
func (handler Driver) Delete(ctx context.Context, files []string) ([]string, error) {
// 封装接口请求正文
reqBody := serializer.RemoteDeleteRequest{
Files: files,
@@ -204,7 +235,7 @@ func (handler *Driver) Delete(ctx context.Context, files []string) ([]string, er
}
// Thumb 获取文件缩略图
func (handler *Driver) Thumb(ctx context.Context, path string) (*response.ContentResponse, error) {
func (handler Driver) Thumb(ctx context.Context, path string) (*response.ContentResponse, error) {
sourcePath := base64.RawURLEncoding.EncodeToString([]byte(path))
thumbURL := handler.getAPIUrl("thumb") + "/" + sourcePath
ttl := model.GetIntSetting("preview_timeout", 60)
@@ -220,7 +251,7 @@ func (handler *Driver) Thumb(ctx context.Context, path string) (*response.Conten
}
// Source 获取外链URL
func (handler *Driver) Source(
func (handler Driver) Source(
ctx context.Context,
path string,
baseURL url.URL,
@@ -274,32 +305,43 @@ func (handler *Driver) Source(
}
// Token 获取上传策略和认证Token
func (handler *Driver) Token(ctx context.Context, ttl int64, uploadSession *serializer.UploadSession, file fsctx.FileHeader) (*serializer.UploadCredential, error) {
func (handler Driver) Token(ctx context.Context, TTL int64, key string) (serializer.UploadCredential, error) {
// 生成回调地址
siteURL := model.GetSiteURL()
apiBaseURI, _ := url.Parse(path.Join("/api/v3/callback/remote", uploadSession.Key, uploadSession.CallbackSecret))
apiBaseURI, _ := url.Parse("/api/v3/callback/remote/" + key)
apiURL := siteURL.ResolveReference(apiBaseURI)
// 在从机端创建上传会话
uploadSession.Callback = apiURL.String()
if err := handler.uploadClient.CreateUploadSession(ctx, uploadSession, ttl, false); err != nil {
return nil, err
// 生成上传策略
policy := serializer.UploadPolicy{
SavePath: handler.Policy.DirNameRule,
FileName: handler.Policy.FileNameRule,
AutoRename: handler.Policy.AutoRename,
MaxSize: handler.Policy.MaxSize,
AllowedExtension: handler.Policy.OptionsSerialized.FileType,
CallbackURL: apiURL.String(),
}
return handler.getUploadCredential(ctx, policy, TTL)
}
// 获取上传地址
uploadURL, sign, err := handler.uploadClient.GetUploadURL(ttl, uploadSession.Key)
func (handler Driver) getUploadCredential(ctx context.Context, policy serializer.UploadPolicy, TTL int64) (serializer.UploadCredential, error) {
policyEncoded, err := policy.EncodeUploadPolicy()
if err != nil {
return nil, fmt.Errorf("failed to sign upload url: %w", err)
return serializer.UploadCredential{}, err
}
return &serializer.UploadCredential{
SessionID: uploadSession.Key,
ChunkSize: handler.Policy.OptionsSerialized.ChunkSize,
UploadURLs: []string{uploadURL},
Credential: sign,
}, nil
}
// 签名上传策略
uploadRequest, _ := http.NewRequest("POST", "/api/v3/slave/upload", nil)
uploadRequest.Header = map[string][]string{
"X-Cr-Policy": {policyEncoded},
"X-Cr-Overwrite": {"false"},
}
auth.SignRequest(handler.AuthInstance, uploadRequest, TTL)
// 取消上传凭证
func (handler *Driver) CancelToken(ctx context.Context, uploadSession *serializer.UploadSession) error {
return handler.uploadClient.DeleteUploadSession(ctx, uploadSession.Key)
if credential, ok := uploadRequest.Header["Authorization"]; ok && len(credential) == 1 {
return serializer.UploadCredential{
Token: credential[0],
Policy: policyEncoded,
}, nil
}
return serializer.UploadCredential{}, errors.New("无法签名上传策略")
}

View File

@@ -2,9 +2,6 @@ package remote
import (
"context"
"errors"
"github.com/cloudreve/Cloudreve/v3/pkg/mocks/remoteclientmock"
"github.com/cloudreve/Cloudreve/v3/pkg/serializer"
"io"
"io/ioutil"
"net/http"
@@ -17,26 +14,45 @@ import (
"github.com/cloudreve/Cloudreve/v3/pkg/cache"
"github.com/cloudreve/Cloudreve/v3/pkg/filesystem/fsctx"
"github.com/cloudreve/Cloudreve/v3/pkg/request"
"github.com/cloudreve/Cloudreve/v3/pkg/serializer"
"github.com/stretchr/testify/assert"
testMock "github.com/stretchr/testify/mock"
)
func TestNewDriver(t *testing.T) {
a := assert.New(t)
// remoteClient 初始化失败
{
d, err := NewDriver(&model.Policy{Server: string([]byte{0x7f})})
a.Error(err)
a.Nil(d)
func TestHandler_Token(t *testing.T) {
asserts := assert.New(t)
handler := Driver{
Policy: &model.Policy{
MaxSize: 10,
AutoRename: true,
DirNameRule: "dir",
FileNameRule: "file",
OptionsSerialized: model.PolicyOption{
FileType: []string{"txt"},
},
Server: "http://test.com",
},
AuthInstance: auth.HMACAuth{},
}
ctx := context.WithValue(context.Background(), fsctx.DisableOverwrite, true)
auth.General = auth.HMACAuth{SecretKey: []byte("test")}
// 成功
{
d, err := NewDriver(&model.Policy{})
a.NoError(err)
a.NotNil(d)
cache.Set("setting_siteURL", "http://test.cloudreve.org", 0)
credential, err := handler.Token(ctx, 10, "123")
asserts.NoError(err)
policy, err := serializer.DecodeUploadPolicy(credential.Policy)
asserts.NoError(err)
asserts.Equal("http://test.cloudreve.org/api/v3/callback/remote/123", policy.CallbackURL)
asserts.Equal(uint64(10), policy.MaxSize)
asserts.Equal(true, policy.AutoRename)
asserts.Equal("dir", policy.SavePath)
asserts.Equal("file", policy.FileName)
asserts.Equal("file", policy.FileName)
asserts.Equal([]string{"txt"}, policy.AllowedExtension)
}
}
func TestHandler_Source(t *testing.T) {
@@ -353,17 +369,87 @@ func TestHandler_Get(t *testing.T) {
}
func TestHandler_Put(t *testing.T) {
a := assert.New(t)
handler, _ := NewDriver(&model.Policy{
Type: "remote",
SecretKey: "test",
Server: "http://test.com",
})
clientMock := &remoteclientmock.RemoteClientMock{}
handler.uploadClient = clientMock
clientMock.On("Upload", testMock.Anything, testMock.Anything).Return(errors.New("error"))
a.Error(handler.Put(context.Background(), &fsctx.FileStream{}))
clientMock.AssertExpectations(t)
asserts := assert.New(t)
handler := Driver{
Policy: &model.Policy{
Type: "remote",
SecretKey: "test",
Server: "http://test.com",
},
AuthInstance: auth.HMACAuth{},
}
ctx := context.Background()
asserts.NoError(cache.Set("setting_upload_credential_timeout", "3600", 0))
// 成功
{
ctx = context.WithValue(ctx, fsctx.UserCtx, model.User{})
clientMock := ClientMock{}
clientMock.On(
"Request",
"POST",
"http://test.com/api/v3/slave/upload",
testMock.Anything,
testMock.Anything,
).Return(&request.Response{
Err: nil,
Response: &http.Response{
StatusCode: 200,
Body: ioutil.NopCloser(strings.NewReader(`{"code":0}`)),
},
})
handler.Client = clientMock
err := handler.Put(ctx, ioutil.NopCloser(strings.NewReader("test input file")), "/", 15)
clientMock.AssertExpectations(t)
asserts.NoError(err)
}
// 请求失败
{
ctx = context.WithValue(ctx, fsctx.UserCtx, model.User{})
clientMock := ClientMock{}
clientMock.On(
"Request",
"POST",
"http://test.com/api/v3/slave/upload",
testMock.Anything,
testMock.Anything,
).Return(&request.Response{
Err: nil,
Response: &http.Response{
StatusCode: 404,
Body: ioutil.NopCloser(strings.NewReader(`{"code":0}`)),
},
})
handler.Client = clientMock
err := handler.Put(ctx, ioutil.NopCloser(strings.NewReader("test input file")), "/", 15)
clientMock.AssertExpectations(t)
asserts.Error(err)
}
// 返回错误
{
ctx = context.WithValue(ctx, fsctx.UserCtx, model.User{})
clientMock := ClientMock{}
clientMock.On(
"Request",
"POST",
"http://test.com/api/v3/slave/upload",
testMock.Anything,
testMock.Anything,
).Return(&request.Response{
Err: nil,
Response: &http.Response{
StatusCode: 200,
Body: ioutil.NopCloser(strings.NewReader(`{"code":1}`)),
},
})
handler.Client = clientMock
err := handler.Put(ctx, ioutil.NopCloser(strings.NewReader("test input file")), "/", 15)
clientMock.AssertExpectations(t)
asserts.Error(err)
}
}
func TestHandler_Thumb(t *testing.T) {
@@ -382,60 +468,3 @@ func TestHandler_Thumb(t *testing.T) {
asserts.NoError(err)
asserts.True(resp.Redirect)
}
func TestHandler_Token(t *testing.T) {
a := assert.New(t)
handler, _ := NewDriver(&model.Policy{})
// 无法创建上传会话
{
clientMock := &remoteclientmock.RemoteClientMock{}
handler.uploadClient = clientMock
clientMock.On("CreateUploadSession", testMock.Anything, testMock.Anything, int64(10), false).Return(errors.New("error"))
res, err := handler.Token(context.Background(), 10, &serializer.UploadSession{}, &fsctx.FileStream{})
a.Error(err)
a.Contains(err.Error(), "error")
a.Nil(res)
clientMock.AssertExpectations(t)
}
// 无法创建上传地址
{
clientMock := &remoteclientmock.RemoteClientMock{}
handler.uploadClient = clientMock
clientMock.On("CreateUploadSession", testMock.Anything, testMock.Anything, int64(10), false).Return(nil)
clientMock.On("GetUploadURL", int64(10), "").Return("", "", errors.New("error"))
res, err := handler.Token(context.Background(), 10, &serializer.UploadSession{}, &fsctx.FileStream{})
a.Error(err)
a.Contains(err.Error(), "error")
a.Nil(res)
clientMock.AssertExpectations(t)
}
// 成功
{
clientMock := &remoteclientmock.RemoteClientMock{}
handler.uploadClient = clientMock
clientMock.On("CreateUploadSession", testMock.Anything, testMock.Anything, int64(10), false).Return(nil)
clientMock.On("GetUploadURL", int64(10), "").Return("1", "2", nil)
res, err := handler.Token(context.Background(), 10, &serializer.UploadSession{}, &fsctx.FileStream{})
a.NoError(err)
a.NotNil(res)
a.Equal("1", res.UploadURLs[0])
a.Equal("2", res.Credential)
clientMock.AssertExpectations(t)
}
}
func TestDriver_CancelToken(t *testing.T) {
a := assert.New(t)
handler, _ := NewDriver(&model.Policy{})
clientMock := &remoteclientmock.RemoteClientMock{}
handler.uploadClient = clientMock
clientMock.On("DeleteUploadSession", testMock.Anything, "key").Return(errors.New("error"))
err := handler.CancelToken(context.Background(), &serializer.UploadSession{Key: "key"})
a.Error(err)
a.Contains(err.Error(), "error")
clientMock.AssertExpectations(t)
}

View File

@@ -2,10 +2,12 @@ package s3
import (
"context"
"crypto/hmac"
"crypto/sha256"
"encoding/base64"
"encoding/hex"
"encoding/json"
"errors"
"fmt"
"github.com/cloudreve/Cloudreve/v3/pkg/filesystem/chunk"
"github.com/cloudreve/Cloudreve/v3/pkg/filesystem/chunk/backoff"
"github.com/cloudreve/Cloudreve/v3/pkg/util"
"io"
"net/http"
@@ -46,18 +48,6 @@ type MetaData struct {
Etag string
}
func NewDriver(policy *model.Policy) (*Driver, error) {
if policy.OptionsSerialized.ChunkSize == 0 {
policy.OptionsSerialized.ChunkSize = 25 << 20 // 25 MB
}
driver := &Driver{
Policy: policy,
}
return driver, driver.InitS3Client()
}
// InitS3Client 初始化S3会话
func (handler *Driver) InitS3Client() error {
if handler.Policy == nil {
@@ -83,7 +73,13 @@ func (handler *Driver) InitS3Client() error {
}
// List 列出给定路径下的文件
func (handler *Driver) List(ctx context.Context, base string, recursive bool) ([]response.Object, error) {
func (handler Driver) List(ctx context.Context, base string, recursive bool) ([]response.Object, error) {
// 初始化客户端
if err := handler.InitS3Client(); err != nil {
return nil, err
}
// 初始化列目录参数
base = strings.TrimPrefix(base, "/")
if base != "" {
@@ -160,7 +156,8 @@ func (handler *Driver) List(ctx context.Context, base string, recursive bool) ([
}
// Get 获取文件
func (handler *Driver) Get(ctx context.Context, path string) (response.RSCloser, error) {
func (handler Driver) Get(ctx context.Context, path string) (response.RSCloser, error) {
// 获取文件源地址
downloadURL, err := handler.Source(
ctx,
@@ -201,23 +198,19 @@ func (handler *Driver) Get(ctx context.Context, path string) (response.RSCloser,
}
// Put 将文件流保存到指定目录
func (handler *Driver) Put(ctx context.Context, file fsctx.FileHeader) error {
defer file.Close()
func (handler Driver) Put(ctx context.Context, file io.ReadCloser, dst string, size uint64) error {
// 初始化客户端
if err := handler.InitS3Client(); err != nil {
return err
}
uploader := s3manager.NewUploader(handler.sess, func(u *s3manager.Uploader) {
u.PartSize = int64(handler.Policy.OptionsSerialized.ChunkSize)
})
uploader := s3manager.NewUploader(handler.sess)
dst := file.Info().SavePath
_, err := uploader.Upload(&s3manager.UploadInput{
Bucket: &handler.Policy.BucketName,
Key: &dst,
Body: io.LimitReader(file, int64(file.Info().Size)),
Body: file,
})
if err != nil {
@@ -229,7 +222,13 @@ func (handler *Driver) Put(ctx context.Context, file fsctx.FileHeader) error {
// Delete 删除一个或多个文件,
// 返回未删除的文件,及遇到的最后一个错误
func (handler *Driver) Delete(ctx context.Context, files []string) ([]string, error) {
func (handler Driver) Delete(ctx context.Context, files []string) ([]string, error) {
// 初始化客户端
if err := handler.InitS3Client(); err != nil {
return files, err
}
failed := make([]string, 0, len(files))
deleted := make([]string, 0, len(files))
@@ -263,12 +262,12 @@ func (handler *Driver) Delete(ctx context.Context, files []string) ([]string, er
}
// Thumb 获取文件缩略图
func (handler *Driver) Thumb(ctx context.Context, path string) (*response.ContentResponse, error) {
func (handler Driver) Thumb(ctx context.Context, path string) (*response.ContentResponse, error) {
return nil, errors.New("未实现")
}
// Source 获取外链URL
func (handler *Driver) Source(
func (handler Driver) Source(
ctx context.Context,
path string,
baseURL url.URL,
@@ -325,75 +324,48 @@ func (handler *Driver) Source(
}
// Token 获取上传策略和认证Token
func (handler *Driver) Token(ctx context.Context, ttl int64, uploadSession *serializer.UploadSession, file fsctx.FileHeader) (*serializer.UploadCredential, error) {
// 检查文件是否存在
fileInfo := file.Info()
if _, err := handler.Meta(ctx, fileInfo.SavePath); err == nil {
return nil, fmt.Errorf("file already exist")
func (handler Driver) Token(ctx context.Context, TTL int64, key string) (serializer.UploadCredential, error) {
// 读取上下文中生成的存储路径和文件大小
savePath, ok := ctx.Value(fsctx.SavePathCtx).(string)
if !ok {
return serializer.UploadCredential{}, errors.New("无法获取存储路径")
}
// 创建分片上传
expires := time.Now().Add(time.Duration(ttl) * time.Second)
res, err := handler.svc.CreateMultipartUpload(&s3.CreateMultipartUploadInput{
Bucket: &handler.Policy.BucketName,
Key: &fileInfo.SavePath,
Expires: &expires,
})
if err != nil {
return nil, fmt.Errorf("failed to create multipart upload: %w", err)
// 生成回调地址
siteURL := model.GetSiteURL()
apiBaseURI, _ := url.Parse("/api/v3/callback/s3/" + key)
apiURL := siteURL.ResolveReference(apiBaseURI)
// 上传策略
putPolicy := UploadPolicy{
Expiration: time.Now().UTC().Add(time.Duration(TTL) * time.Second).Format(time.RFC3339),
Conditions: []interface{}{
map[string]string{"bucket": handler.Policy.BucketName},
[]string{"starts-with", "$key", savePath},
[]string{"starts-with", "$success_action_redirect", apiURL.String()},
[]string{"starts-with", "$name", ""},
[]string{"starts-with", "$Content-Type", ""},
map[string]string{"x-amz-algorithm": "AWS4-HMAC-SHA256"},
},
}
uploadSession.UploadID = *res.UploadId
// 为每个分片签名上传 URL
chunks := chunk.NewChunkGroup(file, handler.Policy.OptionsSerialized.ChunkSize, &backoff.ConstantBackoff{}, false)
urls := make([]string, chunks.Num())
for chunks.Next() {
err := chunks.Process(func(c *chunk.ChunkGroup, chunk io.Reader) error {
signedReq, _ := handler.svc.UploadPartRequest(&s3.UploadPartInput{
Bucket: &handler.Policy.BucketName,
Key: &fileInfo.SavePath,
PartNumber: aws.Int64(int64(c.Index() + 1)),
UploadId: res.UploadId,
})
signedURL, err := signedReq.Presign(time.Duration(ttl) * time.Second)
if err != nil {
return err
}
urls[c.Index()] = signedURL
return nil
})
if err != nil {
return nil, err
}
}
// 签名完成分片上传的请求URL
signedReq, _ := handler.svc.CompleteMultipartUploadRequest(&s3.CompleteMultipartUploadInput{
Bucket: &handler.Policy.BucketName,
Key: &fileInfo.SavePath,
UploadId: res.UploadId,
})
signedURL, err := signedReq.Presign(time.Duration(ttl) * time.Second)
if err != nil {
return nil, err
if handler.Policy.MaxSize > 0 {
putPolicy.Conditions = append(putPolicy.Conditions,
[]interface{}{"content-length-range", 0, handler.Policy.MaxSize})
}
// 生成上传凭证
return &serializer.UploadCredential{
SessionID: uploadSession.Key,
ChunkSize: handler.Policy.OptionsSerialized.ChunkSize,
UploadID: *res.UploadId,
UploadURLs: urls,
CompleteURL: signedURL,
}, nil
return handler.getUploadCredential(ctx, putPolicy, apiURL)
}
// Meta 获取文件信息
func (handler *Driver) Meta(ctx context.Context, path string) (*MetaData, error) {
func (handler Driver) Meta(ctx context.Context, path string) (*MetaData, error) {
// 初始化客户端
if err := handler.InitS3Client(); err != nil {
return nil, err
}
res, err := handler.svc.GetObject(
&s3.GetObjectInput{
Bucket: &handler.Policy.BucketName,
@@ -411,8 +383,58 @@ func (handler *Driver) Meta(ctx context.Context, path string) (*MetaData, error)
}
func (handler Driver) getUploadCredential(ctx context.Context, policy UploadPolicy, callback *url.URL) (serializer.UploadCredential, error) {
// 读取上下文中生成的存储路径和文件大小
savePath, ok := ctx.Value(fsctx.SavePathCtx).(string)
if !ok {
return serializer.UploadCredential{}, errors.New("无法获取存储路径")
}
longDate := time.Now().UTC().Format("20060102T150405Z")
shortDate := time.Now().UTC().Format("20060102")
credential := handler.Policy.AccessKey + "/" + shortDate + "/" + handler.Policy.OptionsSerialized.Region + "/s3/aws4_request"
policy.Conditions = append(policy.Conditions, map[string]string{"x-amz-credential": credential})
policy.Conditions = append(policy.Conditions, map[string]string{"x-amz-date": longDate})
// 编码上传策略
policyJSON, err := json.Marshal(policy)
if err != nil {
return serializer.UploadCredential{}, err
}
policyEncoded := base64.StdEncoding.EncodeToString(policyJSON)
//签名
signature := getHMAC([]byte("AWS4"+handler.Policy.SecretKey), []byte(shortDate))
signature = getHMAC(signature, []byte(handler.Policy.OptionsSerialized.Region))
signature = getHMAC(signature, []byte("s3"))
signature = getHMAC(signature, []byte("aws4_request"))
signature = getHMAC(signature, []byte(policyEncoded))
return serializer.UploadCredential{
Policy: policyEncoded,
Callback: callback.String(),
Token: hex.EncodeToString(signature),
AccessKey: credential,
Path: savePath,
KeyTime: longDate,
}, nil
}
func getHMAC(key []byte, data []byte) []byte {
hash := hmac.New(sha256.New, key)
hash.Write(data)
return hash.Sum(nil)
}
// CORS 创建跨域策略
func (handler *Driver) CORS() error {
func (handler Driver) CORS() error {
// 初始化客户端
if err := handler.InitS3Client(); err != nil {
return err
}
rule := s3.CORSRule{
AllowedMethods: aws.StringSlice([]string{
"GET",
@@ -423,7 +445,6 @@ func (handler *Driver) CORS() error {
}),
AllowedOrigins: aws.StringSlice([]string{"*"}),
AllowedHeaders: aws.StringSlice([]string{"*"}),
ExposeHeaders: aws.StringSlice([]string{"ETag"}),
MaxAgeSeconds: aws.Int64(3600),
}
@@ -436,13 +457,3 @@ func (handler *Driver) CORS() error {
return err
}
// 取消上传凭证
func (handler *Driver) CancelToken(ctx context.Context, uploadSession *serializer.UploadSession) error {
_, err := handler.svc.AbortMultipartUpload(&s3.AbortMultipartUploadInput{
UploadId: &uploadSession.UploadID,
Bucket: &handler.Policy.BucketName,
Key: &uploadSession.SavePath,
})
return err
}

View File

@@ -5,9 +5,9 @@ import (
model "github.com/cloudreve/Cloudreve/v3/models"
"github.com/cloudreve/Cloudreve/v3/pkg/cluster"
"github.com/cloudreve/Cloudreve/v3/pkg/filesystem/driver"
"github.com/cloudreve/Cloudreve/v3/pkg/filesystem/fsctx"
"github.com/cloudreve/Cloudreve/v3/pkg/filesystem/response"
"github.com/cloudreve/Cloudreve/v3/pkg/serializer"
"io"
"net/url"
)
@@ -27,8 +27,8 @@ func NewDriver(master cluster.Node, handler driver.Handler, policy *model.Policy
}
}
func (d *Driver) Put(ctx context.Context, file fsctx.FileHeader) error {
return d.handler.Put(ctx, file)
func (d *Driver) Put(ctx context.Context, file io.ReadCloser, dst string, size uint64) error {
return d.handler.Put(ctx, file, dst, size)
}
func (d *Driver) Delete(ctx context.Context, files []string) ([]string, error) {
@@ -47,15 +47,10 @@ func (d *Driver) Source(ctx context.Context, path string, url url.URL, ttl int64
return "", ErrNotImplemented
}
func (d *Driver) Token(ctx context.Context, ttl int64, uploadSession *serializer.UploadSession, file fsctx.FileHeader) (*serializer.UploadCredential, error) {
return nil, ErrNotImplemented
func (d *Driver) Token(ctx context.Context, ttl int64, callbackKey string) (serializer.UploadCredential, error) {
return serializer.UploadCredential{}, ErrNotImplemented
}
func (d *Driver) List(ctx context.Context, path string, recursive bool) ([]response.Object, error) {
return nil, ErrNotImplemented
}
// 取消上传凭证
func (handler Driver) CancelToken(ctx context.Context, uploadSession *serializer.UploadSession) error {
return nil
}

View File

@@ -13,6 +13,7 @@ import (
"github.com/cloudreve/Cloudreve/v3/pkg/mq"
"github.com/cloudreve/Cloudreve/v3/pkg/request"
"github.com/cloudreve/Cloudreve/v3/pkg/serializer"
"io"
"net/url"
"time"
)
@@ -30,7 +31,7 @@ func NewDriver(node cluster.Node, handler driver.Handler, policy *model.Policy)
var endpoint *url.URL
if serverURL, err := url.Parse(node.DBModel().Server); err == nil {
var controller *url.URL
controller, _ = url.Parse("/api/v3/slave/")
controller, _ = url.Parse("/api/v3/slave")
endpoint = serverURL.ResolveReference(controller)
}
@@ -49,13 +50,15 @@ func NewDriver(node cluster.Node, handler driver.Handler, policy *model.Policy)
}
// Put 将ctx中指定的从机物理文件由从机上传到目标存储策略
func (d *Driver) Put(ctx context.Context, file fsctx.FileHeader) error {
defer file.Close()
func (d *Driver) Put(ctx context.Context, file io.ReadCloser, dst string, size uint64) error {
src, ok := ctx.Value(fsctx.SlaveSrcPath).(string)
if !ok {
return ErrSlaveSrcPathNotExist
}
fileInfo := file.Info()
req := serializer.SlaveTransferReq{
Src: fileInfo.Src,
Dst: fileInfo.SavePath,
Src: src,
Dst: dst,
Policy: d.policy,
}
@@ -109,15 +112,10 @@ func (d *Driver) Source(ctx context.Context, path string, url url.URL, ttl int64
return "", ErrNotImplemented
}
func (d *Driver) Token(ctx context.Context, ttl int64, uploadSession *serializer.UploadSession, file fsctx.FileHeader) (*serializer.UploadCredential, error) {
return nil, ErrNotImplemented
func (d *Driver) Token(ctx context.Context, ttl int64, callbackKey string) (serializer.UploadCredential, error) {
return serializer.UploadCredential{}, ErrNotImplemented
}
func (d *Driver) List(ctx context.Context, path string, recursive bool) ([]response.Object, error) {
return nil, ErrNotImplemented
}
// 取消上传凭证
func (handler Driver) CancelToken(ctx context.Context, uploadSession *serializer.UploadSession) error {
return nil
}

View File

@@ -0,0 +1,55 @@
package template
import (
"context"
"errors"
"io"
"net/url"
model "github.com/cloudreve/Cloudreve/v3/models"
"github.com/cloudreve/Cloudreve/v3/pkg/filesystem/response"
"github.com/cloudreve/Cloudreve/v3/pkg/serializer"
)
// Driver 适配器模板
type Driver struct {
Policy *model.Policy
}
// Get 获取文件
func (handler Driver) Get(ctx context.Context, path string) (response.RSCloser, error) {
return nil, errors.New("未实现")
}
// Put 将文件流保存到指定目录
func (handler Driver) Put(ctx context.Context, file io.ReadCloser, dst string, size uint64) error {
return errors.New("未实现")
}
// Delete 删除一个或多个文件,
// 返回未删除的文件,及遇到的最后一个错误
func (handler Driver) Delete(ctx context.Context, files []string) ([]string, error) {
return []string{}, errors.New("未实现")
}
// Thumb 获取文件缩略图
func (handler Driver) Thumb(ctx context.Context, path string) (*response.ContentResponse, error) {
return nil, errors.New("未实现")
}
// Source 获取外链URL
func (handler Driver) Source(
ctx context.Context,
path string,
baseURL url.URL,
ttl int64,
isDownload bool,
speed int,
) (string, error) {
return "", errors.New("未实现")
}
// Token 获取上传策略和认证Token
func (handler Driver) Token(ctx context.Context, TTL int64, key string) (serializer.UploadCredential, error) {
return serializer.UploadCredential{}, errors.New("未实现")
}

View File

@@ -9,6 +9,7 @@ import (
"encoding/json"
"errors"
"fmt"
"io"
"net/http"
"net/url"
"path"
@@ -118,7 +119,7 @@ func (handler Driver) Get(ctx context.Context, path string) (response.RSCloser,
}
// 获取文件数据流
client := request.NewClient()
client := request.HTTPClient{}
resp, err := client.Request(
"GET",
downloadURL,
@@ -145,7 +146,7 @@ func (handler Driver) Get(ctx context.Context, path string) (response.RSCloser,
}
// Put 将文件流保存到指定目录
func (handler Driver) Put(ctx context.Context, file fsctx.FileHeader) error {
func (handler Driver) Put(ctx context.Context, file io.ReadCloser, dst string, size uint64) error {
defer file.Close()
up := upyun.NewUpYun(&upyun.UpYunConfig{
@@ -154,7 +155,7 @@ func (handler Driver) Put(ctx context.Context, file fsctx.FileHeader) error {
Password: handler.Policy.SecretKey,
})
err := up.Put(&upyun.PutObjectConfig{
Path: file.Info().SavePath,
Path: dst,
Reader: file,
})
@@ -310,29 +311,45 @@ func (handler Driver) signURL(ctx context.Context, path *url.URL, TTL int64) (st
}
// Token 获取上传策略和认证Token
func (handler Driver) Token(ctx context.Context, ttl int64, uploadSession *serializer.UploadSession, file fsctx.FileHeader) (*serializer.UploadCredential, error) {
func (handler Driver) Token(ctx context.Context, TTL int64, key string) (serializer.UploadCredential, error) {
// 读取上下文中生成的存储路径和文件大小
savePath, ok := ctx.Value(fsctx.SavePathCtx).(string)
if !ok {
return serializer.UploadCredential{}, errors.New("无法获取存储路径")
}
fileSize, ok := ctx.Value(fsctx.FileSizeCtx).(uint64)
if !ok {
return serializer.UploadCredential{}, errors.New("无法获取文件大小")
}
// 检查文件大小
// 生成回调地址
siteURL := model.GetSiteURL()
apiBaseURI, _ := url.Parse("/api/v3/callback/upyun/" + uploadSession.Key)
apiBaseURI, _ := url.Parse("/api/v3/callback/upyun/" + key)
apiURL := siteURL.ResolveReference(apiBaseURI)
// 上传策略
fileInfo := file.Info()
putPolicy := UploadPolicy{
Bucket: handler.Policy.BucketName,
// TODO escape
SaveKey: fileInfo.SavePath,
Expiration: time.Now().Add(time.Duration(ttl) * time.Second).Unix(),
SaveKey: savePath,
Expiration: time.Now().Add(time.Duration(TTL) * time.Second).Unix(),
CallbackURL: apiURL.String(),
ContentLength: fileInfo.Size,
ContentLengthRange: fmt.Sprintf("0,%d", fileInfo.Size),
ContentLength: fileSize,
ContentLengthRange: fmt.Sprintf("0,%d", fileSize),
AllowFileType: strings.Join(handler.Policy.OptionsSerialized.FileType, ","),
}
// 生成上传凭证
policyJSON, err := json.Marshal(putPolicy)
return handler.getUploadCredential(ctx, putPolicy)
}
func (handler Driver) getUploadCredential(ctx context.Context, policy UploadPolicy) (serializer.UploadCredential, error) {
// 生成上传策略
policyJSON, err := json.Marshal(policy)
if err != nil {
return nil, err
return serializer.UploadCredential{}, err
}
policyEncoded := base64.StdEncoding.EncodeToString(policyJSON)
@@ -340,19 +357,12 @@ func (handler Driver) Token(ctx context.Context, ttl int64, uploadSession *seria
elements := []string{"POST", "/" + handler.Policy.BucketName, policyEncoded}
signStr := handler.Sign(ctx, elements)
return &serializer.UploadCredential{
SessionID: uploadSession.Key,
Policy: policyEncoded,
Credential: signStr,
UploadURLs: []string{"https://v0.api.upyun.com/" + handler.Policy.BucketName},
return serializer.UploadCredential{
Policy: policyEncoded,
Token: signStr,
}, nil
}
// 取消上传凭证
func (handler Driver) CancelToken(ctx context.Context, uploadSession *serializer.UploadSession) error {
return nil
}
// Sign 计算又拍云的签名头
func (handler Driver) Sign(ctx context.Context, elements []string) string {
password := fmt.Sprintf("%x", md5.Sum([]byte(handler.Policy.SecretKey)))

View File

@@ -7,20 +7,19 @@ import (
)
var (
ErrUnknownPolicyType = errors.New("未知存储策略类型")
ErrFileSizeTooBig = errors.New("单个文件尺寸太大")
ErrFileExtensionNotAllowed = errors.New("不允许上传此类型的文件")
ErrInsufficientCapacity = errors.New("容量空间不足")
ErrIllegalObjectName = errors.New("目标名称非法")
ErrClientCanceled = errors.New("客户端取消操作")
ErrRootProtected = errors.New("无法对根目录进行操作")
ErrInsertFileRecord = serializer.NewError(serializer.CodeDBError, "无法插入文件记录", nil)
ErrFileExisted = serializer.NewError(serializer.CodeObjectExist, "同名文件或目录已存在", nil)
ErrFileUploadSessionExisted = serializer.NewError(serializer.CodeObjectExist, "当前目录下已经有同名文件正在上传中,请尝试清空上传会话", nil)
ErrFolderExisted = serializer.NewError(serializer.CodeObjectExist, "同名目录已存在", nil)
ErrPathNotExist = serializer.NewError(404, "路径不存在", nil)
ErrObjectNotExist = serializer.NewError(404, "文件不存在", nil)
ErrIO = serializer.NewError(serializer.CodeIOFailed, "无法读取文件数据", nil)
ErrDBListObjects = serializer.NewError(serializer.CodeDBError, "无法列取对象记录", nil)
ErrDBDeleteObjects = serializer.NewError(serializer.CodeDBError, "无法删除对象记录", nil)
ErrUnknownPolicyType = errors.New("未知存储策略类型")
ErrFileSizeTooBig = errors.New("单个文件尺寸太大")
ErrFileExtensionNotAllowed = errors.New("不允许上传此类型的文件")
ErrInsufficientCapacity = errors.New("容量空间不足")
ErrIllegalObjectName = errors.New("目标名称非法")
ErrClientCanceled = errors.New("客户端取消操作")
ErrRootProtected = errors.New("无法对根目录进行操作")
ErrInsertFileRecord = serializer.NewError(serializer.CodeDBError, "无法插入文件记录", nil)
ErrFileExisted = serializer.NewError(serializer.CodeObjectExist, "同名文件或目录已存在", nil)
ErrFolderExisted = serializer.NewError(serializer.CodeObjectExist, "同名目录已存在", nil)
ErrPathNotExist = serializer.NewError(404, "路径不存在", nil)
ErrObjectNotExist = serializer.NewError(404, "文件不存在", nil)
ErrIO = serializer.NewError(serializer.CodeIOFailed, "无法读取文件数据", nil)
ErrDBListObjects = serializer.NewError(serializer.CodeDBError, "无法列取对象记录", nil)
ErrDBDeleteObjects = serializer.NewError(serializer.CodeDBError, "无法删除对象记录", nil)
)

View File

@@ -5,7 +5,6 @@ import (
"io"
model "github.com/cloudreve/Cloudreve/v3/models"
"github.com/cloudreve/Cloudreve/v3/pkg/cache"
"github.com/cloudreve/Cloudreve/v3/pkg/conf"
"github.com/cloudreve/Cloudreve/v3/pkg/filesystem/fsctx"
"github.com/cloudreve/Cloudreve/v3/pkg/filesystem/response"
@@ -44,39 +43,41 @@ func (fs *FileSystem) withSpeedLimit(rs response.RSCloser) response.RSCloser {
}
// AddFile 新增文件记录
func (fs *FileSystem) AddFile(ctx context.Context, parent *model.Folder, file fsctx.FileHeader) (*model.File, error) {
func (fs *FileSystem) AddFile(ctx context.Context, parent *model.Folder) (*model.File, error) {
// 添加文件记录前的钩子
err := fs.Trigger(ctx, "BeforeAddFile", file)
err := fs.Trigger(ctx, "BeforeAddFile")
if err != nil {
if err := fs.Trigger(ctx, "BeforeAddFileFailed"); err != nil {
util.Log().Debug("BeforeAddFileFailed 钩子执行失败,%s", err)
}
return nil, err
}
uploadInfo := file.Info()
file := ctx.Value(fsctx.FileHeaderCtx).(FileHeader)
filePath := ctx.Value(fsctx.SavePathCtx).(string)
newFile := model.File{
Name: uploadInfo.FileName,
SourceName: uploadInfo.SavePath,
UserID: fs.User.ID,
Size: uploadInfo.Size,
FolderID: parent.ID,
PolicyID: fs.Policy.ID,
MetadataSerialized: uploadInfo.Metadata,
UploadSessionID: uploadInfo.UploadSessionID,
Name: file.GetFileName(),
SourceName: filePath,
UserID: fs.User.ID,
Size: file.GetSize(),
FolderID: parent.ID,
PolicyID: fs.User.Policy.ID,
}
if fs.Policy.IsThumbExist(uploadInfo.FileName) {
if fs.User.Policy.IsThumbExist(file.GetFileName()) {
newFile.PicInfo = "1,1"
}
err = newFile.Create()
_, err = newFile.Create()
if err != nil {
if err := fs.Trigger(ctx, "AfterValidateFailed", file); err != nil {
if err := fs.Trigger(ctx, "AfterValidateFailed"); err != nil {
util.Log().Debug("AfterValidateFailed 钩子执行失败,%s", err)
}
return nil, ErrFileExisted.WithError(err)
}
fs.User.Storage += newFile.Size
return &newFile, nil
}
@@ -152,7 +153,14 @@ func (fs *FileSystem) GetDownloadContent(ctx context.Context, id uint) (response
// GetContent 获取文件内容path为虚拟路径
func (fs *FileSystem) GetContent(ctx context.Context, id uint) (response.RSCloser, error) {
err := fs.resetFileIDIfNotExist(ctx, id)
// 触发`下载前`钩子
err := fs.Trigger(ctx, "BeforeFileDownload")
if err != nil {
util.Log().Debug("BeforeFileDownload 钩子执行失败,%s", err)
return nil, err
}
err = fs.resetFileIDIfNotExist(ctx, id)
if err != nil {
return nil, err
}
@@ -176,41 +184,23 @@ func (fs *FileSystem) deleteGroupedFile(ctx context.Context, files map[uint][]*m
for policyID, toBeDeletedFiles := range files {
// 列举出需要物理删除的文件的物理路径
sourceNamesAll := make([]string, 0, len(toBeDeletedFiles))
uploadSessions := make([]*serializer.UploadSession, 0, len(toBeDeletedFiles))
sourceNames := make([]string, 0, len(toBeDeletedFiles))
for i := 0; i < len(toBeDeletedFiles); i++ {
sourceNamesAll = append(sourceNamesAll, toBeDeletedFiles[i].SourceName)
if toBeDeletedFiles[i].UploadSessionID != nil {
if session, ok := cache.Get(UploadSessionCachePrefix + *toBeDeletedFiles[i].UploadSessionID); ok {
uploadSession := session.(serializer.UploadSession)
uploadSessions = append(uploadSessions, &uploadSession)
}
}
sourceNames = append(sourceNames, toBeDeletedFiles[i].SourceName)
}
// 切换上传策略
fs.Policy = toBeDeletedFiles[0].GetPolicy()
err := fs.DispatchHandler()
if err != nil {
failed[policyID] = sourceNamesAll
failed[policyID] = sourceNames
continue
}
// 取消上传会话
for _, upSession := range uploadSessions {
if err := fs.Handler.CancelToken(ctx, upSession); err != nil {
util.Log().Warning("无法取消 [%s] 的上传会话: %s", upSession.Name, err)
}
cache.Deletes([]string{upSession.Key}, UploadSessionCachePrefix)
}
// 执行删除
failedFile, _ := fs.Handler.Delete(ctx, sourceNamesAll)
failedFile, _ := fs.Handler.Delete(ctx, sourceNames)
failed[policyID] = failedFile
}
return failed
@@ -225,7 +215,7 @@ func (fs *FileSystem) GroupFileByPolicy(ctx context.Context, files []model.File)
// 如果已存在分组,直接追加
policyGroup[files[key].PolicyID] = append(file, &files[key])
} else {
// 分不存在,创建
// 分不存在,创建
policyGroup[files[key].PolicyID] = make([]*model.File, 0)
policyGroup[files[key].PolicyID] = append(policyGroup[files[key].PolicyID], &files[key])
}
@@ -360,7 +350,7 @@ func (fs *FileSystem) resetPolicyToFirstFile(ctx context.Context) error {
}
// Search 搜索文件
func (fs *FileSystem) Search(ctx context.Context, keywords ...interface{}) ([]serializer.Object, error) {
func (fs *FileSystem) Search(ctx context.Context, keywords ...interface{}) ([]Object, error) {
files, _ := model.GetFilesByKeywords(fs.User.ID, keywords...)
fs.SetTargetFile(&files)

View File

@@ -2,7 +2,6 @@ package filesystem
import (
"context"
"errors"
"os"
"testing"
@@ -10,6 +9,7 @@ import (
model "github.com/cloudreve/Cloudreve/v3/models"
"github.com/cloudreve/Cloudreve/v3/pkg/auth"
"github.com/cloudreve/Cloudreve/v3/pkg/cache"
"github.com/cloudreve/Cloudreve/v3/pkg/filesystem/driver/local"
"github.com/cloudreve/Cloudreve/v3/pkg/filesystem/fsctx"
"github.com/cloudreve/Cloudreve/v3/pkg/serializer"
"github.com/cloudreve/Cloudreve/v3/pkg/util"
@@ -19,10 +19,9 @@ import (
func TestFileSystem_AddFile(t *testing.T) {
asserts := assert.New(t)
file := fsctx.FileStream{
Size: 5,
Name: "1.png",
SavePath: "/Uploads/1_sad.png",
file := local.FileStream{
Size: 5,
Name: "1.png",
}
folder := model.Folder{
Model: gorm.Model{
@@ -41,55 +40,24 @@ func TestFileSystem_AddFile(t *testing.T) {
},
},
},
Policy: &model.Policy{Type: "cos"},
}
ctx := context.WithValue(context.Background(), fsctx.FileHeaderCtx, file)
ctx = context.WithValue(ctx, fsctx.SavePathCtx, "/Uploads/1_sad.png")
_, err := fs.AddFile(context.Background(), &folder, &file)
_, err := fs.AddFile(ctx, &folder)
asserts.Error(err)
mock.ExpectBegin()
mock.ExpectExec("INSERT(.+)").WillReturnResult(sqlmock.NewResult(1, 1))
mock.ExpectExec("UPDATE(.+)storage(.+)").WillReturnResult(sqlmock.NewResult(1, 1))
mock.ExpectCommit()
f, err := fs.AddFile(context.Background(), &folder, &file)
f, err := fs.AddFile(ctx, &folder)
asserts.NoError(err)
asserts.NoError(mock.ExpectationsWereMet())
asserts.Equal("/Uploads/1_sad.png", f.SourceName)
asserts.NotEmpty(f.PicInfo)
// 前置钩子执行失败
{
hookExecuted := false
fs.Use("BeforeAddFile", func(ctx context.Context, fs *FileSystem, file fsctx.FileHeader) error {
hookExecuted = true
return errors.New("error")
})
f, err := fs.AddFile(context.Background(), &folder, &file)
asserts.Error(err)
asserts.Nil(f)
asserts.True(hookExecuted)
}
// 后置钩子执行失败
{
hookExecuted := false
mock.ExpectBegin()
mock.ExpectExec("INSERT(.+)").WillReturnError(errors.New("error"))
mock.ExpectRollback()
fs.Hooks = map[string][]Hook{}
fs.Use("AfterValidateFailed", func(ctx context.Context, fs *FileSystem, file fsctx.FileHeader) error {
hookExecuted = true
return errors.New("error")
})
f, err := fs.AddFile(context.Background(), &folder, &file)
asserts.Error(err)
asserts.Nil(f)
asserts.True(hookExecuted)
asserts.NoError(mock.ExpectationsWereMet())
}
}
func TestFileSystem_GetContent(t *testing.T) {
@@ -296,22 +264,6 @@ func TestFileSystem_deleteGroupedFile(t *testing.T) {
3: {},
}, failed)
}
// 包含上传会话文件
{
sessionID := "session"
cache.Set(UploadSessionCachePrefix+sessionID, serializer.UploadSession{Key: sessionID}, 0)
files[1].Policy.Type = "local"
files[3].Policy.Type = "local"
files[0].UploadSessionID = &sessionID
failed := fs.deleteGroupedFile(ctx, fs.GroupFileByPolicy(ctx, files))
asserts.Equal(map[uint][]string{
1: {},
2: {},
3: {},
}, failed)
_, ok := cache.Get(UploadSessionCachePrefix + sessionID)
asserts.False(ok)
}
}
func TestFileSystem_GetSource(t *testing.T) {

View File

@@ -3,10 +3,18 @@ package filesystem
import (
"errors"
"fmt"
model "github.com/cloudreve/Cloudreve/v3/models"
"github.com/cloudreve/Cloudreve/v3/pkg/cluster"
"github.com/cloudreve/Cloudreve/v3/pkg/conf"
"github.com/cloudreve/Cloudreve/v3/pkg/filesystem/driver"
"github.com/cloudreve/Cloudreve/v3/pkg/filesystem/driver/shadow/masterinslave"
"github.com/cloudreve/Cloudreve/v3/pkg/filesystem/driver/shadow/slaveinmaster"
"io"
"net/http"
"net/url"
"sync"
model "github.com/cloudreve/Cloudreve/v3/models"
"github.com/cloudreve/Cloudreve/v3/pkg/auth"
"github.com/cloudreve/Cloudreve/v3/pkg/conf"
"github.com/cloudreve/Cloudreve/v3/pkg/filesystem/driver/cos"
"github.com/cloudreve/Cloudreve/v3/pkg/filesystem/driver/local"
"github.com/cloudreve/Cloudreve/v3/pkg/filesystem/driver/onedrive"
@@ -14,16 +22,11 @@ import (
"github.com/cloudreve/Cloudreve/v3/pkg/filesystem/driver/qiniu"
"github.com/cloudreve/Cloudreve/v3/pkg/filesystem/driver/remote"
"github.com/cloudreve/Cloudreve/v3/pkg/filesystem/driver/s3"
"github.com/cloudreve/Cloudreve/v3/pkg/filesystem/driver/shadow/masterinslave"
"github.com/cloudreve/Cloudreve/v3/pkg/filesystem/driver/shadow/slaveinmaster"
"github.com/cloudreve/Cloudreve/v3/pkg/filesystem/driver/upyun"
"github.com/cloudreve/Cloudreve/v3/pkg/request"
"github.com/cloudreve/Cloudreve/v3/pkg/serializer"
"github.com/gin-gonic/gin"
cossdk "github.com/tencentyun/cos-go-sdk-v5"
"net/http"
"net/url"
"sync"
)
// FSPool 文件系统资源池
@@ -33,6 +36,16 @@ var FSPool = sync.Pool{
},
}
// FileHeader 上传来的文件数据处理器
type FileHeader interface {
io.Reader
io.Closer
GetSize() uint64
GetMIMEType() string
GetFileName() string
GetVirtualPath() string
}
// FileSystem 管理文件的文件系统
type FileSystem struct {
// 文件系统所有者
@@ -91,8 +104,6 @@ func (fs *FileSystem) reset() {
func NewFileSystem(user *model.User) (*FileSystem, error) {
fs := getEmptyFS()
fs.User = user
fs.Policy = &fs.User.Policy
// 分配存储策略适配器
err := fs.DispatchHandler()
@@ -121,11 +132,17 @@ func NewAnonymousFileSystem() (*FileSystem, error) {
// DispatchHandler 根据存储策略分配文件适配器
func (fs *FileSystem) DispatchHandler() error {
var policyType string
var currentPolicy *model.Policy
if fs.Policy == nil {
return errors.New("未设置存储策略")
// 如果没有具体指定,就是用用户当前存储策略
policyType = fs.User.Policy.Type
currentPolicy = &fs.User.Policy
} else {
policyType = fs.Policy.Type
currentPolicy = fs.Policy
}
policyType := fs.Policy.Type
currentPolicy := fs.Policy
switch policyType {
case "mock", "anonymous":
@@ -136,19 +153,23 @@ func (fs *FileSystem) DispatchHandler() error {
}
return nil
case "remote":
handler, err := remote.NewDriver(currentPolicy)
if err != nil {
return err
fs.Handler = remote.Driver{
Policy: currentPolicy,
Client: request.NewClient(),
AuthInstance: auth.HMACAuth{[]byte(currentPolicy.SecretKey)},
}
fs.Handler = handler
return nil
case "qiniu":
fs.Handler = qiniu.NewDriver(currentPolicy)
fs.Handler = qiniu.Driver{
Policy: currentPolicy,
}
return nil
case "oss":
handler, err := oss.NewDriver(currentPolicy)
fs.Handler = handler
return err
fs.Handler = oss.Driver{
Policy: currentPolicy,
HTTPClient: request.NewClient(),
}
return nil
case "upyun":
fs.Handler = upyun.Driver{
Policy: currentPolicy,
@@ -173,14 +194,13 @@ func (fs *FileSystem) DispatchHandler() error {
}
return nil
case "s3":
handler, err := s3.NewDriver(currentPolicy)
fs.Handler = handler
return err
fs.Handler = s3.Driver{
Policy: currentPolicy,
}
return nil
default:
return ErrUnknownPolicyType
}
return nil
}
// NewFileSystemFromContext 从gin.Context创建文件系统
@@ -201,14 +221,19 @@ func NewFileSystemFromCallback(c *gin.Context) (*FileSystem, error) {
}
// 获取回调会话
callbackSessionRaw, ok := c.Get(UploadSessionCtx)
callbackSessionRaw, ok := c.Get("callbackSession")
if !ok {
return nil, errors.New("找不到回调会话")
}
callbackSession := callbackSessionRaw.(*serializer.UploadSession)
// 重新指向上传策略
fs.Policy = &callbackSession.Policy
policy, err := model.GetPolicyByID(callbackSession.PolicyID)
if err != nil {
return nil, err
}
fs.Policy = &policy
fs.User.Policy = policy
err = fs.DispatchHandler()
return fs, err
@@ -216,7 +241,7 @@ func NewFileSystemFromCallback(c *gin.Context) (*FileSystem, error) {
// SwitchToSlaveHandler 将负责上传的 Handler 切换为从机节点
func (fs *FileSystem) SwitchToSlaveHandler(node cluster.Node) {
fs.Handler = slaveinmaster.NewDriver(node, fs.Handler, fs.Policy)
fs.Handler = slaveinmaster.NewDriver(node, fs.Handler, &fs.User.Policy)
}
// SwitchToShadowHandler 将负责上传的 Handler 切换为从机节点转存使用的影子处理器

View File

@@ -2,16 +2,16 @@ package filesystem
import (
"github.com/cloudreve/Cloudreve/v3/pkg/cluster"
"github.com/cloudreve/Cloudreve/v3/pkg/conf"
"github.com/cloudreve/Cloudreve/v3/pkg/filesystem/driver/shadow/masterinslave"
"github.com/cloudreve/Cloudreve/v3/pkg/filesystem/driver/shadow/slaveinmaster"
"github.com/cloudreve/Cloudreve/v3/pkg/serializer"
"net/http/httptest"
"github.com/DATA-DOG/go-sqlmock"
model "github.com/cloudreve/Cloudreve/v3/models"
"github.com/cloudreve/Cloudreve/v3/pkg/cache"
"github.com/cloudreve/Cloudreve/v3/pkg/filesystem/driver/local"
"github.com/cloudreve/Cloudreve/v3/pkg/filesystem/driver/remote"
"github.com/cloudreve/Cloudreve/v3/pkg/serializer"
"github.com/gin-gonic/gin"
"github.com/stretchr/testify/assert"
@@ -36,7 +36,7 @@ func TestNewFileSystem(t *testing.T) {
fs, err = NewFileSystem(&user)
asserts.NoError(err)
asserts.NotNil(fs.Handler)
asserts.IsType(&remote.Driver{}, fs.Handler)
asserts.IsType(remote.Driver{}, fs.Handler)
user.Policy.Type = "unknown"
fs, err = NewFileSystem(&user)
@@ -64,10 +64,9 @@ func TestNewFileSystemFromContext(t *testing.T) {
func TestDispatchHandler(t *testing.T) {
asserts := assert.New(t)
fs := &FileSystem{
User: &model.User{},
Policy: &model.Policy{
User: &model.User{Policy: model.Policy{
Type: "local",
},
}},
}
// 未指定,使用用户默认
@@ -96,7 +95,7 @@ func TestDispatchHandler(t *testing.T) {
err = fs.DispatchHandler()
asserts.NoError(err)
fs.Policy = &model.Policy{Type: "oss", Server: "https://s.com", BucketName: "1234"}
fs.Policy = &model.Policy{Type: "oss"}
err = fs.DispatchHandler()
asserts.NoError(err)
@@ -141,6 +140,23 @@ func TestNewFileSystemFromCallback(t *testing.T) {
asserts.Error(err)
}
// 找不到上传策略
{
c, _ := gin.CreateTestContext(httptest.NewRecorder())
c.Set("user", &model.User{
Policy: model.Policy{
Type: "local",
},
})
c.Set("callbackSession", &serializer.UploadSession{PolicyID: 138})
cache.Deletes([]string{"138"}, "policy_")
mock.ExpectQuery("SELECT(.+)").WillReturnRows(sqlmock.NewRows([]string{"id"}))
fs, err := NewFileSystemFromCallback(c)
asserts.NoError(mock.ExpectationsWereMet())
asserts.Nil(fs)
asserts.Error(err)
}
// 成功
{
c, _ := gin.CreateTestContext(httptest.NewRecorder())
@@ -149,8 +165,11 @@ func TestNewFileSystemFromCallback(t *testing.T) {
Type: "local",
},
})
c.Set(UploadSessionCtx, &serializer.UploadSession{Policy: model.Policy{Type: "local"}})
c.Set("callbackSession", &serializer.UploadSession{PolicyID: 138})
cache.Deletes([]string{"138"}, "policy_")
mock.ExpectQuery("SELECT(.+)").WillReturnRows(sqlmock.NewRows([]string{"id", "type", "options"}).AddRow(138, "local", "{}"))
fs, err := NewFileSystemFromCallback(c)
asserts.NoError(mock.ExpectationsWereMet())
asserts.NotNil(fs)
asserts.NoError(err)
}
@@ -215,16 +234,6 @@ func TestNewAnonymousFileSystem(t *testing.T) {
asserts.Error(err)
asserts.Nil(fs)
}
// 从机
{
conf.SystemConfig.Mode = "slave"
fs, err := NewAnonymousFileSystem()
asserts.NoError(mock.ExpectationsWereMet())
asserts.NoError(err)
asserts.NotNil(fs)
asserts.NotNil(fs.Handler)
}
}
func TestFileSystem_Recycle(t *testing.T) {

View File

@@ -5,6 +5,10 @@ type key int
const (
// GinCtx Gin的上下文
GinCtx key = iota
// SavePathCtx 文件物理路径
SavePathCtx
// FileHeaderCtx 上传的文件
FileHeaderCtx
// PathCtx 文件或目录的虚拟路径
PathCtx
// FileModelCtx 文件数据库模型
@@ -33,6 +37,10 @@ const (
ForceUsePublicEndpointCtx
// CancelFuncCtx Context 取消函數
CancelFuncCtx
// ValidateCapacityOnceCtx 限定归还容量的操作只執行一次
ValidateCapacityOnceCtx
// 禁止上传时同名覆盖操作
DisableOverwrite
// 文件在从机节点中的路径
SlaveSrcPath
)

View File

@@ -1,113 +0,0 @@
package fsctx
import (
"errors"
"io"
"time"
)
type WriteMode int
const (
Overwrite WriteMode = 0x00001
// Append 只适用于本地策略
Append WriteMode = 0x00002
Nop WriteMode = 0x00004
)
type UploadTaskInfo struct {
Size uint64
MIMEType string
FileName string
VirtualPath string
Mode WriteMode
Metadata map[string]string
LastModified *time.Time
SavePath string
UploadSessionID *string
AppendStart uint64
Model interface{}
Src string
}
// FileHeader 上传来的文件数据处理器
type FileHeader interface {
io.Reader
io.Closer
io.Seeker
Info() *UploadTaskInfo
SetSize(uint64)
SetModel(fileModel interface{})
Seekable() bool
}
// FileStream 用户传来的文件
type FileStream struct {
Mode WriteMode
LastModified *time.Time
Metadata map[string]string
File io.ReadCloser
Seeker io.Seeker
Size uint64
VirtualPath string
Name string
MIMEType string
SavePath string
UploadSessionID *string
AppendStart uint64
Model interface{}
Src string
}
func (file *FileStream) Read(p []byte) (n int, err error) {
if file.File != nil {
return file.File.Read(p)
}
return 0, io.EOF
}
func (file *FileStream) Close() error {
if file.File != nil {
return file.File.Close()
}
return nil
}
func (file *FileStream) Seek(offset int64, whence int) (int64, error) {
if file.Seekable() {
return file.Seeker.Seek(offset, whence)
}
return 0, errors.New("no seeker")
}
func (file *FileStream) Seekable() bool {
return file.Seeker != nil
}
func (file *FileStream) Info() *UploadTaskInfo {
return &UploadTaskInfo{
Size: file.Size,
MIMEType: file.MIMEType,
FileName: file.Name,
VirtualPath: file.VirtualPath,
Mode: file.Mode,
Metadata: file.Metadata,
LastModified: file.LastModified,
SavePath: file.SavePath,
UploadSessionID: file.UploadSessionID,
AppendStart: file.AppendStart,
Model: file.Model,
Src: file.Src,
}
}
func (file *FileStream) SetSize(size uint64) {
file.Size = size
}
func (file *FileStream) SetModel(fileModel interface{}) {
file.Model = fileModel
}

View File

@@ -1,78 +0,0 @@
package fsctx
import (
model "github.com/cloudreve/Cloudreve/v3/models"
"github.com/stretchr/testify/assert"
"io"
"io/ioutil"
"os"
"strings"
"testing"
)
func TestFileStream_Read(t *testing.T) {
asserts := assert.New(t)
file := FileStream{
File: ioutil.NopCloser(strings.NewReader("123")),
}
var p = make([]byte, 3)
{
n, err := file.Read(p)
asserts.Equal(3, n)
asserts.NoError(err)
}
}
func TestFileStream_Close(t *testing.T) {
asserts := assert.New(t)
{
file := FileStream{
File: ioutil.NopCloser(strings.NewReader("123")),
}
err := file.Close()
asserts.NoError(err)
}
{
file := FileStream{}
err := file.Close()
asserts.NoError(err)
}
}
func TestFileStream_Seek(t *testing.T) {
asserts := assert.New(t)
f, _ := os.CreateTemp("", "*")
defer func() {
f.Close()
os.Remove(f.Name())
}()
{
file := FileStream{
File: f,
Seeker: f,
}
res, err := file.Seek(0, io.SeekStart)
asserts.NoError(err)
asserts.EqualValues(0, res)
}
{
file := FileStream{}
res, err := file.Seek(0, io.SeekStart)
asserts.Error(err)
asserts.EqualValues(0, res)
}
}
func TestFileStream_Info(t *testing.T) {
a := assert.New(t)
file := FileStream{}
a.NotNil(file.Info())
file.SetSize(10)
a.EqualValues(10, file.Info().Size)
file.SetModel(&model.File{})
a.NotNil(file.Info().Model)
}

View File

@@ -2,19 +2,21 @@ package filesystem
import (
"context"
model "github.com/cloudreve/Cloudreve/v3/models"
"github.com/cloudreve/Cloudreve/v3/pkg/cache"
"github.com/cloudreve/Cloudreve/v3/pkg/cluster"
"github.com/cloudreve/Cloudreve/v3/pkg/filesystem/driver/local"
"github.com/cloudreve/Cloudreve/v3/pkg/filesystem/fsctx"
"github.com/cloudreve/Cloudreve/v3/pkg/serializer"
"github.com/cloudreve/Cloudreve/v3/pkg/util"
"errors"
"io/ioutil"
"strings"
"sync"
model "github.com/cloudreve/Cloudreve/v3/models"
"github.com/cloudreve/Cloudreve/v3/pkg/conf"
"github.com/cloudreve/Cloudreve/v3/pkg/filesystem/fsctx"
"github.com/cloudreve/Cloudreve/v3/pkg/request"
"github.com/cloudreve/Cloudreve/v3/pkg/serializer"
"github.com/cloudreve/Cloudreve/v3/pkg/util"
)
// Hook 钩子函数
type Hook func(ctx context.Context, fs *FileSystem, file fsctx.FileHeader) error
type Hook func(ctx context.Context, fs *FileSystem) error
// Use 注入钩子
func (fs *FileSystem) Use(name string, hook Hook) {
@@ -39,10 +41,10 @@ func (fs *FileSystem) CleanHooks(name string) {
// Trigger 触发钩子,遇到第一个错误时
// 返回错误,后续钩子不会继续执行
func (fs *FileSystem) Trigger(ctx context.Context, name string, file fsctx.FileHeader) error {
func (fs *FileSystem) Trigger(ctx context.Context, name string) error {
if hooks, ok := fs.Hooks[name]; ok {
for _, hook := range hooks {
err := hook(ctx, fs, file)
err := hook(ctx, fs)
if err != nil {
util.Log().Warning("钩子执行失败:%s", err)
return err
@@ -52,22 +54,56 @@ func (fs *FileSystem) Trigger(ctx context.Context, name string, file fsctx.FileH
return nil
}
// HookValidateFile 一系列对文件检验的集合
func HookValidateFile(ctx context.Context, fs *FileSystem, file fsctx.FileHeader) error {
fileInfo := file.Info()
// HookIsFileExist 检查虚拟路径文件是否存在
func HookIsFileExist(ctx context.Context, fs *FileSystem) error {
filePath := ctx.Value(fsctx.PathCtx).(string)
if ok, _ := fs.IsFileExist(filePath); ok {
return nil
}
return ErrObjectNotExist
}
// HookSlaveUploadValidate Slave模式下对文件上传的一系列验证
func HookSlaveUploadValidate(ctx context.Context, fs *FileSystem) error {
file := ctx.Value(fsctx.FileHeaderCtx).(FileHeader)
policy := ctx.Value(fsctx.UploadPolicyCtx).(serializer.UploadPolicy)
// 验证单文件尺寸
if !fs.ValidateFileSize(ctx, fileInfo.Size) {
return ErrFileSizeTooBig
if policy.MaxSize > 0 {
if file.GetSize() > policy.MaxSize {
return ErrFileSizeTooBig
}
}
// 验证文件名
if !fs.ValidateLegalName(ctx, fileInfo.FileName) {
if !fs.ValidateLegalName(ctx, file.GetFileName()) {
return ErrIllegalObjectName
}
// 验证扩展名
if !fs.ValidateExtension(ctx, fileInfo.FileName) {
if len(policy.AllowedExtension) > 0 && !IsInExtensionList(policy.AllowedExtension, file.GetFileName()) {
return ErrFileExtensionNotAllowed
}
return nil
}
// HookValidateFile 一系列对文件检验的集合
func HookValidateFile(ctx context.Context, fs *FileSystem) error {
file := ctx.Value(fsctx.FileHeaderCtx).(FileHeader)
// 验证单文件尺寸
if !fs.ValidateFileSize(ctx, file.GetSize()) {
return ErrFileSizeTooBig
}
// 验证文件名
if !fs.ValidateLegalName(ctx, file.GetFileName()) {
return ErrIllegalObjectName
}
// 验证扩展名
if !fs.ValidateExtension(ctx, file.GetFileName()) {
return ErrFileExtensionNotAllowed
}
@@ -76,41 +112,58 @@ func HookValidateFile(ctx context.Context, fs *FileSystem, file fsctx.FileHeader
}
// HookResetPolicy 重设存储策略为上下文已有文件
func HookResetPolicy(ctx context.Context, fs *FileSystem, file fsctx.FileHeader) error {
func HookResetPolicy(ctx context.Context, fs *FileSystem) error {
originFile, ok := ctx.Value(fsctx.FileModelCtx).(model.File)
if !ok {
return ErrObjectNotExist
}
fs.Policy = originFile.GetPolicy()
fs.User.Policy = *fs.Policy
return fs.DispatchHandler()
}
// HookValidateCapacity 验证用户容量
func HookValidateCapacity(ctx context.Context, fs *FileSystem, file fsctx.FileHeader) error {
// HookValidateCapacity 验证并扣除用户容量,包含数据库操作
func HookValidateCapacity(ctx context.Context, fs *FileSystem) error {
file := ctx.Value(fsctx.FileHeaderCtx).(FileHeader)
// 验证并扣除容量
if fs.User.GetRemainingCapacity() < file.Info().Size {
if !fs.ValidateCapacity(ctx, file.GetSize()) {
return ErrInsufficientCapacity
}
return nil
}
// HookValidateCapacityDiff 根据原有文件和新文件的大小验证用户容量
func HookValidateCapacityDiff(ctx context.Context, fs *FileSystem, newFile fsctx.FileHeader) error {
originFile := ctx.Value(fsctx.FileModelCtx).(model.File)
newFileSize := newFile.Info().Size
// HookValidateCapacityWithoutIncrease 验证用户容量,不扣除
func HookValidateCapacityWithoutIncrease(ctx context.Context, fs *FileSystem) error {
file := ctx.Value(fsctx.FileHeaderCtx).(FileHeader)
// 验证并扣除容量
if fs.User.GetRemainingCapacity() < file.GetSize() {
return ErrInsufficientCapacity
}
return nil
}
if newFileSize > originFile.Size {
return HookValidateCapacity(ctx, fs, newFile)
// HookChangeCapacity 根据原有文件和新文件的大小更新用户容量
func HookChangeCapacity(ctx context.Context, fs *FileSystem) error {
newFile := ctx.Value(fsctx.FileHeaderCtx).(FileHeader)
originFile := ctx.Value(fsctx.FileModelCtx).(model.File)
if newFile.GetSize() > originFile.Size {
if !fs.ValidateCapacity(ctx, newFile.GetSize()-originFile.Size) {
return ErrInsufficientCapacity
}
return nil
}
fs.User.DeductionStorage(originFile.Size - newFile.GetSize())
return nil
}
// HookDeleteTempFile 删除已保存的临时文件
func HookDeleteTempFile(ctx context.Context, fs *FileSystem, file fsctx.FileHeader) error {
func HookDeleteTempFile(ctx context.Context, fs *FileSystem) error {
filePath := ctx.Value(fsctx.SavePathCtx).(string)
// 删除临时文件
_, err := fs.Handler.Delete(ctx, []string{file.Info().SavePath})
_, err := fs.Handler.Delete(ctx, []string{filePath})
if err != nil {
util.Log().Warning("无法清理上传临时文件,%s", err)
}
@@ -119,18 +172,14 @@ func HookDeleteTempFile(ctx context.Context, fs *FileSystem, file fsctx.FileHead
}
// HookCleanFileContent 清空文件内容
func HookCleanFileContent(ctx context.Context, fs *FileSystem, file fsctx.FileHeader) error {
func HookCleanFileContent(ctx context.Context, fs *FileSystem) error {
filePath := ctx.Value(fsctx.SavePathCtx).(string)
// 清空内容
return fs.Handler.Put(ctx, &fsctx.FileStream{
File: ioutil.NopCloser(strings.NewReader("")),
SavePath: file.Info().SavePath,
Size: 0,
Mode: fsctx.Overwrite,
})
return fs.Handler.Put(ctx, ioutil.NopCloser(strings.NewReader("")), filePath, 0)
}
// HookClearFileSize 将原始文件的尺寸设为0
func HookClearFileSize(ctx context.Context, fs *FileSystem, file fsctx.FileHeader) error {
func HookClearFileSize(ctx context.Context, fs *FileSystem) error {
originFile, ok := ctx.Value(fsctx.FileModelCtx).(model.File)
if !ok {
return ErrObjectNotExist
@@ -139,7 +188,7 @@ func HookClearFileSize(ctx context.Context, fs *FileSystem, file fsctx.FileHeade
}
// HookCancelContext 取消上下文
func HookCancelContext(ctx context.Context, fs *FileSystem, file fsctx.FileHeader) error {
func HookCancelContext(ctx context.Context, fs *FileSystem) error {
cancelFunc, ok := ctx.Value(fsctx.CancelFuncCtx).(context.CancelFunc)
if ok {
cancelFunc()
@@ -147,8 +196,29 @@ func HookCancelContext(ctx context.Context, fs *FileSystem, file fsctx.FileHeade
return nil
}
// HookGiveBackCapacity 归还用户容量
func HookGiveBackCapacity(ctx context.Context, fs *FileSystem) error {
file := ctx.Value(fsctx.FileHeaderCtx).(FileHeader)
once, ok := ctx.Value(fsctx.ValidateCapacityOnceCtx).(*sync.Once)
if !ok {
once = &sync.Once{}
}
// 归还用户容量
res := true
once.Do(func() {
res = fs.User.DeductionStorage(file.GetSize())
})
if !res {
return errors.New("无法继续降低用户已用存储")
}
return nil
}
// HookUpdateSourceName 更新文件SourceName
func HookUpdateSourceName(ctx context.Context, fs *FileSystem, file fsctx.FileHeader) error {
// TODO测试
func HookUpdateSourceName(ctx context.Context, fs *FileSystem) error {
originFile, ok := ctx.Value(fsctx.FileModelCtx).(model.File)
if !ok {
return ErrObjectNotExist
@@ -157,141 +227,103 @@ func HookUpdateSourceName(ctx context.Context, fs *FileSystem, file fsctx.FileHe
}
// GenericAfterUpdate 文件内容更新后
func GenericAfterUpdate(ctx context.Context, fs *FileSystem, newFile fsctx.FileHeader) error {
func GenericAfterUpdate(ctx context.Context, fs *FileSystem) error {
// 更新文件尺寸
originFile, ok := ctx.Value(fsctx.FileModelCtx).(model.File)
if !ok {
return ErrObjectNotExist
}
newFile.SetModel(&originFile)
fs.SetTargetFile(&[]model.File{originFile})
err := originFile.UpdateSize(newFile.Info().Size)
newFile, ok := ctx.Value(fsctx.FileHeaderCtx).(FileHeader)
if !ok {
return ErrObjectNotExist
}
err := originFile.UpdateSize(newFile.GetSize())
if err != nil {
return err
}
// 尝试清空原有缩略图并重新生成
if originFile.GetPolicy().IsThumbGenerateNeeded() {
fs.recycleLock.Lock()
go func() {
defer fs.recycleLock.Unlock()
if originFile.PicInfo != "" {
_, _ = fs.Handler.Delete(ctx, []string{originFile.SourceName + conf.ThumbConfig.FileSuffix})
fs.GenerateThumbnail(ctx, &originFile)
}
}()
}
return nil
}
// SlaveAfterUpload Slave模式下上传完成钩子
func SlaveAfterUpload(session *serializer.UploadSession) Hook {
return func(ctx context.Context, fs *FileSystem, fileHeader fsctx.FileHeader) error {
fileInfo := fileHeader.Info()
func SlaveAfterUpload(ctx context.Context, fs *FileSystem) error {
fileHeader := ctx.Value(fsctx.FileHeaderCtx).(FileHeader)
policy := ctx.Value(fsctx.UploadPolicyCtx).(serializer.UploadPolicy)
// 构造一个model.File用于生成缩略图
file := model.File{
Name: fileInfo.FileName,
SourceName: fileInfo.SavePath,
}
fs.GenerateThumbnail(ctx, &file)
if session.Callback == "" {
return nil
}
// 发送回调请求
callbackBody := serializer.UploadCallback{
PicInfo: file.PicInfo,
}
return cluster.RemoteCallback(session.Callback, callbackBody)
// 构造一个model.File用于生成缩略图
file := model.File{
Name: fileHeader.GetFileName(),
SourceName: ctx.Value(fsctx.SavePathCtx).(string),
}
fs.GenerateThumbnail(ctx, &file)
if policy.CallbackURL == "" {
return nil
}
// 发送回调请求
callbackBody := serializer.UploadCallback{
Name: file.Name,
SourceName: file.SourceName,
PicInfo: file.PicInfo,
Size: fileHeader.GetSize(),
}
return request.RemoteCallback(policy.CallbackURL, callbackBody)
}
// GenericAfterUpload 文件上传完成后,包含数据库操作
func GenericAfterUpload(ctx context.Context, fs *FileSystem, fileHeader fsctx.FileHeader) error {
fileInfo := fileHeader.Info()
func GenericAfterUpload(ctx context.Context, fs *FileSystem) error {
// 文件存放的虚拟路径
virtualPath := ctx.Value(fsctx.FileHeaderCtx).(FileHeader).GetVirtualPath()
// 创建或查找根目录
folder, err := fs.CreateDirectory(ctx, fileInfo.VirtualPath)
if err != nil {
return err
// 检查路径是否存在,不存在就创建
isExist, folder := fs.IsPathExist(virtualPath)
if !isExist {
newFolder, err := fs.CreateDirectory(ctx, virtualPath)
if err != nil {
return err
}
folder = newFolder
}
// 检查文件是否存在
if ok, file := fs.IsChildFileExist(
if ok, _ := fs.IsChildFileExist(
folder,
fileInfo.FileName,
ctx.Value(fsctx.FileHeaderCtx).(FileHeader).GetFileName(),
); ok {
if file.UploadSessionID != nil {
return ErrFileUploadSessionExisted
}
return ErrFileExisted
}
// 向数据库中插入记录
file, err := fs.AddFile(ctx, folder, fileHeader)
file, err := fs.AddFile(ctx, folder)
if err != nil {
return ErrInsertFileRecord
}
fileHeader.SetModel(file)
fs.SetTargetFile(&[]model.File{*file})
return nil
}
// HookGenerateThumb 生成缩略图
func HookGenerateThumb(ctx context.Context, fs *FileSystem, fileHeader fsctx.FileHeader) error {
// 异步尝试生成缩略图
fileMode := fileHeader.Info().Model.(*model.File)
if fs.Policy.IsThumbGenerateNeeded() {
if fs.User.Policy.IsThumbGenerateNeeded() {
fs.recycleLock.Lock()
go func() {
defer fs.recycleLock.Unlock()
_, _ = fs.Handler.Delete(ctx, []string{fileMode.SourceName + model.GetSettingByNameWithDefault("thumb_file_suffix", "._thumb")})
fs.GenerateThumbnail(ctx, fileMode)
fs.GenerateThumbnail(ctx, file)
}()
}
return nil
}
// HookClearFileHeaderSize 将FileHeader大小设定为0
func HookClearFileHeaderSize(ctx context.Context, fs *FileSystem, fileHeader fsctx.FileHeader) error {
fileHeader.SetSize(0)
return nil
}
// HookTruncateFileTo 将物理文件截断至 size
func HookTruncateFileTo(size uint64) Hook {
return func(ctx context.Context, fs *FileSystem, fileHeader fsctx.FileHeader) error {
if handler, ok := fs.Handler.(local.Driver); ok {
return handler.Truncate(ctx, fileHeader.Info().SavePath, size)
}
return nil
}
}
// HookChunkUploadFinished 单个分片上传结束后
func HookChunkUploaded(ctx context.Context, fs *FileSystem, fileHeader fsctx.FileHeader) error {
fileInfo := fileHeader.Info()
// 更新文件大小
return fileInfo.Model.(*model.File).UpdateSize(fileInfo.AppendStart + fileInfo.Size)
}
// HookChunkUploadFailed 单个分片上传失败后
func HookChunkUploadFailed(ctx context.Context, fs *FileSystem, fileHeader fsctx.FileHeader) error {
fileInfo := fileHeader.Info()
// 更新文件大小
return fileInfo.Model.(*model.File).UpdateSize(fileInfo.AppendStart)
}
// HookPopPlaceholderToFile 将占位文件提升为正式文件
func HookPopPlaceholderToFile(picInfo string) Hook {
return func(ctx context.Context, fs *FileSystem, fileHeader fsctx.FileHeader) error {
fileInfo := fileHeader.Info()
fileModel := fileInfo.Model.(*model.File)
return fileModel.PopChunkToFile(fileInfo.LastModified, picInfo)
}
}
// HookChunkUploadFinished 分片上传结束后处理文件
func HookDeleteUploadSession(id string) Hook {
return func(ctx context.Context, fs *FileSystem, fileHeader fsctx.FileHeader) error {
cache.Deletes([]string{id}, UploadSessionCachePrefix)
return nil
}
}

View File

@@ -3,20 +3,22 @@ package filesystem
import (
"context"
"errors"
"io"
"io/ioutil"
"net/http"
"os"
"strings"
"sync"
"testing"
"github.com/DATA-DOG/go-sqlmock"
model "github.com/cloudreve/Cloudreve/v3/models"
"github.com/cloudreve/Cloudreve/v3/pkg/cache"
"github.com/cloudreve/Cloudreve/v3/pkg/conf"
"github.com/cloudreve/Cloudreve/v3/pkg/filesystem/driver/local"
"github.com/cloudreve/Cloudreve/v3/pkg/filesystem/fsctx"
"github.com/cloudreve/Cloudreve/v3/pkg/mocks/requestmock"
"github.com/cloudreve/Cloudreve/v3/pkg/request"
"github.com/cloudreve/Cloudreve/v3/pkg/serializer"
"io/ioutil"
"net/http"
"strings"
"testing"
model "github.com/cloudreve/Cloudreve/v3/models"
"github.com/jinzhu/gorm"
"github.com/stretchr/testify/assert"
testMock "github.com/stretchr/testify/mock"
@@ -24,72 +26,78 @@ import (
func TestGenericBeforeUpload(t *testing.T) {
asserts := assert.New(t)
file := &fsctx.FileStream{
file := local.FileStream{
Size: 5,
Name: "1.txt",
}
ctx := context.Background()
cache.Set("pack_size_0", uint64(0), 0)
ctx := context.WithValue(context.Background(), fsctx.FileHeaderCtx, file)
fs := FileSystem{
User: &model.User{
Storage: 0,
Group: model.Group{
MaxStorage: 11,
},
},
Policy: &model.Policy{
MaxSize: 4,
OptionsSerialized: model.PolicyOption{
FileType: []string{"txt"},
Policy: model.Policy{
MaxSize: 4,
OptionsSerialized: model.PolicyOption{
FileType: []string{"txt"},
},
},
},
}
asserts.Error(HookValidateFile(ctx, &fs, file))
asserts.Error(HookValidateFile(ctx, &fs))
file.Size = 1
file.Name = "1"
asserts.Error(HookValidateFile(ctx, &fs, file))
ctx = context.WithValue(context.Background(), fsctx.FileHeaderCtx, file)
asserts.Error(HookValidateFile(ctx, &fs))
file.Name = "1.txt"
asserts.NoError(HookValidateFile(ctx, &fs, file))
ctx = context.WithValue(context.Background(), fsctx.FileHeaderCtx, file)
asserts.NoError(HookValidateFile(ctx, &fs))
file.Name = "1.t/xt"
asserts.Error(HookValidateFile(ctx, &fs, file))
ctx = context.WithValue(context.Background(), fsctx.FileHeaderCtx, file)
asserts.Error(HookValidateFile(ctx, &fs))
}
func TestGenericAfterUploadCanceled(t *testing.T) {
asserts := assert.New(t)
file := &fsctx.FileStream{
Size: 5,
Name: "TestGenericAfterUploadCanceled",
SavePath: "TestGenericAfterUploadCanceled",
f, err := os.Create("TestGenericAfterUploadCanceled")
asserts.NoError(err)
f.Close()
file := local.FileStream{
Size: 5,
Name: "TestGenericAfterUploadCanceled",
}
ctx := context.Background()
ctx := context.WithValue(context.Background(), fsctx.SavePathCtx, "TestGenericAfterUploadCanceled")
ctx = context.WithValue(ctx, fsctx.FileHeaderCtx, file)
fs := FileSystem{
User: &model.User{},
User: &model.User{Storage: 5},
Handler: local.Driver{},
}
// 成功
{
mockHandler := &FileHeaderMock{}
fs.Handler = mockHandler
mockHandler.On("Delete", testMock.Anything, testMock.Anything).Return([]string{}, nil)
err := HookDeleteTempFile(ctx, &fs, file)
asserts.NoError(err)
mockHandler.AssertExpectations(t)
}
err = HookDeleteTempFile(ctx, &fs)
asserts.NoError(err)
err = HookGiveBackCapacity(ctx, &fs)
asserts.NoError(err)
asserts.Equal(uint64(0), fs.User.Storage)
// 失败
{
mockHandler := &FileHeaderMock{}
fs.Handler = mockHandler
mockHandler.On("Delete", testMock.Anything, testMock.Anything).Return([]string{}, errors.New(""))
err := HookDeleteTempFile(ctx, &fs, file)
asserts.NoError(err)
mockHandler.AssertExpectations(t)
}
f, err = os.Create("TestGenericAfterUploadCanceled")
asserts.NoError(err)
f.Close()
// 容量不能再降低
err = HookGiveBackCapacity(ctx, &fs)
asserts.Error(err)
//文件不存在
fs.User.Storage = 5
err = HookDeleteTempFile(ctx, &fs)
asserts.NoError(err)
}
func TestGenericAfterUpload(t *testing.T) {
@@ -100,98 +108,69 @@ func TestGenericAfterUpload(t *testing.T) {
ID: 1,
},
},
Policy: &model.Policy{},
}
ctx := context.Background()
file := &fsctx.FileStream{
ctx := context.WithValue(context.Background(), fsctx.FileHeaderCtx, local.FileStream{
VirtualPath: "/我的文件",
Name: "test.txt",
}
})
ctx = context.WithValue(ctx, fsctx.SavePathCtx, "")
// 正常
mock.ExpectQuery("SELECT(.+)").
WithArgs(1).
WillReturnRows(sqlmock.NewRows([]string{"id", "owner_id"}).AddRow(1, 1))
mock.ExpectQuery("SELECT(.+)files").
WithArgs(1, "我的文件").
WillReturnRows(sqlmock.NewRows([]string{"id", "owner_id"}))
// 1
mock.ExpectBegin()
mock.ExpectQuery("SELECT(.+)").
WithArgs("我的文件", 1, 1).
WithArgs(1, 1, "我的文件").
WillReturnRows(sqlmock.NewRows([]string{"id", "owner_id"}).AddRow(2, 1))
mock.ExpectCommit()
mock.ExpectQuery("SELECT(.+)files(.+)").WillReturnError(errors.New("not found"))
mock.ExpectBegin()
mock.ExpectExec("INSERT(.+)files(.+)").WillReturnResult(sqlmock.NewResult(1, 1))
mock.ExpectExec("UPDATE(.+)storage(.+)").WillReturnResult(sqlmock.NewResult(1, 1))
mock.ExpectCommit()
err := GenericAfterUpload(ctx, &fs, file)
err := GenericAfterUpload(ctx, &fs)
asserts.NoError(err)
asserts.NoError(mock.ExpectationsWereMet())
// 路径不存在
mock.ExpectQuery("SELECT(.+)folders(.+)").WillReturnRows(
mock.NewRows([]string{"name"}),
)
err = GenericAfterUpload(ctx, &fs)
asserts.Equal(ErrRootProtected, err)
asserts.NoError(mock.ExpectationsWereMet())
// 文件已存在
mock.ExpectQuery("SELECT(.+)").
WithArgs(1).
WillReturnRows(sqlmock.NewRows([]string{"id", "owner_id"}).AddRow(1, 1))
mock.ExpectQuery("SELECT(.+)files").
WithArgs(1, "我的文件").
WillReturnRows(sqlmock.NewRows([]string{"id", "owner_id"}))
// 1
mock.ExpectBegin()
mock.ExpectQuery("SELECT(.+)").
WithArgs("我的文件", 1, 1).
WithArgs(1, 1, "我的文件").
WillReturnRows(sqlmock.NewRows([]string{"id", "owner_id"}).AddRow(2, 1))
mock.ExpectCommit()
mock.ExpectQuery("SELECT(.+)files(.+)").WillReturnRows(
mock.NewRows([]string{"name"}).AddRow("test.txt"),
)
err = GenericAfterUpload(ctx, &fs, file)
err = GenericAfterUpload(ctx, &fs)
asserts.Equal(ErrFileExisted, err)
asserts.NoError(mock.ExpectationsWereMet())
// 文件已存在, 且为上传占位符
mock.ExpectQuery("SELECT(.+)").
WithArgs(1).
WillReturnRows(sqlmock.NewRows([]string{"id", "owner_id"}).AddRow(1, 1))
mock.ExpectQuery("SELECT(.+)files").
WithArgs(1, "我的文件").
WillReturnRows(sqlmock.NewRows([]string{"id", "owner_id"}))
// 1
mock.ExpectBegin()
mock.ExpectQuery("SELECT(.+)").
WithArgs("我的文件", 1, 1).
WillReturnRows(sqlmock.NewRows([]string{"id", "owner_id"}).AddRow(2, 1))
mock.ExpectCommit()
mock.ExpectQuery("SELECT(.+)files(.+)").WillReturnRows(
mock.NewRows([]string{"name", "upload_session_id"}).AddRow("test.txt", "1"),
)
err = GenericAfterUpload(ctx, &fs, file)
asserts.Equal(ErrFileUploadSessionExisted, err)
asserts.NoError(mock.ExpectationsWereMet())
// 插入失败
mock.ExpectQuery("SELECT(.+)").
WithArgs(1).
WillReturnRows(sqlmock.NewRows([]string{"id", "owner_id"}).AddRow(1, 1))
mock.ExpectQuery("SELECT(.+)files").
WithArgs(1, "我的文件").
WillReturnRows(sqlmock.NewRows([]string{"id", "owner_id"}))
// 1
mock.ExpectBegin()
mock.ExpectQuery("SELECT(.+)").
WithArgs("我的文件", 1, 1).
WithArgs(1, 1, "我的文件").
WillReturnRows(sqlmock.NewRows([]string{"id", "owner_id"}).AddRow(2, 1))
mock.ExpectCommit()
mock.ExpectQuery("SELECT(.+)files(.+)").WillReturnError(errors.New("not found"))
mock.ExpectBegin()
mock.ExpectExec("INSERT(.+)files(.+)").WillReturnError(errors.New("error"))
mock.ExpectRollback()
err = GenericAfterUpload(ctx, &fs, file)
err = GenericAfterUpload(ctx, &fs)
asserts.Equal(ErrInsertFileRecord, err)
asserts.NoError(mock.ExpectationsWereMet())
@@ -201,7 +180,7 @@ func TestFileSystem_Use(t *testing.T) {
asserts := assert.New(t)
fs := FileSystem{}
hook := func(ctx context.Context, fs *FileSystem, fileHeader fsctx.FileHeader) error {
hook := func(ctx context.Context, fs *FileSystem) error {
return nil
}
@@ -236,34 +215,56 @@ func TestFileSystem_Trigger(t *testing.T) {
}
ctx := context.Background()
hook := func(ctx context.Context, fs *FileSystem, fileHeader fsctx.FileHeader) error {
hook := func(ctx context.Context, fs *FileSystem) error {
fs.User.Storage++
return nil
}
// 一个
fs.Use("BeforeUpload", hook)
err := fs.Trigger(ctx, "BeforeUpload", nil)
err := fs.Trigger(ctx, "BeforeUpload")
asserts.NoError(err)
asserts.Equal(uint64(1), fs.User.Storage)
// 多个
fs.Use("BeforeUpload", hook)
fs.Use("BeforeUpload", hook)
err = fs.Trigger(ctx, "BeforeUpload", nil)
err = fs.Trigger(ctx, "BeforeUpload")
asserts.NoError(err)
asserts.Equal(uint64(4), fs.User.Storage)
}
func TestHookIsFileExist(t *testing.T) {
asserts := assert.New(t)
fs := &FileSystem{User: &model.User{
Model: gorm.Model{
ID: 1,
},
}}
ctx := context.WithValue(context.Background(), fsctx.PathCtx, "/test.txt")
{
mock.ExpectQuery("SELECT(.+)").
WithArgs(1).
WillReturnRows(sqlmock.NewRows([]string{"id", "owner_id"}).AddRow(1, 1))
mock.ExpectQuery("SELECT(.+)").WithArgs(uint(1), "test.txt").WillReturnRows(
sqlmock.NewRows([]string{"Name"}).AddRow("s"),
)
err := HookIsFileExist(ctx, fs)
asserts.NoError(mock.ExpectationsWereMet())
asserts.NoError(err)
}
{
mock.ExpectQuery("SELECT(.+)").
WithArgs(1).
WillReturnRows(sqlmock.NewRows([]string{"id", "owner_id"}).AddRow(1, 1))
mock.ExpectQuery("SELECT(.+)").WithArgs(uint(1), "test.txt").WillReturnRows(
sqlmock.NewRows([]string{"Name"}),
)
err := HookIsFileExist(ctx, fs)
asserts.NoError(mock.ExpectationsWereMet())
asserts.Error(err)
}
// 多个,有失败
fs.Use("BeforeUpload", func(ctx context.Context, fs *FileSystem, file fsctx.FileHeader) error {
return errors.New("error")
})
fs.Use("BeforeUpload", func(ctx context.Context, fs *FileSystem, file fsctx.FileHeader) error {
asserts.Fail("following hooks executed")
return nil
})
err = fs.Trigger(ctx, "BeforeUpload", nil)
asserts.Error(err)
}
func TestHookValidateCapacity(t *testing.T) {
@@ -276,41 +277,17 @@ func TestHookValidateCapacity(t *testing.T) {
MaxStorage: 11,
},
}}
ctx := context.Background()
file := &fsctx.FileStream{Size: 11}
ctx := context.WithValue(context.Background(), fsctx.FileHeaderCtx, local.FileStream{Size: 10})
{
err := HookValidateCapacity(ctx, fs, file)
err := HookValidateCapacity(ctx, fs)
asserts.NoError(err)
}
{
file.Size = 12
err := HookValidateCapacity(ctx, fs, file)
err := HookValidateCapacity(ctx, fs)
asserts.Error(err)
}
}
func TestHookValidateCapacityDiff(t *testing.T) {
a := assert.New(t)
fs := &FileSystem{User: &model.User{
Group: model.Group{
MaxStorage: 11,
},
}}
file := model.File{Size: 10}
ctx := context.WithValue(context.Background(), fsctx.FileModelCtx, file)
// 无需操作
{
a.NoError(HookValidateCapacityDiff(ctx, fs, &fsctx.FileStream{Size: 10}))
}
// 需要验证
{
a.Error(HookValidateCapacityDiff(ctx, fs, &fsctx.FileStream{Size: 12}))
}
}
func TestHookResetPolicy(t *testing.T) {
asserts := assert.New(t)
fs := &FileSystem{User: &model.User{
@@ -324,7 +301,7 @@ func TestHookResetPolicy(t *testing.T) {
mock.ExpectQuery("SELECT(.+)policies(.+)").
WillReturnRows(sqlmock.NewRows([]string{"id", "type"}).AddRow(2, "local"))
ctx := context.WithValue(context.Background(), fsctx.FileModelCtx, file)
err := HookResetPolicy(ctx, fs, nil)
err := HookResetPolicy(ctx, fs)
asserts.NoError(mock.ExpectationsWereMet())
asserts.NoError(err)
}
@@ -333,22 +310,76 @@ func TestHookResetPolicy(t *testing.T) {
{
cache.Deletes([]string{"2"}, "policy_")
ctx := context.Background()
err := HookResetPolicy(ctx, fs, nil)
err := HookResetPolicy(ctx, fs)
asserts.Error(err)
}
}
func TestHookChangeCapacity(t *testing.T) {
asserts := assert.New(t)
cache.Set("pack_size_1", uint64(0), 0)
// 容量增加 失败
{
fs := &FileSystem{User: &model.User{
Model: gorm.Model{ID: 1},
}}
newFile := local.FileStream{Size: 10}
oldFile := model.File{Size: 9}
ctx := context.WithValue(context.Background(), fsctx.FileModelCtx, oldFile)
ctx = context.WithValue(ctx, fsctx.FileHeaderCtx, newFile)
err := HookChangeCapacity(ctx, fs)
asserts.Equal(ErrInsufficientCapacity, err)
}
// 容量增加 成功
{
fs := &FileSystem{User: &model.User{
Model: gorm.Model{ID: 1},
Group: model.Group{MaxStorage: 1},
}}
newFile := local.FileStream{Size: 10}
oldFile := model.File{Size: 9}
ctx := context.WithValue(context.Background(), fsctx.FileModelCtx, oldFile)
ctx = context.WithValue(ctx, fsctx.FileHeaderCtx, newFile)
mock.ExpectBegin()
mock.ExpectExec("UPDATE(.+)").WithArgs(1, sqlmock.AnyArg(), 1).WillReturnResult(sqlmock.NewResult(1, 1))
err := HookChangeCapacity(ctx, fs)
asserts.NoError(mock.ExpectationsWereMet())
asserts.NoError(err)
asserts.Equal(uint64(1), fs.User.Storage)
}
// 容量减少
{
fs := &FileSystem{User: &model.User{
Model: gorm.Model{ID: 1},
Storage: 1,
}}
newFile := local.FileStream{Size: 9}
oldFile := model.File{Size: 10}
ctx := context.WithValue(context.Background(), fsctx.FileModelCtx, oldFile)
ctx = context.WithValue(ctx, fsctx.FileHeaderCtx, newFile)
err := HookChangeCapacity(ctx, fs)
asserts.NoError(err)
asserts.Equal(uint64(0), fs.User.Storage)
}
}
func TestHookCleanFileContent(t *testing.T) {
asserts := assert.New(t)
fs := &FileSystem{User: &model.User{
Model: gorm.Model{ID: 1},
}}
file := &fsctx.FileStream{SavePath: "123/123"}
ctx := context.WithValue(context.Background(), fsctx.SavePathCtx, "123/123")
handlerMock := FileHeaderMock{}
handlerMock.On("Put", testMock.Anything, testMock.Anything).Return(errors.New("error"))
handlerMock.On("Put", testMock.Anything, testMock.Anything, "123/123").Return(errors.New("error"))
fs.Handler = handlerMock
err := HookCleanFileContent(context.Background(), fs, file)
err := HookCleanFileContent(ctx, fs)
asserts.Error(err)
handlerMock.AssertExpectations(t)
}
@@ -364,17 +395,14 @@ func TestHookClearFileSize(t *testing.T) {
ctx := context.WithValue(
context.Background(),
fsctx.FileModelCtx,
model.File{Model: gorm.Model{ID: 1}, Size: 10},
model.File{Model: gorm.Model{ID: 1}},
)
mock.ExpectBegin()
mock.ExpectExec("UPDATE(.+)files(.+)").
WithArgs(0, sqlmock.AnyArg(), 1, 10).
WillReturnResult(sqlmock.NewResult(1, 1))
mock.ExpectExec("UPDATE(.+)users(.+)").
WithArgs(10, sqlmock.AnyArg()).
mock.ExpectExec("UPDATE(.+)").
WithArgs(0, sqlmock.AnyArg(), 1).
WillReturnResult(sqlmock.NewResult(1, 1))
mock.ExpectCommit()
err := HookClearFileSize(ctx, fs, nil)
err := HookClearFileSize(ctx, fs)
asserts.NoError(mock.ExpectationsWereMet())
asserts.NoError(err)
}
@@ -382,7 +410,7 @@ func TestHookClearFileSize(t *testing.T) {
// 上下文对象不存在
{
ctx := context.Background()
err := HookClearFileSize(ctx, fs, nil)
err := HookClearFileSize(ctx, fs)
asserts.Error(err)
}
@@ -404,7 +432,7 @@ func TestHookUpdateSourceName(t *testing.T) {
mock.ExpectBegin()
mock.ExpectExec("UPDATE(.+)").WithArgs("new.txt", sqlmock.AnyArg(), 1).WillReturnResult(sqlmock.NewResult(1, 1))
mock.ExpectCommit()
err := HookUpdateSourceName(ctx, fs, nil)
err := HookUpdateSourceName(ctx, fs)
asserts.NoError(mock.ExpectationsWereMet())
asserts.NoError(err)
}
@@ -412,7 +440,7 @@ func TestHookUpdateSourceName(t *testing.T) {
// 上下文错误
{
ctx := context.Background()
err := HookUpdateSourceName(ctx, fs, nil)
err := HookUpdateSourceName(ctx, fs)
asserts.Error(err)
}
}
@@ -429,32 +457,41 @@ func TestGenericAfterUpdate(t *testing.T) {
Model: gorm.Model{ID: 1},
PicInfo: "1,1",
}
newFile := &fsctx.FileStream{Size: 10}
newFile := local.FileStream{Size: 10}
ctx := context.WithValue(context.Background(), fsctx.FileModelCtx, originFile)
ctx = context.WithValue(ctx, fsctx.FileHeaderCtx, newFile)
handlerMock := FileHeaderMock{}
handlerMock.On("Delete", testMock.Anything, []string{"._thumb"}).Return([]string{}, nil)
fs.Handler = handlerMock
mock.ExpectBegin()
mock.ExpectExec("UPDATE(.+)files(.+)").
WithArgs(10, sqlmock.AnyArg(), 1, 0).
WillReturnResult(sqlmock.NewResult(1, 1))
mock.ExpectExec("UPDATE(.+)users(.+)").
WithArgs(10, sqlmock.AnyArg()).
mock.ExpectExec("UPDATE(.+)").
WithArgs(10, sqlmock.AnyArg(), 1).
WillReturnResult(sqlmock.NewResult(1, 1))
mock.ExpectCommit()
err := GenericAfterUpdate(ctx, fs, newFile)
err := GenericAfterUpdate(ctx, fs)
asserts.NoError(mock.ExpectationsWereMet())
asserts.NoError(err)
}
// 新文件上下文不存在
{
originFile := model.File{
Model: gorm.Model{ID: 1},
PicInfo: "1,1",
}
ctx := context.WithValue(context.Background(), fsctx.FileModelCtx, originFile)
err := GenericAfterUpdate(ctx, fs)
asserts.Error(err)
}
// 原始文件上下文不存在
{
newFile := &fsctx.FileStream{Size: 10}
ctx := context.Background()
err := GenericAfterUpdate(ctx, fs, newFile)
newFile := local.FileStream{Size: 10}
ctx := context.WithValue(context.Background(), fsctx.FileHeaderCtx, newFile)
err := GenericAfterUpdate(ctx, fs)
asserts.Error(err)
}
@@ -465,41 +502,91 @@ func TestGenericAfterUpdate(t *testing.T) {
Model: gorm.Model{ID: 1},
PicInfo: "1,1",
}
newFile := &fsctx.FileStream{Size: 10}
newFile := local.FileStream{Size: 10}
ctx := context.WithValue(context.Background(), fsctx.FileModelCtx, originFile)
ctx = context.WithValue(ctx, fsctx.FileHeaderCtx, newFile)
mock.ExpectBegin()
mock.ExpectExec("UPDATE(.+)").
WithArgs(10, sqlmock.AnyArg(), 1, 0).
WithArgs(10, sqlmock.AnyArg(), 1).
WillReturnError(errors.New("error"))
mock.ExpectRollback()
err := GenericAfterUpdate(ctx, fs, newFile)
err := GenericAfterUpdate(ctx, fs)
asserts.NoError(mock.ExpectationsWereMet())
asserts.Error(err)
}
}
func TestHookGenerateThumb(t *testing.T) {
a := assert.New(t)
mockHandler := &FileHeaderMock{}
fs := &FileSystem{
User: &model.User{
Model: gorm.Model{ID: 1},
},
Handler: mockHandler,
Policy: &model.Policy{Type: "local"},
func TestHookSlaveUploadValidate(t *testing.T) {
asserts := assert.New(t)
conf.SystemConfig.Mode = "slave"
fs, err := NewAnonymousFileSystem()
conf.SystemConfig.Mode = "master"
asserts.NoError(err)
// 正常
{
policy := serializer.UploadPolicy{
SavePath: "",
MaxSize: 10,
AllowedExtension: nil,
}
file := local.FileStream{Name: "1.txt", Size: 10}
ctx := context.WithValue(context.Background(), fsctx.UploadPolicyCtx, policy)
ctx = context.WithValue(ctx, fsctx.FileHeaderCtx, file)
asserts.NoError(HookSlaveUploadValidate(ctx, fs))
}
mockHandler.On("Delete", testMock.Anything, []string{"1.txt._thumb"}).Return([]string{}, nil)
a.NoError(HookGenerateThumb(context.Background(), fs, &fsctx.FileStream{
Model: &model.File{
SourceName: "1.txt",
},
}))
fs.Recycle()
mockHandler.AssertExpectations(t)
// 尺寸太大
{
policy := serializer.UploadPolicy{
SavePath: "",
MaxSize: 10,
AllowedExtension: nil,
}
file := local.FileStream{Name: "1.txt", Size: 11}
ctx := context.WithValue(context.Background(), fsctx.UploadPolicyCtx, policy)
ctx = context.WithValue(ctx, fsctx.FileHeaderCtx, file)
asserts.Equal(ErrFileSizeTooBig, HookSlaveUploadValidate(ctx, fs))
}
// 文件名非法
{
policy := serializer.UploadPolicy{
SavePath: "",
MaxSize: 10,
AllowedExtension: nil,
}
file := local.FileStream{Name: "/1.txt", Size: 10}
ctx := context.WithValue(context.Background(), fsctx.UploadPolicyCtx, policy)
ctx = context.WithValue(ctx, fsctx.FileHeaderCtx, file)
asserts.Equal(ErrIllegalObjectName, HookSlaveUploadValidate(ctx, fs))
}
// 扩展名非法
{
policy := serializer.UploadPolicy{
SavePath: "",
MaxSize: 10,
AllowedExtension: []string{"jpg"},
}
file := local.FileStream{Name: "1.txt", Size: 10}
ctx := context.WithValue(context.Background(), fsctx.UploadPolicyCtx, policy)
ctx = context.WithValue(ctx, fsctx.FileHeaderCtx, file)
asserts.Equal(ErrFileExtensionNotAllowed, HookSlaveUploadValidate(ctx, fs))
}
}
type ClientMock struct {
testMock.Mock
}
func (m ClientMock) Request(method, target string, body io.Reader, opts ...request.Option) *request.Response {
args := m.Called(method, target, body, opts)
return args.Get(0).(*request.Response)
}
func TestSlaveAfterUpload(t *testing.T) {
@@ -511,7 +598,7 @@ func TestSlaveAfterUpload(t *testing.T) {
// 成功
{
clientMock := requestmock.RequestMock{}
clientMock := ClientMock{}
clientMock.On(
"Request",
"POST",
@@ -526,28 +613,19 @@ func TestSlaveAfterUpload(t *testing.T) {
},
})
request.GeneralClient = clientMock
file := &fsctx.FileStream{
ctx := context.WithValue(context.Background(), fsctx.FileHeaderCtx, local.FileStream{
Size: 10,
VirtualPath: "/my",
Name: "test.txt",
SavePath: "/not_exist",
}
err := SlaveAfterUpload(&serializer.UploadSession{Callback: "http://test/callbakc"})(context.Background(), fs, file)
})
ctx = context.WithValue(ctx, fsctx.UploadPolicyCtx, serializer.UploadPolicy{
CallbackURL: "http://test/callbakc",
})
ctx = context.WithValue(ctx, fsctx.SavePathCtx, "/not_exist")
err := SlaveAfterUpload(ctx, fs)
clientMock.AssertExpectations(t)
asserts.NoError(err)
}
// 跳过回调
{
file := &fsctx.FileStream{
Size: 10,
VirtualPath: "/my",
Name: "test.txt",
SavePath: "/not_exist",
}
err := SlaveAfterUpload(&serializer.UploadSession{})(context.Background(), fs, file)
asserts.NoError(err)
}
}
func TestFileSystem_CleanHooks(t *testing.T) {
@@ -585,7 +663,7 @@ func TestHookCancelContext(t *testing.T) {
// empty ctx
{
asserts.NoError(HookCancelContext(ctx, fs, nil))
asserts.NoError(HookCancelContext(ctx, fs))
select {
case <-ctx.Done():
t.Errorf("Channel should not be closed")
@@ -597,99 +675,62 @@ func TestHookCancelContext(t *testing.T) {
// with cancel ctx
{
ctx = context.WithValue(ctx, fsctx.CancelFuncCtx, cancel)
asserts.NoError(HookCancelContext(ctx, fs, nil))
asserts.NoError(HookCancelContext(ctx, fs))
_, ok := <-ctx.Done()
asserts.False(ok)
}
}
func TestHookClearFileHeaderSize(t *testing.T) {
a := assert.New(t)
fs := &FileSystem{}
file := &fsctx.FileStream{Size: 10}
a.NoError(HookClearFileHeaderSize(context.Background(), fs, file))
a.EqualValues(0, file.Size)
}
func TestHookTruncateFileTo(t *testing.T) {
a := assert.New(t)
fs := &FileSystem{}
file := &fsctx.FileStream{}
a.NoError(HookTruncateFileTo(0)(context.Background(), fs, file))
fs.Handler = local.Driver{}
a.Error(HookTruncateFileTo(0)(context.Background(), fs, file))
}
func TestHookChunkUploaded(t *testing.T) {
a := assert.New(t)
fs := &FileSystem{}
file := &fsctx.FileStream{
AppendStart: 10,
Size: 10,
Model: &model.File{
Model: gorm.Model{ID: 1},
func TestHookGiveBackCapacity(t *testing.T) {
asserts := assert.New(t)
fs := &FileSystem{
User: &model.User{
Model: gorm.Model{ID: 1},
Storage: 10,
},
}
ctx := context.WithValue(context.Background(), fsctx.FileHeaderCtx, local.FileStream{Size: 1})
mock.ExpectBegin()
mock.ExpectExec("UPDATE(.+)files(.+)").WithArgs(20, sqlmock.AnyArg(), 1, 0).WillReturnResult(sqlmock.NewResult(1, 1))
mock.ExpectExec("UPDATE(.+)users(.+)").
WithArgs(20, sqlmock.AnyArg()).
WillReturnResult(sqlmock.NewResult(1, 1))
mock.ExpectCommit()
a.NoError(HookChunkUploaded(context.Background(), fs, file))
a.NoError(mock.ExpectationsWereMet())
}
func TestHookChunkUploadFailed(t *testing.T) {
a := assert.New(t)
fs := &FileSystem{}
file := &fsctx.FileStream{
AppendStart: 10,
Size: 10,
Model: &model.File{
Model: gorm.Model{ID: 1},
},
// without once limit
{
asserts.NoError(HookGiveBackCapacity(ctx, fs))
asserts.EqualValues(9, fs.User.Storage)
asserts.NoError(HookGiveBackCapacity(ctx, fs))
asserts.EqualValues(8, fs.User.Storage)
}
mock.ExpectBegin()
mock.ExpectExec("UPDATE(.+)files(.+)").WithArgs(10, sqlmock.AnyArg(), 1, 0).WillReturnResult(sqlmock.NewResult(1, 1))
mock.ExpectExec("UPDATE(.+)users(.+)").
WithArgs(10, sqlmock.AnyArg()).
WillReturnResult(sqlmock.NewResult(1, 1))
mock.ExpectCommit()
a.NoError(HookChunkUploadFailed(context.Background(), fs, file))
a.NoError(mock.ExpectationsWereMet())
// with once limit
{
ctx = context.WithValue(ctx, fsctx.ValidateCapacityOnceCtx, &sync.Once{})
asserts.NoError(HookGiveBackCapacity(ctx, fs))
asserts.EqualValues(7, fs.User.Storage)
asserts.NoError(HookGiveBackCapacity(ctx, fs))
asserts.EqualValues(7, fs.User.Storage)
}
}
func TestHookPopPlaceholderToFile(t *testing.T) {
func TestHookValidateCapacityWithoutIncrease(t *testing.T) {
a := assert.New(t)
fs := &FileSystem{}
file := &fsctx.FileStream{
Model: &model.File{
Model: gorm.Model{ID: 1},
fs := &FileSystem{
User: &model.User{
Model: gorm.Model{ID: 1},
Storage: 10,
Group: model.Group{},
},
}
ctx := context.WithValue(context.Background(), fsctx.FileHeaderCtx, local.FileStream{Size: 1})
mock.ExpectBegin()
mock.ExpectExec("UPDATE(.+)files(.+)").WillReturnResult(sqlmock.NewResult(1, 1))
mock.ExpectCommit()
a.NoError(HookPopPlaceholderToFile("1,1")(context.Background(), fs, file))
a.NoError(mock.ExpectationsWereMet())
}
func TestHookDeleteUploadSession(t *testing.T) {
a := assert.New(t)
fs := &FileSystem{}
file := &fsctx.FileStream{
Model: &model.File{
Model: gorm.Model{ID: 1},
},
// not enough
{
fs.User.Group.MaxStorage = 10
a.Error(HookValidateCapacityWithoutIncrease(ctx, fs))
a.EqualValues(10, fs.User.Storage)
}
cache.Set(UploadSessionCachePrefix+"TestHookDeleteUploadSession", "", 0)
a.NoError(HookDeleteUploadSession("TestHookDeleteUploadSession")(context.Background(), fs, file))
_, ok := cache.Get(UploadSessionCachePrefix + "TestHookDeleteUploadSession")
a.False(ok)
// enough
{
fs.User.Group.MaxStorage = 11
a.NoError(HookValidateCapacityWithoutIncrease(ctx, fs))
a.EqualValues(10, fs.User.Storage)
}
}

View File

@@ -3,6 +3,7 @@ package filesystem
import (
"context"
"fmt"
"strconv"
"sync"
"runtime"
@@ -64,7 +65,7 @@ type Pool struct {
// Init 初始化任务池
func getThumbWorker() *Pool {
once.Do(func() {
maxWorker := model.GetIntSetting("thumb_max_task_count", -1)
maxWorker := conf.ThumbConfig.MaxTaskCount
if maxWorker <= 0 {
maxWorker = runtime.GOMAXPROCS(0)
}
@@ -117,9 +118,9 @@ func (fs *FileSystem) GenerateThumbnail(ctx context.Context, file *model.File) {
// 生成缩略图
image.GetThumb(fs.GenerateThumbnailSize(w, h))
// 保存到文件
err = image.Save(util.RelativePath(file.SourceName + model.GetSettingByNameWithDefault("thumb_file_suffix", "._thumb")))
err = image.Save(util.RelativePath(file.SourceName + conf.ThumbConfig.FileSuffix))
image = nil
if model.IsTrueVal(model.GetSettingByName("thumb_gc_after_gen")) {
if conf.ThumbConfig.GCAfterGen {
util.Log().Debug("GenerateThumbnail runtime.GC")
runtime.GC()
}
@@ -138,11 +139,17 @@ func (fs *FileSystem) GenerateThumbnail(ctx context.Context, file *model.File) {
// 失败时删除缩略图文件
if err != nil {
_, _ = fs.Handler.Delete(newCtx, []string{file.SourceName + model.GetSettingByNameWithDefault("thumb_file_suffix", "._thumb")})
_, _ = fs.Handler.Delete(newCtx, []string{file.SourceName + conf.ThumbConfig.FileSuffix})
}
}
// GenerateThumbnailSize 获取要生成的缩略图的尺寸
func (fs *FileSystem) GenerateThumbnailSize(w, h int) (uint, uint) {
return uint(model.GetIntSetting("thumb_width", 400)), uint(model.GetIntSetting("thumb_width", 300))
if conf.SystemConfig.Mode == "master" {
options := model.GetSettingByNames("thumb_width", "thumb_height")
w, _ := strconv.ParseUint(options["thumb_width"], 10, 32)
h, _ := strconv.ParseUint(options["thumb_height"], 10, 32)
return uint(w), uint(h)
}
return conf.ThumbConfig.MaxWidth, conf.ThumbConfig.MaxHeight
}

View File

@@ -2,15 +2,13 @@ package filesystem
import (
"context"
"errors"
"testing"
model "github.com/cloudreve/Cloudreve/v3/models"
"github.com/cloudreve/Cloudreve/v3/pkg/cache"
"github.com/cloudreve/Cloudreve/v3/pkg/filesystem/response"
"github.com/cloudreve/Cloudreve/v3/pkg/request"
testMock "github.com/stretchr/testify/mock"
"testing"
"github.com/stretchr/testify/assert"
testMock "github.com/stretchr/testify/mock"
)
func TestFileSystem_GetThumb(t *testing.T) {
@@ -49,22 +47,3 @@ func TestFileSystem_ThumbWorker(t *testing.T) {
getThumbWorker().releaseWorker()
})
}
func TestFileSystem_GenerateThumbnail(t *testing.T) {
fs := &FileSystem{User: &model.User{}}
// 无法生成缩略图
{
fs.SetTargetFile(&[]model.File{{}})
fs.GenerateThumbnail(context.Background(), &model.File{})
}
// 无法获取文件数据
{
testHandller := new(FileHeaderMock)
testHandller.On("Get", testMock.Anything, "").Return(request.NopRSCloser{}, errors.New("error"))
fs.Handler = testHandller
fs.GenerateThumbnail(context.Background(), &model.File{Name: "test.png"})
testHandller.AssertExpectations(t)
}
}

View File

@@ -5,6 +5,7 @@ import (
"fmt"
"path"
"strings"
"time"
model "github.com/cloudreve/Cloudreve/v3/models"
"github.com/cloudreve/Cloudreve/v3/pkg/filesystem/fsctx"
@@ -18,6 +19,18 @@ import (
=================
*/
// Object 文件或者目录
type Object struct {
ID string `json:"id"`
Name string `json:"name"`
Path string `json:"path"`
Pic string `json:"pic"`
Size uint64 `json:"size"`
Type string `json:"type"`
Date time.Time `json:"date"`
Key string `json:"key,omitempty"`
}
// Rename 重命名对象
func (fs *FileSystem) Rename(ctx context.Context, dir, file []uint, new string) (err error) {
// 验证新名字
@@ -122,12 +135,15 @@ func (fs *FileSystem) Move(ctx context.Context, dirs, files []uint, src, dst str
// Delete 递归删除对象, force 为 true 时强制删除文件记录,忽略物理删除是否成功
func (fs *FileSystem) Delete(ctx context.Context, dirs, files []uint, force bool) error {
// 已删除的总容量,map用于去重
var deletedStorage = make(map[uint]uint64)
var totalStorage = make(map[uint]uint64)
// 已删除的文件ID
var deletedFiles = make([]*model.File, 0, len(fs.FileTarget))
var deletedFileIDs = make([]uint, 0, len(fs.FileTarget))
// 删除失败的文件的父目录ID
// 所有文件的ID
var allFiles = make([]*model.File, 0, len(fs.FileTarget))
var allFileIDs = make([]uint, 0, len(fs.FileTarget))
// 列出要删除的目录
if len(dirs) > 0 {
@@ -161,35 +177,38 @@ func (fs *FileSystem) Delete(ctx context.Context, dirs, files []uint, force bool
for i := 0; i < len(fs.FileTarget); i++ {
if !util.ContainsString(failed[fs.FileTarget[i].PolicyID], fs.FileTarget[i].SourceName) {
// 已成功删除的文件
deletedFiles = append(deletedFiles, &fs.FileTarget[i])
deletedFileIDs = append(deletedFileIDs, fs.FileTarget[i].ID)
deletedStorage[fs.FileTarget[i].ID] = fs.FileTarget[i].Size
}
// 全部文件
allFiles = append(allFiles, &fs.FileTarget[i])
totalStorage[fs.FileTarget[i].ID] = fs.FileTarget[i].Size
allFileIDs = append(allFileIDs, fs.FileTarget[i].ID)
}
// 如果强制删除,则将全部文件视为删除成功
if force {
deletedFiles = allFiles
deletedFileIDs = allFileIDs
deletedStorage = totalStorage
}
// 删除文件记录
err = model.DeleteFiles(deletedFiles, fs.User.ID)
err = model.DeleteFileByIDs(deletedFileIDs)
if err != nil {
return ErrDBDeleteObjects.WithError(err)
}
// 删除文件记录对应的分享记录
// TODO 先取消分享再删除文件
deletedFileIDs := make([]uint, len(deletedFiles))
for k, file := range deletedFiles {
deletedFileIDs[k] = file.ID
}
model.DeleteShareBySourceIDs(deletedFileIDs, false)
// 归还容量
var total uint64
for _, value := range deletedStorage {
total += value
}
fs.User.DeductionStorage(total)
// 如果文件全部删除成功,继续删除目录
if len(deletedFiles) == len(allFiles) {
if len(deletedFileIDs) == len(allFileIDs) {
var allFolderIDs = make([]uint, 0, len(fs.DirTarget))
for _, value := range fs.DirTarget {
allFolderIDs = append(allFolderIDs, value.ID)
@@ -203,7 +222,7 @@ func (fs *FileSystem) Delete(ctx context.Context, dirs, files []uint, force bool
model.DeleteShareBySourceIDs(allFolderIDs, true)
}
if notDeleted := len(fs.FileTarget) - len(deletedFiles); notDeleted > 0 {
if notDeleted := len(fs.FileTarget) - len(deletedFileIDs); notDeleted > 0 {
return serializer.NewError(
serializer.CodeNotFullySuccess,
fmt.Sprintf("有 %d 个文件未能成功删除", notDeleted),
@@ -221,15 +240,6 @@ func (fs *FileSystem) ListDeleteDirs(ctx context.Context, ids []uint) error {
if err != nil {
return ErrDBListObjects.WithError(err)
}
// 忽略根目录
for i := 0; i < len(folders); i++ {
if folders[i].ParentID == nil {
folders = append(folders[:i], folders[i+1:]...)
break
}
}
fs.SetTargetDir(&folders)
// 检索目录下的子文件
@@ -256,7 +266,7 @@ func (fs *FileSystem) ListDeleteFiles(ctx context.Context, ids []uint) error {
// pathProcessor为最终对象路径的处理钩子。
// 有些情况下(如在分享页面列对象)时,
// 路径需要截取掉被分享目录路径之前的部分。
func (fs *FileSystem) List(ctx context.Context, dirPath string, pathProcessor func(string) string) ([]serializer.Object, error) {
func (fs *FileSystem) List(ctx context.Context, dirPath string, pathProcessor func(string) string) ([]Object, error) {
// 获取父目录
isExist, folder := fs.IsPathExist(dirPath)
if !isExist {
@@ -279,7 +289,7 @@ func (fs *FileSystem) List(ctx context.Context, dirPath string, pathProcessor fu
// ListPhysical 列出存储策略中的外部目录
// TODO:测试
func (fs *FileSystem) ListPhysical(ctx context.Context, dirPath string) ([]serializer.Object, error) {
func (fs *FileSystem) ListPhysical(ctx context.Context, dirPath string) ([]Object, error) {
if err := fs.DispatchHandler(); fs.Policy == nil || err != nil {
return nil, ErrUnknownPolicyType
}
@@ -309,7 +319,7 @@ func (fs *FileSystem) ListPhysical(ctx context.Context, dirPath string) ([]seria
return fs.listObjects(ctx, dirPath, nil, folders, nil), nil
}
func (fs *FileSystem) listObjects(ctx context.Context, parent string, files []model.File, folders []model.Folder, pathProcessor func(string) string) []serializer.Object {
func (fs *FileSystem) listObjects(ctx context.Context, parent string, files []model.File, folders []model.Folder, pathProcessor func(string) string) []Object {
// 分享文件的ID
shareKey := ""
if key, ok := ctx.Value(fsctx.ShareKeyCtx).(string); ok {
@@ -317,7 +327,7 @@ func (fs *FileSystem) listObjects(ctx context.Context, parent string, files []mo
}
// 汇总处理结果
objects := make([]serializer.Object, 0, len(files)+len(folders))
objects := make([]Object, 0, len(files)+len(folders))
// 所有对象的父目录
var processedPath string
@@ -333,15 +343,14 @@ func (fs *FileSystem) listObjects(ctx context.Context, parent string, files []mo
}
}
objects = append(objects, serializer.Object{
ID: hashid.HashID(subFolder.ID, hashid.FolderID),
Name: subFolder.Name,
Path: processedPath,
Pic: "",
Size: 0,
Type: "dir",
Date: subFolder.UpdatedAt,
CreateDate: subFolder.CreatedAt,
objects = append(objects, Object{
ID: hashid.HashID(subFolder.ID, hashid.FolderID),
Name: subFolder.Name,
Path: processedPath,
Pic: "",
Size: 0,
Type: "dir",
Date: subFolder.CreatedAt,
})
}
@@ -354,39 +363,30 @@ func (fs *FileSystem) listObjects(ctx context.Context, parent string, files []mo
}
}
if file.UploadSessionID == nil {
newFile := serializer.Object{
ID: hashid.HashID(file.ID, hashid.FileID),
Name: file.Name,
Path: processedPath,
Pic: file.PicInfo,
Size: file.Size,
Type: "file",
Date: file.UpdatedAt,
SourceEnabled: file.GetPolicy().IsOriginLinkEnable,
CreateDate: file.CreatedAt,
}
if shareKey != "" {
newFile.Key = shareKey
}
objects = append(objects, newFile)
newFile := Object{
ID: hashid.HashID(file.ID, hashid.FileID),
Name: file.Name,
Path: processedPath,
Pic: file.PicInfo,
Size: file.Size,
Type: "file",
Date: file.CreatedAt,
}
if shareKey != "" {
newFile.Key = shareKey
}
objects = append(objects, newFile)
}
return objects
}
// CreateDirectory 根据给定的完整创建目录,支持递归创建。如果目录已存在,则直接
// 返回已存在的目录。
// CreateDirectory 根据给定的完整创建目录,支持递归创建
func (fs *FileSystem) CreateDirectory(ctx context.Context, fullPath string) (*model.Folder, error) {
if fullPath == "." || fullPath == "" {
if fullPath == "/" || fullPath == "." || fullPath == "" {
return nil, ErrRootProtected
}
if fullPath == "/" {
return fs.User.Root()
}
// 获取要创建目录的父路径和目录名
fullPath = path.Clean(fullPath)
base := path.Dir(fullPath)
@@ -403,6 +403,10 @@ func (fs *FileSystem) CreateDirectory(ctx context.Context, fullPath string) (*mo
// 父目录是否存在
isExist, parent := fs.IsPathExist(base)
if !isExist {
// 递归创建父目录
if _, ok := ctx.Value(fsctx.IgnoreDirectoryConflictCtx).(bool); !ok {
ctx = context.WithValue(ctx, fsctx.IgnoreDirectoryConflictCtx, true)
}
newParent, err := fs.CreateDirectory(ctx, base)
if err != nil {
return nil, err
@@ -424,9 +428,11 @@ func (fs *FileSystem) CreateDirectory(ctx context.Context, fullPath string) (*mo
_, err := newFolder.Create()
if err != nil {
return nil, fmt.Errorf("failed to create folder: %w", err)
}
if _, ok := ctx.Value(fsctx.IgnoreDirectoryConflictCtx).(bool); !ok {
return nil, ErrFolderExisted
}
}
return &newFolder, nil
}

View File

@@ -3,8 +3,6 @@ package filesystem
import (
"context"
"errors"
"github.com/cloudreve/Cloudreve/v3/pkg/filesystem/response"
testMock "github.com/stretchr/testify/mock"
"os"
"testing"
@@ -13,10 +11,12 @@ import (
"github.com/cloudreve/Cloudreve/v3/pkg/cache"
"github.com/cloudreve/Cloudreve/v3/pkg/conf"
"github.com/cloudreve/Cloudreve/v3/pkg/filesystem/fsctx"
"github.com/cloudreve/Cloudreve/v3/pkg/filesystem/response"
"github.com/cloudreve/Cloudreve/v3/pkg/serializer"
"github.com/cloudreve/Cloudreve/v3/pkg/util"
"github.com/jinzhu/gorm"
"github.com/stretchr/testify/assert"
testMock "github.com/stretchr/testify/mock"
)
func TestFileSystem_ListPhysical(t *testing.T) {
@@ -214,7 +214,7 @@ func TestFileSystem_CreateDirectory(t *testing.T) {
asserts.Equal(ErrFileExisted, err)
asserts.NoError(mock.ExpectationsWereMet())
// 存在同名目录,直接返回
// 存在同名目录
mock.ExpectQuery("SELECT(.+)").
WithArgs(1).
WillReturnRows(sqlmock.NewRows([]string{"id", "owner_id"}).AddRow(1, 1))
@@ -225,101 +225,35 @@ func TestFileSystem_CreateDirectory(t *testing.T) {
mock.ExpectQuery("SELECT(.+)files").WillReturnRows(sqlmock.NewRows([]string{"id", "name"}))
mock.ExpectBegin()
// ab
mock.ExpectQuery("SELECT(.+)").
WithArgs("ab", 2, 1).
WillReturnRows(sqlmock.NewRows([]string{"id", "owner_id"}).AddRow(3, 1))
mock.ExpectCommit()
res, err := fs.CreateDirectory(ctx, "/ad/ab")
asserts.NoError(err)
asserts.EqualValues(3, res.ID)
asserts.NoError(mock.ExpectationsWereMet())
// 成功创建
mock.ExpectQuery("SELECT(.+)").
WithArgs(1).
WillReturnRows(sqlmock.NewRows([]string{"id", "owner_id"}).AddRow(1, 1))
// ad
mock.ExpectQuery("SELECT(.+)").
WithArgs(1, 1, "ad").
WillReturnRows(sqlmock.NewRows([]string{"id", "owner_id"}).AddRow(2, 1))
mock.ExpectQuery("SELECT(.+)files").WillReturnRows(sqlmock.NewRows([]string{"id", "name"}))
mock.ExpectBegin()
mock.ExpectQuery("SELECT(.+)").
WithArgs("ab", 2, 1).
WillReturnRows(sqlmock.NewRows([]string{"id", "owner_id"}))
mock.ExpectExec("INSERT(.+)").WillReturnResult(sqlmock.NewResult(1, 1))
mock.ExpectCommit()
_, err = fs.CreateDirectory(ctx, "/ad/ab")
asserts.NoError(err)
asserts.NoError(mock.ExpectationsWereMet())
// 成功创建, 递归创建父目录
// 根目录
mock.ExpectQuery("SELECT(.+)").
WithArgs(1).
WillReturnRows(sqlmock.NewRows([]string{"id", "owner_id"}).AddRow(1, 1))
// ad
mock.ExpectQuery("SELECT(.+)").
WithArgs(1, 1, "ad").
WillReturnRows(sqlmock.NewRows([]string{"id", "owner_id"}))
// 根目录
mock.ExpectQuery("SELECT(.+)").
WithArgs(1).
WillReturnRows(sqlmock.NewRows([]string{"id", "owner_id"}).AddRow(1, 1))
mock.ExpectQuery("SELECT(.+)files").WillReturnRows(sqlmock.NewRows([]string{"id", "name"}))
// 创建ad
mock.ExpectBegin()
mock.ExpectQuery("SELECT(.+)").
WithArgs("ad", 1, 1).
WillReturnRows(sqlmock.NewRows([]string{"id", "owner_id"}))
mock.ExpectExec("INSERT(.+)").WillReturnResult(sqlmock.NewResult(2, 1))
mock.ExpectCommit()
mock.ExpectQuery("SELECT(.+)files").WillReturnRows(sqlmock.NewRows([]string{"id", "name"}))
// 创建ab
mock.ExpectBegin()
mock.ExpectQuery("SELECT(.+)").
WithArgs("ab", 2, 1).
WillReturnRows(sqlmock.NewRows([]string{"id", "owner_id"}))
mock.ExpectExec("INSERT(.+)").WillReturnResult(sqlmock.NewResult(1, 1))
mock.ExpectCommit()
_, err = fs.CreateDirectory(ctx, "/ad/ab")
asserts.NoError(err)
asserts.NoError(mock.ExpectationsWereMet())
// 底层创建失败
// 成功创建
mock.ExpectQuery("SELECT(.+)").
WithArgs(1).
WillReturnRows(sqlmock.NewRows([]string{"id", "owner_id"}).AddRow(1, 1))
// ad
mock.ExpectQuery("SELECT(.+)").
WithArgs(1, 1, "ad").
WillReturnRows(sqlmock.NewRows([]string{"id", "owner_id"}))
// 根目录
mock.ExpectQuery("SELECT(.+)").
WithArgs(1).
WillReturnRows(sqlmock.NewRows([]string{"id", "owner_id"}).AddRow(1, 1))
mock.ExpectQuery("SELECT(.+)files").WillReturnRows(sqlmock.NewRows([]string{"id", "name"}))
// 创建ad
mock.ExpectBegin()
mock.ExpectQuery("SELECT(.+)").
WithArgs("ad", 1, 1).
WillReturnRows(sqlmock.NewRows([]string{"id", "owner_id"}))
mock.ExpectExec("INSERT(.+)").WillReturnResult(sqlmock.NewResult(2, 1)).WillReturnError(errors.New("error"))
mock.ExpectExec("INSERT(.+)").WillReturnError(errors.New("s"))
mock.ExpectRollback()
_, err = fs.CreateDirectory(ctx, "/ad/ab")
asserts.Error(err)
asserts.NoError(mock.ExpectationsWereMet())
// 直接创建根目录
// 成功创建
// 根目录
mock.ExpectQuery("SELECT(.+)").
WithArgs(1).
WillReturnRows(sqlmock.NewRows([]string{"id", "owner_id"}).AddRow(1, 1))
_, err = fs.CreateDirectory(ctx, "/")
// ad
mock.ExpectQuery("SELECT(.+)").
WithArgs(1, 1, "ad").
WillReturnRows(sqlmock.NewRows([]string{"id", "owner_id"}).AddRow(2, 1))
mock.ExpectQuery("SELECT(.+)files").WillReturnRows(sqlmock.NewRows([]string{"id", "name"}))
mock.ExpectBegin()
mock.ExpectExec("INSERT(.+)").WillReturnResult(sqlmock.NewResult(1, 1))
mock.ExpectCommit()
_, err = fs.CreateDirectory(ctx, "/ad/ab")
asserts.NoError(err)
asserts.NoError(mock.ExpectationsWereMet())
// 父目录不存在
mock.ExpectQuery("SELECT(.+)folders").WillReturnRows(sqlmock.NewRows([]string{"id", "name"}))
_, err = fs.CreateDirectory(ctx, "/ad")
asserts.Equal(ErrRootProtected, err)
asserts.NoError(mock.ExpectationsWereMet())
}
func TestFileSystem_ListDeleteFiles(t *testing.T) {
@@ -362,12 +296,12 @@ func TestFileSystem_ListDeleteDirs(t *testing.T) {
{
mock.ExpectQuery("SELECT(.+)").
WillReturnRows(
sqlmock.NewRows([]string{"id", "parent_id"}).
AddRow(1, 0).
AddRow(2, 0).
AddRow(3, 0),
sqlmock.NewRows([]string{"id"}).
AddRow(1).
AddRow(2).
AddRow(3),
)
mock.ExpectQuery("SELECT(.+)files(.+)").
mock.ExpectQuery("SELECT(.+)").
WithArgs(1, 2, 3).
WillReturnRows(
sqlmock.NewRows([]string{"id", "name"}).
@@ -382,47 +316,21 @@ func TestFileSystem_ListDeleteDirs(t *testing.T) {
asserts.NoError(mock.ExpectationsWereMet())
}
// 成功,忽略根目录
{
mock.ExpectQuery("SELECT(.+)").
WillReturnRows(
sqlmock.NewRows([]string{"id", "parent_id"}).
AddRow(1, 0).
AddRow(2, nil).
AddRow(3, 0),
)
mock.ExpectQuery("SELECT(.+)files(.+)").
WithArgs(1, 3).
WillReturnRows(
sqlmock.NewRows([]string{"id", "name"}).
AddRow(4, "1.txt").
AddRow(5, "2.txt").
AddRow(6, "3.txt"),
)
fs.CleanTargets()
err := fs.ListDeleteDirs(context.Background(), []uint{1})
asserts.NoError(err)
asserts.Len(fs.FileTarget, 3)
asserts.Len(fs.DirTarget, 2)
asserts.NoError(mock.ExpectationsWereMet())
}
// 检索文件发生错误
{
mock.ExpectQuery("SELECT(.+)").
WillReturnRows(
sqlmock.NewRows([]string{"id", "parent_id"}).
AddRow(1, 0).
AddRow(2, 0).
AddRow(3, 0),
sqlmock.NewRows([]string{"id"}).
AddRow(1).
AddRow(2).
AddRow(3),
)
mock.ExpectQuery("SELECT(.+)").
WithArgs(1, 2, 3).
WillReturnError(errors.New("error"))
fs.CleanTargets()
err := fs.ListDeleteDirs(context.Background(), []uint{1})
asserts.Error(err)
asserts.Len(fs.DirTarget, 3)
asserts.Len(fs.DirTarget, 6)
asserts.NoError(mock.ExpectationsWereMet())
}
// 检索目录发生错误
@@ -441,7 +349,7 @@ func TestFileSystem_Delete(t *testing.T) {
cache.Set("pack_size_1", uint64(0), 0)
fs := &FileSystem{User: &model.User{
Model: gorm.Model{
ID: 0,
ID: 1,
},
Storage: 3,
Group: model.Group{MaxStorage: 3},
@@ -453,10 +361,10 @@ func TestFileSystem_Delete(t *testing.T) {
fs.CleanTargets()
mock.ExpectQuery("SELECT(.+)").
WillReturnRows(
sqlmock.NewRows([]string{"id", "parent_id"}).
AddRow(1, 0).
AddRow(2, 0).
AddRow(3, 0),
sqlmock.NewRows([]string{"id"}).
AddRow(1).
AddRow(2).
AddRow(3),
)
mock.ExpectQuery("SELECT(.+)").
WithArgs(1, 2, 3).
@@ -472,16 +380,18 @@ func TestFileSystem_Delete(t *testing.T) {
// 删除文件记录
mock.ExpectBegin()
mock.ExpectExec("DELETE(.+)").
WillReturnResult(sqlmock.NewResult(0, 1))
mock.ExpectExec("DELETE(.+)").
WillReturnResult(sqlmock.NewResult(0, 1))
mock.ExpectExec("UPDATE(.+)users(.+)storage(.+)").WillReturnResult(sqlmock.NewResult(1, 1))
WillReturnResult(sqlmock.NewResult(0, 3))
mock.ExpectCommit()
// 删除对应分享
mock.ExpectBegin()
mock.ExpectExec("UPDATE(.+)shares").
WillReturnResult(sqlmock.NewResult(0, 3))
mock.ExpectCommit()
// 归还容量
mock.ExpectBegin()
mock.ExpectExec("UPDATE(.+)").
WillReturnResult(sqlmock.NewResult(0, 3))
mock.ExpectCommit()
// 删除目录
mock.ExpectBegin()
mock.ExpectExec("DELETE(.+)").
@@ -497,6 +407,7 @@ func TestFileSystem_Delete(t *testing.T) {
fs.DirTarget = []model.Folder{}
err := fs.Delete(ctx, []uint{1}, []uint{1}, true)
asserts.NoError(err)
asserts.Equal(uint64(0), fs.User.Storage)
}
//全部成功
{
@@ -508,10 +419,10 @@ func TestFileSystem_Delete(t *testing.T) {
asserts.NoError(err)
mock.ExpectQuery("SELECT(.+)").
WillReturnRows(
sqlmock.NewRows([]string{"id", "parent_id"}).
AddRow(1, 0).
AddRow(2, 0).
AddRow(3, 0),
sqlmock.NewRows([]string{"id"}).
AddRow(1).
AddRow(2).
AddRow(3),
)
mock.ExpectQuery("SELECT(.+)").
WithArgs(1, 2, 3).
@@ -527,16 +438,18 @@ func TestFileSystem_Delete(t *testing.T) {
// 删除文件记录
mock.ExpectBegin()
mock.ExpectExec("DELETE(.+)").
WillReturnResult(sqlmock.NewResult(0, 1))
mock.ExpectExec("DELETE(.+)").
WillReturnResult(sqlmock.NewResult(0, 1))
mock.ExpectExec("UPDATE(.+)users(.+)storage(.+)").WillReturnResult(sqlmock.NewResult(1, 1))
WillReturnResult(sqlmock.NewResult(0, 3))
mock.ExpectCommit()
// 删除对应分享
mock.ExpectBegin()
mock.ExpectExec("UPDATE(.+)shares").
WillReturnResult(sqlmock.NewResult(0, 3))
mock.ExpectCommit()
// 归还容量
mock.ExpectBegin()
mock.ExpectExec("UPDATE(.+)").
WillReturnResult(sqlmock.NewResult(0, 3))
mock.ExpectCommit()
// 删除目录
mock.ExpectBegin()
mock.ExpectExec("DELETE(.+)").
@@ -552,6 +465,7 @@ func TestFileSystem_Delete(t *testing.T) {
fs.DirTarget = []model.Folder{}
err = fs.Delete(ctx, []uint{1}, []uint{1}, false)
asserts.NoError(err)
asserts.Equal(uint64(0), fs.User.Storage)
}
}
@@ -659,9 +573,7 @@ func TestFileSystem_Rename(t *testing.T) {
Model: gorm.Model{
ID: 1,
},
},
Policy: &model.Policy{},
}
}}
ctx := context.Background()
// 重命名文件 成功
@@ -771,7 +683,7 @@ func TestFileSystem_Rename(t *testing.T) {
// 新名字是文件,扩展名不合法
{
fs.Policy.OptionsSerialized.FileType = []string{"txt"}
fs.User.Policy.OptionsSerialized.FileType = []string{"txt"}
err := fs.Rename(ctx, []uint{}, []uint{10}, "1.jpg")
asserts.Error(err)
asserts.Equal(ErrIllegalObjectName, err)
@@ -779,7 +691,7 @@ func TestFileSystem_Rename(t *testing.T) {
// 新名字是目录,不应该检测扩展名
{
fs.Policy.OptionsSerialized.FileType = []string{"txt"}
fs.User.Policy.OptionsSerialized.FileType = []string{"txt"}
mock.ExpectQuery("SELECT(.+)folders(.+)").
WithArgs(10, 1).
WillReturnRows(sqlmock.NewRows([]string{"id", "name"}))

View File

@@ -2,18 +2,18 @@ package filesystem
import (
"context"
"io"
"os"
"path"
"time"
model "github.com/cloudreve/Cloudreve/v3/models"
"github.com/cloudreve/Cloudreve/v3/pkg/cache"
"github.com/cloudreve/Cloudreve/v3/pkg/filesystem/driver/local"
"github.com/cloudreve/Cloudreve/v3/pkg/filesystem/fsctx"
"github.com/cloudreve/Cloudreve/v3/pkg/request"
"github.com/cloudreve/Cloudreve/v3/pkg/serializer"
"github.com/cloudreve/Cloudreve/v3/pkg/util"
"github.com/gin-gonic/gin"
"github.com/gofrs/uuid"
)
/* ================
@@ -21,17 +21,12 @@ import (
================
*/
const (
UploadSessionMetaKey = "upload_session"
UploadSessionCtx = "uploadSession"
UserCtx = "user"
UploadSessionCachePrefix = "callback_"
)
// Upload 上传文件
func (fs *FileSystem) Upload(ctx context.Context, file *fsctx.FileStream) (err error) {
func (fs *FileSystem) Upload(ctx context.Context, file FileHeader) (err error) {
ctx = context.WithValue(ctx, fsctx.FileHeaderCtx, file)
// 上传前的钩子
err = fs.Trigger(ctx, "BeforeUpload", file)
err = fs.Trigger(ctx, "BeforeUpload")
if err != nil {
request.BlackHole(file)
return err
@@ -39,34 +34,30 @@ func (fs *FileSystem) Upload(ctx context.Context, file *fsctx.FileStream) (err e
// 生成文件名和路径,
var savePath string
if file.SavePath == "" {
// 如果是更新操作就从上下文中获取
if originFile, ok := ctx.Value(fsctx.FileModelCtx).(model.File); ok {
savePath = originFile.SourceName
} else {
savePath = fs.GenerateSavePath(ctx, file)
}
file.SavePath = savePath
// 如果是更新操作就从上下文中获取
if originFile, ok := ctx.Value(fsctx.FileModelCtx).(model.File); ok {
savePath = originFile.SourceName
} else {
savePath = fs.GenerateSavePath(ctx, file)
}
ctx = context.WithValue(ctx, fsctx.SavePathCtx, savePath)
// 处理客户端未完成上传时,关闭连接
go fs.CancelUpload(ctx, savePath, file)
// 保存文件
if file.Mode&fsctx.Nop != fsctx.Nop {
// 处理客户端未完成上传时,关闭连接
go fs.CancelUpload(ctx, savePath, file)
err = fs.Handler.Put(ctx, file)
if err != nil {
fs.Trigger(ctx, "AfterUploadFailed", file)
return err
}
err = fs.Handler.Put(ctx, file, savePath, file.GetSize())
if err != nil {
fs.Trigger(ctx, "AfterUploadFailed")
return err
}
// 上传完成后的钩子
err = fs.Trigger(ctx, "AfterUpload", file)
err = fs.Trigger(ctx, "AfterUpload")
if err != nil {
// 上传完成后续处理失败
followUpErr := fs.Trigger(ctx, "AfterValidateFailed", file)
followUpErr := fs.Trigger(ctx, "AfterValidateFailed")
// 失败后再失败...
if followUpErr != nil {
util.Log().Debug("AfterValidateFailed 钩子执行失败,%s", followUpErr)
@@ -75,28 +66,56 @@ func (fs *FileSystem) Upload(ctx context.Context, file *fsctx.FileStream) (err e
return err
}
util.Log().Info(
"新文件PUT:%s , 大小:%d, 上传者:%s",
file.GetFileName(),
file.GetSize(),
fs.User.Nick,
)
return nil
}
// GenerateSavePath 生成要存放文件的路径
// TODO 完善测试
func (fs *FileSystem) GenerateSavePath(ctx context.Context, file fsctx.FileHeader) string {
fileInfo := file.Info()
func (fs *FileSystem) GenerateSavePath(ctx context.Context, file FileHeader) string {
if fs.User.Model.ID != 0 {
return path.Join(
fs.User.Policy.GeneratePath(
fs.User.Model.ID,
file.GetVirtualPath(),
),
fs.User.Policy.GenerateFileName(
fs.User.Model.ID,
file.GetFileName(),
),
)
}
// 匿名文件系统尝试根据上下文中的上传策略生成路径
var anonymousPolicy model.Policy
if policy, ok := ctx.Value(fsctx.UploadPolicyCtx).(serializer.UploadPolicy); ok {
anonymousPolicy = model.Policy{
Type: "remote",
AutoRename: policy.AutoRename,
DirNameRule: policy.SavePath,
FileNameRule: policy.FileName,
}
}
return path.Join(
fs.Policy.GeneratePath(
fs.User.Model.ID,
fileInfo.VirtualPath,
anonymousPolicy.GeneratePath(
0,
"",
),
fs.Policy.GenerateFileName(
fs.User.Model.ID,
fileInfo.FileName,
anonymousPolicy.GenerateFileName(
0,
file.GetFileName(),
),
)
}
// CancelUpload 监测客户端取消上传
func (fs *FileSystem) CancelUpload(ctx context.Context, path string, file fsctx.FileHeader) {
func (fs *FileSystem) CancelUpload(ctx context.Context, path string, file FileHeader) {
var reqContext context.Context
if ginCtx, ok := ctx.Value(fsctx.GinCtx).(*gin.Context); ok {
reqContext = ginCtx.Request.Context()
@@ -117,7 +136,8 @@ func (fs *FileSystem) CancelUpload(ctx context.Context, path string, file fsctx.
if fs.Hooks["AfterUploadCanceled"] == nil {
return
}
err := fs.Trigger(ctx, "AfterUploadCanceled", file)
ctx = context.WithValue(ctx, fsctx.SavePathCtx, path)
err := fs.Trigger(ctx, "AfterUploadCanceled")
if err != nil {
util.Log().Debug("执行 AfterUploadCanceled 钩子出错,%s", err)
}
@@ -126,81 +146,67 @@ func (fs *FileSystem) CancelUpload(ctx context.Context, path string, file fsctx.
}
}
// CreateUploadSession 创建上传会话
func (fs *FileSystem) CreateUploadSession(ctx context.Context, file *fsctx.FileStream) (*serializer.UploadCredential, error) {
// GetUploadToken 生成新的上传凭证
func (fs *FileSystem) GetUploadToken(ctx context.Context, path string, size uint64, name string) (*serializer.UploadCredential, error) {
// 获取相关有效期设置
credentialTTL := model.GetIntSetting("upload_credential_timeout", 3600)
callBackSessionTTL := model.GetIntSetting("upload_session_timeout", 86400)
callbackKey := uuid.Must(uuid.NewV4()).String()
fileSize := file.Size
var err error
// 创建占位的文件,同时校验文件信息
file.Mode = fsctx.Nop
if callbackKey != "" {
file.UploadSessionID = &callbackKey
// 检查文件大小
if fs.User.Policy.MaxSize != 0 {
if size > fs.User.Policy.MaxSize {
return nil, ErrFileSizeTooBig
}
}
fs.Use("BeforeUpload", HookValidateFile)
fs.Use("BeforeUpload", HookValidateCapacity)
// 验证文件规格
if err := fs.Upload(ctx, file); err != nil {
return nil, err
}
uploadSession := &serializer.UploadSession{
Key: callbackKey,
UID: fs.User.ID,
Policy: *fs.Policy,
VirtualPath: file.VirtualPath,
Name: file.Name,
Size: fileSize,
SavePath: file.SavePath,
LastModified: file.LastModified,
CallbackSecret: util.RandStringRunes(32),
// 是否需要预先生成存储路径
var savePath string
if fs.User.Policy.IsPathGenerateNeeded() {
savePath = fs.GenerateSavePath(ctx, local.FileStream{Name: name, VirtualPath: path})
ctx = context.WithValue(ctx, fsctx.SavePathCtx, savePath)
}
ctx = context.WithValue(ctx, fsctx.FileSizeCtx, size)
// 获取上传凭证
credential, err := fs.Handler.Token(ctx, int64(callBackSessionTTL), uploadSession, file)
callbackKey := util.RandStringRunes(32)
credential, err := fs.Handler.Token(ctx, int64(credentialTTL), callbackKey)
if err != nil {
return nil, err
}
// 创建占位符
if !fs.Policy.IsUploadPlaceholderWithSize() {
fs.Use("AfterUpload", HookClearFileHeaderSize)
}
fs.Use("AfterUpload", GenericAfterUpload)
ctx = context.WithValue(ctx, fsctx.IgnoreDirectoryConflictCtx, true)
if err := fs.Upload(ctx, file); err != nil {
return nil, err
return nil, serializer.NewError(serializer.CodeEncryptError, "无法获取上传凭证", err)
}
// 创建回调会话
err = cache.Set(
UploadSessionCachePrefix+callbackKey,
*uploadSession,
"callback_"+callbackKey,
serializer.UploadSession{
Key: callbackKey,
UID: fs.User.ID,
PolicyID: fs.User.GetPolicyID(0),
VirtualPath: path,
Name: name,
Size: size,
SavePath: savePath,
},
callBackSessionTTL,
)
if err != nil {
return nil, err
}
// 补全上传凭证其他信息
credential.Expires = time.Now().Add(time.Duration(callBackSessionTTL) * time.Second).Unix()
return credential, nil
return &credential, nil
}
// UploadFromStream 从文件流上传文件
func (fs *FileSystem) UploadFromStream(ctx context.Context, file *fsctx.FileStream, resetPolicy bool) error {
if resetPolicy {
// 重设存储策略
fs.Policy = &fs.User.Policy
err := fs.DispatchHandler()
if err != nil {
return err
}
func (fs *FileSystem) UploadFromStream(ctx context.Context, src io.ReadCloser, dst string, size uint64) error {
// 构建文件头
fileName := path.Base(dst)
filePath := path.Dir(dst)
fileData := local.FileStream{
File: src,
Size: size,
Name: fileName,
VirtualPath: filePath,
}
// 给文件系统分配钩子
@@ -209,18 +215,29 @@ func (fs *FileSystem) UploadFromStream(ctx context.Context, file *fsctx.FileStre
fs.Use("BeforeUpload", HookValidateFile)
fs.Use("BeforeUpload", HookValidateCapacity)
fs.Use("AfterUploadCanceled", HookDeleteTempFile)
fs.Use("AfterUploadCanceled", HookGiveBackCapacity)
fs.Use("AfterUpload", GenericAfterUpload)
fs.Use("AfterUpload", HookGenerateThumb)
fs.Use("AfterValidateFailed", HookDeleteTempFile)
fs.Use("AfterValidateFailed", HookGiveBackCapacity)
fs.Use("AfterUploadFailed", HookGiveBackCapacity)
}
fs.Lock.Unlock()
// 开始上传
return fs.Upload(ctx, file)
return fs.Upload(ctx, fileData)
}
// UploadFromPath 将本机已有文件上传到用户的文件系统
func (fs *FileSystem) UploadFromPath(ctx context.Context, src, dst string, mode fsctx.WriteMode) error {
func (fs *FileSystem) UploadFromPath(ctx context.Context, src, dst string, resetPolicy bool) error {
// 重设存储策略
if resetPolicy {
fs.Policy = &fs.User.Policy
err := fs.DispatchHandler()
if err != nil {
return err
}
}
file, err := os.Open(util.RelativePath(src))
if err != nil {
return err
@@ -235,12 +252,5 @@ func (fs *FileSystem) UploadFromPath(ctx context.Context, src, dst string, mode
size := fi.Size()
// 开始上传
return fs.UploadFromStream(ctx, &fsctx.FileStream{
File: file,
Seeker: file,
Size: uint64(size),
Name: path.Base(dst),
VirtualPath: path.Dir(dst),
Mode: mode,
}, true)
return fs.UploadFromStream(ctx, file, dst, uint64(size))
}

View File

@@ -3,9 +3,17 @@ package filesystem
import (
"context"
"errors"
"github.com/DATA-DOG/go-sqlmock"
"io"
"io/ioutil"
"net/http"
"net/http/httptest"
"net/url"
"strings"
"testing"
model "github.com/cloudreve/Cloudreve/v3/models"
"github.com/cloudreve/Cloudreve/v3/pkg/cache"
"github.com/cloudreve/Cloudreve/v3/pkg/filesystem/driver/local"
"github.com/cloudreve/Cloudreve/v3/pkg/filesystem/fsctx"
"github.com/cloudreve/Cloudreve/v3/pkg/filesystem/response"
"github.com/cloudreve/Cloudreve/v3/pkg/serializer"
@@ -13,33 +21,12 @@ import (
"github.com/jinzhu/gorm"
"github.com/stretchr/testify/assert"
testMock "github.com/stretchr/testify/mock"
"io/ioutil"
"net/http"
"net/http/httptest"
"net/url"
"strings"
"testing"
)
type FileHeaderMock struct {
testMock.Mock
}
func (m FileHeaderMock) Put(ctx context.Context, file fsctx.FileHeader) error {
args := m.Called(ctx, file)
return args.Error(0)
}
func (m FileHeaderMock) Token(ctx context.Context, ttl int64, uploadSession *serializer.UploadSession, file fsctx.FileHeader) (*serializer.UploadCredential, error) {
args := m.Called(ctx, ttl, uploadSession, file)
return args.Get(0).(*serializer.UploadCredential), args.Error(1)
}
func (m FileHeaderMock) CancelToken(ctx context.Context, uploadSession *serializer.UploadSession) error {
args := m.Called(ctx, uploadSession)
return args.Error(0)
}
func (m FileHeaderMock) List(ctx context.Context, path string, recursive bool) ([]response.Object, error) {
args := m.Called(ctx, path, recursive)
return args.Get(0).([]response.Object), args.Error(1)
@@ -50,6 +37,11 @@ func (m FileHeaderMock) Get(ctx context.Context, path string) (response.RSCloser
return args.Get(0).(response.RSCloser), args.Error(1)
}
func (m FileHeaderMock) Put(ctx context.Context, file io.ReadCloser, dst string, size uint64) error {
args := m.Called(ctx, file, dst)
return args.Error(0)
}
func (m FileHeaderMock) Delete(ctx context.Context, files []string) ([]string, error) {
args := m.Called(ctx, files)
return args.Get(0).([]string), args.Error(1)
@@ -65,6 +57,11 @@ func (m FileHeaderMock) Source(ctx context.Context, path string, url url.URL, ex
return args.Get(0).(string), args.Error(1)
}
func (m FileHeaderMock) Token(ctx context.Context, expires int64, key string) (serializer.UploadCredential, error) {
args := m.Called(ctx, expires, key)
return args.Get(0).(serializer.UploadCredential), args.Error(1)
}
func TestFileSystem_Upload(t *testing.T) {
asserts := assert.New(t)
@@ -77,10 +74,10 @@ func TestFileSystem_Upload(t *testing.T) {
Model: gorm.Model{
ID: 1,
},
},
Policy: &model.Policy{
AutoRename: false,
DirNameRule: "{path}",
Policy: model.Policy{
AutoRename: false,
DirNameRule: "{path}",
},
},
}
ctx, cancel := context.WithCancel(context.Background())
@@ -88,7 +85,7 @@ func TestFileSystem_Upload(t *testing.T) {
c.Request, _ = http.NewRequest("POST", "/", nil)
ctx = context.WithValue(ctx, fsctx.GinCtx, c)
cancel()
file := &fsctx.FileStream{
file := local.FileStream{
Size: 5,
VirtualPath: "/",
Name: "1.txt",
@@ -98,17 +95,17 @@ func TestFileSystem_Upload(t *testing.T) {
// 正常,上下文已指定源文件
testHandler = new(FileHeaderMock)
testHandler.On("Put", testMock.Anything, testMock.Anything).Return(nil)
testHandler.On("Put", testMock.Anything, testMock.Anything, "123/123.txt").Return(nil)
fs = &FileSystem{
Handler: testHandler,
User: &model.User{
Model: gorm.Model{
ID: 1,
},
},
Policy: &model.Policy{
AutoRename: false,
DirNameRule: "{path}",
Policy: model.Policy{
AutoRename: false,
DirNameRule: "{path}",
},
},
}
ctx, cancel = context.WithCancel(context.Background())
@@ -117,7 +114,7 @@ func TestFileSystem_Upload(t *testing.T) {
ctx = context.WithValue(ctx, fsctx.GinCtx, c)
ctx = context.WithValue(ctx, fsctx.FileModelCtx, model.File{SourceName: "123/123.txt"})
cancel()
file = &fsctx.FileStream{
file = local.FileStream{
Size: 5,
VirtualPath: "/",
Name: "1.txt",
@@ -127,7 +124,7 @@ func TestFileSystem_Upload(t *testing.T) {
asserts.NoError(err)
// BeforeUpload 返回错误
fs.Use("BeforeUpload", func(ctx context.Context, fs *FileSystem, file fsctx.FileHeader) error {
fs.Use("BeforeUpload", func(ctx context.Context, fs *FileSystem) error {
return errors.New("error")
})
err = fs.Upload(ctx, file)
@@ -137,7 +134,7 @@ func TestFileSystem_Upload(t *testing.T) {
// 上传文件失败
testHandler2 := new(FileHeaderMock)
testHandler2.On("Put", testMock.Anything, testMock.Anything).Return(errors.New("error"))
testHandler2.On("Put", testMock.Anything, testMock.Anything, testMock.Anything).Return(errors.New("error"))
fs.Handler = testHandler2
err = fs.Upload(ctx, file)
asserts.Error(err)
@@ -145,12 +142,12 @@ func TestFileSystem_Upload(t *testing.T) {
// AfterUpload失败
testHandler3 := new(FileHeaderMock)
testHandler3.On("Put", testMock.Anything, testMock.Anything).Return(nil)
testHandler3.On("Put", testMock.Anything, testMock.Anything, testMock.Anything).Return(nil)
fs.Handler = testHandler3
fs.Use("AfterUpload", func(ctx context.Context, fs *FileSystem, file fsctx.FileHeader) error {
fs.Use("AfterUpload", func(ctx context.Context, fs *FileSystem) error {
return errors.New("error")
})
fs.Use("AfterValidateFailed", func(ctx context.Context, fs *FileSystem, file fsctx.FileHeader) error {
fs.Use("AfterValidateFailed", func(ctx context.Context, fs *FileSystem) error {
return errors.New("error")
})
err = fs.Upload(ctx, file)
@@ -159,39 +156,43 @@ func TestFileSystem_Upload(t *testing.T) {
}
func TestFileSystem_GenerateSavePath_Anonymous(t *testing.T) {
asserts := assert.New(t)
fs := FileSystem{User: &model.User{}}
ctx := context.WithValue(
context.Background(),
fsctx.UploadPolicyCtx,
serializer.UploadPolicy{
SavePath: "{randomkey16}",
AutoRename: false,
},
)
savePath := fs.GenerateSavePath(ctx, local.FileStream{
Name: "test.test",
})
asserts.Len(savePath, 26)
asserts.Contains(savePath, "test.test")
}
func TestFileSystem_GetUploadToken(t *testing.T) {
asserts := assert.New(t)
fs := FileSystem{
User: &model.User{Model: gorm.Model{ID: 1}},
Policy: &model.Policy{},
}
fs := FileSystem{User: &model.User{Model: gorm.Model{ID: 1}}}
ctx := context.Background()
// 成功
{
cache.SetSettings(map[string]string{
"upload_session_timeout": "10",
"upload_credential_timeout": "10",
"upload_session_timeout": "10",
}, "setting_")
testHandler := new(FileHeaderMock)
testHandler.On("Token", testMock.Anything, int64(10), testMock.Anything, testMock.Anything).Return(&serializer.UploadCredential{Credential: "test"}, nil)
testHandler.On("Token", testMock.Anything, int64(10), testMock.Anything).Return(serializer.UploadCredential{Token: "test"}, nil)
fs.Handler = testHandler
mock.ExpectQuery("SELECT(.+)").
WithArgs(1).
WillReturnRows(sqlmock.NewRows([]string{"id", "owner_id"}).AddRow(1, 1))
mock.ExpectQuery("SELECT(.+)files(.+)").WillReturnError(errors.New("not found"))
mock.ExpectBegin()
mock.ExpectExec("INSERT(.+)files(.+)").WillReturnResult(sqlmock.NewResult(1, 1))
mock.ExpectExec("UPDATE(.+)storage(.+)").WillReturnResult(sqlmock.NewResult(1, 1))
mock.ExpectCommit()
res, err := fs.CreateUploadSession(ctx, &fsctx.FileStream{
Size: 0,
Name: "file",
VirtualPath: "/",
})
asserts.NoError(mock.ExpectationsWereMet())
res, err := fs.GetUploadToken(ctx, "/", 10, "123")
testHandler.AssertExpectations(t)
asserts.NoError(err)
asserts.Equal("test", res.Credential)
asserts.Equal("test", res.Token)
}
// 无法获取上传凭证
@@ -201,22 +202,9 @@ func TestFileSystem_GetUploadToken(t *testing.T) {
"upload_session_timeout": "10",
}, "setting_")
testHandler := new(FileHeaderMock)
testHandler.On("Token", testMock.Anything, int64(10), testMock.Anything, testMock.Anything).Return(&serializer.UploadCredential{}, errors.New("error"))
testHandler.On("Token", testMock.Anything, int64(10), testMock.Anything).Return(serializer.UploadCredential{}, errors.New("error"))
fs.Handler = testHandler
mock.ExpectQuery("SELECT(.+)").
WithArgs(1).
WillReturnRows(sqlmock.NewRows([]string{"id", "owner_id"}).AddRow(1, 1))
mock.ExpectQuery("SELECT(.+)files(.+)").WillReturnError(errors.New("not found"))
mock.ExpectBegin()
mock.ExpectExec("INSERT(.+)files(.+)").WillReturnResult(sqlmock.NewResult(1, 1))
mock.ExpectExec("UPDATE(.+)storage(.+)").WillReturnResult(sqlmock.NewResult(1, 1))
mock.ExpectCommit()
_, err := fs.CreateUploadSession(ctx, &fsctx.FileStream{
Size: 0,
Name: "file",
VirtualPath: "/",
})
asserts.NoError(mock.ExpectationsWereMet())
_, err := fs.GetUploadToken(ctx, "/", 10, "123")
testHandler.AssertExpectations(t)
asserts.Error(err)
}
@@ -224,41 +212,27 @@ func TestFileSystem_GetUploadToken(t *testing.T) {
func TestFileSystem_UploadFromStream(t *testing.T) {
asserts := assert.New(t)
fs := FileSystem{
User: &model.User{
Model: gorm.Model{ID: 1},
Policy: model.Policy{Type: "mock"},
},
Policy: &model.Policy{Type: "mock"},
}
fs := FileSystem{User: &model.User{Model: gorm.Model{ID: 1}}}
ctx := context.Background()
err := fs.UploadFromStream(ctx, &fsctx.FileStream{
File: ioutil.NopCloser(strings.NewReader("123")),
}, true)
err := fs.UploadFromStream(ctx, ioutil.NopCloser(strings.NewReader("123")), "/1.txt", 1)
asserts.Error(err)
}
func TestFileSystem_UploadFromPath(t *testing.T) {
asserts := assert.New(t)
fs := FileSystem{
User: &model.User{
Model: gorm.Model{ID: 1},
Policy: model.Policy{Type: "mock"},
},
Policy: &model.Policy{Type: "mock"},
}
fs := FileSystem{User: &model.User{Policy: model.Policy{Type: "mock"}, Model: gorm.Model{ID: 1}}}
ctx := context.Background()
// 文件不存在
{
err := fs.UploadFromPath(ctx, "test/not_exist", "/", fsctx.Overwrite)
err := fs.UploadFromPath(ctx, "test/not_exist", "/", true)
asserts.Error(err)
}
// 文存在,上传失败
{
err := fs.UploadFromPath(ctx, "tests/test.zip", "/", fsctx.Overwrite)
err := fs.UploadFromPath(ctx, "tests/test.zip", "/", true)
asserts.Error(err)
}
}

View File

@@ -45,10 +45,10 @@ func (fs *FileSystem) ValidateLegalName(ctx context.Context, name string) bool {
// ValidateFileSize 验证上传的文件大小是否超出限制
func (fs *FileSystem) ValidateFileSize(ctx context.Context, size uint64) bool {
if fs.Policy.MaxSize == 0 {
if fs.User.Policy.MaxSize == 0 {
return true
}
return size <= fs.Policy.MaxSize
return size <= fs.User.Policy.MaxSize
}
// ValidateCapacity 验证并扣除用户容量
@@ -59,11 +59,11 @@ func (fs *FileSystem) ValidateCapacity(ctx context.Context, size uint64) bool {
// ValidateExtension 验证文件扩展名
func (fs *FileSystem) ValidateExtension(ctx context.Context, fileName string) bool {
// 不需要验证
if len(fs.Policy.OptionsSerialized.FileType) == 0 {
if len(fs.User.Policy.OptionsSerialized.FileType) == 0 {
return true
}
return IsInExtensionList(fs.Policy.OptionsSerialized.FileType, fileName)
return IsInExtensionList(fs.User.Policy.OptionsSerialized.FileType, fileName)
}
// IsInExtensionList 返回文件的扩展名是否在给定的列表范围内

View File

@@ -68,9 +68,10 @@ func TestFileSystem_ValidateFileSize(t *testing.T) {
asserts := assert.New(t)
ctx := context.Background()
fs := FileSystem{
User: &model.User{},
Policy: &model.Policy{
MaxSize: 10,
User: &model.User{
Policy: model.Policy{
MaxSize: 10,
},
},
}
@@ -79,7 +80,7 @@ func TestFileSystem_ValidateFileSize(t *testing.T) {
asserts.False(fs.ValidateFileSize(ctx, 11))
// 无限制
fs.Policy.MaxSize = 0
fs.User.Policy.MaxSize = 0
asserts.True(fs.ValidateFileSize(ctx, 11))
}
@@ -87,10 +88,11 @@ func TestFileSystem_ValidateExtension(t *testing.T) {
asserts := assert.New(t)
ctx := context.Background()
fs := FileSystem{
User: &model.User{},
Policy: &model.Policy{
OptionsSerialized: model.PolicyOption{
FileType: nil,
User: &model.User{
Policy: model.Policy{
OptionsSerialized: model.PolicyOption{
FileType: nil,
},
},
},
}
@@ -98,11 +100,11 @@ func TestFileSystem_ValidateExtension(t *testing.T) {
asserts.True(fs.ValidateExtension(ctx, "1"))
asserts.True(fs.ValidateExtension(ctx, "1.txt"))
fs.Policy.OptionsSerialized.FileType = []string{}
fs.User.Policy.OptionsSerialized.FileType = []string{}
asserts.True(fs.ValidateExtension(ctx, "1"))
asserts.True(fs.ValidateExtension(ctx, "1.txt"))
fs.Policy.OptionsSerialized.FileType = []string{"txt", "jpg"}
fs.User.Policy.OptionsSerialized.FileType = []string{"txt", "jpg"}
asserts.False(fs.ValidateExtension(ctx, "1"))
asserts.False(fs.ValidateExtension(ctx, "1.jpg.png"))
asserts.True(fs.ValidateExtension(ctx, "1.txt"))

View File

@@ -7,9 +7,11 @@ import (
"github.com/cloudreve/Cloudreve/v3/pkg/auth"
"github.com/cloudreve/Cloudreve/v3/pkg/balancer"
"github.com/cloudreve/Cloudreve/v3/pkg/cluster"
"github.com/cloudreve/Cloudreve/v3/pkg/request"
"github.com/cloudreve/Cloudreve/v3/pkg/serializer"
"github.com/cloudreve/Cloudreve/v3/pkg/task"
testMock "github.com/stretchr/testify/mock"
"io"
)
type NodePoolMock struct {
@@ -149,3 +151,11 @@ func (t TaskPoolMock) Add(num int) {
func (t TaskPoolMock) Submit(job task.Job) {
t.Called(job)
}
type RequestMock struct {
testMock.Mock
}
func (r RequestMock) Request(method, target string, body io.Reader, opts ...request.Option) *request.Response {
return r.Called(method, target, body, opts).Get(0).(*request.Response)
}

View File

@@ -1,32 +0,0 @@
package remoteclientmock
import (
"context"
"github.com/cloudreve/Cloudreve/v3/pkg/filesystem/fsctx"
"github.com/cloudreve/Cloudreve/v3/pkg/serializer"
"github.com/stretchr/testify/mock"
)
type RemoteClientMock struct {
mock.Mock
}
func (r *RemoteClientMock) CreateUploadSession(ctx context.Context, session *serializer.UploadSession, ttl int64, overwrite bool) error {
return r.Called(ctx, session, ttl, overwrite).Error(0)
}
func (r *RemoteClientMock) GetUploadURL(ttl int64, sessionID string) (string, string, error) {
args := r.Called(ttl, sessionID)
return args.String(0), args.String(1), args.Error(2)
}
func (r *RemoteClientMock) Upload(ctx context.Context, file fsctx.FileHeader) error {
args := r.Called(ctx, file)
return args.Error(0)
}
func (r *RemoteClientMock) DeleteUploadSession(ctx context.Context, sessionID string) error {
args := r.Called(ctx, sessionID)
return args.Error(0)
}

View File

@@ -1,15 +0,0 @@
package requestmock
import (
"github.com/cloudreve/Cloudreve/v3/pkg/request"
"github.com/stretchr/testify/mock"
"io"
)
type RequestMock struct {
mock.Mock
}
func (r RequestMock) Request(method, target string, body io.Reader, opts ...request.Option) *request.Response {
return r.Called(method, target, body, opts).Get(0).(*request.Response)
}

Some files were not shown because too many files have changed in this diff Show More