mirror of
https://github.com/cloudreve/cloudreve.git
synced 2026-03-06 23:37:02 +00:00
Compare commits
92 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
d0779f564e | ||
|
|
350954911e | ||
|
|
b8bc5bed13 | ||
|
|
91377f4676 | ||
|
|
b1803fa51f | ||
|
|
f8b7e086ba | ||
|
|
ff22f5c8b9 | ||
|
|
aaf8a793ee | ||
|
|
2ab2662fcd | ||
|
|
71df067a76 | ||
|
|
d34cb3e5d3 | ||
|
|
b5e8e4843f | ||
|
|
86877aef4b | ||
|
|
3d9b9ae5d6 | ||
|
|
8741c3cc78 | ||
|
|
6c93e37777 | ||
|
|
841a2e258d | ||
|
|
da2f6c5b07 | ||
|
|
a26183875f | ||
|
|
79913a5dfa | ||
|
|
4f6989f1b8 | ||
|
|
fcc29e31eb | ||
|
|
00e2b26294 | ||
|
|
4f65d0e859 | ||
|
|
3804efd792 | ||
|
|
0c9383e329 | ||
|
|
13d36c25d4 | ||
|
|
18f5bffed1 | ||
|
|
478d390867 | ||
|
|
febbd0c5a0 | ||
|
|
32a655f84e | ||
|
|
0a18d984ab | ||
|
|
265bc099b2 | ||
|
|
90a47c9ec0 | ||
|
|
6451e4c903 | ||
|
|
b50756dbcb | ||
|
|
23dc7e370e | ||
|
|
1f3c1d7ce2 | ||
|
|
84807be1ca | ||
|
|
20e90e3963 | ||
|
|
ace398d87b | ||
|
|
ec776ac837 | ||
|
|
d117080991 | ||
|
|
1c0a735df8 | ||
|
|
c6130ab078 | ||
|
|
31315c86ee | ||
|
|
636ac52a3f | ||
|
|
1821923b74 | ||
|
|
a568e5e45a | ||
|
|
e51c5cd70d | ||
|
|
5a3ea89866 | ||
|
|
eaa8c9e12d | ||
|
|
d54ca151b2 | ||
|
|
7eb8173101 | ||
|
|
d3016b60af | ||
|
|
9e5713b139 | ||
|
|
07f13cc350 | ||
|
|
0df9529b32 | ||
|
|
015ccd5026 | ||
|
|
5802161102 | ||
|
|
b6efca1878 | ||
|
|
15e3e3db5c | ||
|
|
24dfb2c24e | ||
|
|
dd4c3e05d3 | ||
|
|
5bda037d74 | ||
|
|
c89327631e | ||
|
|
9136f3caec | ||
|
|
0650684dd9 | ||
|
|
effbc8607e | ||
|
|
b96019be7c | ||
|
|
081e75146c | ||
|
|
e0714fdd53 | ||
|
|
4925a356e3 | ||
|
|
050a68a359 | ||
|
|
7214e59c25 | ||
|
|
118d738797 | ||
|
|
285611baf7 | ||
|
|
521c5c8dc4 | ||
|
|
285e80ba76 | ||
|
|
2811ee3285 | ||
|
|
7dd636da74 | ||
|
|
3444b4a75e | ||
|
|
c301bd6045 | ||
|
|
72173bf894 | ||
|
|
6fdf77e00e | ||
|
|
e37e93a7b6 | ||
|
|
868a88e5fc | ||
|
|
8a222e7df4 | ||
|
|
8443a30fb1 | ||
|
|
de9c41082c | ||
|
|
855c9d92c4 | ||
|
|
c84d0114ae |
61
.github/stale.yml
vendored
Normal file
61
.github/stale.yml
vendored
Normal file
@@ -0,0 +1,61 @@
|
||||
# Configuration for probot-stale - https://github.com/probot/stale
|
||||
|
||||
# Number of days of inactivity before an Issue or Pull Request becomes stale
|
||||
daysUntilStale: 360
|
||||
|
||||
# Number of days of inactivity before an Issue or Pull Request with the stale label is closed.
|
||||
# Set to false to disable. If disabled, issues still need to be closed manually, but will remain marked as stale.
|
||||
daysUntilClose: 30
|
||||
|
||||
# Only issues or pull requests with all of these labels are check if stale. Defaults to `[]` (disabled)
|
||||
onlyLabels: []
|
||||
|
||||
# Issues or Pull Requests with these labels will never be considered stale. Set to `[]` to disable
|
||||
exemptLabels:
|
||||
- pinned
|
||||
- security
|
||||
- "[Status] Maybe Later"
|
||||
|
||||
# Set to true to ignore issues in a project (defaults to false)
|
||||
exemptProjects: true
|
||||
|
||||
# Set to true to ignore issues in a milestone (defaults to false)
|
||||
exemptMilestones: true
|
||||
|
||||
# Set to true to ignore issues with an assignee (defaults to false)
|
||||
exemptAssignees: true
|
||||
|
||||
# Label to use when marking as stale
|
||||
staleLabel: wontfix
|
||||
|
||||
# Comment to post when marking as stale. Set to `false` to disable
|
||||
markComment: >
|
||||
This issue has been automatically marked as stale because it has not had
|
||||
recent activity. It will be closed if no further activity occurs. Thank you
|
||||
for your contributions.
|
||||
|
||||
# Comment to post when removing the stale label.
|
||||
# unmarkComment: >
|
||||
# Your comment here.
|
||||
|
||||
# Comment to post when closing a stale Issue or Pull Request.
|
||||
# closeComment: >
|
||||
# Your comment here.
|
||||
|
||||
# Limit the number of actions per hour, from 1-30. Default is 30
|
||||
limitPerRun: 30
|
||||
|
||||
# Limit to only `issues` or `pulls`
|
||||
# only: issues
|
||||
|
||||
# Optionally, specify configuration settings that are specific to just 'issues' or 'pulls':
|
||||
# pulls:
|
||||
# daysUntilStale: 30
|
||||
# markComment: >
|
||||
# This pull request has been automatically marked as stale because it has not had
|
||||
# recent activity. It will be closed if no further activity occurs. Thank you
|
||||
# for your contributions.
|
||||
|
||||
# issues:
|
||||
# exemptLabels:
|
||||
# - confirmed
|
||||
88
.github/workflows/build.yml
vendored
88
.github/workflows/build.yml
vendored
@@ -1,61 +1,55 @@
|
||||
name: Build
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [ master ]
|
||||
on: workflow_dispatch
|
||||
|
||||
jobs:
|
||||
build:
|
||||
name: Build
|
||||
runs-on: ubuntu-18.04
|
||||
steps:
|
||||
- name: Set up Go 1.17
|
||||
uses: actions/setup-go@v2
|
||||
with:
|
||||
go-version: "1.17"
|
||||
id: go
|
||||
|
||||
- name: Set up Go 1.13
|
||||
uses: actions/setup-go@v1
|
||||
with:
|
||||
go-version: 1.13
|
||||
id: go
|
||||
- name: Check out code into the Go module directory
|
||||
uses: actions/checkout@v2
|
||||
with:
|
||||
clean: false
|
||||
submodules: "recursive"
|
||||
- run: |
|
||||
git fetch --prune --unshallow --tags
|
||||
|
||||
- name: Check out code into the Go module directory
|
||||
uses: actions/checkout@v2
|
||||
with:
|
||||
clean: false
|
||||
submodules: 'recursive'
|
||||
- run: |
|
||||
git fetch --prune --unshallow --tags
|
||||
- name: Get dependencies and build
|
||||
run: |
|
||||
sudo apt-get update
|
||||
sudo apt-get -y install gcc-mingw-w64-x86-64
|
||||
sudo apt-get -y install gcc-arm-linux-gnueabihf libc6-dev-armhf-cross
|
||||
sudo apt-get -y install gcc-aarch64-linux-gnu libc6-dev-arm64-cross
|
||||
chmod +x ./build.sh
|
||||
./build.sh -r b
|
||||
|
||||
- name: Get dependencies and build
|
||||
run: |
|
||||
go get github.com/rakyll/statik
|
||||
export PATH=$PATH:~/go/bin/
|
||||
statik -src=models -f
|
||||
sudo apt-get update
|
||||
sudo apt-get -y install gcc-mingw-w64-x86-64
|
||||
sudo apt-get -y install gcc-arm-linux-gnueabihf libc6-dev-armhf-cross
|
||||
sudo apt-get -y install gcc-aarch64-linux-gnu libc6-dev-arm64-cross
|
||||
chmod +x ./build.sh
|
||||
./build.sh -r b
|
||||
- name: Upload binary files (windows_amd64)
|
||||
uses: actions/upload-artifact@v2
|
||||
with:
|
||||
name: cloudreve_windows_amd64
|
||||
path: release/cloudreve*windows_amd64.*
|
||||
|
||||
- name: Upload binary files (windows_amd64)
|
||||
uses: actions/upload-artifact@v2
|
||||
with:
|
||||
name: cloudreve_windows_amd64
|
||||
path: release/cloudreve*windows_amd64.*
|
||||
- name: Upload binary files (linux_amd64)
|
||||
uses: actions/upload-artifact@v2
|
||||
with:
|
||||
name: cloudreve_linux_amd64
|
||||
path: release/cloudreve*linux_amd64.*
|
||||
|
||||
- name: Upload binary files (linux_amd64)
|
||||
uses: actions/upload-artifact@v2
|
||||
with:
|
||||
name: cloudreve_linux_amd64
|
||||
path: release/cloudreve*linux_amd64.*
|
||||
- name: Upload binary files (linux_arm)
|
||||
uses: actions/upload-artifact@v2
|
||||
with:
|
||||
name: cloudreve_linux_arm
|
||||
path: release/cloudreve*linux_arm.*
|
||||
|
||||
- name: Upload binary files (linux_arm)
|
||||
uses: actions/upload-artifact@v2
|
||||
with:
|
||||
name: cloudreve_linux_arm
|
||||
path: release/cloudreve*linux_arm.*
|
||||
|
||||
- name: Upload binary files (linux_arm64)
|
||||
uses: actions/upload-artifact@v2
|
||||
with:
|
||||
name: cloudreve_linux_arm64
|
||||
path: release/cloudreve*linux_arm64.*
|
||||
- name: Upload binary files (linux_arm64)
|
||||
uses: actions/upload-artifact@v2
|
||||
with:
|
||||
name: cloudreve_linux_arm64
|
||||
path: release/cloudreve*linux_arm64.*
|
||||
|
||||
50
.github/workflows/docker-release.yml
vendored
Normal file
50
.github/workflows/docker-release.yml
vendored
Normal file
@@ -0,0 +1,50 @@
|
||||
name: Build and push docker image
|
||||
|
||||
on:
|
||||
push:
|
||||
tags:
|
||||
- 3.* # triggered on every push with tag 3.*
|
||||
workflow_dispatch: # or just on button clicked
|
||||
|
||||
jobs:
|
||||
docker-build:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v2
|
||||
- run: git fetch --prune --unshallow
|
||||
- name: Setup Environments
|
||||
id: envs
|
||||
run: |
|
||||
CLOUDREVE_LATEST_TAG=$(git describe --tags --abbrev=0)
|
||||
DOCKER_IMAGE="cloudreve/cloudreve"
|
||||
|
||||
echo "RELEASE_VERSION=${GITHUB_REF#refs}"
|
||||
TAGS="${DOCKER_IMAGE}:latest,${DOCKER_IMAGE}:${CLOUDREVE_LATEST_TAG}"
|
||||
|
||||
echo "CLOUDREVE_LATEST_TAG:${CLOUDREVE_LATEST_TAG}"
|
||||
echo ::set-output name=tags::${TAGS}
|
||||
- name: Setup QEMU Emulator
|
||||
uses: docker/setup-qemu-action@master
|
||||
with:
|
||||
platforms: all
|
||||
- name: Setup Docker Buildx Command
|
||||
id: buildx
|
||||
uses: docker/setup-buildx-action@master
|
||||
- name: Login to Dockerhub
|
||||
uses: docker/login-action@v1
|
||||
with:
|
||||
username: ${{ secrets.DOCKERHUB_USERNAME }}
|
||||
password: ${{ secrets.DOCKERHUB_PASSWORD }}
|
||||
- name: Build Docker Image and Push
|
||||
id: docker_build
|
||||
uses: docker/build-push-action@v2
|
||||
with:
|
||||
push: true
|
||||
builder: ${{ steps.buildx.outputs.name }}
|
||||
context: .
|
||||
file: ./Dockerfile
|
||||
platforms: linux/amd64,linux/arm64,linux/arm/v7
|
||||
tags: ${{ steps.envs.outputs.tags }}
|
||||
- name: Image Digest
|
||||
run: echo ${{ steps.docker_build.outputs.digest }}
|
||||
50
.github/workflows/test.yml
vendored
50
.github/workflows/test.yml
vendored
@@ -2,46 +2,34 @@ name: Test
|
||||
|
||||
on:
|
||||
pull_request:
|
||||
branches:
|
||||
branches:
|
||||
- master
|
||||
push:
|
||||
branches: [ master ]
|
||||
branches: [master]
|
||||
|
||||
jobs:
|
||||
|
||||
test:
|
||||
name: Test
|
||||
runs-on: ubuntu-18.04
|
||||
steps:
|
||||
- name: Set up Go 1.17
|
||||
uses: actions/setup-go@v2
|
||||
with:
|
||||
go-version: "1.17"
|
||||
id: go
|
||||
|
||||
- name: Set up Go 1.13
|
||||
uses: actions/setup-go@v1
|
||||
with:
|
||||
go-version: 1.13
|
||||
id: go
|
||||
- name: Check out code into the Go module directory
|
||||
uses: actions/checkout@v2
|
||||
with:
|
||||
submodules: "recursive"
|
||||
|
||||
- name: Check out code into the Go module directory
|
||||
uses: actions/checkout@v2
|
||||
with:
|
||||
submodules: 'recursive'
|
||||
- name: Build static files
|
||||
run: |
|
||||
mkdir assets/build
|
||||
touch assets/build/test.html
|
||||
|
||||
- name: Get dependencies
|
||||
run: |
|
||||
go get github.com/rakyll/statik
|
||||
export PATH=$PATH:~/go/bin/
|
||||
statik -src=models -f
|
||||
- name: Test
|
||||
run: go test -coverprofile=coverage.txt -covermode=atomic ./...
|
||||
|
||||
- name: Test
|
||||
run: go test -coverprofile=coverage.txt -covermode=atomic ./...
|
||||
|
||||
- name: Upload binary files (linux_arm)
|
||||
uses: actions/upload-artifact@v2
|
||||
with:
|
||||
name: cloudreve_linux_arm
|
||||
path: release/cloudreve*linux_arm.*
|
||||
|
||||
- name: Upload binary files (linux_arm64)
|
||||
uses: actions/upload-artifact@v2
|
||||
with:
|
||||
name: cloudreve_linux_arm64
|
||||
path: release/cloudreve*linux_arm64.*
|
||||
- name: Upload coverage reports to Codecov
|
||||
uses: codecov/codecov-action@v2
|
||||
|
||||
11
.travis.yml
11
.travis.yml
@@ -1,13 +1,12 @@
|
||||
language: go
|
||||
go:
|
||||
- 1.13.x
|
||||
- 1.17.x
|
||||
node_js: "12.16.3"
|
||||
git:
|
||||
depth: 1
|
||||
install:
|
||||
- go get github.com/rakyll/statik
|
||||
before_script:
|
||||
- statik -src=models -f
|
||||
before_install:
|
||||
- mkdir assets/build
|
||||
- touch assets/build/test.html
|
||||
script:
|
||||
- go test -coverprofile=coverage.txt -covermode=atomic ./...
|
||||
after_success:
|
||||
@@ -27,4 +26,4 @@ deploy:
|
||||
draft: true
|
||||
skip_cleanup: true
|
||||
on:
|
||||
tags: true
|
||||
tags: true
|
||||
|
||||
87
Dockerfile
87
Dockerfile
@@ -1,72 +1,41 @@
|
||||
FROM golang:1.17-alpine as cloudreve_builder
|
||||
|
||||
|
||||
# install dependencies and build tools
|
||||
RUN apk update && apk add --no-cache wget curl git yarn build-base gcc abuild binutils binutils-doc gcc-doc
|
||||
|
||||
WORKDIR /cloudreve_builder
|
||||
RUN git clone --recurse-submodules https://github.com/cloudreve/Cloudreve.git
|
||||
|
||||
# build frontend
|
||||
FROM node:lts-buster AS fe-builder
|
||||
|
||||
COPY ./assets /assets
|
||||
|
||||
WORKDIR /assets
|
||||
|
||||
# If encountered problems like JavaScript heap out of memory, please uncomment the following options
|
||||
ENV NODE_OPTIONS --max_old_space_size=4096
|
||||
|
||||
# yarn repo connection is unstable, adjust the network timeout to 10 min.
|
||||
RUN set -ex \
|
||||
&& yarn install --network-timeout 600000 \
|
||||
&& yarn run build
|
||||
WORKDIR /cloudreve_builder/Cloudreve/assets
|
||||
RUN yarn install --network-timeout 1000000
|
||||
RUN yarn run build && rm -rf build/*.map
|
||||
|
||||
# build backend
|
||||
FROM golang:1.15.1-alpine3.12 AS be-builder
|
||||
|
||||
ENV GO111MODULE on
|
||||
|
||||
COPY . /go/src/github.com/cloudreve/Cloudreve/v3
|
||||
COPY --from=fe-builder /assets/build/ /go/src/github.com/cloudreve/Cloudreve/v3/assets/build/
|
||||
|
||||
WORKDIR /go/src/github.com/cloudreve/Cloudreve/v3
|
||||
|
||||
RUN set -ex \
|
||||
&& apk upgrade \
|
||||
&& apk add gcc libc-dev git \
|
||||
WORKDIR /cloudreve_builder/Cloudreve
|
||||
RUN tag_name=$(git describe --tags) \
|
||||
&& export COMMIT_SHA=$(git rev-parse --short HEAD) \
|
||||
&& export VERSION=$(git describe --tags) \
|
||||
&& (cd && go get github.com/rakyll/statik) \
|
||||
&& statik -src=assets/build/ -include=*.html,*.js,*.json,*.css,*.png,*.svg,*.ico -f \
|
||||
&& go install -ldflags "-X 'github.com/cloudreve/Cloudreve/v3/pkg/conf.BackendVersion=${VERSION}' \
|
||||
-X 'github.com/cloudreve/Cloudreve/v3/pkg/conf.LastCommit=${COMMIT_SHA}'\
|
||||
-w -s"
|
||||
&& go build -a -o cloudreve -ldflags " -X 'github.com/HFO4/cloudreve/pkg/conf.BackendVersion=$tag_name' -X 'github.com/HFO4/cloudreve/pkg/conf.LastCommit=$COMMIT_SHA'"
|
||||
|
||||
|
||||
# build final image
|
||||
FROM alpine:3.12 AS dist
|
||||
FROM alpine:latest
|
||||
|
||||
LABEL maintainer="mritd <mritd@linux.com>"
|
||||
WORKDIR /cloudreve
|
||||
|
||||
# we use the Asia/Shanghai timezone by default, you can be modified
|
||||
# by `docker build --build-arg=TZ=Other_Timezone ...`
|
||||
ARG TZ="Asia/Shanghai"
|
||||
RUN apk update && apk add --no-cache tzdata
|
||||
|
||||
ENV TZ ${TZ}
|
||||
# we using the `Asia/Shanghai` timezone by default, you can do modification at your will
|
||||
RUN cp /usr/share/zoneinfo/Asia/Shanghai /etc/localtime \
|
||||
&& echo "Asia/Shanghai" > /etc/timezone
|
||||
|
||||
COPY --from=be-builder /go/bin/Cloudreve /cloudreve/cloudreve
|
||||
COPY docker-bootstrap.sh /cloudreve/bootstrap.sh
|
||||
COPY --from=cloudreve_builder /cloudreve_builder/Cloudreve/cloudreve ./
|
||||
|
||||
RUN apk upgrade \
|
||||
&& apk add bash tzdata aria2 \
|
||||
&& ln -s /cloudreve/cloudreve /usr/bin/cloudreve \
|
||||
&& ln -sf /usr/share/zoneinfo/${TZ} /etc/localtime \
|
||||
&& echo ${TZ} > /etc/timezone \
|
||||
&& rm -rf /var/cache/apk/* \
|
||||
&& mkdir /etc/cloudreve \
|
||||
&& ln -s /etc/cloudreve/cloureve.db /cloudreve/cloudreve.db \
|
||||
&& ln -s /etc/cloudreve/conf.ini /cloudreve/conf.ini
|
||||
# prepare permissions and aria2 dir
|
||||
RUN chmod +x ./cloudreve && mkdir -p /data/aria2 && chmod -R 766 /data/aria2
|
||||
|
||||
# cloudreve use tcp 5212 port by default
|
||||
EXPOSE 5212/tcp
|
||||
EXPOSE 5212
|
||||
VOLUME ["/cloudreve/uploads", "/cloudreve/avatar", "/data"]
|
||||
|
||||
# cloudreve stores all files(including executable file) in the `/cloudreve`
|
||||
# directory by default; users should mount the configfile to the `/etc/cloudreve`
|
||||
# directory by themselves for persistence considerations, and the data storage
|
||||
# directory recommends using `/data` directory.
|
||||
VOLUME /etc/cloudreve
|
||||
|
||||
VOLUME /data
|
||||
|
||||
ENTRYPOINT ["sh", "/cloudreve/bootstrap.sh"]
|
||||
ENTRYPOINT ["./cloudreve"]
|
||||
27
README.md
27
README.md
@@ -9,16 +9,19 @@
|
||||
<h4 align="center">支持多家云存储驱动的公有云文件系统.</h4>
|
||||
|
||||
<p align="center">
|
||||
<a href="https://travis-ci.com/github/cloudreve/Cloudreve/">
|
||||
<img src="https://img.shields.io/travis/com/cloudreve/Cloudreve?style=flat-square"
|
||||
alt="travis">
|
||||
<a href="https://github.com/cloudreve/Cloudreve/actions/workflows/test.yml">
|
||||
<img src="https://img.shields.io/github/workflow/status/cloudreve/Cloudreve/Test?style=flat-square"
|
||||
alt="GitHub Test Workflow">
|
||||
</a>
|
||||
<a href="https://codecov.io/gh/cloudreve/Cloudreve"><img src="https://img.shields.io/codecov/c/github/cloudreve/Cloudreve?style=flat-square"></a>
|
||||
<a href="https://goreportcard.com/report/github.com/cloudreve/Cloudreve">
|
||||
<img src="https://goreportcard.com/badge/github.com/cloudreve/Cloudreve?style=flat-square">
|
||||
</a>
|
||||
<a href="https://github.com/cloudreve/Cloudreve/releases">
|
||||
<img src="https://img.shields.io/github/v/release/cloudreve/Cloudreve?include_prereleases&style=flat-square">
|
||||
<img src="https://img.shields.io/github/v/release/cloudreve/Cloudreve?include_prereleases&style=flat-square" />
|
||||
</a>
|
||||
<a href="https://hub.docker.com/r/cloudreve/cloudreve">
|
||||
<img src="https://img.shields.io/docker/image-size/cloudreve/cloudreve?style=flat-square"/>
|
||||
</a>
|
||||
</p>
|
||||
|
||||
@@ -68,7 +71,7 @@ chmod +x ./cloudreve
|
||||
|
||||
## :gear: 构建
|
||||
|
||||
自行构建前需要拥有 `Go >= 1.13`、`yarn`等必要依赖。
|
||||
自行构建前需要拥有 `Go >= 1.17`、`yarn`等必要依赖。
|
||||
|
||||
#### 克隆代码
|
||||
|
||||
@@ -85,19 +88,7 @@ cd assets
|
||||
yarn install
|
||||
# 开始构建
|
||||
yarn run build
|
||||
```
|
||||
|
||||
#### 嵌入静态资源
|
||||
|
||||
```shell
|
||||
# 回到项目主目录
|
||||
cd ../
|
||||
|
||||
# 安装 statik, 用于嵌入静态资源
|
||||
go get github.com/rakyll/statik
|
||||
|
||||
# 开始嵌入
|
||||
statik -src=assets/build/ -include=*.html,*.js,*.json,*.css,*.png,*.svg,*.ico -f
|
||||
```
|
||||
|
||||
#### 编译项目
|
||||
@@ -108,7 +99,7 @@ export COMMIT_SHA=$(git rev-parse --short HEAD)
|
||||
export VERSION=$(git describe --tags)
|
||||
|
||||
# 开始编译
|
||||
go build -a -o cloudreve -ldflags " -X 'github.com/cloudreve/Cloudreve/v3/pkg/conf.BackendVersion=$VERSION' -X 'github.com/cloudreve/Cloudreve/v3/pkg/conf.LastCommit=$COMMIT_SHA'"
|
||||
go build -a -o cloudreve -ldflags "-s -w -X 'github.com/cloudreve/Cloudreve/v3/pkg/conf.BackendVersion=$VERSION' -X 'github.com/cloudreve/Cloudreve/v3/pkg/conf.LastCommit=$COMMIT_SHA'"
|
||||
```
|
||||
|
||||
你也可以使用项目根目录下的`build.sh`快速开始构建:
|
||||
|
||||
2
assets
2
assets
Submodule assets updated: 88c1133306...ac3af6097f
@@ -1,6 +1,7 @@
|
||||
package bootstrap
|
||||
|
||||
import (
|
||||
"embed"
|
||||
model "github.com/cloudreve/Cloudreve/v3/models"
|
||||
"github.com/cloudreve/Cloudreve/v3/models/scripts"
|
||||
"github.com/cloudreve/Cloudreve/v3/pkg/aria2"
|
||||
@@ -16,7 +17,7 @@ import (
|
||||
)
|
||||
|
||||
// Init 初始化启动
|
||||
func Init(path string) {
|
||||
func Init(path string, statics embed.FS) {
|
||||
InitApplication()
|
||||
conf.Init(path)
|
||||
// Debug 关闭时,切换为生产模式
|
||||
@@ -37,7 +38,7 @@ func Init(path string) {
|
||||
{
|
||||
"both",
|
||||
func() {
|
||||
cache.Init()
|
||||
cache.Init(conf.SystemConfig.Mode == "slave")
|
||||
},
|
||||
},
|
||||
{
|
||||
@@ -79,7 +80,7 @@ func Init(path string) {
|
||||
{
|
||||
"master",
|
||||
func() {
|
||||
InitStatic()
|
||||
InitStatic(statics)
|
||||
},
|
||||
},
|
||||
{
|
||||
|
||||
@@ -1,17 +1,20 @@
|
||||
package bootstrap
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"embed"
|
||||
"encoding/json"
|
||||
"io"
|
||||
"io/ioutil"
|
||||
"io/fs"
|
||||
"net/http"
|
||||
"path"
|
||||
"path/filepath"
|
||||
|
||||
"github.com/pkg/errors"
|
||||
|
||||
"github.com/cloudreve/Cloudreve/v3/pkg/conf"
|
||||
"github.com/cloudreve/Cloudreve/v3/pkg/util"
|
||||
_ "github.com/cloudreve/Cloudreve/v3/statik"
|
||||
|
||||
"github.com/gin-contrib/static"
|
||||
"github.com/rakyll/statik/fs"
|
||||
)
|
||||
|
||||
const StaticFolder = "statics"
|
||||
@@ -35,124 +38,100 @@ func (b *GinFS) Open(name string) (http.File, error) {
|
||||
|
||||
// Exists 文件是否存在
|
||||
func (b *GinFS) Exists(prefix string, filepath string) bool {
|
||||
|
||||
if _, err := b.FS.Open(filepath); err != nil {
|
||||
return false
|
||||
}
|
||||
return true
|
||||
|
||||
}
|
||||
|
||||
// InitStatic 初始化静态资源文件
|
||||
func InitStatic() {
|
||||
var err error
|
||||
|
||||
func InitStatic(statics embed.FS) {
|
||||
if util.Exists(util.RelativePath(StaticFolder)) {
|
||||
util.Log().Info("检测到 statics 目录存在,将使用此目录下的静态资源文件")
|
||||
StaticFS = static.LocalFile(util.RelativePath("statics"), false)
|
||||
|
||||
// 检查静态资源的版本
|
||||
f, err := StaticFS.Open("version.json")
|
||||
if err != nil {
|
||||
util.Log().Warning("静态资源版本标识文件不存在,请重新构建或删除 statics 目录")
|
||||
return
|
||||
}
|
||||
|
||||
b, err := ioutil.ReadAll(f)
|
||||
if err != nil {
|
||||
util.Log().Warning("无法读取静态资源文件版本,请重新构建或删除 statics 目录")
|
||||
return
|
||||
}
|
||||
|
||||
var v staticVersion
|
||||
if err := json.Unmarshal(b, &v); err != nil {
|
||||
util.Log().Warning("无法解析静态资源文件版本, %s", err)
|
||||
return
|
||||
}
|
||||
|
||||
staticName := "cloudreve-frontend"
|
||||
if conf.IsPro == "true" {
|
||||
staticName += "-pro"
|
||||
}
|
||||
|
||||
if v.Name != staticName {
|
||||
util.Log().Warning("静态资源版本不匹配,请重新构建或删除 statics 目录")
|
||||
return
|
||||
}
|
||||
|
||||
if v.Version != conf.RequiredStaticVersion {
|
||||
util.Log().Warning("静态资源版本不匹配 [当前 %s, 需要: %s],请重新构建或删除 statics 目录", v.Version, conf.RequiredStaticVersion)
|
||||
return
|
||||
}
|
||||
|
||||
} else {
|
||||
StaticFS = &GinFS{}
|
||||
StaticFS.(*GinFS).FS, err = fs.New()
|
||||
// 初始化静态资源
|
||||
embedFS, err := fs.Sub(statics, "assets/build")
|
||||
if err != nil {
|
||||
util.Log().Panic("无法初始化静态资源, %s", err)
|
||||
}
|
||||
|
||||
StaticFS = &GinFS{
|
||||
FS: http.FS(embedFS),
|
||||
}
|
||||
}
|
||||
// 检查静态资源的版本
|
||||
f, err := StaticFS.Open("version.json")
|
||||
if err != nil {
|
||||
util.Log().Warning("静态资源版本标识文件不存在,请重新构建或删除 statics 目录")
|
||||
return
|
||||
}
|
||||
|
||||
b, err := io.ReadAll(f)
|
||||
if err != nil {
|
||||
util.Log().Warning("无法读取静态资源文件版本,请重新构建或删除 statics 目录")
|
||||
return
|
||||
}
|
||||
|
||||
var v staticVersion
|
||||
if err := json.Unmarshal(b, &v); err != nil {
|
||||
util.Log().Warning("无法解析静态资源文件版本, %s", err)
|
||||
return
|
||||
}
|
||||
|
||||
staticName := "cloudreve-frontend"
|
||||
if conf.IsPro == "true" {
|
||||
staticName += "-pro"
|
||||
}
|
||||
|
||||
if v.Name != staticName {
|
||||
util.Log().Warning("静态资源版本不匹配,请重新构建或删除 statics 目录")
|
||||
return
|
||||
}
|
||||
|
||||
if v.Version != conf.RequiredStaticVersion {
|
||||
util.Log().Warning("静态资源版本不匹配 [当前 %s, 需要: %s],请重新构建或删除 statics 目录", v.Version, conf.RequiredStaticVersion)
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
// Eject 抽离内置静态资源
|
||||
func Eject() {
|
||||
staticFS, err := fs.New()
|
||||
func Eject(statics embed.FS) {
|
||||
// 初始化静态资源
|
||||
embedFS, err := fs.Sub(statics, "assets/build")
|
||||
if err != nil {
|
||||
util.Log().Panic("无法初始化静态资源, %s", err)
|
||||
}
|
||||
|
||||
root, err := staticFS.Open("/")
|
||||
if err != nil {
|
||||
util.Log().Panic("根目录不存在, %s", err)
|
||||
}
|
||||
|
||||
var walk func(relPath string, object http.File)
|
||||
walk = func(relPath string, object http.File) {
|
||||
stat, err := object.Stat()
|
||||
var walk func(relPath string, d fs.DirEntry, err error) error
|
||||
walk = func(relPath string, d fs.DirEntry, err error) error {
|
||||
if err != nil {
|
||||
util.Log().Error("无法获取[%s]的信息, %s, 跳过...", relPath, err)
|
||||
return
|
||||
return errors.Errorf("无法获取[%s]的信息, %s, 跳过...", relPath, err)
|
||||
}
|
||||
|
||||
if !stat.IsDir() {
|
||||
if !d.IsDir() {
|
||||
// 写入文件
|
||||
out, err := util.CreatNestedFile(util.RelativePath(StaticFolder + relPath))
|
||||
out, err := util.CreatNestedFile(filepath.Join(util.RelativePath(""), StaticFolder, relPath))
|
||||
defer out.Close()
|
||||
|
||||
if err != nil {
|
||||
util.Log().Error("无法创建文件[%s], %s, 跳过...", relPath, err)
|
||||
return
|
||||
return errors.Errorf("无法创建文件[%s], %s, 跳过...", relPath, err)
|
||||
}
|
||||
|
||||
util.Log().Info("导出 [%s]...", relPath)
|
||||
if _, err := io.Copy(out, object); err != nil {
|
||||
util.Log().Error("无法写入文件[%s], %s, 跳过...", relPath, err)
|
||||
return
|
||||
obj, _ := embedFS.Open(relPath)
|
||||
if _, err := io.Copy(out, bufio.NewReader(obj)); err != nil {
|
||||
return errors.Errorf("无法写入文件[%s], %s, 跳过...", relPath, err)
|
||||
}
|
||||
} else {
|
||||
// 列出目录
|
||||
objects, err := object.Readdir(0)
|
||||
if err != nil {
|
||||
util.Log().Error("无法步入子目录[%s], %s, 跳过...", relPath, err)
|
||||
return
|
||||
}
|
||||
|
||||
// 递归遍历子目录
|
||||
for _, newObject := range objects {
|
||||
newPath := path.Join(relPath, newObject.Name())
|
||||
newRoot, err := staticFS.Open(newPath)
|
||||
if err != nil {
|
||||
util.Log().Error("无法打开对象[%s], %s, 跳过...", newPath, err)
|
||||
continue
|
||||
}
|
||||
walk(newPath, newRoot)
|
||||
}
|
||||
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
util.Log().Info("开始导出内置静态资源...")
|
||||
walk("/", root)
|
||||
// util.Log().Info("开始导出内置静态资源...")
|
||||
err = fs.WalkDir(embedFS, ".", walk)
|
||||
if err != nil {
|
||||
util.Log().Error("导出内置静态资源遇到错误:%s", err)
|
||||
return
|
||||
}
|
||||
util.Log().Info("内置静态资源导出完成")
|
||||
}
|
||||
|
||||
122
build.sh
122
build.sh
@@ -1,13 +1,16 @@
|
||||
#!/bin/bash
|
||||
|
||||
REPO=$(cd $(dirname $0); pwd)
|
||||
REPO=$(
|
||||
cd $(dirname $0)
|
||||
pwd
|
||||
)
|
||||
COMMIT_SHA=$(git rev-parse --short HEAD)
|
||||
VERSION=$(git describe --tags)
|
||||
ASSETS="false"
|
||||
BINARY="false"
|
||||
RELEASE="false"
|
||||
|
||||
debugInfo () {
|
||||
debugInfo() {
|
||||
echo "Repo: $REPO"
|
||||
echo "Build assets: $ASSETS"
|
||||
echo "Build binary: $BINARY"
|
||||
@@ -16,10 +19,9 @@ debugInfo () {
|
||||
echo "Commit: $COMMIT_SHA"
|
||||
}
|
||||
|
||||
buildAssets () {
|
||||
buildAssets() {
|
||||
cd $REPO
|
||||
rm -rf assets/build
|
||||
rm -f statik/statik.go
|
||||
|
||||
export CI=false
|
||||
|
||||
@@ -27,94 +29,88 @@ buildAssets () {
|
||||
|
||||
yarn install
|
||||
yarn run build
|
||||
|
||||
if ! [ -x "$(command -v statik)" ]; then
|
||||
export CGO_ENABLED=0
|
||||
go get github.com/rakyll/statik
|
||||
fi
|
||||
|
||||
cd $REPO
|
||||
statik -src=assets/build/ -include=*.html,*.js,*.json,*.css,*.png,*.svg,*.ico,*.ttf -f
|
||||
cd build
|
||||
rm -rf *.map
|
||||
}
|
||||
|
||||
buildBinary () {
|
||||
buildBinary() {
|
||||
cd $REPO
|
||||
go build -a -o cloudreve -ldflags " -X 'github.com/cloudreve/Cloudreve/v3/pkg/conf.BackendVersion=$VERSION' -X 'github.com/cloudreve/Cloudreve/v3/pkg/conf.LastCommit=$COMMIT_SHA'"
|
||||
}
|
||||
|
||||
_build() {
|
||||
local osarch=$1
|
||||
IFS=/ read -r -a arr <<<"$osarch"
|
||||
os="${arr[0]}"
|
||||
arch="${arr[1]}"
|
||||
gcc="${arr[2]}"
|
||||
local osarch=$1
|
||||
IFS=/ read -r -a arr <<<"$osarch"
|
||||
os="${arr[0]}"
|
||||
arch="${arr[1]}"
|
||||
gcc="${arr[2]}"
|
||||
|
||||
# Go build to build the binary.
|
||||
export GOOS=$os
|
||||
export GOARCH=$arch
|
||||
export CC=$gcc
|
||||
export CGO_ENABLED=1
|
||||
# Go build to build the binary.
|
||||
export GOOS=$os
|
||||
export GOARCH=$arch
|
||||
export CC=$gcc
|
||||
export CGO_ENABLED=1
|
||||
|
||||
if [ -n "$VERSION" ]; then
|
||||
out="release/cloudreve_${VERSION}_${os}_${arch}"
|
||||
else
|
||||
out="release/cloudreve_${COMMIT_SHA}_${os}_${arch}"
|
||||
fi
|
||||
if [ -n "$VERSION" ]; then
|
||||
out="release/cloudreve_${VERSION}_${os}_${arch}"
|
||||
else
|
||||
out="release/cloudreve_${COMMIT_SHA}_${os}_${arch}"
|
||||
fi
|
||||
|
||||
go build -a -o "${out}" -ldflags " -X 'github.com/cloudreve/Cloudreve/v3/pkg/conf.BackendVersion=$VERSION' -X 'github.com/cloudreve/Cloudreve/v3/pkg/conf.LastCommit=$COMMIT_SHA'"
|
||||
go build -a -o "${out}" -ldflags " -X 'github.com/cloudreve/Cloudreve/v3/pkg/conf.BackendVersion=$VERSION' -X 'github.com/cloudreve/Cloudreve/v3/pkg/conf.LastCommit=$COMMIT_SHA'"
|
||||
|
||||
if [ "$os" = "windows" ]; then
|
||||
mv $out release/cloudreve.exe
|
||||
zip -j -q "${out}.zip" release/cloudreve.exe
|
||||
rm -f "release/cloudreve.exe"
|
||||
else
|
||||
mv $out release/cloudreve
|
||||
tar -zcvf "${out}.tar.gz" -C release cloudreve
|
||||
rm -f "release/cloudreve"
|
||||
fi
|
||||
if [ "$os" = "windows" ]; then
|
||||
mv $out release/cloudreve.exe
|
||||
zip -j -q "${out}.zip" release/cloudreve.exe
|
||||
rm -f "release/cloudreve.exe"
|
||||
else
|
||||
mv $out release/cloudreve
|
||||
tar -zcvf "${out}.tar.gz" -C release cloudreve
|
||||
rm -f "release/cloudreve"
|
||||
fi
|
||||
}
|
||||
|
||||
release(){
|
||||
release() {
|
||||
cd $REPO
|
||||
## List of architectures and OS to test coss compilation.
|
||||
SUPPORTED_OSARCH="linux/amd64/gcc linux/arm/arm-linux-gnueabihf-gcc windows/amd64/x86_64-w64-mingw32-gcc linux/arm64/aarch64-linux-gnu-gcc"
|
||||
|
||||
echo "Release builds for OS/Arch/CC: ${SUPPORTED_OSARCH}"
|
||||
for each_osarch in ${SUPPORTED_OSARCH}; do
|
||||
_build "${each_osarch}"
|
||||
_build "${each_osarch}"
|
||||
done
|
||||
}
|
||||
|
||||
usage() {
|
||||
echo "Usage: $0 [-a] [-c] [-b] [-r]" 1>&2;
|
||||
exit 1;
|
||||
echo "Usage: $0 [-a] [-c] [-b] [-r]" 1>&2
|
||||
exit 1
|
||||
}
|
||||
|
||||
while getopts "bacr:d" o; do
|
||||
case "${o}" in
|
||||
b)
|
||||
ASSETS="true"
|
||||
BINARY="true"
|
||||
;;
|
||||
a)
|
||||
ASSETS="true"
|
||||
;;
|
||||
c)
|
||||
BINARY="true"
|
||||
;;
|
||||
r)
|
||||
ASSETS="true"
|
||||
RELEASE="true"
|
||||
;;
|
||||
d)
|
||||
DEBUG="true"
|
||||
;;
|
||||
*)
|
||||
usage
|
||||
;;
|
||||
b)
|
||||
ASSETS="true"
|
||||
BINARY="true"
|
||||
;;
|
||||
a)
|
||||
ASSETS="true"
|
||||
;;
|
||||
c)
|
||||
BINARY="true"
|
||||
;;
|
||||
r)
|
||||
ASSETS="true"
|
||||
RELEASE="true"
|
||||
;;
|
||||
d)
|
||||
DEBUG="true"
|
||||
;;
|
||||
*)
|
||||
usage
|
||||
;;
|
||||
esac
|
||||
done
|
||||
shift $((OPTIND-1))
|
||||
shift $((OPTIND - 1))
|
||||
|
||||
if [ "$DEBUG" = "true" ]; then
|
||||
debugInfo
|
||||
|
||||
33
docker-compose.yml
Normal file
33
docker-compose.yml
Normal file
@@ -0,0 +1,33 @@
|
||||
version: "3.8"
|
||||
services:
|
||||
cloudreve:
|
||||
container_name: cloudreve
|
||||
image: cloudreve/cloudreve:latest
|
||||
restart: unless-stopped
|
||||
ports:
|
||||
- "5212:5212"
|
||||
volumes:
|
||||
- temp_data:/data
|
||||
- ./cloudreve/uploads:/cloudreve/uploads
|
||||
- ./cloudreve/conf.ini:/cloudreve/conf.ini
|
||||
- ./cloudreve/cloudreve.db:/cloudreve/cloudreve.db
|
||||
- ./cloudreve/avatar:/cloudreve/avatar
|
||||
depends_on:
|
||||
- aria2
|
||||
aria2:
|
||||
container_name: aria2
|
||||
image: p3terx/aria2-pro # third party image, please keep notice what you are doing
|
||||
restart: unless-stopped
|
||||
environment:
|
||||
- RPC_SECRET=your_aria_rpc_token # aria rpc token, customize your own
|
||||
- RPC_PORT=6800
|
||||
volumes:
|
||||
- ./aria2/config:/config
|
||||
- temp_data:/data
|
||||
volumes:
|
||||
temp_data:
|
||||
driver: local
|
||||
driver_opts:
|
||||
type: none
|
||||
device: $PWD/data
|
||||
o: bind
|
||||
69
go.mod
69
go.mod
@@ -1,47 +1,90 @@
|
||||
module github.com/cloudreve/Cloudreve/v3
|
||||
|
||||
go 1.13
|
||||
go 1.17
|
||||
|
||||
require (
|
||||
github.com/DATA-DOG/go-sqlmock v1.3.3
|
||||
github.com/aliyun/aliyun-oss-go-sdk v2.0.5+incompatible
|
||||
github.com/HFO4/aliyun-oss-go-sdk v2.2.3+incompatible
|
||||
github.com/aws/aws-sdk-go v1.31.5
|
||||
github.com/baiyubin/aliyun-sts-go-sdk v0.0.0-20180326062324-cfa1a18b161f // indirect
|
||||
github.com/duo-labs/webauthn v0.0.0-20191119193225-4bf9a0f776d4
|
||||
github.com/fatih/color v1.7.0
|
||||
github.com/gin-contrib/cors v1.3.0
|
||||
github.com/gin-contrib/gzip v0.0.2-0.20200226035851-25bef2ef21e8
|
||||
github.com/gin-contrib/sessions v0.0.1
|
||||
github.com/gin-contrib/static v0.0.0-20191128031702-f81c604d8ac2
|
||||
github.com/gin-gonic/gin v1.5.0
|
||||
github.com/gin-gonic/gin v1.7.7
|
||||
github.com/go-ini/ini v1.50.0
|
||||
github.com/go-mail/mail v2.3.1+incompatible
|
||||
github.com/gofrs/uuid v4.0.0+incompatible
|
||||
github.com/gomodule/redigo v2.0.0+incompatible
|
||||
github.com/google/go-querystring v1.0.0
|
||||
github.com/gorilla/websocket v1.4.1
|
||||
github.com/hashicorp/go-version v1.2.0
|
||||
github.com/hashicorp/go-version v1.3.0
|
||||
github.com/jinzhu/gorm v1.9.11
|
||||
github.com/juju/ratelimit v1.0.1
|
||||
github.com/mattn/go-colorable v0.1.4 // indirect
|
||||
github.com/mojocn/base64Captcha v0.0.0-20190801020520-752b1cd608b2
|
||||
github.com/nfnt/resize v0.0.0-20180221191011-83c6a9932646
|
||||
github.com/pkg/errors v0.9.1
|
||||
github.com/pquerna/otp v1.2.0
|
||||
github.com/qiniu/api.v7/v7 v7.4.0
|
||||
github.com/qiniu/go-sdk/v7 v7.11.1
|
||||
github.com/rafaeljusto/redigomock v0.0.0-20191117212112-00b2509252a1
|
||||
github.com/rakyll/statik v0.1.7
|
||||
github.com/robfig/cron/v3 v3.0.1
|
||||
github.com/smartystreets/goconvey v1.6.4 // indirect
|
||||
github.com/speps/go-hashids v2.0.0+incompatible
|
||||
github.com/stretchr/testify v1.5.1
|
||||
github.com/stretchr/testify v1.6.1
|
||||
github.com/tencentcloud/tencentcloud-sdk-go v3.0.125+incompatible
|
||||
github.com/tencentyun/cos-go-sdk-v5 v0.0.0-20200120023323-87ff3bc489ac
|
||||
github.com/upyun/go-sdk v2.1.0+incompatible
|
||||
golang.org/x/image v0.0.0-20211028202545-6944b10bf410
|
||||
golang.org/x/text v0.3.6
|
||||
gopkg.in/alexcesaro/quotedprintable.v3 v3.0.0-20150716171945-2caba252f4dc // indirect
|
||||
golang.org/x/text v0.3.7
|
||||
gopkg.in/go-playground/validator.v9 v9.29.1
|
||||
)
|
||||
|
||||
require (
|
||||
cloud.google.com/go v0.37.4 // indirect
|
||||
github.com/agl/ed25519 v0.0.0-20170116200512-5312a6153412 // indirect
|
||||
github.com/baiyubin/aliyun-sts-go-sdk v0.0.0-20180326062324-cfa1a18b161f // indirect
|
||||
github.com/boj/redistore v0.0.0-20180917114910-cd5dcc76aeff // indirect
|
||||
github.com/boombuler/barcode v1.0.1-0.20190219062509-6c824513bacc // indirect
|
||||
github.com/cloudflare/cfssl v0.0.0-20190726000631-633726f6bcb7 // indirect
|
||||
github.com/davecgh/go-spew v1.1.1 // indirect
|
||||
github.com/denisenkom/go-mssqldb v0.0.0-20190515213511-eb9f6a1743f3 // indirect
|
||||
github.com/dgrijalva/jwt-go v3.2.0+incompatible // indirect
|
||||
github.com/gin-contrib/sse v0.1.0 // indirect
|
||||
github.com/go-playground/locales v0.14.0 // indirect
|
||||
github.com/go-playground/universal-translator v0.18.0 // indirect
|
||||
github.com/go-playground/validator/v10 v10.8.0 // indirect
|
||||
github.com/go-sql-driver/mysql v1.5.0 // indirect
|
||||
github.com/golang/freetype v0.0.0-20170609003504-e2365dfdc4a0 // indirect
|
||||
github.com/golang/protobuf v1.3.3 // indirect
|
||||
github.com/google/certificate-transparency-go v1.0.21 // indirect
|
||||
github.com/gorilla/context v1.1.1 // indirect
|
||||
github.com/gorilla/securecookie v1.1.1 // indirect
|
||||
github.com/gorilla/sessions v1.1.3 // indirect
|
||||
github.com/jinzhu/inflection v1.0.0 // indirect
|
||||
github.com/jmespath/go-jmespath v0.3.0 // indirect
|
||||
github.com/json-iterator/go v1.1.9 // indirect
|
||||
github.com/katzenpost/core v0.0.7 // indirect
|
||||
github.com/leodido/go-urn v1.2.1 // indirect
|
||||
github.com/lib/pq v1.1.1 // indirect
|
||||
github.com/mattn/go-colorable v0.1.4 // indirect
|
||||
github.com/mattn/go-isatty v0.0.12 // indirect
|
||||
github.com/mattn/go-sqlite3 v1.11.0 // indirect
|
||||
github.com/mitchellh/mapstructure v1.1.2 // indirect
|
||||
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
|
||||
github.com/modern-go/reflect2 v1.0.1 // indirect
|
||||
github.com/mozillazg/go-httpheader v0.2.1 // indirect
|
||||
github.com/pmezard/go-difflib v1.0.0 // indirect
|
||||
github.com/quasoft/memstore v0.0.0-20180925164028-84a050167438 // indirect
|
||||
github.com/satori/go.uuid v1.2.0 // indirect
|
||||
github.com/smartystreets/goconvey v1.6.4 // indirect
|
||||
github.com/stretchr/objx v0.2.0 // indirect
|
||||
github.com/ugorji/go/codec v1.1.7 // indirect
|
||||
golang.org/x/crypto v0.0.0-20210921155107-089bfa567519 // indirect
|
||||
golang.org/x/sync v0.0.0-20201207232520-09787c993a3a // indirect
|
||||
golang.org/x/sys v0.0.0-20211020174200-9d6173849985 // indirect
|
||||
golang.org/x/time v0.0.0-20181108054448-85acf8d2951c // indirect
|
||||
gopkg.in/alexcesaro/quotedprintable.v3 v3.0.0-20150716171945-2caba252f4dc // indirect
|
||||
gopkg.in/ini.v1 v1.51.0 // indirect
|
||||
gopkg.in/mail.v2 v2.3.1 // indirect
|
||||
gopkg.in/yaml.v2 v2.2.8 // indirect
|
||||
gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b // indirect
|
||||
)
|
||||
|
||||
97
go.sum
97
go.sum
@@ -5,6 +5,8 @@ cloud.google.com/go v0.37.4/go.mod h1:NHPJ89PdicEuT9hdPXMROBD91xc5uRDxsMtSB16k7h
|
||||
github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
|
||||
github.com/DATA-DOG/go-sqlmock v1.3.3 h1:CWUqKXe0s8A2z6qCgkP4Kru7wC11YoAnoupUKFDnH08=
|
||||
github.com/DATA-DOG/go-sqlmock v1.3.3/go.mod h1:f/Ixk793poVmq4qj/V1dPUg2JEAKC73Q5eFN3EC/SaM=
|
||||
github.com/HFO4/aliyun-oss-go-sdk v2.2.3+incompatible h1:aX/+gJM2dAMDDy3JqWS0DJn3JfOUchf4k37P5TbBKU8=
|
||||
github.com/HFO4/aliyun-oss-go-sdk v2.2.3+incompatible/go.mod h1:8KDiKVrHK/UbXAhj+iQGp1m40rQa+UAvzBi7m22KywI=
|
||||
github.com/QcloudApi/qcloud_sign_golang v0.0.0-20141224014652-e4130a326409/go.mod h1:1pk82RBxDY/JZnPQrtqHlUFfCctgdorsd9M06fMynOM=
|
||||
github.com/Shopify/sarama v1.19.0/go.mod h1:FVkBWblsNy7DGZRfXLU0O9RCGt5g3g3yEuWXgklEdEo=
|
||||
github.com/Shopify/toxiproxy v2.1.4+incompatible/go.mod h1:OXgGpZ6Cli1/URJOF1DMxUHB2q5Ap20/P/eIdh4G0pI=
|
||||
@@ -12,9 +14,6 @@ github.com/agl/ed25519 v0.0.0-20170116200512-5312a6153412 h1:w1UutsfOrms1J05zt7I
|
||||
github.com/agl/ed25519 v0.0.0-20170116200512-5312a6153412/go.mod h1:WPjqKcmVOxf0XSf3YxCJs6N6AOSrOx3obionmG7T0y0=
|
||||
github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
|
||||
github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
|
||||
github.com/aliyun/aliyun-oss-go-sdk v2.0.0/go.mod h1:T/Aws4fEfogEE9v+HPhhw+CntffsBHJ8nXQCwKr0/g8=
|
||||
github.com/aliyun/aliyun-oss-go-sdk v2.0.5+incompatible h1:A3oZlWPD/Poa19FvNbw+Zu4yKAurDBTjlRDilYGBiS4=
|
||||
github.com/aliyun/aliyun-oss-go-sdk v2.0.5+incompatible/go.mod h1:T/Aws4fEfogEE9v+HPhhw+CntffsBHJ8nXQCwKr0/g8=
|
||||
github.com/apache/thrift v0.12.0/go.mod h1:cp2SuWMxlEZw2r+iP2GNCdIi4C1qmUzdZFSVb+bacwQ=
|
||||
github.com/aws/aws-sdk-go v1.31.5 h1:DFA7BzTydO4etqsTja+x7UfkOKQUv1xzEluLvNk81L0=
|
||||
github.com/aws/aws-sdk-go v1.31.5/go.mod h1:5zCpMtNQVjRREroY7sYe8lOMRSxkhG6MZveU8YkpAk0=
|
||||
@@ -31,6 +30,7 @@ github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDk
|
||||
github.com/cloudflare/cfssl v0.0.0-20190726000631-633726f6bcb7 h1:Puu1hUwfps3+1CUzYdAZXijuvLuRMirgiXdf3zsM2Ig=
|
||||
github.com/cloudflare/cfssl v0.0.0-20190726000631-633726f6bcb7/go.mod h1:yMWuSON2oQp+43nFtAV/uvKQIFpSPerB57DCt9t8sSA=
|
||||
github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7DoTY=
|
||||
github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
|
||||
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
|
||||
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||
@@ -61,8 +61,9 @@ github.com/gin-contrib/sse v0.1.0/go.mod h1:RHrZQHXnP2xjPF+u1gW/2HnVO7nvIa9PG3Gm
|
||||
github.com/gin-contrib/static v0.0.0-20191128031702-f81c604d8ac2 h1:xLG16iua01X7Gzms9045s2Y2niNpvSY/Zb1oBwgNYZY=
|
||||
github.com/gin-contrib/static v0.0.0-20191128031702-f81c604d8ac2/go.mod h1:VhW/Ch/3FhimwZb8Oj+qJmdMmoB8r7lmJ5auRjm50oQ=
|
||||
github.com/gin-gonic/gin v1.4.0/go.mod h1:OW2EZn3DO8Ln9oIKOvM++LBO+5UPHJJDH72/q/3rZdM=
|
||||
github.com/gin-gonic/gin v1.5.0 h1:fi+bqFAx/oLK54somfCtEZs9HeH1LHVoEPUgARpTqyc=
|
||||
github.com/gin-gonic/gin v1.5.0/go.mod h1:Nd6IXA8m5kNZdNEHMBd93KT+mdY3+bewLgRvmCsR2Do=
|
||||
github.com/gin-gonic/gin v1.7.7 h1:3DoBmSbJbZAWqXJC3SLjAPfutPJJRN1U5pALB7EeTTs=
|
||||
github.com/gin-gonic/gin v1.7.7/go.mod h1:axIBovoeJpVj8S3BwE0uPMTeReE4+AfFtqpqaZ1qq1U=
|
||||
github.com/globalsign/mgo v0.0.0-20181015135952-eeefdecb41b8/go.mod h1:xkRDCp4j0OGD1HRkm4kmhM+pmpv3AKq5SU7GMg4oO/Q=
|
||||
github.com/go-ini/ini v1.50.0 h1:ogX6RS8VstVN8MJcwhEP78hHhWaI3klN02+97bByabY=
|
||||
github.com/go-ini/ini v1.50.0/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3Ies8=
|
||||
@@ -70,10 +71,19 @@ github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2
|
||||
github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE=
|
||||
github.com/go-mail/mail v2.3.1+incompatible h1:UzNOn0k5lpfVtO31cK3hn6I4VEVGhe3lX8AJBAxXExM=
|
||||
github.com/go-mail/mail v2.3.1+incompatible/go.mod h1:VPWjmmNyRsWXQZHVHT3g0YbIINUkSmuKOiLIDkWbL6M=
|
||||
github.com/go-playground/locales v0.12.1 h1:2FITxuFt/xuCNP1Acdhv62OzaCiviiE4kotfhkmOqEc=
|
||||
github.com/go-playground/assert/v2 v2.0.1 h1:MsBgLAaY856+nPRTKrp3/OZK38U/wa0CcBYNjji3q3A=
|
||||
github.com/go-playground/assert/v2 v2.0.1/go.mod h1:VDjEfimB/XKnb+ZQfWdccd7VUvScMdVu0Titje2rxJ4=
|
||||
github.com/go-playground/locales v0.12.1/go.mod h1:IUMDtCfWo/w/mtMfIE/IG2K+Ey3ygWanZIBtBW0W2TM=
|
||||
github.com/go-playground/universal-translator v0.16.0 h1:X++omBR/4cE2MNg91AoC3rmGrCjJ8eAeUP/K/EKx4DM=
|
||||
github.com/go-playground/locales v0.13.0/go.mod h1:taPMhCMXrRLJO55olJkUXHZBHCxTMfnGwq/HNwmWNS8=
|
||||
github.com/go-playground/locales v0.14.0 h1:u50s323jtVGugKlcYeyzC0etD1HifMjqmJqb8WugfUU=
|
||||
github.com/go-playground/locales v0.14.0/go.mod h1:sawfccIbzZTqEDETgFXqTho0QybSa7l++s0DH+LDiLs=
|
||||
github.com/go-playground/universal-translator v0.16.0/go.mod h1:1AnU7NaIRDWWzGEKwgtJRd2xk99HeFyHw3yid4rvQIY=
|
||||
github.com/go-playground/universal-translator v0.17.0/go.mod h1:UkSxE5sNxxRwHyU+Scu5vgOQjsIJAF8j9muTVoKLVtA=
|
||||
github.com/go-playground/universal-translator v0.18.0 h1:82dyy6p4OuJq4/CByFNOn/jYrnRPArHwAcmLoJZxyho=
|
||||
github.com/go-playground/universal-translator v0.18.0/go.mod h1:UvRDBj+xPUEGrFYl+lu/H90nyDXpg0fqeB/AQUGNTVA=
|
||||
github.com/go-playground/validator/v10 v10.4.1/go.mod h1:nlOn6nFhuKACm19sB/8EGNn9GlaMV7XkbRSipzJ0Ii4=
|
||||
github.com/go-playground/validator/v10 v10.8.0 h1:1kAa0fCrnpv+QYdkdcRzrRM7AyYs5o8+jZdJCz9xj6k=
|
||||
github.com/go-playground/validator/v10 v10.8.0/go.mod h1:9JhgTzTaE31GZDpH/HSvHiRJrJ3iKAgqqH0Bl/Ocjdk=
|
||||
github.com/go-sql-driver/mysql v1.4.1/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w=
|
||||
github.com/go-sql-driver/mysql v1.5.0 h1:ozyZYNQW3x3HtqT1jira07DN2PArx2v7/mN66gGcHOs=
|
||||
github.com/go-sql-driver/mysql v1.5.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg=
|
||||
@@ -89,8 +99,9 @@ github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfb
|
||||
github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
|
||||
github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
|
||||
github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
|
||||
github.com/golang/protobuf v1.3.2 h1:6nsPYzhq5kReh6QImI3k5qWzO4PEbvbIW2cwSfR/6xs=
|
||||
github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
|
||||
github.com/golang/protobuf v1.3.3 h1:gyjaxf+svBWX08ZjK86iN9geUJF0H6gp2IRKX6Nf6/I=
|
||||
github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw=
|
||||
github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
|
||||
github.com/gomodule/redigo v2.0.0+incompatible h1:K/R+8tc58AaqLkqG2Ol3Qk+DR/TlNuhuh457pBFPtt0=
|
||||
github.com/gomodule/redigo v2.0.0+incompatible/go.mod h1:B4C85qUVwatsJoIUNIfCRsp7qO0iAmpGFZ4EELWSbC4=
|
||||
@@ -119,8 +130,8 @@ github.com/gorilla/sessions v1.1.3 h1:uXoZdcdA5XdXF3QzuSlheVRUvjl+1rKY7zBXL68L9R
|
||||
github.com/gorilla/sessions v1.1.3/go.mod h1:8KCfur6+4Mqcc6S0FEfKuN15Vl5MgXW92AE8ovaJD0w=
|
||||
github.com/gorilla/websocket v1.4.1 h1:q7AeDBpnBk8AogcD4DSag/Ukw/KV+YhzLj2bP5HvKCM=
|
||||
github.com/gorilla/websocket v1.4.1/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=
|
||||
github.com/hashicorp/go-version v1.2.0 h1:3vNe/fWF5CBgRIguda1meWhsZHy3m8gCJ5wx+dIzX/E=
|
||||
github.com/hashicorp/go-version v1.2.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA=
|
||||
github.com/hashicorp/go-version v1.3.0 h1:McDWVJIU/y+u1BRV06dPaLfLCaT7fUTJLp5r04x7iNw=
|
||||
github.com/hashicorp/go-version v1.3.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA=
|
||||
github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
|
||||
github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU=
|
||||
github.com/jinzhu/gorm v1.9.11 h1:gaHGvE+UnWGlbWG4Y3FUwY1EcZ5n6S9WtqBA/uySMLE=
|
||||
@@ -132,8 +143,9 @@ github.com/jinzhu/now v1.0.1/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/
|
||||
github.com/jmespath/go-jmespath v0.3.0 h1:OS12ieG61fsCg5+qLJ+SsW9NicxNkg3b25OyT2yCeUc=
|
||||
github.com/jmespath/go-jmespath v0.3.0/go.mod h1:9QtRXoHjLGCJ5IBSaohpXITPlowMeeYCZ7fLUTSywik=
|
||||
github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU=
|
||||
github.com/json-iterator/go v1.1.7 h1:KfgG9LzI+pYjr4xvmz/5H4FXjokeP+rlHLhv3iH62Fo=
|
||||
github.com/json-iterator/go v1.1.7/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
|
||||
github.com/json-iterator/go v1.1.9 h1:9yzud/Ht36ygwatGx56VwCZtlI/2AD15T1X2sjSuGns=
|
||||
github.com/json-iterator/go v1.1.9/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
|
||||
github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU=
|
||||
github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo=
|
||||
github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU=
|
||||
@@ -146,22 +158,28 @@ github.com/kidstuff/mongostore v0.0.0-20181113001930-e650cd85ee4b/go.mod h1:g2nV
|
||||
github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
|
||||
github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
|
||||
github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc=
|
||||
github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI=
|
||||
github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
|
||||
github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI=
|
||||
github.com/kr/pretty v0.3.0 h1:WgNl7dwNpEZ6jJ9k1snq4pZsg7DOEN8hP9Xw0Tsjwk0=
|
||||
github.com/kr/pretty v0.3.0/go.mod h1:640gp4NfQd8pI5XOwp5fnNeVWj67G7CFk/SaSQn7NBk=
|
||||
github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
|
||||
github.com/kr/pty v1.1.8/go.mod h1:O1sed60cT9XZ5uDucP5qwvh+TE3NnUj51EiZO/lmSfw=
|
||||
github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE=
|
||||
github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
|
||||
github.com/leodido/go-urn v1.1.0 h1:Sm1gr51B1kKyfD2BlRcLSiEkffoG96g6TPv6eRoEiB8=
|
||||
github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
|
||||
github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
|
||||
github.com/leodido/go-urn v1.1.0/go.mod h1:+cyI34gQWZcE1eQU7NVgKkkzdXDQHr1dBMtdAPozLkw=
|
||||
github.com/leodido/go-urn v1.2.0/go.mod h1:+8+nEpDfqqsY+g338gtMEUOtuK+4dEMhiQEgxpxOKII=
|
||||
github.com/leodido/go-urn v1.2.1 h1:BqpAaACuzVSgi/VLzGZIobT2z4v53pjosyNd9Yv6n/w=
|
||||
github.com/leodido/go-urn v1.2.1/go.mod h1:zt4jvISO2HfUBqxjfIshjdMTYS56ZS/qv49ictyFfxY=
|
||||
github.com/lib/pq v1.1.1 h1:sJZmqHoEaY7f+NPP8pgLB/WxulyR3fewgCM2qaSlBb4=
|
||||
github.com/lib/pq v1.1.1/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo=
|
||||
github.com/mattn/go-colorable v0.1.4 h1:snbPLB8fVfU9iwbbo30TPtbLRzwWu6aJS6Xh4eaaviA=
|
||||
github.com/mattn/go-colorable v0.1.4/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE=
|
||||
github.com/mattn/go-isatty v0.0.7/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s=
|
||||
github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s=
|
||||
github.com/mattn/go-isatty v0.0.9 h1:d5US/mDsogSGW37IV293h//ZFaeajb69h+EHFsv2xGg=
|
||||
github.com/mattn/go-isatty v0.0.9/go.mod h1:YNRxwqDuOph6SZLI9vUUz6OYw3QyUt7WiY2yME+cCiQ=
|
||||
github.com/mattn/go-isatty v0.0.12 h1:wuysRhFDzyxgEmMf5xjvJ2M9dZoWAXNNr5LSBS7uHXY=
|
||||
github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU=
|
||||
github.com/mattn/go-sqlite3 v1.11.0 h1:LDdKkqtYlom37fkvqs8rMPFKAMe8+SgjbwZ6ex1/A/Q=
|
||||
github.com/mattn/go-sqlite3 v1.11.0/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc=
|
||||
github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0=
|
||||
@@ -179,13 +197,12 @@ github.com/mojocn/base64Captcha v0.0.0-20190801020520-752b1cd608b2/go.mod h1:wAQ
|
||||
github.com/mozillazg/go-httpheader v0.2.1 h1:geV7TrjbL8KXSyvghnFm+NyTux/hxwueTSrwhe88TQQ=
|
||||
github.com/mozillazg/go-httpheader v0.2.1/go.mod h1:jJ8xECTlalr6ValeXYdOF8fFUISeBAdw6E61aqQma60=
|
||||
github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
|
||||
github.com/nfnt/resize v0.0.0-20180221191011-83c6a9932646 h1:zYyBkD/k9seD2A7fsi6Oo2LfFZAehjjQMERAvZLEDnQ=
|
||||
github.com/nfnt/resize v0.0.0-20180221191011-83c6a9932646/go.mod h1:jpp1/29i3P1S/RLdc7JQKbRpFeM1dOBd8T9ki5s+AY8=
|
||||
github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
|
||||
github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
|
||||
github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
|
||||
github.com/openzipkin/zipkin-go v0.1.6/go.mod h1:QgAqvLzwWbR/WpD4A3cGpPtJrZXNIiJc5AZX7/PBEpw=
|
||||
github.com/pierrec/lz4 v2.0.5+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY=
|
||||
github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA=
|
||||
github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
|
||||
github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
|
||||
github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
|
||||
@@ -200,17 +217,20 @@ github.com/prometheus/client_model v0.0.0-20190115171406-56726106282f/go.mod h1:
|
||||
github.com/prometheus/common v0.2.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4=
|
||||
github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk=
|
||||
github.com/prometheus/procfs v0.0.0-20190117184657-bf6a532e95b1/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk=
|
||||
github.com/qiniu/api.v7/v7 v7.4.0 h1:9dZMVQifh31QGFLVaHls6akCaS2rlj3du8MnEFd7XjQ=
|
||||
github.com/qiniu/api.v7/v7 v7.4.0/go.mod h1:VE5oC5rkE1xul0u1S2N0b2Uxq9/6hZzhyqjgK25XDcM=
|
||||
github.com/qiniu/dyn v1.3.0/go.mod h1:E8oERcm8TtwJiZvkQPbcAh0RL8jO1G0VXJMW3FAWdkk=
|
||||
github.com/qiniu/go-sdk/v7 v7.11.1 h1:/LZ9rvFS4p6SnszhGv11FNB1+n4OZvBCwFg7opH5Ovs=
|
||||
github.com/qiniu/go-sdk/v7 v7.11.1/go.mod h1:btsaOc8CA3hdVloULfFdDgDc+g4f3TDZEFsDY0BLE+w=
|
||||
github.com/qiniu/x v1.10.5/go.mod h1:03Ni9tj+N2h2aKnAz+6N0Xfl8FwMEDRC2PAlxekASDs=
|
||||
github.com/quasoft/memstore v0.0.0-20180925164028-84a050167438 h1:jnz/4VenymvySjE+Ez511s0pqVzkUOmr1fwCVytNNWk=
|
||||
github.com/quasoft/memstore v0.0.0-20180925164028-84a050167438/go.mod h1:wTPjTepVu7uJBYgZ0SdWHQlIas582j6cn2jgk4DDdlg=
|
||||
github.com/rafaeljusto/redigomock v0.0.0-20191117212112-00b2509252a1 h1:leEwA4MD1ew0lNgzz6Q4G76G3AEfeci+TMggN6WuFRs=
|
||||
github.com/rafaeljusto/redigomock v0.0.0-20191117212112-00b2509252a1/go.mod h1:JaY6n2sDr+z2WTsXkOmNRUfDy6FN0L6Nk7x06ndm4tY=
|
||||
github.com/rakyll/statik v0.1.7 h1:OF3QCZUuyPxuGEP7B4ypUa7sB/iHtqOTDYZXGM8KOdQ=
|
||||
github.com/rakyll/statik v0.1.7/go.mod h1:AlZONWzMtEnMs7W4e/1LURLiI49pIMmp6V9Unghqrcc=
|
||||
github.com/rcrowley/go-metrics v0.0.0-20181016184325-3113b8401b8a/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4=
|
||||
github.com/robfig/cron/v3 v3.0.1 h1:WdRxkvbJztn8LMz/QEvLN5sBU+xKpSqwwUO1Pjr4qDs=
|
||||
github.com/robfig/cron/v3 v3.0.1/go.mod h1:eQICP3HwyT7UooqI/z+Ov+PtYAWygg1TEWWzGIFLtro=
|
||||
github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc=
|
||||
github.com/rogpeppe/go-internal v1.8.0 h1:FCbCCtXNOY3UtUuHUYaghJg4y7Fd14rXifAYUAtL9R8=
|
||||
github.com/rogpeppe/go-internal v1.8.0/go.mod h1:WmiCO8CzOY8rg0OYDC4/i/2WRWAB6poM+XZ2dLUbcbE=
|
||||
github.com/satori/go.uuid v1.2.0 h1:0uYX9dsZ2yD7q2RtLRtPSdGDWzjeM3TbMJP9utgA0ww=
|
||||
github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0=
|
||||
github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo=
|
||||
@@ -227,8 +247,9 @@ github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoH
|
||||
github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
|
||||
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
|
||||
github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
|
||||
github.com/stretchr/testify v1.5.1 h1:nOGnQDM7FYENwehXlg/kFVnos3rEvtKTjRvOWSzb6H4=
|
||||
github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA=
|
||||
github.com/stretchr/testify v1.6.1 h1:hDPOHmpOpP40lSULcqw7IrRb/u7w6RpDC9399XyoNd0=
|
||||
github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
|
||||
github.com/tencentcloud/tencentcloud-sdk-go v3.0.125+incompatible h1:dqpmYaez7VBT7PCRBcBxkzlDOiTk7Td8ATiia1b1GuE=
|
||||
github.com/tencentcloud/tencentcloud-sdk-go v3.0.125+incompatible/go.mod h1:0PfYow01SHPMhKY31xa+EFz2RStxIqj6JFAJS+IkCi4=
|
||||
github.com/tencentyun/cos-go-sdk-v5 v0.0.0-20200120023323-87ff3bc489ac h1:PSBhZblOjdwH7SIVgcue+7OlnLHkM45KuScLZ+PiVbQ=
|
||||
@@ -244,10 +265,12 @@ go.opencensus.io v0.20.1/go.mod h1:6WKK9ahsWS3RSO+PY9ZHZUfv2irvY6gN279GOPZjmmk=
|
||||
golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
|
||||
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
|
||||
golang.org/x/crypto v0.0.0-20190325154230-a5d413f7728c/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
|
||||
golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4 h1:HuIa8hRrWRSrqYzx1qI49NNxhdi2PrY7gxVSq1JjLDc=
|
||||
golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
|
||||
golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
|
||||
golang.org/x/crypto v0.0.0-20210711020723-a769d52b0f97/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
|
||||
golang.org/x/crypto v0.0.0-20210921155107-089bfa567519 h1:7I4JAnoQBe7ZtJcBaYHi5UtiO8tQHbUSXxL+pnGRANg=
|
||||
golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
|
||||
golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
|
||||
golang.org/x/image v0.0.0-20190501045829-6d32002ffd75 h1:TbGuee8sSq15Iguxu4deQ7+Bqq/d2rsQejGcEtADAMQ=
|
||||
golang.org/x/image v0.0.0-20190501045829-6d32002ffd75/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js=
|
||||
golang.org/x/image v0.0.0-20211028202545-6944b10bf410 h1:hTftEOvwiOq2+O8k2D5/Q7COC7k5Qcrgc2TFURJYnvQ=
|
||||
golang.org/x/image v0.0.0-20211028202545-6944b10bf410/go.mod h1:023OzeP/+EPmXeapQh35lcL3II3LrY8Ic+EFFKVhULM=
|
||||
@@ -266,8 +289,9 @@ golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn
|
||||
golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
|
||||
golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
|
||||
golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
|
||||
golang.org/x/net v0.0.0-20200202094626-16171245cfb2 h1:CCH4IOTTfewWjGOlSp+zGcjutRKlBEZQ6wTn8ozI/nI=
|
||||
golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
|
||||
golang.org/x/net v0.0.0-20210226172049-e18ecbb05110 h1:qWPm9rbaAMKs8Bq/9LRpbMqxWRVUAQwMI9fVrssnTfw=
|
||||
golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
|
||||
golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
|
||||
golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
|
||||
golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
@@ -275,6 +299,8 @@ golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJ
|
||||
golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.0.0-20201207232520-09787c993a3a h1:DcqTD9SDLc+1P/r1EmRBwnVsrOwW+kk2vWf9n+1sGhs=
|
||||
golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||
golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||
golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||
@@ -285,14 +311,21 @@ golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5h
|
||||
golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20190813064441-fde4db37ae7a h1:aYOabOQFp6Vj6W1F80affTUvO9UxmJRx8K0gsfABByQ=
|
||||
golang.org/x/sys v0.0.0-20190813064441-fde4db37ae7a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.0.0-20211020174200-9d6173849985 h1:LOlKVhfDyahgmqa97awczplwkjzNaELFg3zRIJ13RYo=
|
||||
golang.org/x/sys v0.0.0-20211020174200-9d6173849985/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
|
||||
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
|
||||
golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
|
||||
golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs=
|
||||
golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
|
||||
golang.org/x/text v0.3.6 h1:aRYxNxv6iGQlyVaZmk6ZgYEDa+Jg18DxebPSrd6bg1M=
|
||||
golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
|
||||
golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
|
||||
golang.org/x/text v0.3.7 h1:olpwvP2KacW1ZWvsR7uQhoyTYvKAupfQrRGBFM352Gk=
|
||||
golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
|
||||
golang.org/x/time v0.0.0-20181108054448-85acf8d2951c h1:fqgJT0MGcGpPgpWU7VRdRjuArfcOvC4AoJmILihzhDg=
|
||||
golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
|
||||
golang.org/x/tools v0.0.0-20180828015842-6cd1fcedba52/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
|
||||
@@ -314,8 +347,10 @@ gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLks
|
||||
gopkg.in/alexcesaro/quotedprintable.v3 v3.0.0-20150716171945-2caba252f4dc h1:2gGKlE2+asNV9m7xrywl36YYNnBG5ZQ0r/BOOxqPpmk=
|
||||
gopkg.in/alexcesaro/quotedprintable.v3 v3.0.0-20150716171945-2caba252f4dc/go.mod h1:m7x9LTH6d71AHyAX77c9yqWCCa3UKHcVEj9y7hAtKDk=
|
||||
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
||||
gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY=
|
||||
gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
||||
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk=
|
||||
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q=
|
||||
gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI=
|
||||
gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys=
|
||||
gopkg.in/go-playground/assert.v1 v1.2.1 h1:xoYuJVE7KT85PYWrN730RguIQO0ePzVRfFMXadIrXTM=
|
||||
gopkg.in/go-playground/assert.v1 v1.2.1/go.mod h1:9RXL0bg/zibRAgZUYszZSwO/z8Y/a8bDuhia5mkpMnE=
|
||||
@@ -328,8 +363,12 @@ gopkg.in/mail.v2 v2.3.1 h1:WYFn/oANrAGP2C0dcV6/pbkPzv8yGzqTjPmTeO7qoXk=
|
||||
gopkg.in/mail.v2 v2.3.1/go.mod h1:htwXN1Qh09vZJ1NVKxQqHPBaCBbzKhp5GzuJEA4VJWw=
|
||||
gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw=
|
||||
gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
|
||||
gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw=
|
||||
gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
|
||||
gopkg.in/yaml.v2 v2.2.8 h1:obN1ZagJSUGI0Ek/LBmuj4SNLPfIny3KsKFopxRdj10=
|
||||
gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
|
||||
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
|
||||
gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b h1:h8qDotaEPuJATrMmW04NCwg7v22aHH28wwpauUhK9Oo=
|
||||
gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
|
||||
honnef.co/go/tools v0.0.0-20180728063816-88497007e858/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
|
||||
honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
|
||||
honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
|
||||
|
||||
8
main.go
8
main.go
@@ -1,6 +1,7 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"embed"
|
||||
"flag"
|
||||
|
||||
"github.com/cloudreve/Cloudreve/v3/bootstrap"
|
||||
@@ -15,18 +16,21 @@ var (
|
||||
scriptName string
|
||||
)
|
||||
|
||||
//go:embed assets/build
|
||||
var staticEmbed embed.FS
|
||||
|
||||
func init() {
|
||||
flag.StringVar(&confPath, "c", util.RelativePath("conf.ini"), "配置文件路径")
|
||||
flag.BoolVar(&isEject, "eject", false, "导出内置静态资源")
|
||||
flag.StringVar(&scriptName, "database-script", "", "运行内置数据库助手脚本")
|
||||
flag.Parse()
|
||||
bootstrap.Init(confPath)
|
||||
bootstrap.Init(confPath, staticEmbed)
|
||||
}
|
||||
|
||||
func main() {
|
||||
if isEject {
|
||||
// 开始导出内置静态资源文件
|
||||
bootstrap.Eject()
|
||||
bootstrap.Eject(staticEmbed)
|
||||
return
|
||||
}
|
||||
|
||||
|
||||
@@ -5,20 +5,25 @@ import (
|
||||
"context"
|
||||
"crypto/md5"
|
||||
"fmt"
|
||||
"github.com/cloudreve/Cloudreve/v3/pkg/filesystem"
|
||||
"github.com/cloudreve/Cloudreve/v3/pkg/filesystem/driver/oss"
|
||||
"github.com/cloudreve/Cloudreve/v3/pkg/filesystem/driver/upyun"
|
||||
"github.com/cloudreve/Cloudreve/v3/pkg/mq"
|
||||
"github.com/cloudreve/Cloudreve/v3/pkg/util"
|
||||
"github.com/qiniu/go-sdk/v7/auth/qbox"
|
||||
"io/ioutil"
|
||||
"net/http"
|
||||
|
||||
model "github.com/cloudreve/Cloudreve/v3/models"
|
||||
"github.com/cloudreve/Cloudreve/v3/pkg/auth"
|
||||
"github.com/cloudreve/Cloudreve/v3/pkg/cache"
|
||||
"github.com/cloudreve/Cloudreve/v3/pkg/filesystem/driver/onedrive"
|
||||
"github.com/cloudreve/Cloudreve/v3/pkg/filesystem/driver/oss"
|
||||
"github.com/cloudreve/Cloudreve/v3/pkg/filesystem/driver/upyun"
|
||||
"github.com/cloudreve/Cloudreve/v3/pkg/serializer"
|
||||
"github.com/cloudreve/Cloudreve/v3/pkg/util"
|
||||
"github.com/gin-contrib/sessions"
|
||||
"github.com/gin-gonic/gin"
|
||||
"github.com/qiniu/api.v7/v7/auth/qbox"
|
||||
)
|
||||
|
||||
const (
|
||||
CallbackFailedStatusCode = http.StatusUnauthorized
|
||||
)
|
||||
|
||||
// SignRequired 验证请求签名
|
||||
@@ -117,48 +122,60 @@ func WebDAVAuth() gin.HandlerFunc {
|
||||
}
|
||||
}
|
||||
|
||||
// 对上传会话进行验证
|
||||
func UseUploadSession(policyType string) gin.HandlerFunc {
|
||||
return func(c *gin.Context) {
|
||||
// 验证key并查找用户
|
||||
resp := uploadCallbackCheck(c, policyType)
|
||||
if resp.Code != 0 {
|
||||
c.JSON(CallbackFailedStatusCode, resp)
|
||||
c.Abort()
|
||||
return
|
||||
}
|
||||
|
||||
c.Next()
|
||||
}
|
||||
}
|
||||
|
||||
// uploadCallbackCheck 对上传回调请求的 callback key 进行验证,如果成功则返回上传用户
|
||||
func uploadCallbackCheck(c *gin.Context) (serializer.Response, *model.User) {
|
||||
func uploadCallbackCheck(c *gin.Context, policyType string) serializer.Response {
|
||||
// 验证 Callback Key
|
||||
callbackKey := c.Param("key")
|
||||
if callbackKey == "" {
|
||||
return serializer.ParamErr("Callback Key 不能为空", nil), nil
|
||||
sessionID := c.Param("sessionID")
|
||||
if sessionID == "" {
|
||||
return serializer.ParamErr("Session ID 不能为空", nil)
|
||||
}
|
||||
callbackSessionRaw, exist := cache.Get("callback_" + callbackKey)
|
||||
|
||||
callbackSessionRaw, exist := cache.Get(filesystem.UploadSessionCachePrefix + sessionID)
|
||||
if !exist {
|
||||
return serializer.ParamErr("回调会话不存在或已过期", nil), nil
|
||||
return serializer.ParamErr("上传会话不存在或已过期", nil)
|
||||
}
|
||||
|
||||
callbackSession := callbackSessionRaw.(serializer.UploadSession)
|
||||
c.Set("callbackSession", &callbackSession)
|
||||
c.Set(filesystem.UploadSessionCtx, &callbackSession)
|
||||
if callbackSession.Policy.Type != policyType {
|
||||
return serializer.Err(serializer.CodePolicyNotAllowed, "Policy not supported", nil)
|
||||
}
|
||||
|
||||
// 清理回调会话
|
||||
_ = cache.Deletes([]string{callbackKey}, "callback_")
|
||||
_ = cache.Deletes([]string{sessionID}, filesystem.UploadSessionCachePrefix)
|
||||
|
||||
// 查找用户
|
||||
user, err := model.GetActiveUserByID(callbackSession.UID)
|
||||
if err != nil {
|
||||
return serializer.Err(serializer.CodeCheckLogin, "找不到用户", err), nil
|
||||
return serializer.Err(serializer.CodeCheckLogin, "找不到用户", err)
|
||||
}
|
||||
c.Set("user", &user)
|
||||
|
||||
return serializer.Response{}, &user
|
||||
c.Set(filesystem.UserCtx, &user)
|
||||
return serializer.Response{}
|
||||
}
|
||||
|
||||
// RemoteCallbackAuth 远程回调签名验证
|
||||
func RemoteCallbackAuth() gin.HandlerFunc {
|
||||
return func(c *gin.Context) {
|
||||
// 验证key并查找用户
|
||||
resp, user := uploadCallbackCheck(c)
|
||||
if resp.Code != 0 {
|
||||
c.JSON(200, resp)
|
||||
c.Abort()
|
||||
return
|
||||
}
|
||||
|
||||
// 验证签名
|
||||
authInstance := auth.HMACAuth{SecretKey: []byte(user.Policy.SecretKey)}
|
||||
session := c.MustGet(filesystem.UploadSessionCtx).(*serializer.UploadSession)
|
||||
authInstance := auth.HMACAuth{SecretKey: []byte(session.Policy.SecretKey)}
|
||||
if err := auth.CheckRequest(authInstance, c.Request); err != nil {
|
||||
c.JSON(200, serializer.Err(serializer.CodeCheckLogin, err.Error(), err))
|
||||
c.JSON(CallbackFailedStatusCode, serializer.Err(serializer.CodeCredentialInvalid, err.Error(), err))
|
||||
c.Abort()
|
||||
return
|
||||
}
|
||||
@@ -171,16 +188,10 @@ func RemoteCallbackAuth() gin.HandlerFunc {
|
||||
// QiniuCallbackAuth 七牛回调签名验证
|
||||
func QiniuCallbackAuth() gin.HandlerFunc {
|
||||
return func(c *gin.Context) {
|
||||
// 验证key并查找用户
|
||||
resp, user := uploadCallbackCheck(c)
|
||||
if resp.Code != 0 {
|
||||
c.JSON(401, serializer.GeneralUploadCallbackFailed{Error: resp.Msg})
|
||||
c.Abort()
|
||||
return
|
||||
}
|
||||
session := c.MustGet(filesystem.UploadSessionCtx).(*serializer.UploadSession)
|
||||
|
||||
// 验证回调是否来自qiniu
|
||||
mac := qbox.NewMac(user.Policy.AccessKey, user.Policy.SecretKey)
|
||||
mac := qbox.NewMac(session.Policy.AccessKey, session.Policy.SecretKey)
|
||||
ok, err := mac.VerifyCallback(c.Request)
|
||||
if err != nil {
|
||||
util.Log().Debug("无法验证回调请求,%s", err)
|
||||
@@ -188,6 +199,7 @@ func QiniuCallbackAuth() gin.HandlerFunc {
|
||||
c.Abort()
|
||||
return
|
||||
}
|
||||
|
||||
if !ok {
|
||||
c.JSON(401, serializer.GeneralUploadCallbackFailed{Error: "回调签名无效"})
|
||||
c.Abort()
|
||||
@@ -201,14 +213,6 @@ func QiniuCallbackAuth() gin.HandlerFunc {
|
||||
// OSSCallbackAuth 阿里云OSS回调签名验证
|
||||
func OSSCallbackAuth() gin.HandlerFunc {
|
||||
return func(c *gin.Context) {
|
||||
// 验证key并查找用户
|
||||
resp, _ := uploadCallbackCheck(c)
|
||||
if resp.Code != 0 {
|
||||
c.JSON(401, serializer.GeneralUploadCallbackFailed{Error: resp.Msg})
|
||||
c.Abort()
|
||||
return
|
||||
}
|
||||
|
||||
err := oss.VerifyCallbackSignature(c.Request)
|
||||
if err != nil {
|
||||
util.Log().Debug("回调签名验证失败,%s", err)
|
||||
@@ -224,13 +228,7 @@ func OSSCallbackAuth() gin.HandlerFunc {
|
||||
// UpyunCallbackAuth 又拍云回调签名验证
|
||||
func UpyunCallbackAuth() gin.HandlerFunc {
|
||||
return func(c *gin.Context) {
|
||||
// 验证key并查找用户
|
||||
resp, user := uploadCallbackCheck(c)
|
||||
if resp.Code != 0 {
|
||||
c.JSON(401, serializer.GeneralUploadCallbackFailed{Error: resp.Msg})
|
||||
c.Abort()
|
||||
return
|
||||
}
|
||||
session := c.MustGet(filesystem.UploadSessionCtx).(*serializer.UploadSession)
|
||||
|
||||
// 获取请求正文
|
||||
body, err := ioutil.ReadAll(c.Request.Body)
|
||||
@@ -244,7 +242,7 @@ func UpyunCallbackAuth() gin.HandlerFunc {
|
||||
c.Request.Body = ioutil.NopCloser(bytes.NewReader(body))
|
||||
|
||||
// 准备验证Upyun回调签名
|
||||
handler := upyun.Driver{Policy: &user.Policy}
|
||||
handler := upyun.Driver{Policy: &session.Policy}
|
||||
contentMD5 := c.Request.Header.Get("Content-Md5")
|
||||
date := c.Request.Header.Get("Date")
|
||||
actualSignature := c.Request.Header.Get("Authorization")
|
||||
@@ -277,50 +275,10 @@ func UpyunCallbackAuth() gin.HandlerFunc {
|
||||
}
|
||||
|
||||
// OneDriveCallbackAuth OneDrive回调签名验证
|
||||
// TODO 解耦
|
||||
func OneDriveCallbackAuth() gin.HandlerFunc {
|
||||
return func(c *gin.Context) {
|
||||
// 验证key并查找用户
|
||||
resp, _ := uploadCallbackCheck(c)
|
||||
if resp.Code != 0 {
|
||||
c.JSON(401, serializer.GeneralUploadCallbackFailed{Error: resp.Msg})
|
||||
c.Abort()
|
||||
return
|
||||
}
|
||||
|
||||
// 发送回调结束信号
|
||||
onedrive.FinishCallback(c.Param("key"))
|
||||
|
||||
c.Next()
|
||||
}
|
||||
}
|
||||
|
||||
// COSCallbackAuth 腾讯云COS回调签名验证
|
||||
// TODO 解耦 测试
|
||||
func COSCallbackAuth() gin.HandlerFunc {
|
||||
return func(c *gin.Context) {
|
||||
// 验证key并查找用户
|
||||
resp, _ := uploadCallbackCheck(c)
|
||||
if resp.Code != 0 {
|
||||
c.JSON(401, serializer.GeneralUploadCallbackFailed{Error: resp.Msg})
|
||||
c.Abort()
|
||||
return
|
||||
}
|
||||
|
||||
c.Next()
|
||||
}
|
||||
}
|
||||
|
||||
// S3CallbackAuth Amazon S3回调签名验证
|
||||
func S3CallbackAuth() gin.HandlerFunc {
|
||||
return func(c *gin.Context) {
|
||||
// 验证key并查找用户
|
||||
resp, _ := uploadCallbackCheck(c)
|
||||
if resp.Code != 0 {
|
||||
c.JSON(401, serializer.GeneralUploadCallbackFailed{Error: resp.Msg})
|
||||
c.Abort()
|
||||
return
|
||||
}
|
||||
mq.GlobalMQ.Publish(c.Param("sessionID"), mq.Message{})
|
||||
|
||||
c.Next()
|
||||
}
|
||||
|
||||
@@ -3,21 +3,24 @@ package middleware
|
||||
import (
|
||||
"database/sql"
|
||||
"errors"
|
||||
"github.com/cloudreve/Cloudreve/v3/pkg/cache"
|
||||
"github.com/cloudreve/Cloudreve/v3/pkg/filesystem"
|
||||
"github.com/cloudreve/Cloudreve/v3/pkg/mq"
|
||||
"github.com/cloudreve/Cloudreve/v3/pkg/serializer"
|
||||
"github.com/qiniu/go-sdk/v7/auth/qbox"
|
||||
"io/ioutil"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/DATA-DOG/go-sqlmock"
|
||||
model "github.com/cloudreve/Cloudreve/v3/models"
|
||||
"github.com/cloudreve/Cloudreve/v3/pkg/auth"
|
||||
"github.com/cloudreve/Cloudreve/v3/pkg/cache"
|
||||
"github.com/cloudreve/Cloudreve/v3/pkg/serializer"
|
||||
"github.com/cloudreve/Cloudreve/v3/pkg/util"
|
||||
"github.com/gin-gonic/gin"
|
||||
"github.com/jinzhu/gorm"
|
||||
"github.com/qiniu/api.v7/v7/auth/qbox"
|
||||
"github.com/stretchr/testify/assert"
|
||||
)
|
||||
|
||||
@@ -223,19 +226,31 @@ func TestWebDAVAuth(t *testing.T) {
|
||||
|
||||
}
|
||||
|
||||
func TestRemoteCallbackAuth(t *testing.T) {
|
||||
func TestUseUploadSession(t *testing.T) {
|
||||
asserts := assert.New(t)
|
||||
rec := httptest.NewRecorder()
|
||||
AuthFunc := RemoteCallbackAuth()
|
||||
AuthFunc := UseUploadSession("local")
|
||||
|
||||
// sessionID 为空
|
||||
{
|
||||
|
||||
c, _ := gin.CreateTestContext(rec)
|
||||
c.Params = []gin.Param{}
|
||||
c.Request, _ = http.NewRequest("POST", "/api/v3/callback/remote/sessionID", nil)
|
||||
authInstance := auth.HMACAuth{SecretKey: []byte("123")}
|
||||
auth.SignRequest(authInstance, c.Request, 0)
|
||||
AuthFunc(c)
|
||||
asserts.True(c.IsAborted())
|
||||
}
|
||||
|
||||
// 成功
|
||||
{
|
||||
cache.Set(
|
||||
"callback_testCallBackRemote",
|
||||
filesystem.UploadSessionCachePrefix+"testCallBackRemote",
|
||||
serializer.UploadSession{
|
||||
UID: 1,
|
||||
PolicyID: 513,
|
||||
VirtualPath: "/",
|
||||
Policy: model.Policy{Type: "local"},
|
||||
},
|
||||
0,
|
||||
)
|
||||
@@ -248,7 +263,7 @@ func TestRemoteCallbackAuth(t *testing.T) {
|
||||
WillReturnRows(sqlmock.NewRows([]string{"id", "secret_key"}).AddRow(2, "123"))
|
||||
c, _ := gin.CreateTestContext(rec)
|
||||
c.Params = []gin.Param{
|
||||
{"key", "testCallBackRemote"},
|
||||
{"sessionID", "testCallBackRemote"},
|
||||
}
|
||||
c.Request, _ = http.NewRequest("POST", "/api/v3/callback/remote/testCallBackRemote", nil)
|
||||
authInstance := auth.HMACAuth{SecretKey: []byte("123")}
|
||||
@@ -257,80 +272,96 @@ func TestRemoteCallbackAuth(t *testing.T) {
|
||||
asserts.NoError(mock.ExpectationsWereMet())
|
||||
asserts.False(c.IsAborted())
|
||||
}
|
||||
}
|
||||
|
||||
// Callback Key 不存在
|
||||
func TestUploadCallbackCheck(t *testing.T) {
|
||||
a := assert.New(t)
|
||||
rec := httptest.NewRecorder()
|
||||
|
||||
// 上传会话不存在
|
||||
{
|
||||
|
||||
c, _ := gin.CreateTestContext(rec)
|
||||
c.Params = []gin.Param{
|
||||
{"key", "testCallBackRemote"},
|
||||
{"sessionID", "testSessionNotExist"},
|
||||
}
|
||||
c.Request, _ = http.NewRequest("POST", "/api/v3/callback/remote/testCallBackRemote", nil)
|
||||
authInstance := auth.HMACAuth{SecretKey: []byte("123")}
|
||||
auth.SignRequest(authInstance, c.Request, 0)
|
||||
AuthFunc(c)
|
||||
asserts.True(c.IsAborted())
|
||||
res := uploadCallbackCheck(c, "local")
|
||||
a.Contains("上传会话不存在或已过期", res.Msg)
|
||||
}
|
||||
|
||||
// 上传策略不一致
|
||||
{
|
||||
c, _ := gin.CreateTestContext(rec)
|
||||
c.Params = []gin.Param{
|
||||
{"sessionID", "testPolicyNotMatch"},
|
||||
}
|
||||
cache.Set(
|
||||
filesystem.UploadSessionCachePrefix+"testPolicyNotMatch",
|
||||
serializer.UploadSession{
|
||||
UID: 1,
|
||||
VirtualPath: "/",
|
||||
Policy: model.Policy{Type: "remote"},
|
||||
},
|
||||
0,
|
||||
)
|
||||
res := uploadCallbackCheck(c, "local")
|
||||
a.Contains("Policy not supported", res.Msg)
|
||||
}
|
||||
|
||||
// 用户不存在
|
||||
{
|
||||
c, _ := gin.CreateTestContext(rec)
|
||||
c.Params = []gin.Param{
|
||||
{"sessionID", "testUserNotExist"},
|
||||
}
|
||||
cache.Set(
|
||||
"callback_testCallBackRemote",
|
||||
filesystem.UploadSessionCachePrefix+"testUserNotExist",
|
||||
serializer.UploadSession{
|
||||
UID: 1,
|
||||
PolicyID: 550,
|
||||
UID: 313,
|
||||
VirtualPath: "/",
|
||||
Policy: model.Policy{Type: "remote"},
|
||||
},
|
||||
0,
|
||||
)
|
||||
cache.Deletes([]string{"1"}, "policy_")
|
||||
mock.ExpectQuery("SELECT(.+)users(.+)").
|
||||
WillReturnRows(sqlmock.NewRows([]string{"id", "group_id"}))
|
||||
res := uploadCallbackCheck(c, "remote")
|
||||
a.Contains("找不到用户", res.Msg)
|
||||
a.NoError(mock.ExpectationsWereMet())
|
||||
_, ok := cache.Get(filesystem.UploadSessionCachePrefix + "testUserNotExist")
|
||||
a.False(ok)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRemoteCallbackAuth(t *testing.T) {
|
||||
asserts := assert.New(t)
|
||||
rec := httptest.NewRecorder()
|
||||
AuthFunc := RemoteCallbackAuth()
|
||||
|
||||
// 成功
|
||||
{
|
||||
c, _ := gin.CreateTestContext(rec)
|
||||
c.Params = []gin.Param{
|
||||
{"key", "testCallBackRemote"},
|
||||
}
|
||||
c.Set(filesystem.UploadSessionCtx, &serializer.UploadSession{
|
||||
UID: 1,
|
||||
VirtualPath: "/",
|
||||
Policy: model.Policy{SecretKey: "123"},
|
||||
})
|
||||
c.Request, _ = http.NewRequest("POST", "/api/v3/callback/remote/testCallBackRemote", nil)
|
||||
authInstance := auth.HMACAuth{SecretKey: []byte("123")}
|
||||
auth.SignRequest(authInstance, c.Request, 0)
|
||||
AuthFunc(c)
|
||||
asserts.NoError(mock.ExpectationsWereMet())
|
||||
asserts.True(c.IsAborted())
|
||||
asserts.False(c.IsAborted())
|
||||
}
|
||||
|
||||
// 签名错误
|
||||
{
|
||||
cache.Set(
|
||||
"callback_testCallBackRemote",
|
||||
serializer.UploadSession{
|
||||
UID: 1,
|
||||
PolicyID: 514,
|
||||
VirtualPath: "/",
|
||||
},
|
||||
0,
|
||||
)
|
||||
cache.Deletes([]string{"1"}, "policy_")
|
||||
mock.ExpectQuery("SELECT(.+)users(.+)").
|
||||
WillReturnRows(sqlmock.NewRows([]string{"id", "group_id"}).AddRow(1, 1))
|
||||
mock.ExpectQuery("SELECT(.+)groups(.+)").
|
||||
WillReturnRows(sqlmock.NewRows([]string{"id", "policies"}).AddRow(1, "[514]"))
|
||||
mock.ExpectQuery("SELECT(.+)policies(.+)").
|
||||
WillReturnRows(sqlmock.NewRows([]string{"id", "secret_key"}).AddRow(2, "123"))
|
||||
c, _ := gin.CreateTestContext(rec)
|
||||
c.Params = []gin.Param{
|
||||
{"key", "testCallBackRemote"},
|
||||
}
|
||||
c.Set(filesystem.UploadSessionCtx, &serializer.UploadSession{
|
||||
UID: 1,
|
||||
VirtualPath: "/",
|
||||
Policy: model.Policy{SecretKey: "123"},
|
||||
})
|
||||
c.Request, _ = http.NewRequest("POST", "/api/v3/callback/remote/testCallBackRemote", nil)
|
||||
AuthFunc(c)
|
||||
asserts.NoError(mock.ExpectationsWereMet())
|
||||
asserts.True(c.IsAborted())
|
||||
}
|
||||
|
||||
// Callback Key 为空
|
||||
{
|
||||
c, _ := gin.CreateTestContext(rec)
|
||||
c.Request, _ = http.NewRequest("POST", "/api/v3/callback/remote", nil)
|
||||
AuthFunc(c)
|
||||
asserts.True(c.IsAborted())
|
||||
}
|
||||
}
|
||||
@@ -340,39 +371,17 @@ func TestQiniuCallbackAuth(t *testing.T) {
|
||||
rec := httptest.NewRecorder()
|
||||
AuthFunc := QiniuCallbackAuth()
|
||||
|
||||
// Callback Key 相关验证失败
|
||||
{
|
||||
c, _ := gin.CreateTestContext(rec)
|
||||
c.Params = []gin.Param{
|
||||
{"key", "testQiniuBackRemote"},
|
||||
}
|
||||
c.Request, _ = http.NewRequest("POST", "/api/v3/callback/remote/testQiniuBackRemote", nil)
|
||||
AuthFunc(c)
|
||||
asserts.True(c.IsAborted())
|
||||
}
|
||||
|
||||
// 成功
|
||||
{
|
||||
cache.Set(
|
||||
"callback_testCallBackQiniu",
|
||||
serializer.UploadSession{
|
||||
UID: 1,
|
||||
PolicyID: 515,
|
||||
VirtualPath: "/",
|
||||
},
|
||||
0,
|
||||
)
|
||||
cache.Deletes([]string{"1"}, "policy_")
|
||||
mock.ExpectQuery("SELECT(.+)users(.+)").
|
||||
WillReturnRows(sqlmock.NewRows([]string{"id", "group_id"}).AddRow(1, 1))
|
||||
mock.ExpectQuery("SELECT(.+)groups(.+)").
|
||||
WillReturnRows(sqlmock.NewRows([]string{"id", "policies"}).AddRow(1, "[515]"))
|
||||
mock.ExpectQuery("SELECT(.+)policies(.+)").
|
||||
WillReturnRows(sqlmock.NewRows([]string{"id", "access_key", "secret_key"}).AddRow(2, "123", "123"))
|
||||
c, _ := gin.CreateTestContext(rec)
|
||||
c.Params = []gin.Param{
|
||||
{"key", "testCallBackQiniu"},
|
||||
}
|
||||
c.Set(filesystem.UploadSessionCtx, &serializer.UploadSession{
|
||||
UID: 1,
|
||||
VirtualPath: "/",
|
||||
Policy: model.Policy{
|
||||
SecretKey: "123",
|
||||
AccessKey: "123",
|
||||
},
|
||||
})
|
||||
c.Request, _ = http.NewRequest("POST", "/api/v3/callback/qiniu/testCallBackQiniu", nil)
|
||||
mac := qbox.NewMac("123", "123")
|
||||
token, err := mac.SignRequest(c.Request)
|
||||
@@ -385,33 +394,21 @@ func TestQiniuCallbackAuth(t *testing.T) {
|
||||
|
||||
// 验证失败
|
||||
{
|
||||
cache.Set(
|
||||
"callback_testCallBackQiniu",
|
||||
serializer.UploadSession{
|
||||
UID: 1,
|
||||
PolicyID: 516,
|
||||
VirtualPath: "/",
|
||||
},
|
||||
0,
|
||||
)
|
||||
cache.Deletes([]string{"1"}, "policy_")
|
||||
mock.ExpectQuery("SELECT(.+)users(.+)").
|
||||
WillReturnRows(sqlmock.NewRows([]string{"id", "group_id"}).AddRow(1, 1))
|
||||
mock.ExpectQuery("SELECT(.+)groups(.+)").
|
||||
WillReturnRows(sqlmock.NewRows([]string{"id", "policies"}).AddRow(1, "[516]"))
|
||||
mock.ExpectQuery("SELECT(.+)policies(.+)").
|
||||
WillReturnRows(sqlmock.NewRows([]string{"id", "access_key", "secret_key"}).AddRow(2, "123", "123"))
|
||||
c, _ := gin.CreateTestContext(rec)
|
||||
c.Params = []gin.Param{
|
||||
{"key", "testCallBackQiniu"},
|
||||
}
|
||||
c.Set(filesystem.UploadSessionCtx, &serializer.UploadSession{
|
||||
UID: 1,
|
||||
VirtualPath: "/",
|
||||
Policy: model.Policy{
|
||||
SecretKey: "123",
|
||||
AccessKey: "123",
|
||||
},
|
||||
})
|
||||
c.Request, _ = http.NewRequest("POST", "/api/v3/callback/qiniu/testCallBackQiniu", nil)
|
||||
mac := qbox.NewMac("123", "123")
|
||||
mac := qbox.NewMac("123", "1213")
|
||||
token, err := mac.SignRequest(c.Request)
|
||||
asserts.NoError(err)
|
||||
c.Request.Header["Authorization"] = []string{"QBox " + token + " "}
|
||||
c.Request.Header["Authorization"] = []string{"QBox " + token}
|
||||
AuthFunc(c)
|
||||
asserts.NoError(mock.ExpectationsWereMet())
|
||||
asserts.True(c.IsAborted())
|
||||
}
|
||||
}
|
||||
@@ -421,76 +418,41 @@ func TestOSSCallbackAuth(t *testing.T) {
|
||||
rec := httptest.NewRecorder()
|
||||
AuthFunc := OSSCallbackAuth()
|
||||
|
||||
// Callback Key 相关验证失败
|
||||
{
|
||||
c, _ := gin.CreateTestContext(rec)
|
||||
c.Params = []gin.Param{
|
||||
{"key", "testOSSBackRemote"},
|
||||
}
|
||||
c.Request, _ = http.NewRequest("POST", "/api/v3/callback/oss/testQiniuBackRemote", nil)
|
||||
AuthFunc(c)
|
||||
asserts.True(c.IsAborted())
|
||||
}
|
||||
|
||||
// 签名验证失败
|
||||
{
|
||||
cache.Set(
|
||||
"callback_testCallBackOSS",
|
||||
serializer.UploadSession{
|
||||
UID: 1,
|
||||
PolicyID: 517,
|
||||
VirtualPath: "/",
|
||||
},
|
||||
0,
|
||||
)
|
||||
cache.Deletes([]string{"1"}, "policy_")
|
||||
mock.ExpectQuery("SELECT(.+)users(.+)").
|
||||
WillReturnRows(sqlmock.NewRows([]string{"id", "group_id"}).AddRow(1, 1))
|
||||
mock.ExpectQuery("SELECT(.+)groups(.+)").
|
||||
WillReturnRows(sqlmock.NewRows([]string{"id", "policies"}).AddRow(1, "[517]"))
|
||||
mock.ExpectQuery("SELECT(.+)policies(.+)").
|
||||
WillReturnRows(sqlmock.NewRows([]string{"id", "access_key", "secret_key"}).AddRow(2, "123", "123"))
|
||||
c, _ := gin.CreateTestContext(rec)
|
||||
c.Params = []gin.Param{
|
||||
{"key", "testCallBackOSS"},
|
||||
}
|
||||
c.Set(filesystem.UploadSessionCtx, &serializer.UploadSession{
|
||||
UID: 1,
|
||||
VirtualPath: "/",
|
||||
Policy: model.Policy{
|
||||
SecretKey: "123",
|
||||
AccessKey: "123",
|
||||
},
|
||||
})
|
||||
c.Request, _ = http.NewRequest("POST", "/api/v3/callback/oss/testCallBackOSS", nil)
|
||||
mac := qbox.NewMac("123", "123")
|
||||
token, err := mac.SignRequest(c.Request)
|
||||
asserts.NoError(err)
|
||||
c.Request.Header["Authorization"] = []string{"QBox " + token}
|
||||
AuthFunc(c)
|
||||
asserts.NoError(mock.ExpectationsWereMet())
|
||||
asserts.True(c.IsAborted())
|
||||
}
|
||||
|
||||
// 成功
|
||||
{
|
||||
cache.Set(
|
||||
"callback_TnXx5E5VyfJUyM1UdkdDu1rtnJ34EbmH",
|
||||
serializer.UploadSession{
|
||||
UID: 1,
|
||||
PolicyID: 518,
|
||||
VirtualPath: "/",
|
||||
},
|
||||
0,
|
||||
)
|
||||
cache.Deletes([]string{"1"}, "policy_")
|
||||
mock.ExpectQuery("SELECT(.+)users(.+)").
|
||||
WillReturnRows(sqlmock.NewRows([]string{"id", "group_id"}).AddRow(1, 1))
|
||||
mock.ExpectQuery("SELECT(.+)groups(.+)").
|
||||
WillReturnRows(sqlmock.NewRows([]string{"id", "policies"}).AddRow(1, "[518]"))
|
||||
mock.ExpectQuery("SELECT(.+)policies(.+)").
|
||||
WillReturnRows(sqlmock.NewRows([]string{"id", "access_key", "secret_key"}).AddRow(2, "123", "123"))
|
||||
c, _ := gin.CreateTestContext(rec)
|
||||
c.Params = []gin.Param{
|
||||
{"key", "TnXx5E5VyfJUyM1UdkdDu1rtnJ34EbmH"},
|
||||
}
|
||||
c.Set(filesystem.UploadSessionCtx, &serializer.UploadSession{
|
||||
UID: 1,
|
||||
VirtualPath: "/",
|
||||
Policy: model.Policy{
|
||||
SecretKey: "123",
|
||||
AccessKey: "123",
|
||||
},
|
||||
})
|
||||
c.Request, _ = http.NewRequest("POST", "/api/v3/callback/oss/TnXx5E5VyfJUyM1UdkdDu1rtnJ34EbmH", ioutil.NopCloser(strings.NewReader(`{"name":"2f7b2ccf30e9270ea920f1ab8a4037a546a2f0d5.jpg","source_name":"1/1_hFRtDLgM_2f7b2ccf30e9270ea920f1ab8a4037a546a2f0d5.jpg","size":114020,"pic_info":"810,539"}`)))
|
||||
c.Request.Header["Authorization"] = []string{"e5LwzwTkP9AFAItT4YzvdJOHd0Y0wqTMWhsV/h5SG90JYGAmMd+8LQyj96R+9qUfJWjMt6suuUh7LaOryR87Dw=="}
|
||||
c.Request.Header["X-Oss-Pub-Key-Url"] = []string{"aHR0cHM6Ly9nb3NzcHVibGljLmFsaWNkbi5jb20vY2FsbGJhY2tfcHViX2tleV92MS5wZW0="}
|
||||
AuthFunc(c)
|
||||
asserts.NoError(mock.ExpectationsWereMet())
|
||||
asserts.False(c.IsAborted())
|
||||
}
|
||||
|
||||
@@ -507,130 +469,71 @@ func TestUpyunCallbackAuth(t *testing.T) {
|
||||
rec := httptest.NewRecorder()
|
||||
AuthFunc := UpyunCallbackAuth()
|
||||
|
||||
// Callback Key 相关验证失败
|
||||
{
|
||||
c, _ := gin.CreateTestContext(rec)
|
||||
c.Params = []gin.Param{
|
||||
{"key", "testUpyunBackRemote"},
|
||||
}
|
||||
c.Request, _ = http.NewRequest("POST", "/api/v3/callback/upyun/testUpyunBackRemote", nil)
|
||||
AuthFunc(c)
|
||||
asserts.True(c.IsAborted())
|
||||
}
|
||||
|
||||
// 无法获取请求正文
|
||||
{
|
||||
cache.Set(
|
||||
"callback_testCallBackUpyun",
|
||||
serializer.UploadSession{
|
||||
UID: 1,
|
||||
PolicyID: 509,
|
||||
VirtualPath: "/",
|
||||
},
|
||||
0,
|
||||
)
|
||||
cache.Deletes([]string{"1"}, "policy_")
|
||||
mock.ExpectQuery("SELECT(.+)users(.+)").
|
||||
WillReturnRows(sqlmock.NewRows([]string{"id", "group_id"}).AddRow(1, 1))
|
||||
mock.ExpectQuery("SELECT(.+)groups(.+)").
|
||||
WillReturnRows(sqlmock.NewRows([]string{"id", "policies"}).AddRow(1, "[519]"))
|
||||
mock.ExpectQuery("SELECT(.+)policies(.+)").
|
||||
WillReturnRows(sqlmock.NewRows([]string{"id", "access_key", "secret_key"}).AddRow(2, "123", "123"))
|
||||
c, _ := gin.CreateTestContext(rec)
|
||||
c.Params = []gin.Param{
|
||||
{"key", "testCallBackUpyun"},
|
||||
}
|
||||
c.Set(filesystem.UploadSessionCtx, &serializer.UploadSession{
|
||||
UID: 1,
|
||||
VirtualPath: "/",
|
||||
Policy: model.Policy{
|
||||
SecretKey: "123",
|
||||
AccessKey: "123",
|
||||
},
|
||||
})
|
||||
c.Request, _ = http.NewRequest("POST", "/api/v3/callback/upyun/testCallBackUpyun", ioutil.NopCloser(fakeRead("")))
|
||||
AuthFunc(c)
|
||||
asserts.NoError(mock.ExpectationsWereMet())
|
||||
asserts.True(c.IsAborted())
|
||||
}
|
||||
|
||||
// 正文MD5不一致
|
||||
{
|
||||
cache.Set(
|
||||
"callback_testCallBackUpyun",
|
||||
serializer.UploadSession{
|
||||
UID: 1,
|
||||
PolicyID: 510,
|
||||
VirtualPath: "/",
|
||||
},
|
||||
0,
|
||||
)
|
||||
cache.Deletes([]string{"1"}, "policy_")
|
||||
mock.ExpectQuery("SELECT(.+)users(.+)").
|
||||
WillReturnRows(sqlmock.NewRows([]string{"id", "group_id"}).AddRow(1, 1))
|
||||
mock.ExpectQuery("SELECT(.+)groups(.+)").
|
||||
WillReturnRows(sqlmock.NewRows([]string{"id", "policies"}).AddRow(1, "[520]"))
|
||||
mock.ExpectQuery("SELECT(.+)policies(.+)").
|
||||
WillReturnRows(sqlmock.NewRows([]string{"id", "access_key", "secret_key"}).AddRow(2, "123", "123"))
|
||||
c, _ := gin.CreateTestContext(rec)
|
||||
c.Params = []gin.Param{
|
||||
{"key", "testCallBackUpyun"},
|
||||
}
|
||||
c.Set(filesystem.UploadSessionCtx, &serializer.UploadSession{
|
||||
UID: 1,
|
||||
VirtualPath: "/",
|
||||
Policy: model.Policy{
|
||||
SecretKey: "123",
|
||||
AccessKey: "123",
|
||||
},
|
||||
})
|
||||
c.Request, _ = http.NewRequest("POST", "/api/v3/callback/upyun/testCallBackUpyun", ioutil.NopCloser(strings.NewReader("1")))
|
||||
c.Request.Header["Content-Md5"] = []string{"123"}
|
||||
AuthFunc(c)
|
||||
asserts.NoError(mock.ExpectationsWereMet())
|
||||
asserts.True(c.IsAborted())
|
||||
}
|
||||
|
||||
// 签名不一致
|
||||
{
|
||||
cache.Set(
|
||||
"callback_testCallBackUpyun",
|
||||
serializer.UploadSession{
|
||||
UID: 1,
|
||||
PolicyID: 511,
|
||||
VirtualPath: "/",
|
||||
},
|
||||
0,
|
||||
)
|
||||
cache.Deletes([]string{"1"}, "policy_")
|
||||
mock.ExpectQuery("SELECT(.+)users(.+)").
|
||||
WillReturnRows(sqlmock.NewRows([]string{"id", "group_id"}).AddRow(1, 1))
|
||||
mock.ExpectQuery("SELECT(.+)groups(.+)").
|
||||
WillReturnRows(sqlmock.NewRows([]string{"id", "policies"}).AddRow(1, "[521]"))
|
||||
mock.ExpectQuery("SELECT(.+)policies(.+)").
|
||||
WillReturnRows(sqlmock.NewRows([]string{"id", "access_key", "secret_key"}).AddRow(2, "123", "123"))
|
||||
c, _ := gin.CreateTestContext(rec)
|
||||
c.Params = []gin.Param{
|
||||
{"key", "testCallBackUpyun"},
|
||||
}
|
||||
c.Set(filesystem.UploadSessionCtx, &serializer.UploadSession{
|
||||
UID: 1,
|
||||
VirtualPath: "/",
|
||||
Policy: model.Policy{
|
||||
SecretKey: "123",
|
||||
AccessKey: "123",
|
||||
},
|
||||
})
|
||||
c.Request, _ = http.NewRequest("POST", "/api/v3/callback/upyun/testCallBackUpyun", ioutil.NopCloser(strings.NewReader("1")))
|
||||
c.Request.Header["Content-Md5"] = []string{"c4ca4238a0b923820dcc509a6f75849b"}
|
||||
AuthFunc(c)
|
||||
asserts.NoError(mock.ExpectationsWereMet())
|
||||
asserts.True(c.IsAborted())
|
||||
}
|
||||
|
||||
// 成功
|
||||
{
|
||||
cache.Set(
|
||||
"callback_testCallBackUpyun",
|
||||
serializer.UploadSession{
|
||||
UID: 1,
|
||||
PolicyID: 512,
|
||||
VirtualPath: "/",
|
||||
},
|
||||
0,
|
||||
)
|
||||
cache.Deletes([]string{"1"}, "policy_")
|
||||
mock.ExpectQuery("SELECT(.+)users(.+)").
|
||||
WillReturnRows(sqlmock.NewRows([]string{"id", "group_id"}).AddRow(1, 1))
|
||||
mock.ExpectQuery("SELECT(.+)groups(.+)").
|
||||
WillReturnRows(sqlmock.NewRows([]string{"id", "policies"}).AddRow(1, "[522]"))
|
||||
mock.ExpectQuery("SELECT(.+)policies(.+)").
|
||||
WillReturnRows(sqlmock.NewRows([]string{"id", "access_key", "secret_key"}).AddRow(2, "123", "123"))
|
||||
c, _ := gin.CreateTestContext(rec)
|
||||
c.Params = []gin.Param{
|
||||
{"key", "testCallBackUpyun"},
|
||||
}
|
||||
c.Set(filesystem.UploadSessionCtx, &serializer.UploadSession{
|
||||
UID: 1,
|
||||
VirtualPath: "/",
|
||||
Policy: model.Policy{
|
||||
SecretKey: "123",
|
||||
AccessKey: "123",
|
||||
},
|
||||
})
|
||||
c.Request, _ = http.NewRequest("POST", "/api/v3/callback/upyun/testCallBackUpyun", ioutil.NopCloser(strings.NewReader("1")))
|
||||
c.Request.Header["Content-Md5"] = []string{"c4ca4238a0b923820dcc509a6f75849b"}
|
||||
c.Request.Header["Authorization"] = []string{"UPYUN 123:GWueK9x493BKFFk5gmfdO2Mn6EM="}
|
||||
AuthFunc(c)
|
||||
asserts.NoError(mock.ExpectationsWereMet())
|
||||
asserts.False(c.IsAborted())
|
||||
}
|
||||
}
|
||||
@@ -640,87 +543,28 @@ func TestOneDriveCallbackAuth(t *testing.T) {
|
||||
rec := httptest.NewRecorder()
|
||||
AuthFunc := OneDriveCallbackAuth()
|
||||
|
||||
// Callback Key 相关验证失败
|
||||
{
|
||||
c, _ := gin.CreateTestContext(rec)
|
||||
c.Params = []gin.Param{
|
||||
{"key", "testUpyunBackRemote"},
|
||||
}
|
||||
c.Request, _ = http.NewRequest("POST", "/api/v3/callback/upyun/testUpyunBackRemote", nil)
|
||||
AuthFunc(c)
|
||||
asserts.True(c.IsAborted())
|
||||
}
|
||||
|
||||
// 成功
|
||||
{
|
||||
cache.Set(
|
||||
"callback_testCallBackUpyun",
|
||||
serializer.UploadSession{
|
||||
UID: 1,
|
||||
PolicyID: 512,
|
||||
VirtualPath: "/",
|
||||
c, _ := gin.CreateTestContext(rec)
|
||||
c.Params = []gin.Param{
|
||||
{"sessionID", "TestOneDriveCallbackAuth"},
|
||||
}
|
||||
c.Set(filesystem.UploadSessionCtx, &serializer.UploadSession{
|
||||
UID: 1,
|
||||
VirtualPath: "/",
|
||||
Policy: model.Policy{
|
||||
SecretKey: "123",
|
||||
AccessKey: "123",
|
||||
},
|
||||
0,
|
||||
)
|
||||
cache.Deletes([]string{"1"}, "policy_")
|
||||
mock.ExpectQuery("SELECT(.+)users(.+)").
|
||||
WillReturnRows(sqlmock.NewRows([]string{"id", "group_id"}).AddRow(1, 1))
|
||||
mock.ExpectQuery("SELECT(.+)groups(.+)").
|
||||
WillReturnRows(sqlmock.NewRows([]string{"id", "policies"}).AddRow(1, "[657]"))
|
||||
mock.ExpectQuery("SELECT(.+)policies(.+)").
|
||||
WillReturnRows(sqlmock.NewRows([]string{"id", "access_key", "secret_key"}).AddRow(2, "123", "123"))
|
||||
c, _ := gin.CreateTestContext(rec)
|
||||
c.Params = []gin.Param{
|
||||
{"key", "testCallBackUpyun"},
|
||||
}
|
||||
c.Request, _ = http.NewRequest("POST", "/api/v3/callback/upyun/testCallBackUpyun", ioutil.NopCloser(strings.NewReader("1")))
|
||||
})
|
||||
c.Request, _ = http.NewRequest("POST", "/api/v3/callback/upyun/TestOneDriveCallbackAuth", ioutil.NopCloser(strings.NewReader("1")))
|
||||
res := mq.GlobalMQ.Subscribe("TestOneDriveCallbackAuth", 1)
|
||||
AuthFunc(c)
|
||||
asserts.NoError(mock.ExpectationsWereMet())
|
||||
asserts.False(c.IsAborted())
|
||||
}
|
||||
}
|
||||
|
||||
func TestCOSCallbackAuth(t *testing.T) {
|
||||
asserts := assert.New(t)
|
||||
rec := httptest.NewRecorder()
|
||||
AuthFunc := COSCallbackAuth()
|
||||
|
||||
// Callback Key 相关验证失败
|
||||
{
|
||||
c, _ := gin.CreateTestContext(rec)
|
||||
c.Params = []gin.Param{
|
||||
{"key", "testUpyunBackRemote"},
|
||||
select {
|
||||
case <-res:
|
||||
case <-time.After(time.Millisecond * 500):
|
||||
asserts.Fail("mq message should be published")
|
||||
}
|
||||
c.Request, _ = http.NewRequest("POST", "/api/v3/callback/upyun/testUpyunBackRemote", nil)
|
||||
AuthFunc(c)
|
||||
asserts.True(c.IsAborted())
|
||||
}
|
||||
|
||||
// 成功
|
||||
{
|
||||
cache.Set(
|
||||
"callback_testCallBackUpyun",
|
||||
serializer.UploadSession{
|
||||
UID: 1,
|
||||
PolicyID: 512,
|
||||
VirtualPath: "/",
|
||||
},
|
||||
0,
|
||||
)
|
||||
cache.Deletes([]string{"1"}, "policy_")
|
||||
mock.ExpectQuery("SELECT(.+)users(.+)").
|
||||
WillReturnRows(sqlmock.NewRows([]string{"id", "group_id"}).AddRow(1, 1))
|
||||
mock.ExpectQuery("SELECT(.+)groups(.+)").
|
||||
WillReturnRows(sqlmock.NewRows([]string{"id", "policies"}).AddRow(1, "[702]"))
|
||||
mock.ExpectQuery("SELECT(.+)policies(.+)").
|
||||
WillReturnRows(sqlmock.NewRows([]string{"id", "access_key", "secret_key"}).AddRow(2, "123", "123"))
|
||||
c, _ := gin.CreateTestContext(rec)
|
||||
c.Params = []gin.Param{
|
||||
{"key", "testCallBackUpyun"},
|
||||
}
|
||||
c.Request, _ = http.NewRequest("POST", "/api/v3/callback/upyun/testCallBackUpyun", ioutil.NopCloser(strings.NewReader("1")))
|
||||
AuthFunc(c)
|
||||
asserts.NoError(mock.ExpectationsWereMet())
|
||||
asserts.False(c.IsAborted())
|
||||
}
|
||||
}
|
||||
@@ -759,46 +603,3 @@ func TestIsAdmin(t *testing.T) {
|
||||
asserts.False(c.IsAborted())
|
||||
}
|
||||
}
|
||||
|
||||
func TestS3CallbackAuth(t *testing.T) {
|
||||
asserts := assert.New(t)
|
||||
rec := httptest.NewRecorder()
|
||||
AuthFunc := S3CallbackAuth()
|
||||
|
||||
// Callback Key 相关验证失败
|
||||
{
|
||||
c, _ := gin.CreateTestContext(rec)
|
||||
c.Params = []gin.Param{
|
||||
{"key", "testUpyunBackRemote"},
|
||||
}
|
||||
c.Request, _ = http.NewRequest("POST", "/api/v3/callback/upyun/testUpyunBackRemote", nil)
|
||||
AuthFunc(c)
|
||||
asserts.True(c.IsAborted())
|
||||
}
|
||||
|
||||
// 成功
|
||||
{
|
||||
cache.Set(
|
||||
"callback_testCallBackUpyun",
|
||||
serializer.UploadSession{
|
||||
UID: 1,
|
||||
PolicyID: 512,
|
||||
VirtualPath: "/",
|
||||
},
|
||||
0,
|
||||
)
|
||||
cache.Deletes([]string{"1"}, "policy_")
|
||||
mock.ExpectQuery("SELECT(.+)users(.+)").
|
||||
WillReturnRows(sqlmock.NewRows([]string{"id", "group_id"}).AddRow(1, 1))
|
||||
mock.ExpectQuery("SELECT(.+)groups(.+)").
|
||||
WillReturnRows(sqlmock.NewRows([]string{"id", "policies"}).AddRow(1, "[702]"))
|
||||
c, _ := gin.CreateTestContext(rec)
|
||||
c.Params = []gin.Param{
|
||||
{"key", "testCallBackUpyun"},
|
||||
}
|
||||
c.Request, _ = http.NewRequest("POST", "/api/v3/callback/upyun/testCallBackUpyun", ioutil.NopCloser(strings.NewReader("1")))
|
||||
AuthFunc(c)
|
||||
asserts.False(c.IsAborted())
|
||||
asserts.NoError(mock.ExpectationsWereMet())
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
package middleware
|
||||
|
||||
import (
|
||||
"github.com/cloudreve/Cloudreve/v3/pkg/auth"
|
||||
"github.com/cloudreve/Cloudreve/v3/pkg/cluster"
|
||||
"github.com/cloudreve/Cloudreve/v3/pkg/serializer"
|
||||
"github.com/gin-gonic/gin"
|
||||
@@ -10,9 +11,9 @@ import (
|
||||
// MasterMetadata 解析主机节点发来请求的包含主机节点信息的元数据
|
||||
func MasterMetadata() gin.HandlerFunc {
|
||||
return func(c *gin.Context) {
|
||||
c.Set("MasterSiteID", c.GetHeader("X-Cr-Site-Id"))
|
||||
c.Set("MasterSiteURL", c.GetHeader("X-Cr-Site-Url"))
|
||||
c.Set("MasterVersion", c.GetHeader("X-Cr-Cloudreve-Version"))
|
||||
c.Set("MasterSiteID", c.GetHeader(auth.CrHeaderPrefix+"Site-Id"))
|
||||
c.Set("MasterSiteURL", c.GetHeader(auth.CrHeaderPrefix+"Site-Url"))
|
||||
c.Set("MasterVersion", c.GetHeader(auth.CrHeaderPrefix+"Cloudreve-Version"))
|
||||
c.Next()
|
||||
}
|
||||
}
|
||||
@@ -41,7 +42,7 @@ func UseSlaveAria2Instance(clusterController cluster.Controller) gin.HandlerFunc
|
||||
|
||||
func SlaveRPCSignRequired(nodePool cluster.Pool) gin.HandlerFunc {
|
||||
return func(c *gin.Context) {
|
||||
nodeID, err := strconv.ParseUint(c.GetHeader("X-Cr-Node-Id"), 10, 64)
|
||||
nodeID, err := strconv.ParseUint(c.GetHeader(auth.CrHeaderPrefix+"Node-Id"), 10, 64)
|
||||
if err != nil {
|
||||
c.JSON(200, serializer.ParamErr("未知的主机节点ID", err))
|
||||
c.Abort()
|
||||
|
||||
@@ -38,3 +38,10 @@ func IsFunctionEnabled(key string) gin.HandlerFunc {
|
||||
c.Next()
|
||||
}
|
||||
}
|
||||
|
||||
// CacheControl 屏蔽客户端缓存
|
||||
func CacheControl() gin.HandlerFunc {
|
||||
return func(c *gin.Context) {
|
||||
c.Header("Cache-Control", "private, no-cache")
|
||||
}
|
||||
}
|
||||
@@ -76,3 +76,12 @@ func TestIsFunctionEnabled(t *testing.T) {
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
func TestCacheControl(t *testing.T) {
|
||||
a := assert.New(t)
|
||||
TestFunc := CacheControl()
|
||||
rec := httptest.NewRecorder()
|
||||
c, _ := gin.CreateTestContext(rec)
|
||||
TestFunc(c)
|
||||
a.Contains(c.Writer.Header().Get("Cache-Control"), "no-cache")
|
||||
}
|
||||
@@ -62,6 +62,10 @@ func FrontendFileHandler() gin.HandlerFunc {
|
||||
return
|
||||
}
|
||||
|
||||
if path == "/service-worker.js" {
|
||||
c.Header("Cache-Control", "public, no-cache")
|
||||
}
|
||||
|
||||
// 存在的静态文件
|
||||
fileServer.ServeHTTP(c.Writer, c.Request)
|
||||
c.Abort()
|
||||
|
||||
117
models/defaults.go
Normal file
117
models/defaults.go
Normal file
@@ -0,0 +1,117 @@
|
||||
package model
|
||||
|
||||
import (
|
||||
"github.com/cloudreve/Cloudreve/v3/pkg/conf"
|
||||
"github.com/cloudreve/Cloudreve/v3/pkg/util"
|
||||
"github.com/gofrs/uuid"
|
||||
)
|
||||
|
||||
var defaultSettings = []Setting{
|
||||
{Name: "siteURL", Value: `http://localhost`, Type: "basic"},
|
||||
{Name: "siteName", Value: `Cloudreve`, Type: "basic"},
|
||||
{Name: "siteICPId", Value: ``, Type: "basic"},
|
||||
{Name: "register_enabled", Value: `1`, Type: "register"},
|
||||
{Name: "default_group", Value: `2`, Type: "register"},
|
||||
{Name: "siteKeywords", Value: `网盘,网盘`, Type: "basic"},
|
||||
{Name: "siteDes", Value: `Cloudreve`, Type: "basic"},
|
||||
{Name: "siteTitle", Value: `平步云端`, Type: "basic"},
|
||||
{Name: "siteScript", Value: ``, Type: "basic"},
|
||||
{Name: "siteID", Value: uuid.Must(uuid.NewV4()).String(), Type: "basic"},
|
||||
{Name: "fromName", Value: `Cloudreve`, Type: "mail"},
|
||||
{Name: "mail_keepalive", Value: `30`, Type: "mail"},
|
||||
{Name: "fromAdress", Value: `no-reply@acg.blue`, Type: "mail"},
|
||||
{Name: "smtpHost", Value: `smtp.mxhichina.com`, Type: "mail"},
|
||||
{Name: "smtpPort", Value: `25`, Type: "mail"},
|
||||
{Name: "replyTo", Value: `abslant@126.com`, Type: "mail"},
|
||||
{Name: "smtpUser", Value: `no-reply@acg.blue`, Type: "mail"},
|
||||
{Name: "smtpPass", Value: ``, Type: "mail"},
|
||||
{Name: "smtpEncryption", Value: `0`, Type: "mail"},
|
||||
{Name: "maxEditSize", Value: `4194304`, Type: "file_edit"},
|
||||
{Name: "archive_timeout", Value: `600`, Type: "timeout"},
|
||||
{Name: "download_timeout", Value: `600`, Type: "timeout"},
|
||||
{Name: "preview_timeout", Value: `600`, Type: "timeout"},
|
||||
{Name: "doc_preview_timeout", Value: `600`, Type: "timeout"},
|
||||
{Name: "upload_session_timeout", Value: `86400`, Type: "timeout"},
|
||||
{Name: "slave_api_timeout", Value: `60`, Type: "timeout"},
|
||||
{Name: "slave_node_retry", Value: `3`, Type: "slave"},
|
||||
{Name: "slave_ping_interval", Value: `60`, Type: "slave"},
|
||||
{Name: "slave_recover_interval", Value: `120`, Type: "slave"},
|
||||
{Name: "slave_transfer_timeout", Value: `172800`, Type: "timeout"},
|
||||
{Name: "onedrive_monitor_timeout", Value: `600`, Type: "timeout"},
|
||||
{Name: "share_download_session_timeout", Value: `2073600`, Type: "timeout"},
|
||||
{Name: "onedrive_callback_check", Value: `20`, Type: "timeout"},
|
||||
{Name: "folder_props_timeout", Value: `300`, Type: "timeout"},
|
||||
{Name: "chunk_retries", Value: `5`, Type: "retry"},
|
||||
{Name: "onedrive_source_timeout", Value: `1800`, Type: "timeout"},
|
||||
{Name: "reset_after_upload_failed", Value: `0`, Type: "upload"},
|
||||
{Name: "use_temp_chunk_buffer", Value: `1`, Type: "upload"},
|
||||
{Name: "login_captcha", Value: `0`, Type: "login"},
|
||||
{Name: "reg_captcha", Value: `0`, Type: "login"},
|
||||
{Name: "email_active", Value: `0`, Type: "register"},
|
||||
{Name: "mail_activation_template", Value: `<!DOCTYPE html PUBLIC"-//W3C//DTD XHTML 1.0 Transitional//EN""http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd"><html xmlns="http://www.w3.org/1999/xhtml"style="font-family: 'Helvetica Neue', Helvetica, Arial, sans-serif; box-sizing: border-box;
|
||||
font-size: 14px; margin: 0;"><head><meta name="viewport"content="width=device-width"/><meta http-equiv="Content-Type"content="text/html; charset=UTF-8"/><title>激活您的账户</title><style type="text/css">img{max-width:100%}body{-webkit-font-smoothing:antialiased;-webkit-text-size-adjust:none;width:100%!important;height:100%;line-height:1.6em}body{background-color:#f6f6f6}@media only screen and(max-width:640px){body{padding:0!important}h1{font-weight:800!important;margin:20px 0 5px!important}h2{font-weight:800!important;margin:20px 0 5px!important}h3{font-weight:800!important;margin:20px 0 5px!important}h4{font-weight:800!important;margin:20px 0 5px!important}h1{font-size:22px!important}h2{font-size:18px!important}h3{font-size:16px!important}.container{padding:0!important;width:100%!important}.content{padding:0!important}.content-wrap{padding:10px!important}.invoice{width:100%!important}}</style></head><body itemscope itemtype="http://schema.org/EmailMessage"style="font-family: 'Helvetica Neue',Helvetica,Arial,sans-serif; box-sizing:
|
||||
border-box; font-size: 14px; -webkit-font-smoothing: antialiased; -webkit-text-size-adjust: none; width: 100% !important; height: 100%; line-height: 1.6em; background-color: #f6f6f6; margin: 0;"bgcolor="#f6f6f6"><table class="body-wrap"style="font-family: 'Helvetica Neue',Helvetica,Arial,sans-serif; box-sizing: border-box; font-size: 14px; width: 100%; background-color: #f6f6f6; margin: 0;"bgcolor="#f6f6f6"><tr style="font-family: 'Helvetica Neue',Helvetica,Arial,sans-serif;
|
||||
box-sizing: border-box; font-size: 14px; margin: 0;"><td style="font-family: 'Helvetica Neue',Helvetica,Arial,sans-serif; box-sizing: border-box; font-size: 14px; vertical-align: top; margin: 0;"valign="top"></td><td class="container"width="600"style="font-family: 'Helvetica Neue',Helvetica,Arial,sans-serif; box-sizing: border-box; font-size: 14px; vertical-align: top; display: block !important; max-width: 600px !important; clear: both !important; margin: 0 auto;"valign="top"><div class="content"style="font-family: 'Helvetica Neue',Helvetica,Arial,sans-serif; box-sizing: border-box; font-size: 14px; max-width: 600px; display: block; margin: 0 auto; padding: 20px;"><table class="main"width="100%"cellpadding="0"cellspacing="0"style="font-family: 'Helvetica Neue',Helvetica,Arial,sans-serif; box-sizing: border-box; font-size: 14px; border-radius: 3px; background-color: #fff; margin: 0; border: 1px
|
||||
solid #e9e9e9;"bgcolor="#fff"><tr style="font-family: 'Helvetica Neue',Helvetica,Arial,sans-serif; box-sizing: border-box; font-size:
|
||||
14px; margin: 0;"><td class="alert alert-warning"style="font-family: 'Helvetica Neue',Helvetica,Arial,sans-serif; box-sizing: border-box; font-size: 16px; vertical-align: top; color: #fff; font-weight: 500; text-align: center; border-radius: 3px 3px 0 0; background-color: #009688; margin: 0; padding: 20px;"align="center"bgcolor="#FF9F00"valign="top">激活{siteTitle}账户</td></tr><tr style="font-family: 'Helvetica Neue',Helvetica,Arial,sans-serif; box-sizing: border-box; font-size: 14px; margin: 0;"><td class="content-wrap"style="font-family: 'Helvetica Neue',Helvetica,Arial,sans-serif; box-sizing: border-box; font-size: 14px; vertical-align: top; margin: 0; padding: 20px;"valign="top"><table width="100%"cellpadding="0"cellspacing="0"style="font-family: 'Helvetica Neue',Helvetica,Arial,sans-serif; box-sizing: border-box; font-size: 14px; margin: 0;"><tr style="font-family: 'Helvetica Neue',Helvetica,Arial,sans-serif; box-sizing: border-box; font-size: 14px; margin: 0;"><td class="content-block"style="font-family: 'Helvetica
|
||||
Neue',Helvetica,Arial,sans-serif; box-sizing: border-box; font-size: 14px; vertical-align: top; margin: 0; padding: 0 0 20px;"valign="top">亲爱的<strong style="font-family: 'Helvetica Neue',Helvetica,Arial,sans-serif; box-sizing: border-box; font-size: 14px; margin: 0;">{userName}</strong>:</td></tr><tr style="font-family: 'Helvetica Neue',Helvetica,Arial,sans-serif; box-sizing: border-box; font-size: 14px; margin: 0;"><td class="content-block"style="font-family: 'Helvetica Neue',Helvetica,Arial,sans-serif; box-sizing: border-box; font-size: 14px; vertical-align: top; margin: 0; padding: 0 0 20px;"valign="top">感谢您注册{siteTitle},请点击下方按钮完成账户激活。</td></tr><tr style="font-family: 'Helvetica Neue',Helvetica,Arial,sans-serif; box-sizing: border-box; font-size: 14px; margin: 0;"><td class="content-block"style="font-family: 'Helvetica Neue',Helvetica,Arial,sans-serif; box-sizing: border-box; font-size: 14px; vertical-align: top; margin: 0; padding: 0 0 20px;"valign="top"><a href="{activationUrl}"class="btn-primary"style="font-family: 'Helvetica Neue',Helvetica,Arial,sans-serif; box-sizing: border-box; font-size: 14px; color: #FFF; text-decoration: none; line-height: 2em; font-weight: bold; text-align: center; cursor: pointer; display: inline-block; border-radius: 5px; text-transform: capitalize; background-color: #009688; margin: 0; border-color: #009688; border-style: solid; border-width: 10px 20px;">激活账户</a></td></tr><tr style="font-family: 'Helvetica Neue',Helvetica,Arial,sans-serif; box-sizing: border-box; font-size: 14px; margin: 0;"><td class="content-block"style="font-family: 'Helvetica Neue',Helvetica,Arial,sans-serif; box-sizing: border-box; font-size: 14px; vertical-align: top; margin: 0; padding: 0 0 20px;"valign="top">感谢您选择{siteTitle}。</td></tr></table></td></tr></table><div class="footer"style="font-family: 'Helvetica Neue',Helvetica,Arial,sans-serif; box-sizing: border-box; font-size: 14px; width: 100%; clear: both; color: #999; margin: 0; padding: 20px;"><table width="100%"style="font-family: 'Helvetica Neue',Helvetica,Arial,sans-serif; box-sizing: border-box; font-size: 14px; margin: 0;"><tr style="font-family: 'Helvetica Neue',Helvetica,Arial,sans-serif; box-sizing: border-box; font-size: 14px; margin: 0;"><td class="aligncenter content-block"style="font-family: 'Helvetica Neue',Helvetica,Arial,sans-serif; box-sizing: border-box; font-size: 12px; vertical-align: top; color: #999; text-align: center; margin: 0; padding: 0 0 20px;"align="center"valign="top">此邮件由系统自动发送,请不要直接回复。</td></tr></table></div></div></td><td style="font-family: 'Helvetica Neue',Helvetica,Arial,sans-serif; box-sizing: border-box; font-size: 14px; vertical-align: top; margin: 0;"valign="top"></td></tr></table></body></html>`, Type: "mail_template"},
|
||||
{Name: "forget_captcha", Value: `0`, Type: "login"},
|
||||
{Name: "mail_reset_pwd_template", Value: `<!DOCTYPE html PUBLIC"-//W3C//DTD XHTML 1.0 Transitional//EN""http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd"><html xmlns="http://www.w3.org/1999/xhtml"style="font-family: 'Helvetica Neue', Helvetica, Arial, sans-serif; box-sizing: border-box;
|
||||
font-size: 14px; margin: 0;"><head><meta name="viewport"content="width=device-width"/><meta http-equiv="Content-Type"content="text/html; charset=UTF-8"/><title>重设密码</title><style type="text/css">img{max-width:100%}body{-webkit-font-smoothing:antialiased;-webkit-text-size-adjust:none;width:100%!important;height:100%;line-height:1.6em}body{background-color:#f6f6f6}@media only screen and(max-width:640px){body{padding:0!important}h1{font-weight:800!important;margin:20px 0 5px!important}h2{font-weight:800!important;margin:20px 0 5px!important}h3{font-weight:800!important;margin:20px 0 5px!important}h4{font-weight:800!important;margin:20px 0 5px!important}h1{font-size:22px!important}h2{font-size:18px!important}h3{font-size:16px!important}.container{padding:0!important;width:100%!important}.content{padding:0!important}.content-wrap{padding:10px!important}.invoice{width:100%!important}}</style></head><body itemscope itemtype="http://schema.org/EmailMessage"style="font-family: 'Helvetica Neue',Helvetica,Arial,sans-serif; box-sizing:
|
||||
border-box; font-size: 14px; -webkit-font-smoothing: antialiased; -webkit-text-size-adjust: none; width: 100% !important; height: 100%; line-height: 1.6em; background-color: #f6f6f6; margin: 0;"bgcolor="#f6f6f6"><table class="body-wrap"style="font-family: 'Helvetica Neue',Helvetica,Arial,sans-serif; box-sizing: border-box; font-size: 14px; width: 100%; background-color: #f6f6f6; margin: 0;"bgcolor="#f6f6f6"><tr style="font-family: 'Helvetica Neue',Helvetica,Arial,sans-serif;
|
||||
box-sizing: border-box; font-size: 14px; margin: 0;"><td style="font-family: 'Helvetica Neue',Helvetica,Arial,sans-serif; box-sizing: border-box; font-size: 14px; vertical-align: top; margin: 0;"valign="top"></td><td class="container"width="600"style="font-family: 'Helvetica Neue',Helvetica,Arial,sans-serif; box-sizing: border-box; font-size: 14px; vertical-align: top; display: block !important; max-width: 600px !important; clear: both !important; margin: 0 auto;"valign="top"><div class="content"style="font-family: 'Helvetica Neue',Helvetica,Arial,sans-serif; box-sizing: border-box; font-size: 14px; max-width: 600px; display: block; margin: 0 auto; padding: 20px;"><table class="main"width="100%"cellpadding="0"cellspacing="0"style="font-family: 'Helvetica Neue',Helvetica,Arial,sans-serif; box-sizing: border-box; font-size: 14px; border-radius: 3px; background-color: #fff; margin: 0; border: 1px
|
||||
solid #e9e9e9;"bgcolor="#fff"><tr style="font-family: 'Helvetica Neue',Helvetica,Arial,sans-serif; box-sizing: border-box; font-size:
|
||||
14px; margin: 0;"><td class="alert alert-warning"style="font-family: 'Helvetica Neue',Helvetica,Arial,sans-serif; box-sizing: border-box; font-size: 16px; vertical-align: top; color: #fff; font-weight: 500; text-align: center; border-radius: 3px 3px 0 0; background-color: #2196F3; margin: 0; padding: 20px;"align="center"bgcolor="#FF9F00"valign="top">重设{siteTitle}密码</td></tr><tr style="font-family: 'Helvetica Neue',Helvetica,Arial,sans-serif; box-sizing: border-box; font-size: 14px; margin: 0;"><td class="content-wrap"style="font-family: 'Helvetica Neue',Helvetica,Arial,sans-serif; box-sizing: border-box; font-size: 14px; vertical-align: top; margin: 0; padding: 20px;"valign="top"><table width="100%"cellpadding="0"cellspacing="0"style="font-family: 'Helvetica Neue',Helvetica,Arial,sans-serif; box-sizing: border-box; font-size: 14px; margin: 0;"><tr style="font-family: 'Helvetica Neue',Helvetica,Arial,sans-serif; box-sizing: border-box; font-size: 14px; margin: 0;"><td class="content-block"style="font-family: 'Helvetica
|
||||
Neue',Helvetica,Arial,sans-serif; box-sizing: border-box; font-size: 14px; vertical-align: top; margin: 0; padding: 0 0 20px;"valign="top">亲爱的<strong style="font-family: 'Helvetica Neue',Helvetica,Arial,sans-serif; box-sizing: border-box; font-size: 14px; margin: 0;">{userName}</strong>:</td></tr><tr style="font-family: 'Helvetica Neue',Helvetica,Arial,sans-serif; box-sizing: border-box; font-size: 14px; margin: 0;"><td class="content-block"style="font-family: 'Helvetica Neue',Helvetica,Arial,sans-serif; box-sizing: border-box; font-size: 14px; vertical-align: top; margin: 0; padding: 0 0 20px;"valign="top">请点击下方按钮完成密码重设。如果非你本人操作,请忽略此邮件。</td></tr><tr style="font-family: 'Helvetica Neue',Helvetica,Arial,sans-serif; box-sizing: border-box; font-size: 14px; margin: 0;"><td class="content-block"style="font-family: 'Helvetica Neue',Helvetica,Arial,sans-serif; box-sizing: border-box; font-size: 14px; vertical-align: top; margin: 0; padding: 0 0 20px;"valign="top"><a href="{resetUrl}"class="btn-primary"style="font-family: 'Helvetica Neue',Helvetica,Arial,sans-serif; box-sizing: border-box; font-size: 14px; color: #FFF; text-decoration: none; line-height: 2em; font-weight: bold; text-align: center; cursor: pointer; display: inline-block; border-radius: 5px; text-transform: capitalize; background-color: #2196F3; margin: 0; border-color: #2196F3; border-style: solid; border-width: 10px 20px;">重设密码</a></td></tr><tr style="font-family: 'Helvetica Neue',Helvetica,Arial,sans-serif; box-sizing: border-box; font-size: 14px; margin: 0;"><td class="content-block"style="font-family: 'Helvetica Neue',Helvetica,Arial,sans-serif; box-sizing: border-box; font-size: 14px; vertical-align: top; margin: 0; padding: 0 0 20px;"valign="top">感谢您选择{siteTitle}。</td></tr></table></td></tr></table><div class="footer"style="font-family: 'Helvetica Neue',Helvetica,Arial,sans-serif; box-sizing: border-box; font-size: 14px; width: 100%; clear: both; color: #999; margin: 0; padding: 20px;"><table width="100%"style="font-family: 'Helvetica Neue',Helvetica,Arial,sans-serif; box-sizing: border-box; font-size: 14px; margin: 0;"><tr style="font-family: 'Helvetica Neue',Helvetica,Arial,sans-serif; box-sizing: border-box; font-size: 14px; margin: 0;"><td class="aligncenter content-block"style="font-family: 'Helvetica Neue',Helvetica,Arial,sans-serif; box-sizing: border-box; font-size: 12px; vertical-align: top; color: #999; text-align: center; margin: 0; padding: 0 0 20px;"align="center"valign="top">此邮件由系统自动发送,请不要直接回复。</td></tr></table></div></div></td><td style="font-family: 'Helvetica Neue',Helvetica,Arial,sans-serif; box-sizing: border-box; font-size: 14px; vertical-align: top; margin: 0;"valign="top"></td></tr></table></body></html>`, Type: "mail_template"},
|
||||
{Name: "db_version_" + conf.RequiredDBVersion, Value: `installed`, Type: "version"},
|
||||
{Name: "hot_share_num", Value: `10`, Type: "share"},
|
||||
{Name: "gravatar_server", Value: `https://www.gravatar.com/`, Type: "avatar"},
|
||||
{Name: "defaultTheme", Value: `#3f51b5`, Type: "basic"},
|
||||
{Name: "themes", Value: `{"#3f51b5":{"palette":{"primary":{"main":"#3f51b5"},"secondary":{"main":"#f50057"}}},"#2196f3":{"palette":{"primary":{"main":"#2196f3"},"secondary":{"main":"#FFC107"}}},"#673AB7":{"palette":{"primary":{"main":"#673AB7"},"secondary":{"main":"#2196F3"}}},"#E91E63":{"palette":{"primary":{"main":"#E91E63"},"secondary":{"main":"#42A5F5","contrastText":"#fff"}}},"#FF5722":{"palette":{"primary":{"main":"#FF5722"},"secondary":{"main":"#3F51B5"}}},"#FFC107":{"palette":{"primary":{"main":"#FFC107"},"secondary":{"main":"#26C6DA"}}},"#8BC34A":{"palette":{"primary":{"main":"#8BC34A","contrastText":"#fff"},"secondary":{"main":"#FF8A65","contrastText":"#fff"}}},"#009688":{"palette":{"primary":{"main":"#009688"},"secondary":{"main":"#4DD0E1","contrastText":"#fff"}}},"#607D8B":{"palette":{"primary":{"main":"#607D8B"},"secondary":{"main":"#F06292"}}},"#795548":{"palette":{"primary":{"main":"#795548"},"secondary":{"main":"#4CAF50","contrastText":"#fff"}}}}`, Type: "basic"},
|
||||
{Name: "max_worker_num", Value: `10`, Type: "task"},
|
||||
{Name: "max_parallel_transfer", Value: `4`, Type: "task"},
|
||||
{Name: "secret_key", Value: util.RandStringRunes(256), Type: "auth"},
|
||||
{Name: "temp_path", Value: "temp", Type: "path"},
|
||||
{Name: "avatar_path", Value: "avatar", Type: "path"},
|
||||
{Name: "avatar_size", Value: "2097152", Type: "avatar"},
|
||||
{Name: "avatar_size_l", Value: "200", Type: "avatar"},
|
||||
{Name: "avatar_size_m", Value: "130", Type: "avatar"},
|
||||
{Name: "avatar_size_s", Value: "50", Type: "avatar"},
|
||||
{Name: "home_view_method", Value: "icon", Type: "view"},
|
||||
{Name: "share_view_method", Value: "list", Type: "view"},
|
||||
{Name: "cron_garbage_collect", Value: "@hourly", Type: "cron"},
|
||||
{Name: "cron_recycle_upload_session", Value: "@every 1h30m", Type: "cron"},
|
||||
{Name: "authn_enabled", Value: "0", Type: "authn"},
|
||||
{Name: "captcha_type", Value: "normal", Type: "captcha"},
|
||||
{Name: "captcha_height", Value: "60", Type: "captcha"},
|
||||
{Name: "captcha_width", Value: "240", Type: "captcha"},
|
||||
{Name: "captcha_mode", Value: "3", Type: "captcha"},
|
||||
{Name: "captcha_ComplexOfNoiseText", Value: "0", Type: "captcha"},
|
||||
{Name: "captcha_ComplexOfNoiseDot", Value: "0", Type: "captcha"},
|
||||
{Name: "captcha_IsShowHollowLine", Value: "0", Type: "captcha"},
|
||||
{Name: "captcha_IsShowNoiseDot", Value: "1", Type: "captcha"},
|
||||
{Name: "captcha_IsShowNoiseText", Value: "0", Type: "captcha"},
|
||||
{Name: "captcha_IsShowSlimeLine", Value: "1", Type: "captcha"},
|
||||
{Name: "captcha_IsShowSineLine", Value: "0", Type: "captcha"},
|
||||
{Name: "captcha_CaptchaLen", Value: "6", Type: "captcha"},
|
||||
{Name: "captcha_ReCaptchaKey", Value: "defaultKey", Type: "captcha"},
|
||||
{Name: "captcha_ReCaptchaSecret", Value: "defaultSecret", Type: "captcha"},
|
||||
{Name: "captcha_TCaptcha_CaptchaAppId", Value: "", Type: "captcha"},
|
||||
{Name: "captcha_TCaptcha_AppSecretKey", Value: "", Type: "captcha"},
|
||||
{Name: "captcha_TCaptcha_SecretId", Value: "", Type: "captcha"},
|
||||
{Name: "captcha_TCaptcha_SecretKey", Value: "", Type: "captcha"},
|
||||
{Name: "thumb_width", Value: "400", Type: "thumb"},
|
||||
{Name: "thumb_height", Value: "300", Type: "thumb"},
|
||||
{Name: "thumb_file_suffix", Value: "._thumb", Type: "thumb"},
|
||||
{Name: "thumb_max_task_count", Value: "-1", Type: "thumb"},
|
||||
{Name: "thumb_encode_method", Value: "jpg", Type: "thumb"},
|
||||
{Name: "thumb_gc_after_gen", Value: "0", Type: "thumb"},
|
||||
{Name: "thumb_encode_quality", Value: "85", Type: "thumb"},
|
||||
{Name: "pwa_small_icon", Value: "/static/img/favicon.ico", Type: "pwa"},
|
||||
{Name: "pwa_medium_icon", Value: "/static/img/logo192.png", Type: "pwa"},
|
||||
{Name: "pwa_large_icon", Value: "/static/img/logo512.png", Type: "pwa"},
|
||||
{Name: "pwa_display", Value: "standalone", Type: "pwa"},
|
||||
{Name: "pwa_theme_color", Value: "#000000", Type: "pwa"},
|
||||
{Name: "pwa_background_color", Value: "#ffffff", Type: "pwa"},
|
||||
{Name: "office_preview_service", Value: "https://view.officeapps.live.com/op/view.aspx?src={$src}", Type: "preview"},
|
||||
}
|
||||
162
models/file.go
162
models/file.go
@@ -2,6 +2,8 @@ package model
|
||||
|
||||
import (
|
||||
"encoding/gob"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"path"
|
||||
"time"
|
||||
|
||||
@@ -13,19 +15,22 @@ import (
|
||||
type File struct {
|
||||
// 表字段
|
||||
gorm.Model
|
||||
Name string `gorm:"unique_index:idx_only_one"`
|
||||
SourceName string `gorm:"type:text"`
|
||||
UserID uint `gorm:"index:user_id;unique_index:idx_only_one"`
|
||||
Size uint64
|
||||
PicInfo string
|
||||
FolderID uint `gorm:"index:folder_id;unique_index:idx_only_one"`
|
||||
PolicyID uint
|
||||
Name string `gorm:"unique_index:idx_only_one"`
|
||||
SourceName string `gorm:"type:text"`
|
||||
UserID uint `gorm:"index:user_id;unique_index:idx_only_one"`
|
||||
Size uint64
|
||||
PicInfo string
|
||||
FolderID uint `gorm:"index:folder_id;unique_index:idx_only_one"`
|
||||
PolicyID uint
|
||||
UploadSessionID *string `gorm:"index:session_id;unique_index:session_only_one"`
|
||||
Metadata string `gorm:"type:text"`
|
||||
|
||||
// 关联模型
|
||||
Policy Policy `gorm:"PRELOAD:false,association_autoupdate:false"`
|
||||
|
||||
// 数据库忽略字段
|
||||
Position string `gorm:"-"`
|
||||
Position string `gorm:"-"`
|
||||
MetadataSerialized map[string]string `gorm:"-"`
|
||||
}
|
||||
|
||||
func init() {
|
||||
@@ -34,12 +39,40 @@ func init() {
|
||||
}
|
||||
|
||||
// Create 创建文件记录
|
||||
func (file *File) Create() (uint, error) {
|
||||
if err := DB.Create(file).Error; err != nil {
|
||||
func (file *File) Create() error {
|
||||
tx := DB.Begin()
|
||||
|
||||
if err := tx.Create(file).Error; err != nil {
|
||||
util.Log().Warning("无法插入文件记录, %s", err)
|
||||
return 0, err
|
||||
tx.Rollback()
|
||||
return err
|
||||
}
|
||||
return file.ID, nil
|
||||
|
||||
user := &User{}
|
||||
user.ID = file.UserID
|
||||
if err := user.ChangeStorage(tx, "+", file.Size); err != nil {
|
||||
tx.Rollback()
|
||||
return err
|
||||
}
|
||||
|
||||
return tx.Commit().Error
|
||||
}
|
||||
|
||||
// AfterFind 找到文件后的钩子
|
||||
func (file *File) AfterFind() (err error) {
|
||||
// 反序列化文件元数据
|
||||
if file.Metadata != "" {
|
||||
err = json.Unmarshal([]byte(file.Metadata), &file.MetadataSerialized)
|
||||
}
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
// BeforeSave Save策略前的钩子
|
||||
func (file *File) BeforeSave() (err error) {
|
||||
metaValue, err := json.Marshal(&file.MetadataSerialized)
|
||||
file.Metadata = string(metaValue)
|
||||
return err
|
||||
}
|
||||
|
||||
// GetChildFile 查找目录下名为name的子文件
|
||||
@@ -118,6 +151,19 @@ func GetChildFilesOfFolders(folders *[]Folder) ([]File, error) {
|
||||
return files, result.Error
|
||||
}
|
||||
|
||||
// GetUploadPlaceholderFiles 获取所有上传占位文件
|
||||
// UID为0表示忽略用户
|
||||
func GetUploadPlaceholderFiles(uid uint) []*File {
|
||||
query := DB
|
||||
if uid != 0 {
|
||||
query = query.Where("user_id = ?", uid)
|
||||
}
|
||||
|
||||
var files []*File
|
||||
query.Where("upload_session_id is not NULL").Find(&files)
|
||||
return files
|
||||
}
|
||||
|
||||
// GetPolicy 获取文件所属策略
|
||||
func (file *File) GetPolicy() *Policy {
|
||||
if file.Policy.Model.ID == 0 {
|
||||
@@ -166,10 +212,38 @@ func RemoveFilesWithSoftLinks(files []File) ([]File, error) {
|
||||
|
||||
}
|
||||
|
||||
// DeleteFileByIDs 根据给定ID批量删除文件记录
|
||||
func DeleteFileByIDs(ids []uint) error {
|
||||
result := DB.Where("id in (?)", ids).Unscoped().Delete(&File{})
|
||||
return result.Error
|
||||
// DeleteFiles 批量删除文件记录并归还容量
|
||||
func DeleteFiles(files []*File, uid uint) error {
|
||||
tx := DB.Begin()
|
||||
user := &User{}
|
||||
user.ID = uid
|
||||
var size uint64
|
||||
for _, file := range files {
|
||||
if file.UserID != uid {
|
||||
tx.Rollback()
|
||||
return errors.New("user id not consistent")
|
||||
}
|
||||
|
||||
result := tx.Unscoped().Where("size = ?", file.Size).Delete(file)
|
||||
if result.Error != nil {
|
||||
tx.Rollback()
|
||||
return result.Error
|
||||
}
|
||||
|
||||
if result.RowsAffected == 0 {
|
||||
tx.Rollback()
|
||||
return errors.New("file size is dirty")
|
||||
}
|
||||
|
||||
size += file.Size
|
||||
}
|
||||
|
||||
if err := user.ChangeStorage(tx, "-", size); err != nil {
|
||||
tx.Rollback()
|
||||
return err
|
||||
}
|
||||
|
||||
return tx.Commit().Error
|
||||
}
|
||||
|
||||
// GetFilesByParentIDs 根据父目录ID查找文件
|
||||
@@ -179,6 +253,13 @@ func GetFilesByParentIDs(ids []uint, uid uint) ([]File, error) {
|
||||
return files, result.Error
|
||||
}
|
||||
|
||||
// GetFilesByUploadSession 查找上传会话对应的文件
|
||||
func GetFilesByUploadSession(sessionID string, uid uint) (*File, error) {
|
||||
file := File{}
|
||||
result := DB.Where("user_id = ? and upload_session_id = ?", uid, sessionID).Find(&file)
|
||||
return &file, result.Error
|
||||
}
|
||||
|
||||
// Rename 重命名文件
|
||||
func (file *File) Rename(new string) error {
|
||||
return DB.Model(&file).Update("name", new).Error
|
||||
@@ -186,12 +267,39 @@ func (file *File) Rename(new string) error {
|
||||
|
||||
// UpdatePicInfo 更新文件的图像信息
|
||||
func (file *File) UpdatePicInfo(value string) error {
|
||||
return DB.Model(&file).Set("gorm:association_autoupdate", false).Update("pic_info", value).Error
|
||||
return DB.Model(&file).Set("gorm:association_autoupdate", false).UpdateColumns(File{PicInfo: value}).Error
|
||||
}
|
||||
|
||||
// UpdateSize 更新文件的大小信息
|
||||
// TODO: 全局锁
|
||||
func (file *File) UpdateSize(value uint64) error {
|
||||
return DB.Model(&file).Set("gorm:association_autoupdate", false).Update("size", value).Error
|
||||
tx := DB.Begin()
|
||||
var sizeDelta uint64
|
||||
operator := "+"
|
||||
user := User{}
|
||||
user.ID = file.UserID
|
||||
if value > file.Size {
|
||||
sizeDelta = value - file.Size
|
||||
} else {
|
||||
operator = "-"
|
||||
sizeDelta = file.Size - value
|
||||
}
|
||||
|
||||
if res := tx.Model(&file).
|
||||
Where("size = ?", file.Size).
|
||||
Set("gorm:association_autoupdate", false).
|
||||
Update("size", value); res.Error != nil {
|
||||
tx.Rollback()
|
||||
return res.Error
|
||||
}
|
||||
|
||||
if err := user.ChangeStorage(tx, operator, sizeDelta); err != nil {
|
||||
tx.Rollback()
|
||||
return err
|
||||
}
|
||||
|
||||
file.Size = value
|
||||
return tx.Commit().Error
|
||||
}
|
||||
|
||||
// UpdateSourceName 更新文件的源文件名
|
||||
@@ -199,6 +307,24 @@ func (file *File) UpdateSourceName(value string) error {
|
||||
return DB.Model(&file).Set("gorm:association_autoupdate", false).Update("source_name", value).Error
|
||||
}
|
||||
|
||||
func (file *File) PopChunkToFile(lastModified *time.Time, picInfo string) error {
|
||||
file.UploadSessionID = nil
|
||||
if lastModified != nil {
|
||||
file.UpdatedAt = *lastModified
|
||||
}
|
||||
|
||||
return DB.Model(file).UpdateColumns(map[string]interface{}{
|
||||
"upload_session_id": file.UploadSessionID,
|
||||
"updated_at": file.UpdatedAt,
|
||||
"pic_info": picInfo,
|
||||
}).Error
|
||||
}
|
||||
|
||||
// CanCopy 返回文件是否可被复制
|
||||
func (file *File) CanCopy() bool {
|
||||
return file.UploadSessionID == nil
|
||||
}
|
||||
|
||||
/*
|
||||
实现 webdav.FileInfo 接口
|
||||
*/
|
||||
|
||||
@@ -15,22 +15,62 @@ func TestFile_Create(t *testing.T) {
|
||||
Name: "123",
|
||||
}
|
||||
|
||||
mock.ExpectBegin()
|
||||
mock.ExpectExec("INSERT(.+)").WillReturnResult(sqlmock.NewResult(5, 1))
|
||||
mock.ExpectCommit()
|
||||
fileID, err := file.Create()
|
||||
asserts.NoError(err)
|
||||
asserts.Equal(uint(5), fileID)
|
||||
asserts.Equal(uint(5), file.ID)
|
||||
asserts.NoError(mock.ExpectationsWereMet())
|
||||
// 无法插入文件记录
|
||||
{
|
||||
mock.ExpectBegin()
|
||||
mock.ExpectExec("INSERT(.+)").WillReturnError(errors.New("error"))
|
||||
mock.ExpectRollback()
|
||||
err := file.Create()
|
||||
asserts.Error(err)
|
||||
asserts.NoError(mock.ExpectationsWereMet())
|
||||
}
|
||||
|
||||
mock.ExpectBegin()
|
||||
mock.ExpectExec("INSERT(.+)").WillReturnError(errors.New("error"))
|
||||
mock.ExpectRollback()
|
||||
fileID, err = file.Create()
|
||||
asserts.Error(err)
|
||||
asserts.NoError(mock.ExpectationsWereMet())
|
||||
// 无法更新用户容量
|
||||
{
|
||||
mock.ExpectBegin()
|
||||
mock.ExpectExec("INSERT(.+)").WillReturnResult(sqlmock.NewResult(5, 1))
|
||||
mock.ExpectExec("UPDATE(.+)").WillReturnError(errors.New("error"))
|
||||
mock.ExpectRollback()
|
||||
err := file.Create()
|
||||
asserts.Error(err)
|
||||
asserts.NoError(mock.ExpectationsWereMet())
|
||||
}
|
||||
|
||||
// 成功
|
||||
{
|
||||
mock.ExpectBegin()
|
||||
mock.ExpectExec("INSERT(.+)").WillReturnResult(sqlmock.NewResult(5, 1))
|
||||
mock.ExpectExec("UPDATE(.+)storage(.+)").WillReturnResult(sqlmock.NewResult(0, 1))
|
||||
mock.ExpectCommit()
|
||||
err := file.Create()
|
||||
asserts.NoError(err)
|
||||
asserts.Equal(uint(5), file.ID)
|
||||
asserts.NoError(mock.ExpectationsWereMet())
|
||||
}
|
||||
}
|
||||
|
||||
func TestFile_AfterFind(t *testing.T) {
|
||||
a := assert.New(t)
|
||||
file := File{
|
||||
Name: "123",
|
||||
Metadata: "{\"name\":\"123\"}",
|
||||
}
|
||||
|
||||
a.NoError(file.AfterFind())
|
||||
a.Equal("123", file.MetadataSerialized["name"])
|
||||
}
|
||||
|
||||
func TestFile_BeforeSave(t *testing.T) {
|
||||
a := assert.New(t)
|
||||
file := File{
|
||||
Name: "123",
|
||||
MetadataSerialized: map[string]string{
|
||||
"name": "123",
|
||||
},
|
||||
}
|
||||
|
||||
a.NoError(file.BeforeSave())
|
||||
a.Equal("{\"name\":\"123\"}", file.Metadata)
|
||||
}
|
||||
|
||||
func TestFolder_GetChildFile(t *testing.T) {
|
||||
@@ -175,6 +215,17 @@ func TestGetChildFilesOfFolders(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestGetUploadPlaceholderFiles(t *testing.T) {
|
||||
a := assert.New(t)
|
||||
|
||||
mock.ExpectQuery("SELECT(.+)upload_session_id(.+)").
|
||||
WithArgs(1).
|
||||
WillReturnRows(sqlmock.NewRows([]string{"id", "name"}).AddRow(1, "1"))
|
||||
files := GetUploadPlaceholderFiles(1)
|
||||
a.NoError(mock.ExpectationsWereMet())
|
||||
a.Len(files, 1)
|
||||
}
|
||||
|
||||
func TestFile_GetPolicy(t *testing.T) {
|
||||
asserts := assert.New(t)
|
||||
|
||||
@@ -282,28 +333,62 @@ func TestRemoveFilesWithSoftLinks(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestDeleteFileByIDs(t *testing.T) {
|
||||
asserts := assert.New(t)
|
||||
func TestDeleteFiles(t *testing.T) {
|
||||
a := assert.New(t)
|
||||
|
||||
// 出错
|
||||
// uid 不一致
|
||||
{
|
||||
err := DeleteFiles([]*File{{}}, 1)
|
||||
a.Contains("user id not consistent", err.Error())
|
||||
}
|
||||
|
||||
// 删除失败
|
||||
{
|
||||
mock.ExpectBegin()
|
||||
mock.ExpectExec("DELETE(.+)").
|
||||
WillReturnError(errors.New("error"))
|
||||
mock.ExpectRollback()
|
||||
err := DeleteFileByIDs([]uint{1, 2, 3})
|
||||
asserts.NoError(mock.ExpectationsWereMet())
|
||||
asserts.Error(err)
|
||||
err := DeleteFiles([]*File{{}}, 0)
|
||||
a.NoError(mock.ExpectationsWereMet())
|
||||
a.Error(err)
|
||||
}
|
||||
|
||||
// 无法变更用户容量
|
||||
{
|
||||
mock.ExpectBegin()
|
||||
mock.ExpectExec("DELETE(.+)").
|
||||
WillReturnResult(sqlmock.NewResult(1, 1))
|
||||
mock.ExpectExec("UPDATE(.+)storage(.+)").WillReturnError(errors.New("error"))
|
||||
mock.ExpectRollback()
|
||||
err := DeleteFiles([]*File{{}}, 0)
|
||||
a.NoError(mock.ExpectationsWereMet())
|
||||
a.Error(err)
|
||||
}
|
||||
|
||||
// 文件脏读
|
||||
{
|
||||
mock.ExpectBegin()
|
||||
mock.ExpectExec("DELETE(.+)").
|
||||
WillReturnResult(sqlmock.NewResult(1, 0))
|
||||
mock.ExpectRollback()
|
||||
err := DeleteFiles([]*File{{Size: 1}, {Size: 2}}, 0)
|
||||
a.NoError(mock.ExpectationsWereMet())
|
||||
a.Error(err)
|
||||
a.Contains("file size is dirty", err.Error())
|
||||
}
|
||||
|
||||
// 成功
|
||||
{
|
||||
mock.ExpectBegin()
|
||||
mock.ExpectExec("DELETE(.+)").
|
||||
WillReturnResult(sqlmock.NewResult(0, 3))
|
||||
WillReturnResult(sqlmock.NewResult(2, 1))
|
||||
mock.ExpectExec("DELETE(.+)").
|
||||
WillReturnResult(sqlmock.NewResult(2, 1))
|
||||
mock.ExpectExec("UPDATE(.+)storage(.+)").WithArgs(uint64(3), sqlmock.AnyArg()).WillReturnResult(sqlmock.NewResult(1, 1))
|
||||
mock.ExpectCommit()
|
||||
err := DeleteFileByIDs([]uint{1, 2, 3})
|
||||
asserts.NoError(mock.ExpectationsWereMet())
|
||||
asserts.NoError(err)
|
||||
err := DeleteFiles([]*File{{Size: 1}, {Size: 2}}, 0)
|
||||
a.NoError(mock.ExpectationsWereMet())
|
||||
a.NoError(err)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -324,6 +409,19 @@ func TestGetFilesByParentIDs(t *testing.T) {
|
||||
asserts.Len(files, 3)
|
||||
}
|
||||
|
||||
func TestGetFilesByUploadSession(t *testing.T) {
|
||||
a := assert.New(t)
|
||||
|
||||
mock.ExpectQuery("SELECT(.+)").
|
||||
WithArgs(1, "sessionID").
|
||||
WillReturnRows(
|
||||
sqlmock.NewRows([]string{"id", "name"}).AddRow(4, "4.txt"))
|
||||
files, err := GetFilesByUploadSession("sessionID", 1)
|
||||
a.NoError(err)
|
||||
a.NoError(mock.ExpectationsWereMet())
|
||||
a.Equal("4.txt", files.Name)
|
||||
}
|
||||
|
||||
func TestFile_Updates(t *testing.T) {
|
||||
asserts := assert.New(t)
|
||||
file := File{Model: gorm.Model{ID: 1}}
|
||||
@@ -340,22 +438,91 @@ func TestFile_Updates(t *testing.T) {
|
||||
// UpdatePicInfo
|
||||
{
|
||||
mock.ExpectBegin()
|
||||
mock.ExpectExec("UPDATE(.+)").WithArgs(10, sqlmock.AnyArg(), 1).WillReturnResult(sqlmock.NewResult(1, 1))
|
||||
mock.ExpectCommit()
|
||||
err := file.UpdateSize(10)
|
||||
asserts.NoError(mock.ExpectationsWereMet())
|
||||
asserts.NoError(err)
|
||||
}
|
||||
|
||||
// UpdatePicInfo
|
||||
{
|
||||
mock.ExpectBegin()
|
||||
mock.ExpectExec("UPDATE(.+)").WithArgs("1,1", sqlmock.AnyArg(), 1).WillReturnResult(sqlmock.NewResult(1, 1))
|
||||
mock.ExpectExec("UPDATE(.+)").WithArgs("1,1", 1).WillReturnResult(sqlmock.NewResult(1, 1))
|
||||
mock.ExpectCommit()
|
||||
err := file.UpdatePicInfo("1,1")
|
||||
asserts.NoError(mock.ExpectationsWereMet())
|
||||
asserts.NoError(err)
|
||||
}
|
||||
|
||||
// UpdateSourceName
|
||||
{
|
||||
mock.ExpectBegin()
|
||||
mock.ExpectExec("UPDATE(.+)").WithArgs("newName", sqlmock.AnyArg(), 1).WillReturnResult(sqlmock.NewResult(1, 1))
|
||||
mock.ExpectCommit()
|
||||
err := file.UpdateSourceName("newName")
|
||||
asserts.NoError(mock.ExpectationsWereMet())
|
||||
asserts.NoError(err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestFile_UpdateSize(t *testing.T) {
|
||||
a := assert.New(t)
|
||||
|
||||
// 增加成功
|
||||
{
|
||||
file := File{Size: 10}
|
||||
mock.ExpectBegin()
|
||||
mock.ExpectExec("UPDATE(.+)files(.+)").WithArgs(11, sqlmock.AnyArg(), 10).WillReturnResult(sqlmock.NewResult(1, 1))
|
||||
mock.ExpectExec("UPDATE(.+)storage(.+)+(.+)").WithArgs(uint64(1), sqlmock.AnyArg()).WillReturnResult(sqlmock.NewResult(1, 1))
|
||||
mock.ExpectCommit()
|
||||
|
||||
a.NoError(file.UpdateSize(11))
|
||||
a.NoError(mock.ExpectationsWereMet())
|
||||
}
|
||||
|
||||
// 减少成功
|
||||
{
|
||||
file := File{Size: 10}
|
||||
mock.ExpectBegin()
|
||||
mock.ExpectExec("UPDATE(.+)files(.+)").WithArgs(8, sqlmock.AnyArg(), 10).WillReturnResult(sqlmock.NewResult(1, 1))
|
||||
mock.ExpectExec("UPDATE(.+)storage(.+)-(.+)").WithArgs(uint64(2), sqlmock.AnyArg()).WillReturnResult(sqlmock.NewResult(1, 1))
|
||||
mock.ExpectCommit()
|
||||
|
||||
a.NoError(file.UpdateSize(8))
|
||||
a.NoError(mock.ExpectationsWereMet())
|
||||
}
|
||||
|
||||
// 文件更新失败
|
||||
{
|
||||
file := File{Size: 10}
|
||||
mock.ExpectBegin()
|
||||
mock.ExpectExec("UPDATE(.+)files(.+)").WithArgs(8, sqlmock.AnyArg(), 10).WillReturnError(errors.New("error"))
|
||||
mock.ExpectRollback()
|
||||
|
||||
a.Error(file.UpdateSize(8))
|
||||
a.NoError(mock.ExpectationsWereMet())
|
||||
}
|
||||
|
||||
// 用户容量更新失败
|
||||
{
|
||||
file := File{Size: 10}
|
||||
mock.ExpectBegin()
|
||||
mock.ExpectExec("UPDATE(.+)files(.+)").WithArgs(8, sqlmock.AnyArg(), 10).WillReturnResult(sqlmock.NewResult(1, 1))
|
||||
mock.ExpectExec("UPDATE(.+)storage(.+)-(.+)").WithArgs(uint64(2), sqlmock.AnyArg()).WillReturnError(errors.New("error"))
|
||||
mock.ExpectRollback()
|
||||
|
||||
a.Error(file.UpdateSize(8))
|
||||
a.NoError(mock.ExpectationsWereMet())
|
||||
}
|
||||
}
|
||||
|
||||
func TestFile_PopChunkToFile(t *testing.T) {
|
||||
a := assert.New(t)
|
||||
timeNow := time.Now()
|
||||
file := File{}
|
||||
mock.ExpectBegin()
|
||||
mock.ExpectExec("UPDATE(.+)files(.+)").WillReturnResult(sqlmock.NewResult(1, 1))
|
||||
mock.ExpectCommit()
|
||||
a.NoError(file.PopChunkToFile(&timeNow, "1,1"))
|
||||
}
|
||||
|
||||
func TestFile_CanCopy(t *testing.T) {
|
||||
a := assert.New(t)
|
||||
file := File{}
|
||||
a.True(file.CanCopy())
|
||||
file.UploadSessionID = &file.Name
|
||||
a.False(file.CanCopy())
|
||||
}
|
||||
|
||||
func TestFile_FileInfoInterface(t *testing.T) {
|
||||
|
||||
@@ -23,11 +23,15 @@ type Folder struct {
|
||||
|
||||
// Create 创建目录
|
||||
func (folder *Folder) Create() (uint, error) {
|
||||
if err := DB.Create(folder).Error; err != nil {
|
||||
tx := DB.Begin()
|
||||
if err := tx.FirstOrCreate(folder, *folder).Error; err != nil {
|
||||
tx.Rollback()
|
||||
util.Log().Warning("无法插入目录记录, %s", err)
|
||||
return 0, err
|
||||
}
|
||||
return folder.ID, nil
|
||||
|
||||
err := tx.Commit().Error
|
||||
return folder.ID, err
|
||||
}
|
||||
|
||||
// GetChild 返回folder下名为name的子目录,不存在则返回错误
|
||||
@@ -158,6 +162,11 @@ func (folder *Folder) MoveOrCopyFileTo(files []uint, dstFolder *Folder, isCopy b
|
||||
|
||||
// 复制文件记录
|
||||
for _, oldFile := range originFiles {
|
||||
if !oldFile.CanCopy() {
|
||||
util.Log().Warning("无法复制正在上传中的文件 [%s], 跳过...", oldFile.Name)
|
||||
continue
|
||||
}
|
||||
|
||||
oldFile.Model = gorm.Model{}
|
||||
oldFile.FolderID = dstFolder.ID
|
||||
oldFile.UserID = dstFolder.OwnerID
|
||||
@@ -246,6 +255,11 @@ func (folder *Folder) CopyFolderTo(folderID uint, dstFolder *Folder) (size uint6
|
||||
|
||||
// 复制文件记录
|
||||
for _, oldFile := range originFiles {
|
||||
if !oldFile.CanCopy() {
|
||||
util.Log().Warning("无法复制正在上传中的文件 [%s], 跳过...", oldFile.Name)
|
||||
continue
|
||||
}
|
||||
|
||||
oldFile.Model = gorm.Model{}
|
||||
oldFile.FolderID = newIDCache[oldFile.FolderID]
|
||||
oldFile.UserID = dstFolder.OwnerID
|
||||
@@ -263,6 +277,13 @@ func (folder *Folder) CopyFolderTo(folderID uint, dstFolder *Folder) (size uint6
|
||||
// MoveFolderTo 将folder目录下的dirs子目录复制或移动到dstFolder,
|
||||
// 返回此过程中增加的容量
|
||||
func (folder *Folder) MoveFolderTo(dirs []uint, dstFolder *Folder) error {
|
||||
|
||||
// 如果目标位置为待移动的目录,会导致 parent 为自己
|
||||
// 造成死循环且无法被除搜索以外的组件展示
|
||||
if folder.OwnerID == dstFolder.OwnerID && util.ContainsUint(dirs, dstFolder.ID) {
|
||||
return errors.New("cannot move a folder into itself")
|
||||
}
|
||||
|
||||
// 更改顶级要移动目录的父目录指向
|
||||
err := DB.Model(Folder{}).Where(
|
||||
"id in (?) and owner_id = ? and parent_id = ?",
|
||||
|
||||
@@ -17,8 +17,9 @@ func TestFolder_Create(t *testing.T) {
|
||||
Name: "new folder",
|
||||
}
|
||||
|
||||
// 插入成功
|
||||
// 不存在,插入成功
|
||||
mock.ExpectBegin()
|
||||
mock.ExpectQuery("SELECT(.+)folders(.+)").WillReturnRows(sqlmock.NewRows([]string{"id"}))
|
||||
mock.ExpectExec("INSERT(.+)").WillReturnResult(sqlmock.NewResult(5, 1))
|
||||
mock.ExpectCommit()
|
||||
fid, err := folder.Create()
|
||||
@@ -28,12 +29,22 @@ func TestFolder_Create(t *testing.T) {
|
||||
|
||||
// 插入失败
|
||||
mock.ExpectBegin()
|
||||
mock.ExpectQuery("SELECT(.+)folders(.+)").WillReturnRows(sqlmock.NewRows([]string{"id"}))
|
||||
mock.ExpectExec("INSERT(.+)").WillReturnError(errors.New("error"))
|
||||
mock.ExpectRollback()
|
||||
fid, err = folder.Create()
|
||||
asserts.Error(err)
|
||||
asserts.Equal(uint(0), fid)
|
||||
asserts.NoError(mock.ExpectationsWereMet())
|
||||
|
||||
// 存在,直接返回
|
||||
mock.ExpectBegin()
|
||||
mock.ExpectQuery("SELECT(.+)folders(.+)").WillReturnRows(sqlmock.NewRows([]string{"id"}).AddRow(5))
|
||||
mock.ExpectCommit()
|
||||
fid, err = folder.Create()
|
||||
asserts.NoError(err)
|
||||
asserts.Equal(uint(5), fid)
|
||||
asserts.NoError(mock.ExpectationsWereMet())
|
||||
}
|
||||
|
||||
func TestFolder_GetChild(t *testing.T) {
|
||||
@@ -212,12 +223,14 @@ func TestFolder_MoveOrCopyFileTo(t *testing.T) {
|
||||
WithArgs(
|
||||
1,
|
||||
2,
|
||||
3,
|
||||
1,
|
||||
1,
|
||||
).WillReturnRows(
|
||||
sqlmock.NewRows([]string{"id", "size"}).
|
||||
AddRow(1, 10).
|
||||
AddRow(2, 20),
|
||||
sqlmock.NewRows([]string{"id", "size", "upload_session_id"}).
|
||||
AddRow(1, 10, nil).
|
||||
AddRow(2, 20, nil).
|
||||
AddRow(2, 20, &folder.Name),
|
||||
)
|
||||
mock.ExpectBegin()
|
||||
mock.ExpectExec("INSERT(.+)").WillReturnResult(sqlmock.NewResult(1, 1))
|
||||
@@ -226,7 +239,7 @@ func TestFolder_MoveOrCopyFileTo(t *testing.T) {
|
||||
mock.ExpectExec("INSERT(.+)").WillReturnResult(sqlmock.NewResult(1, 1))
|
||||
mock.ExpectCommit()
|
||||
storage, err := folder.MoveOrCopyFileTo(
|
||||
[]uint{1, 2},
|
||||
[]uint{1, 2, 3},
|
||||
&dstFolder,
|
||||
true,
|
||||
)
|
||||
@@ -335,7 +348,7 @@ func TestFolder_CopyFolderTo(t *testing.T) {
|
||||
// 测试复制目录结构
|
||||
// test(2)(5)
|
||||
// 1(3)(6) 2.txt
|
||||
// 3(4)(7) 4.txt
|
||||
// 3(4)(7) 4.txt 5.txt(上传中)
|
||||
|
||||
// 正常情况 成功
|
||||
{
|
||||
@@ -360,9 +373,10 @@ func TestFolder_CopyFolderTo(t *testing.T) {
|
||||
mock.ExpectQuery("SELECT(.+)").
|
||||
WithArgs(1, 2, 3, 4).
|
||||
WillReturnRows(
|
||||
sqlmock.NewRows([]string{"id", "name", "folder_id", "size"}).
|
||||
AddRow(1, "2.txt", 2, 10).
|
||||
AddRow(2, "3.txt", 3, 20),
|
||||
sqlmock.NewRows([]string{"id", "name", "folder_id", "size", "upload_session_id"}).
|
||||
AddRow(1, "2.txt", 2, 10, nil).
|
||||
AddRow(2, "3.txt", 3, 20, nil).
|
||||
AddRow(3, "5.txt", 3, 20, &dstFolder.Name),
|
||||
)
|
||||
|
||||
// 复制子文件
|
||||
@@ -493,7 +507,8 @@ func TestFolder_MoveOrCopyFolderTo_Move(t *testing.T) {
|
||||
}
|
||||
// 目标目录
|
||||
dstFolder := Folder{
|
||||
Model: gorm.Model{ID: 10},
|
||||
Model: gorm.Model{ID: 10},
|
||||
OwnerID: 1,
|
||||
}
|
||||
|
||||
// 成功
|
||||
@@ -507,6 +522,12 @@ func TestFolder_MoveOrCopyFolderTo_Move(t *testing.T) {
|
||||
asserts.NoError(mock.ExpectationsWereMet())
|
||||
asserts.NoError(err)
|
||||
}
|
||||
|
||||
// 移动自己到自己内部,失败
|
||||
{
|
||||
err := parFolder.MoveFolderTo([]uint{10, 2}, &dstFolder)
|
||||
asserts.Error(err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestFolder_FileInfoInterface(t *testing.T) {
|
||||
|
||||
@@ -73,10 +73,13 @@ func Init() {
|
||||
}
|
||||
|
||||
//设置连接池
|
||||
//空闲
|
||||
db.DB().SetMaxIdleConns(50)
|
||||
//打开
|
||||
db.DB().SetMaxOpenConns(100)
|
||||
if conf.DatabaseConfig.Type == "sqlite" || conf.DatabaseConfig.Type == "sqlite3" || conf.DatabaseConfig.Type == "UNSET" {
|
||||
db.DB().SetMaxOpenConns(1)
|
||||
} else {
|
||||
db.DB().SetMaxOpenConns(100)
|
||||
}
|
||||
|
||||
//超时
|
||||
db.DB().SetConnMaxLifetime(time.Second * 30)
|
||||
|
||||
|
||||
@@ -7,7 +7,6 @@ import (
|
||||
"github.com/cloudreve/Cloudreve/v3/pkg/conf"
|
||||
"github.com/cloudreve/Cloudreve/v3/pkg/util"
|
||||
"github.com/fatih/color"
|
||||
"github.com/gofrs/uuid"
|
||||
"github.com/hashicorp/go-version"
|
||||
"github.com/jinzhu/gorm"
|
||||
"sort"
|
||||
@@ -78,6 +77,9 @@ func addDefaultPolicy() {
|
||||
DirNameRule: "uploads/{uid}/{path}",
|
||||
FileNameRule: "{uid}_{randomkey8}_{originname}",
|
||||
IsOriginLinkEnable: false,
|
||||
OptionsSerialized: PolicyOption{
|
||||
ChunkSize: 25 << 20, // 25MB
|
||||
},
|
||||
}
|
||||
if err := DB.Create(&defaultPolicy).Error; err != nil {
|
||||
util.Log().Panic("无法创建初始存储策略, %s", err)
|
||||
@@ -86,112 +88,6 @@ func addDefaultPolicy() {
|
||||
}
|
||||
|
||||
func addDefaultSettings() {
|
||||
siteID, _ := uuid.NewV4()
|
||||
|
||||
defaultSettings := []Setting{
|
||||
{Name: "siteURL", Value: `http://localhost`, Type: "basic"},
|
||||
{Name: "siteName", Value: `Cloudreve`, Type: "basic"},
|
||||
{Name: "siteICPId", Value: ``, Type: "basic"},
|
||||
{Name: "register_enabled", Value: `1`, Type: "register"},
|
||||
{Name: "default_group", Value: `2`, Type: "register"},
|
||||
{Name: "siteKeywords", Value: `网盘,网盘`, Type: "basic"},
|
||||
{Name: "siteDes", Value: `Cloudreve`, Type: "basic"},
|
||||
{Name: "siteTitle", Value: `平步云端`, Type: "basic"},
|
||||
{Name: "siteScript", Value: ``, Type: "basic"},
|
||||
{Name: "siteID", Value: siteID.String(), Type: "basic"},
|
||||
{Name: "fromName", Value: `Cloudreve`, Type: "mail"},
|
||||
{Name: "mail_keepalive", Value: `30`, Type: "mail"},
|
||||
{Name: "fromAdress", Value: `no-reply@acg.blue`, Type: "mail"},
|
||||
{Name: "smtpHost", Value: `smtp.mxhichina.com`, Type: "mail"},
|
||||
{Name: "smtpPort", Value: `25`, Type: "mail"},
|
||||
{Name: "replyTo", Value: `abslant@126.com`, Type: "mail"},
|
||||
{Name: "smtpUser", Value: `no-reply@acg.blue`, Type: "mail"},
|
||||
{Name: "smtpPass", Value: ``, Type: "mail"},
|
||||
{Name: "smtpEncryption", Value: `0`, Type: "mail"},
|
||||
{Name: "maxEditSize", Value: `4194304`, Type: "file_edit"},
|
||||
{Name: "archive_timeout", Value: `60`, Type: "timeout"},
|
||||
{Name: "download_timeout", Value: `60`, Type: "timeout"},
|
||||
{Name: "preview_timeout", Value: `60`, Type: "timeout"},
|
||||
{Name: "doc_preview_timeout", Value: `60`, Type: "timeout"},
|
||||
{Name: "upload_credential_timeout", Value: `1800`, Type: "timeout"},
|
||||
{Name: "upload_session_timeout", Value: `86400`, Type: "timeout"},
|
||||
{Name: "slave_api_timeout", Value: `60`, Type: "timeout"},
|
||||
{Name: "slave_node_retry", Value: `3`, Type: "slave"},
|
||||
{Name: "slave_ping_interval", Value: `60`, Type: "slave"},
|
||||
{Name: "slave_recover_interval", Value: `120`, Type: "slave"},
|
||||
{Name: "slave_transfer_timeout", Value: `172800`, Type: "timeout"},
|
||||
{Name: "onedrive_monitor_timeout", Value: `600`, Type: "timeout"},
|
||||
{Name: "share_download_session_timeout", Value: `2073600`, Type: "timeout"},
|
||||
{Name: "onedrive_callback_check", Value: `20`, Type: "timeout"},
|
||||
{Name: "folder_props_timeout", Value: `300`, Type: "timeout"},
|
||||
{Name: "onedrive_chunk_retries", Value: `1`, Type: "retry"},
|
||||
{Name: "onedrive_source_timeout", Value: `1800`, Type: "timeout"},
|
||||
{Name: "reset_after_upload_failed", Value: `0`, Type: "upload"},
|
||||
{Name: "login_captcha", Value: `0`, Type: "login"},
|
||||
{Name: "reg_captcha", Value: `0`, Type: "login"},
|
||||
{Name: "email_active", Value: `0`, Type: "register"},
|
||||
{Name: "mail_activation_template", Value: `<!DOCTYPE html PUBLIC"-//W3C//DTD XHTML 1.0 Transitional//EN""http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd"><html xmlns="http://www.w3.org/1999/xhtml"style="font-family: 'Helvetica Neue', Helvetica, Arial, sans-serif; box-sizing: border-box;
|
||||
font-size: 14px; margin: 0;"><head><meta name="viewport"content="width=device-width"/><meta http-equiv="Content-Type"content="text/html; charset=UTF-8"/><title>激活您的账户</title><style type="text/css">img{max-width:100%}body{-webkit-font-smoothing:antialiased;-webkit-text-size-adjust:none;width:100%!important;height:100%;line-height:1.6em}body{background-color:#f6f6f6}@media only screen and(max-width:640px){body{padding:0!important}h1{font-weight:800!important;margin:20px 0 5px!important}h2{font-weight:800!important;margin:20px 0 5px!important}h3{font-weight:800!important;margin:20px 0 5px!important}h4{font-weight:800!important;margin:20px 0 5px!important}h1{font-size:22px!important}h2{font-size:18px!important}h3{font-size:16px!important}.container{padding:0!important;width:100%!important}.content{padding:0!important}.content-wrap{padding:10px!important}.invoice{width:100%!important}}</style></head><body itemscope itemtype="http://schema.org/EmailMessage"style="font-family: 'Helvetica Neue',Helvetica,Arial,sans-serif; box-sizing:
|
||||
border-box; font-size: 14px; -webkit-font-smoothing: antialiased; -webkit-text-size-adjust: none; width: 100% !important; height: 100%; line-height: 1.6em; background-color: #f6f6f6; margin: 0;"bgcolor="#f6f6f6"><table class="body-wrap"style="font-family: 'Helvetica Neue',Helvetica,Arial,sans-serif; box-sizing: border-box; font-size: 14px; width: 100%; background-color: #f6f6f6; margin: 0;"bgcolor="#f6f6f6"><tr style="font-family: 'Helvetica Neue',Helvetica,Arial,sans-serif;
|
||||
box-sizing: border-box; font-size: 14px; margin: 0;"><td style="font-family: 'Helvetica Neue',Helvetica,Arial,sans-serif; box-sizing: border-box; font-size: 14px; vertical-align: top; margin: 0;"valign="top"></td><td class="container"width="600"style="font-family: 'Helvetica Neue',Helvetica,Arial,sans-serif; box-sizing: border-box; font-size: 14px; vertical-align: top; display: block !important; max-width: 600px !important; clear: both !important; margin: 0 auto;"valign="top"><div class="content"style="font-family: 'Helvetica Neue',Helvetica,Arial,sans-serif; box-sizing: border-box; font-size: 14px; max-width: 600px; display: block; margin: 0 auto; padding: 20px;"><table class="main"width="100%"cellpadding="0"cellspacing="0"style="font-family: 'Helvetica Neue',Helvetica,Arial,sans-serif; box-sizing: border-box; font-size: 14px; border-radius: 3px; background-color: #fff; margin: 0; border: 1px
|
||||
solid #e9e9e9;"bgcolor="#fff"><tr style="font-family: 'Helvetica Neue',Helvetica,Arial,sans-serif; box-sizing: border-box; font-size:
|
||||
14px; margin: 0;"><td class="alert alert-warning"style="font-family: 'Helvetica Neue',Helvetica,Arial,sans-serif; box-sizing: border-box; font-size: 16px; vertical-align: top; color: #fff; font-weight: 500; text-align: center; border-radius: 3px 3px 0 0; background-color: #009688; margin: 0; padding: 20px;"align="center"bgcolor="#FF9F00"valign="top">激活{siteTitle}账户</td></tr><tr style="font-family: 'Helvetica Neue',Helvetica,Arial,sans-serif; box-sizing: border-box; font-size: 14px; margin: 0;"><td class="content-wrap"style="font-family: 'Helvetica Neue',Helvetica,Arial,sans-serif; box-sizing: border-box; font-size: 14px; vertical-align: top; margin: 0; padding: 20px;"valign="top"><table width="100%"cellpadding="0"cellspacing="0"style="font-family: 'Helvetica Neue',Helvetica,Arial,sans-serif; box-sizing: border-box; font-size: 14px; margin: 0;"><tr style="font-family: 'Helvetica Neue',Helvetica,Arial,sans-serif; box-sizing: border-box; font-size: 14px; margin: 0;"><td class="content-block"style="font-family: 'Helvetica
|
||||
Neue',Helvetica,Arial,sans-serif; box-sizing: border-box; font-size: 14px; vertical-align: top; margin: 0; padding: 0 0 20px;"valign="top">亲爱的<strong style="font-family: 'Helvetica Neue',Helvetica,Arial,sans-serif; box-sizing: border-box; font-size: 14px; margin: 0;">{userName}</strong>:</td></tr><tr style="font-family: 'Helvetica Neue',Helvetica,Arial,sans-serif; box-sizing: border-box; font-size: 14px; margin: 0;"><td class="content-block"style="font-family: 'Helvetica Neue',Helvetica,Arial,sans-serif; box-sizing: border-box; font-size: 14px; vertical-align: top; margin: 0; padding: 0 0 20px;"valign="top">感谢您注册{siteTitle},请点击下方按钮完成账户激活。</td></tr><tr style="font-family: 'Helvetica Neue',Helvetica,Arial,sans-serif; box-sizing: border-box; font-size: 14px; margin: 0;"><td class="content-block"style="font-family: 'Helvetica Neue',Helvetica,Arial,sans-serif; box-sizing: border-box; font-size: 14px; vertical-align: top; margin: 0; padding: 0 0 20px;"valign="top"><a href="{activationUrl}"class="btn-primary"style="font-family: 'Helvetica Neue',Helvetica,Arial,sans-serif; box-sizing: border-box; font-size: 14px; color: #FFF; text-decoration: none; line-height: 2em; font-weight: bold; text-align: center; cursor: pointer; display: inline-block; border-radius: 5px; text-transform: capitalize; background-color: #009688; margin: 0; border-color: #009688; border-style: solid; border-width: 10px 20px;">激活账户</a></td></tr><tr style="font-family: 'Helvetica Neue',Helvetica,Arial,sans-serif; box-sizing: border-box; font-size: 14px; margin: 0;"><td class="content-block"style="font-family: 'Helvetica Neue',Helvetica,Arial,sans-serif; box-sizing: border-box; font-size: 14px; vertical-align: top; margin: 0; padding: 0 0 20px;"valign="top">感谢您选择{siteTitle}。</td></tr></table></td></tr></table><div class="footer"style="font-family: 'Helvetica Neue',Helvetica,Arial,sans-serif; box-sizing: border-box; font-size: 14px; width: 100%; clear: both; color: #999; margin: 0; padding: 20px;"><table width="100%"style="font-family: 'Helvetica Neue',Helvetica,Arial,sans-serif; box-sizing: border-box; font-size: 14px; margin: 0;"><tr style="font-family: 'Helvetica Neue',Helvetica,Arial,sans-serif; box-sizing: border-box; font-size: 14px; margin: 0;"><td class="aligncenter content-block"style="font-family: 'Helvetica Neue',Helvetica,Arial,sans-serif; box-sizing: border-box; font-size: 12px; vertical-align: top; color: #999; text-align: center; margin: 0; padding: 0 0 20px;"align="center"valign="top">此邮件由系统自动发送,请不要直接回复。</td></tr></table></div></div></td><td style="font-family: 'Helvetica Neue',Helvetica,Arial,sans-serif; box-sizing: border-box; font-size: 14px; vertical-align: top; margin: 0;"valign="top"></td></tr></table></body></html>`, Type: "mail_template"},
|
||||
{Name: "forget_captcha", Value: `0`, Type: "login"},
|
||||
{Name: "mail_reset_pwd_template", Value: `<!DOCTYPE html PUBLIC"-//W3C//DTD XHTML 1.0 Transitional//EN""http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd"><html xmlns="http://www.w3.org/1999/xhtml"style="font-family: 'Helvetica Neue', Helvetica, Arial, sans-serif; box-sizing: border-box;
|
||||
font-size: 14px; margin: 0;"><head><meta name="viewport"content="width=device-width"/><meta http-equiv="Content-Type"content="text/html; charset=UTF-8"/><title>重设密码</title><style type="text/css">img{max-width:100%}body{-webkit-font-smoothing:antialiased;-webkit-text-size-adjust:none;width:100%!important;height:100%;line-height:1.6em}body{background-color:#f6f6f6}@media only screen and(max-width:640px){body{padding:0!important}h1{font-weight:800!important;margin:20px 0 5px!important}h2{font-weight:800!important;margin:20px 0 5px!important}h3{font-weight:800!important;margin:20px 0 5px!important}h4{font-weight:800!important;margin:20px 0 5px!important}h1{font-size:22px!important}h2{font-size:18px!important}h3{font-size:16px!important}.container{padding:0!important;width:100%!important}.content{padding:0!important}.content-wrap{padding:10px!important}.invoice{width:100%!important}}</style></head><body itemscope itemtype="http://schema.org/EmailMessage"style="font-family: 'Helvetica Neue',Helvetica,Arial,sans-serif; box-sizing:
|
||||
border-box; font-size: 14px; -webkit-font-smoothing: antialiased; -webkit-text-size-adjust: none; width: 100% !important; height: 100%; line-height: 1.6em; background-color: #f6f6f6; margin: 0;"bgcolor="#f6f6f6"><table class="body-wrap"style="font-family: 'Helvetica Neue',Helvetica,Arial,sans-serif; box-sizing: border-box; font-size: 14px; width: 100%; background-color: #f6f6f6; margin: 0;"bgcolor="#f6f6f6"><tr style="font-family: 'Helvetica Neue',Helvetica,Arial,sans-serif;
|
||||
box-sizing: border-box; font-size: 14px; margin: 0;"><td style="font-family: 'Helvetica Neue',Helvetica,Arial,sans-serif; box-sizing: border-box; font-size: 14px; vertical-align: top; margin: 0;"valign="top"></td><td class="container"width="600"style="font-family: 'Helvetica Neue',Helvetica,Arial,sans-serif; box-sizing: border-box; font-size: 14px; vertical-align: top; display: block !important; max-width: 600px !important; clear: both !important; margin: 0 auto;"valign="top"><div class="content"style="font-family: 'Helvetica Neue',Helvetica,Arial,sans-serif; box-sizing: border-box; font-size: 14px; max-width: 600px; display: block; margin: 0 auto; padding: 20px;"><table class="main"width="100%"cellpadding="0"cellspacing="0"style="font-family: 'Helvetica Neue',Helvetica,Arial,sans-serif; box-sizing: border-box; font-size: 14px; border-radius: 3px; background-color: #fff; margin: 0; border: 1px
|
||||
solid #e9e9e9;"bgcolor="#fff"><tr style="font-family: 'Helvetica Neue',Helvetica,Arial,sans-serif; box-sizing: border-box; font-size:
|
||||
14px; margin: 0;"><td class="alert alert-warning"style="font-family: 'Helvetica Neue',Helvetica,Arial,sans-serif; box-sizing: border-box; font-size: 16px; vertical-align: top; color: #fff; font-weight: 500; text-align: center; border-radius: 3px 3px 0 0; background-color: #2196F3; margin: 0; padding: 20px;"align="center"bgcolor="#FF9F00"valign="top">重设{siteTitle}密码</td></tr><tr style="font-family: 'Helvetica Neue',Helvetica,Arial,sans-serif; box-sizing: border-box; font-size: 14px; margin: 0;"><td class="content-wrap"style="font-family: 'Helvetica Neue',Helvetica,Arial,sans-serif; box-sizing: border-box; font-size: 14px; vertical-align: top; margin: 0; padding: 20px;"valign="top"><table width="100%"cellpadding="0"cellspacing="0"style="font-family: 'Helvetica Neue',Helvetica,Arial,sans-serif; box-sizing: border-box; font-size: 14px; margin: 0;"><tr style="font-family: 'Helvetica Neue',Helvetica,Arial,sans-serif; box-sizing: border-box; font-size: 14px; margin: 0;"><td class="content-block"style="font-family: 'Helvetica
|
||||
Neue',Helvetica,Arial,sans-serif; box-sizing: border-box; font-size: 14px; vertical-align: top; margin: 0; padding: 0 0 20px;"valign="top">亲爱的<strong style="font-family: 'Helvetica Neue',Helvetica,Arial,sans-serif; box-sizing: border-box; font-size: 14px; margin: 0;">{userName}</strong>:</td></tr><tr style="font-family: 'Helvetica Neue',Helvetica,Arial,sans-serif; box-sizing: border-box; font-size: 14px; margin: 0;"><td class="content-block"style="font-family: 'Helvetica Neue',Helvetica,Arial,sans-serif; box-sizing: border-box; font-size: 14px; vertical-align: top; margin: 0; padding: 0 0 20px;"valign="top">请点击下方按钮完成密码重设。如果非你本人操作,请忽略此邮件。</td></tr><tr style="font-family: 'Helvetica Neue',Helvetica,Arial,sans-serif; box-sizing: border-box; font-size: 14px; margin: 0;"><td class="content-block"style="font-family: 'Helvetica Neue',Helvetica,Arial,sans-serif; box-sizing: border-box; font-size: 14px; vertical-align: top; margin: 0; padding: 0 0 20px;"valign="top"><a href="{resetUrl}"class="btn-primary"style="font-family: 'Helvetica Neue',Helvetica,Arial,sans-serif; box-sizing: border-box; font-size: 14px; color: #FFF; text-decoration: none; line-height: 2em; font-weight: bold; text-align: center; cursor: pointer; display: inline-block; border-radius: 5px; text-transform: capitalize; background-color: #2196F3; margin: 0; border-color: #2196F3; border-style: solid; border-width: 10px 20px;">重设密码</a></td></tr><tr style="font-family: 'Helvetica Neue',Helvetica,Arial,sans-serif; box-sizing: border-box; font-size: 14px; margin: 0;"><td class="content-block"style="font-family: 'Helvetica Neue',Helvetica,Arial,sans-serif; box-sizing: border-box; font-size: 14px; vertical-align: top; margin: 0; padding: 0 0 20px;"valign="top">感谢您选择{siteTitle}。</td></tr></table></td></tr></table><div class="footer"style="font-family: 'Helvetica Neue',Helvetica,Arial,sans-serif; box-sizing: border-box; font-size: 14px; width: 100%; clear: both; color: #999; margin: 0; padding: 20px;"><table width="100%"style="font-family: 'Helvetica Neue',Helvetica,Arial,sans-serif; box-sizing: border-box; font-size: 14px; margin: 0;"><tr style="font-family: 'Helvetica Neue',Helvetica,Arial,sans-serif; box-sizing: border-box; font-size: 14px; margin: 0;"><td class="aligncenter content-block"style="font-family: 'Helvetica Neue',Helvetica,Arial,sans-serif; box-sizing: border-box; font-size: 12px; vertical-align: top; color: #999; text-align: center; margin: 0; padding: 0 0 20px;"align="center"valign="top">此邮件由系统自动发送,请不要直接回复。</td></tr></table></div></div></td><td style="font-family: 'Helvetica Neue',Helvetica,Arial,sans-serif; box-sizing: border-box; font-size: 14px; vertical-align: top; margin: 0;"valign="top"></td></tr></table></body></html>`, Type: "mail_template"},
|
||||
{Name: "db_version_" + conf.RequiredDBVersion, Value: `installed`, Type: "version"},
|
||||
{Name: "hot_share_num", Value: `10`, Type: "share"},
|
||||
{Name: "gravatar_server", Value: `https://www.gravatar.com/`, Type: "avatar"},
|
||||
{Name: "defaultTheme", Value: `#3f51b5`, Type: "basic"},
|
||||
{Name: "themes", Value: `{"#3f51b5":{"palette":{"primary":{"main":"#3f51b5"},"secondary":{"main":"#f50057"}}},"#2196f3":{"palette":{"primary":{"main":"#2196f3"},"secondary":{"main":"#FFC107"}}},"#673AB7":{"palette":{"primary":{"main":"#673AB7"},"secondary":{"main":"#2196F3"}}},"#E91E63":{"palette":{"primary":{"main":"#E91E63"},"secondary":{"main":"#42A5F5","contrastText":"#fff"}}},"#FF5722":{"palette":{"primary":{"main":"#FF5722"},"secondary":{"main":"#3F51B5"}}},"#FFC107":{"palette":{"primary":{"main":"#FFC107"},"secondary":{"main":"#26C6DA"}}},"#8BC34A":{"palette":{"primary":{"main":"#8BC34A","contrastText":"#fff"},"secondary":{"main":"#FF8A65","contrastText":"#fff"}}},"#009688":{"palette":{"primary":{"main":"#009688"},"secondary":{"main":"#4DD0E1","contrastText":"#fff"}}},"#607D8B":{"palette":{"primary":{"main":"#607D8B"},"secondary":{"main":"#F06292"}}},"#795548":{"palette":{"primary":{"main":"#795548"},"secondary":{"main":"#4CAF50","contrastText":"#fff"}}}}`, Type: "basic"},
|
||||
{Name: "max_worker_num", Value: `10`, Type: "task"},
|
||||
{Name: "max_parallel_transfer", Value: `4`, Type: "task"},
|
||||
{Name: "secret_key", Value: util.RandStringRunes(256), Type: "auth"},
|
||||
{Name: "temp_path", Value: "temp", Type: "path"},
|
||||
{Name: "avatar_path", Value: "avatar", Type: "path"},
|
||||
{Name: "avatar_size", Value: "2097152", Type: "avatar"},
|
||||
{Name: "avatar_size_l", Value: "200", Type: "avatar"},
|
||||
{Name: "avatar_size_m", Value: "130", Type: "avatar"},
|
||||
{Name: "avatar_size_s", Value: "50", Type: "avatar"},
|
||||
{Name: "home_view_method", Value: "icon", Type: "view"},
|
||||
{Name: "share_view_method", Value: "list", Type: "view"},
|
||||
{Name: "cron_garbage_collect", Value: "@hourly", Type: "cron"},
|
||||
{Name: "authn_enabled", Value: "0", Type: "authn"},
|
||||
{Name: "captcha_type", Value: "normal", Type: "captcha"},
|
||||
{Name: "captcha_height", Value: "60", Type: "captcha"},
|
||||
{Name: "captcha_width", Value: "240", Type: "captcha"},
|
||||
{Name: "captcha_mode", Value: "3", Type: "captcha"},
|
||||
{Name: "captcha_ComplexOfNoiseText", Value: "0", Type: "captcha"},
|
||||
{Name: "captcha_ComplexOfNoiseDot", Value: "0", Type: "captcha"},
|
||||
{Name: "captcha_IsShowHollowLine", Value: "0", Type: "captcha"},
|
||||
{Name: "captcha_IsShowNoiseDot", Value: "1", Type: "captcha"},
|
||||
{Name: "captcha_IsShowNoiseText", Value: "0", Type: "captcha"},
|
||||
{Name: "captcha_IsShowSlimeLine", Value: "1", Type: "captcha"},
|
||||
{Name: "captcha_IsShowSineLine", Value: "0", Type: "captcha"},
|
||||
{Name: "captcha_CaptchaLen", Value: "6", Type: "captcha"},
|
||||
{Name: "captcha_ReCaptchaKey", Value: "defaultKey", Type: "captcha"},
|
||||
{Name: "captcha_ReCaptchaSecret", Value: "defaultSecret", Type: "captcha"},
|
||||
{Name: "captcha_TCaptcha_CaptchaAppId", Value: "", Type: "captcha"},
|
||||
{Name: "captcha_TCaptcha_AppSecretKey", Value: "", Type: "captcha"},
|
||||
{Name: "captcha_TCaptcha_SecretId", Value: "", Type: "captcha"},
|
||||
{Name: "captcha_TCaptcha_SecretKey", Value: "", Type: "captcha"},
|
||||
{Name: "thumb_width", Value: "400", Type: "thumb"},
|
||||
{Name: "thumb_height", Value: "300", Type: "thumb"},
|
||||
{Name: "pwa_small_icon", Value: "/static/img/favicon.ico", Type: "pwa"},
|
||||
{Name: "pwa_medium_icon", Value: "/static/img/logo192.png", Type: "pwa"},
|
||||
{Name: "pwa_large_icon", Value: "/static/img/logo512.png", Type: "pwa"},
|
||||
{Name: "pwa_display", Value: "standalone", Type: "pwa"},
|
||||
{Name: "pwa_theme_color", Value: "#000000", Type: "pwa"},
|
||||
{Name: "pwa_background_color", Value: "#ffffff", Type: "pwa"},
|
||||
{Name: "office_preview_service", Value: "https://view.officeapps.live.com/op/view.aspx?src={$src}", Type: "preview"},
|
||||
}
|
||||
|
||||
for _, value := range defaultSettings {
|
||||
DB.Where(Setting{Name: value.Name}).Create(&value)
|
||||
}
|
||||
|
||||
@@ -3,8 +3,7 @@ package model
|
||||
import (
|
||||
"encoding/gob"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"net/url"
|
||||
"github.com/gofrs/uuid"
|
||||
"path"
|
||||
"path/filepath"
|
||||
"strconv"
|
||||
@@ -58,6 +57,10 @@ type PolicyOption struct {
|
||||
Region string `json:"region,omitempty"`
|
||||
// ServerSideEndpoint 服务端请求使用的 Endpoint,为空时使用 Policy.Server 字段
|
||||
ServerSideEndpoint string `json:"server_side_endpoint,omitempty"`
|
||||
// 分片上传的分片大小
|
||||
ChunkSize uint64 `json:"chunk_size,omitempty"`
|
||||
// 分片上传时是否需要预留空间
|
||||
PlaceholderWithSize bool `json:"placeholder_with_size,omitempty"`
|
||||
}
|
||||
|
||||
var thumbSuffix = map[string][]string{
|
||||
@@ -148,7 +151,7 @@ func (policy *Policy) GeneratePath(uid uint, origin string) string {
|
||||
func (policy *Policy) GenerateFileName(uid uint, origin string) string {
|
||||
// 未开启自动重命名时,直接返回原始文件名
|
||||
if !policy.AutoRename {
|
||||
return policy.getOriginNameRule(origin)
|
||||
return origin
|
||||
}
|
||||
|
||||
fileRule := policy.FileNameRule
|
||||
@@ -167,35 +170,15 @@ func (policy *Policy) GenerateFileName(uid uint, origin string) string {
|
||||
"{hour}": time.Now().Format("15"),
|
||||
"{minute}": time.Now().Format("04"),
|
||||
"{second}": time.Now().Format("05"),
|
||||
"{originname}": origin,
|
||||
"{ext}": filepath.Ext(origin),
|
||||
"{uuid}": uuid.Must(uuid.NewV4()).String(),
|
||||
}
|
||||
|
||||
replaceTable["{originname}"] = policy.getOriginNameRule(origin)
|
||||
|
||||
fileRule = util.Replace(replaceTable, fileRule)
|
||||
return fileRule
|
||||
}
|
||||
|
||||
func (policy Policy) getOriginNameRule(origin string) string {
|
||||
// 部分存储策略可以使用{origin}代表原始文件名
|
||||
if origin == "" {
|
||||
// 如果上游未传回原始文件名,则使用占位符,让云存储端替换
|
||||
switch policy.Type {
|
||||
case "qiniu":
|
||||
// 七牛会将$(fname)自动替换为原始文件名
|
||||
return "$(fname)"
|
||||
case "local", "remote":
|
||||
return origin
|
||||
case "oss", "cos":
|
||||
// OSS会将${filename}自动替换为原始文件名
|
||||
return "${filename}"
|
||||
case "upyun":
|
||||
// Upyun会将{filename}{.suffix}自动替换为原始文件名
|
||||
return "{filename}{.suffix}"
|
||||
}
|
||||
}
|
||||
return origin
|
||||
}
|
||||
|
||||
// IsDirectlyPreview 返回此策略下文件是否可以直接预览(不需要重定向)
|
||||
func (policy *Policy) IsDirectlyPreview() bool {
|
||||
return policy.Type == "local"
|
||||
@@ -214,18 +197,7 @@ func (policy *Policy) IsThumbExist(name string) bool {
|
||||
|
||||
// IsTransitUpload 返回此策略上传给定size文件时是否需要服务端中转
|
||||
func (policy *Policy) IsTransitUpload(size uint64) bool {
|
||||
if policy.Type == "local" {
|
||||
return true
|
||||
}
|
||||
if policy.Type == "onedrive" && size < 4*1024*1024 {
|
||||
return true
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// IsPathGenerateNeeded 返回此策略是否需要在生成上传凭证时生成存储路径
|
||||
func (policy *Policy) IsPathGenerateNeeded() bool {
|
||||
return policy.Type != "remote"
|
||||
return policy.Type == "local"
|
||||
}
|
||||
|
||||
// IsThumbGenerateNeeded 返回此策略是否需要在上传后生成缩略图
|
||||
@@ -233,44 +205,24 @@ func (policy *Policy) IsThumbGenerateNeeded() bool {
|
||||
return policy.Type == "local"
|
||||
}
|
||||
|
||||
// IsUploadPlaceholderWithSize 返回此策略创建上传会话时是否需要预留空间
|
||||
func (policy *Policy) IsUploadPlaceholderWithSize() bool {
|
||||
if policy.Type == "remote" {
|
||||
return true
|
||||
}
|
||||
|
||||
if util.ContainsString([]string{"onedrive", "oss", "qiniu", "cos", "s3"}, policy.Type) {
|
||||
return policy.OptionsSerialized.PlaceholderWithSize
|
||||
}
|
||||
|
||||
return false
|
||||
}
|
||||
|
||||
// CanStructureBeListed 返回存储策略是否能被前台列物理目录
|
||||
func (policy *Policy) CanStructureBeListed() bool {
|
||||
return policy.Type != "local" && policy.Type != "remote"
|
||||
}
|
||||
|
||||
// GetUploadURL 获取文件上传服务API地址
|
||||
func (policy *Policy) GetUploadURL() string {
|
||||
server, err := url.Parse(policy.Server)
|
||||
if err != nil {
|
||||
return policy.Server
|
||||
}
|
||||
|
||||
controller, _ := url.Parse("")
|
||||
switch policy.Type {
|
||||
case "local", "onedrive":
|
||||
return "/api/v3/file/upload"
|
||||
case "remote":
|
||||
controller, _ = url.Parse("/api/v3/slave/upload")
|
||||
case "oss":
|
||||
return "https://" + policy.BucketName + "." + policy.Server
|
||||
case "cos":
|
||||
return policy.Server
|
||||
case "upyun":
|
||||
return "https://v0.api.upyun.com/" + policy.BucketName
|
||||
case "s3":
|
||||
if policy.Server == "" {
|
||||
return fmt.Sprintf("https://%s.s3.%s.amazonaws.com/", policy.BucketName,
|
||||
policy.OptionsSerialized.Region)
|
||||
}
|
||||
|
||||
if !strings.Contains(policy.Server, policy.BucketName) {
|
||||
controller, _ = url.Parse("/" + policy.BucketName)
|
||||
}
|
||||
}
|
||||
|
||||
return server.ResolveReference(controller).String()
|
||||
}
|
||||
|
||||
// SaveAndClearCache 更新并清理缓存
|
||||
func (policy *Policy) SaveAndClearCache() error {
|
||||
err := DB.Save(policy).Error
|
||||
|
||||
@@ -104,7 +104,7 @@ func TestPolicy_GenerateFileName(t *testing.T) {
|
||||
asserts.Equal("123.txt", testPolicy.GenerateFileName(1, "123.txt"))
|
||||
|
||||
testPolicy.Type = "oss"
|
||||
asserts.Equal("${filename}", testPolicy.GenerateFileName(1, ""))
|
||||
asserts.Equal("origin", testPolicy.GenerateFileName(1, "origin"))
|
||||
}
|
||||
|
||||
// 重命名开启
|
||||
@@ -145,19 +145,23 @@ func TestPolicy_GenerateFileName(t *testing.T) {
|
||||
|
||||
testPolicy.Type = "oss"
|
||||
testPolicy.FileNameRule = "{uid}123{originname}"
|
||||
asserts.Equal("1123${filename}", testPolicy.GenerateFileName(1, ""))
|
||||
asserts.Equal("1123123321", testPolicy.GenerateFileName(1, "123321"))
|
||||
|
||||
testPolicy.Type = "upyun"
|
||||
testPolicy.FileNameRule = "{uid}123{originname}"
|
||||
asserts.Equal("1123{filename}{.suffix}", testPolicy.GenerateFileName(1, ""))
|
||||
asserts.Equal("1123123321", testPolicy.GenerateFileName(1, "123321"))
|
||||
|
||||
testPolicy.Type = "qiniu"
|
||||
testPolicy.FileNameRule = "{uid}123{originname}"
|
||||
asserts.Equal("1123$(fname)", testPolicy.GenerateFileName(1, ""))
|
||||
asserts.Equal("1123123321", testPolicy.GenerateFileName(1, "123321"))
|
||||
|
||||
testPolicy.Type = "local"
|
||||
testPolicy.FileNameRule = "{uid}123{originname}"
|
||||
asserts.Equal("1123", testPolicy.GenerateFileName(1, ""))
|
||||
|
||||
testPolicy.Type = "local"
|
||||
testPolicy.FileNameRule = "{ext}123{uuid}"
|
||||
asserts.Contains(testPolicy.GenerateFileName(1, "123.txt"), ".txt123")
|
||||
}
|
||||
|
||||
}
|
||||
@@ -170,78 +174,6 @@ func TestPolicy_IsDirectlyPreview(t *testing.T) {
|
||||
asserts.False(policy.IsDirectlyPreview())
|
||||
}
|
||||
|
||||
func TestPolicy_GetUploadURL(t *testing.T) {
|
||||
asserts := assert.New(t)
|
||||
|
||||
// 本地
|
||||
{
|
||||
cache.Set("setting_siteURL", "http://127.0.0.1", 0)
|
||||
policy := Policy{Type: "local", Server: "http://127.0.0.1"}
|
||||
asserts.Equal("/api/v3/file/upload", policy.GetUploadURL())
|
||||
}
|
||||
|
||||
// 远程
|
||||
{
|
||||
policy := Policy{Type: "remote", Server: "http://127.0.0.1"}
|
||||
asserts.Equal("http://127.0.0.1/api/v3/slave/upload", policy.GetUploadURL())
|
||||
}
|
||||
|
||||
// OSS
|
||||
{
|
||||
policy := Policy{Type: "oss", BucketName: "base", Server: "127.0.0.1"}
|
||||
asserts.Equal("https://base.127.0.0.1", policy.GetUploadURL())
|
||||
}
|
||||
|
||||
// cos
|
||||
{
|
||||
policy := Policy{Type: "cos", BaseURL: "base", Server: "http://127.0.0.1"}
|
||||
asserts.Equal("http://127.0.0.1", policy.GetUploadURL())
|
||||
}
|
||||
|
||||
// upyun
|
||||
{
|
||||
policy := Policy{Type: "upyun", BucketName: "base", Server: "http://127.0.0.1"}
|
||||
asserts.Equal("https://v0.api.upyun.com/base", policy.GetUploadURL())
|
||||
}
|
||||
|
||||
// 未知
|
||||
{
|
||||
policy := Policy{Type: "unknown", Server: "http://127.0.0.1"}
|
||||
asserts.Equal("http://127.0.0.1", policy.GetUploadURL())
|
||||
}
|
||||
|
||||
// S3 未填写自动生成
|
||||
{
|
||||
policy := Policy{
|
||||
Type: "s3",
|
||||
Server: "",
|
||||
BucketName: "bucket",
|
||||
OptionsSerialized: PolicyOption{Region: "us-east"},
|
||||
}
|
||||
asserts.Equal("https://bucket.s3.us-east.amazonaws.com/", policy.GetUploadURL())
|
||||
}
|
||||
|
||||
// s3 自己指定
|
||||
{
|
||||
policy := Policy{
|
||||
Type: "s3",
|
||||
Server: "https://s3.us-east.amazonaws.com/",
|
||||
BucketName: "bucket",
|
||||
OptionsSerialized: PolicyOption{Region: "us-east"},
|
||||
}
|
||||
asserts.Equal("https://s3.us-east.amazonaws.com/bucket", policy.GetUploadURL())
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
func TestPolicy_IsPathGenerateNeeded(t *testing.T) {
|
||||
asserts := assert.New(t)
|
||||
policy := Policy{Type: "qiniu"}
|
||||
asserts.True(policy.IsPathGenerateNeeded())
|
||||
policy.Type = "remote"
|
||||
asserts.False(policy.IsPathGenerateNeeded())
|
||||
}
|
||||
|
||||
func TestPolicy_ClearCache(t *testing.T) {
|
||||
asserts := assert.New(t)
|
||||
cache.Set("policy_202", 1, 0)
|
||||
@@ -266,15 +198,18 @@ func TestPolicy_UpdateAccessKey(t *testing.T) {
|
||||
func TestPolicy_Props(t *testing.T) {
|
||||
asserts := assert.New(t)
|
||||
policy := Policy{Type: "onedrive"}
|
||||
policy.OptionsSerialized.PlaceholderWithSize = true
|
||||
asserts.False(policy.IsThumbGenerateNeeded())
|
||||
asserts.True(policy.IsPathGenerateNeeded())
|
||||
asserts.True(policy.IsTransitUpload(4))
|
||||
asserts.False(policy.IsTransitUpload(4))
|
||||
asserts.False(policy.IsTransitUpload(5 * 1024 * 1024))
|
||||
asserts.True(policy.CanStructureBeListed())
|
||||
asserts.True(policy.IsUploadPlaceholderWithSize())
|
||||
policy.Type = "local"
|
||||
asserts.True(policy.IsThumbGenerateNeeded())
|
||||
asserts.True(policy.IsPathGenerateNeeded())
|
||||
asserts.False(policy.CanStructureBeListed())
|
||||
asserts.False(policy.IsUploadPlaceholderWithSize())
|
||||
policy.Type = "remote"
|
||||
asserts.True(policy.IsUploadPlaceholderWithSize())
|
||||
}
|
||||
|
||||
func TestPolicy_IsThumbExist(t *testing.T) {
|
||||
|
||||
@@ -43,6 +43,15 @@ func GetSettingByName(name string) string {
|
||||
return ""
|
||||
}
|
||||
|
||||
// GetSettingByNameWithDefault 用 Name 获取设置值, 取不到时使用缺省值
|
||||
func GetSettingByNameWithDefault(name, fallback string) string {
|
||||
res := GetSettingByName(name)
|
||||
if res == "" {
|
||||
return fallback
|
||||
}
|
||||
return res
|
||||
}
|
||||
|
||||
// GetSettingByNames 用多个 Name 获取设置值
|
||||
func GetSettingByNames(names ...string) map[string]string {
|
||||
var queryRes []Setting
|
||||
|
||||
@@ -59,6 +59,15 @@ func TestGetSettingByType(t *testing.T) {
|
||||
asserts.Equal(map[string]string{}, settings)
|
||||
}
|
||||
|
||||
func TestGetSettingByNameWithDefault(t *testing.T) {
|
||||
a := assert.New(t)
|
||||
|
||||
rows := sqlmock.NewRows([]string{"name", "value", "type"})
|
||||
mock.ExpectQuery("^SELECT \\* FROM `(.+)` WHERE `(.+)`\\.`deleted_at` IS NULL AND(.+)$").WillReturnRows(rows)
|
||||
settings := GetSettingByNameWithDefault("123", "123321")
|
||||
a.Equal("123321", settings)
|
||||
}
|
||||
|
||||
func TestGetSettingByNames(t *testing.T) {
|
||||
cache.Store = cache.NewMemoStore()
|
||||
asserts := assert.New(t)
|
||||
|
||||
@@ -56,8 +56,8 @@ func TestGetTagsByUID(t *testing.T) {
|
||||
func TestGetTagsByID(t *testing.T) {
|
||||
asserts := assert.New(t)
|
||||
mock.ExpectQuery("SELECT(.+)").WillReturnRows(sqlmock.NewRows([]string{"name"}).AddRow("tag"))
|
||||
res, err := GetTagsByUID(1)
|
||||
res, err := GetTagsByID(1, 1)
|
||||
asserts.NoError(mock.ExpectationsWereMet())
|
||||
asserts.NoError(err)
|
||||
asserts.EqualValues("tag", res[0].Name)
|
||||
asserts.EqualValues("tag", res.Name)
|
||||
}
|
||||
|
||||
@@ -64,7 +64,7 @@ func ListTasks(uid uint, page, pageSize int, order string) ([]Task, int) {
|
||||
dbChain = dbChain.Where("user_id = ?", uid)
|
||||
|
||||
// 计算总数用于分页
|
||||
dbChain.Model(&Share{}).Count(&total)
|
||||
dbChain.Model(&Task{}).Count(&total)
|
||||
|
||||
// 查询记录
|
||||
dbChain.Limit(pageSize).Offset((page - 1) * pageSize).Order(order).Find(&tasks)
|
||||
|
||||
@@ -3,6 +3,7 @@ package model
|
||||
import (
|
||||
"crypto/md5"
|
||||
"crypto/sha1"
|
||||
"encoding/gob"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"strings"
|
||||
@@ -46,6 +47,10 @@ type User struct {
|
||||
OptionsSerialized UserOption `gorm:"-"`
|
||||
}
|
||||
|
||||
func init() {
|
||||
gob.Register(User{})
|
||||
}
|
||||
|
||||
// UserOption 用户个性化配置字段
|
||||
type UserOption struct {
|
||||
ProfileOff bool `json:"profile_off,omitempty"`
|
||||
@@ -89,6 +94,11 @@ func (user *User) IncreaseStorage(size uint64) bool {
|
||||
return false
|
||||
}
|
||||
|
||||
// ChangeStorage 更新用户容量
|
||||
func (user *User) ChangeStorage(tx *gorm.DB, operator string, size uint64) error {
|
||||
return tx.Model(user).Update("storage", gorm.Expr("storage "+operator+" ?", size)).Error
|
||||
}
|
||||
|
||||
// IncreaseStorageWithoutCheck 忽略可用容量,增加用户已用容量
|
||||
func (user *User) IncreaseStorageWithoutCheck(size uint64) {
|
||||
if size == 0 {
|
||||
|
||||
@@ -13,7 +13,6 @@ import (
|
||||
"github.com/cloudreve/Cloudreve/v3/pkg/aria2/rpc"
|
||||
"github.com/cloudreve/Cloudreve/v3/pkg/cluster"
|
||||
"github.com/cloudreve/Cloudreve/v3/pkg/filesystem"
|
||||
"github.com/cloudreve/Cloudreve/v3/pkg/filesystem/driver/local"
|
||||
"github.com/cloudreve/Cloudreve/v3/pkg/filesystem/fsctx"
|
||||
"github.com/cloudreve/Cloudreve/v3/pkg/mq"
|
||||
"github.com/cloudreve/Cloudreve/v3/pkg/task"
|
||||
@@ -191,12 +190,12 @@ func (monitor *Monitor) ValidateFile() error {
|
||||
defer fs.Recycle()
|
||||
|
||||
// 创建上下文环境
|
||||
ctx := context.WithValue(context.Background(), fsctx.FileHeaderCtx, local.FileStream{
|
||||
file := &fsctx.FileStream{
|
||||
Size: monitor.Task.TotalSize,
|
||||
})
|
||||
}
|
||||
|
||||
// 验证用户容量
|
||||
if err := filesystem.HookValidateCapacityWithoutIncrease(ctx, fs); err != nil {
|
||||
if err := filesystem.HookValidateCapacity(context.Background(), fs, file); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
@@ -205,11 +204,11 @@ func (monitor *Monitor) ValidateFile() error {
|
||||
if fileInfo.Selected == "true" {
|
||||
// 创建上下文环境
|
||||
fileSize, _ := strconv.ParseUint(fileInfo.Length, 10, 64)
|
||||
ctx := context.WithValue(context.Background(), fsctx.FileHeaderCtx, local.FileStream{
|
||||
file := &fsctx.FileStream{
|
||||
Size: fileSize,
|
||||
Name: filepath.Base(fileInfo.Path),
|
||||
})
|
||||
if err := filesystem.HookValidateFile(ctx, fs); err != nil {
|
||||
}
|
||||
if err := filesystem.HookValidateFile(context.Background(), fs, file); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
@@ -11,7 +11,7 @@ type StatusInfo struct {
|
||||
UploadLength string `json:"uploadLength"` // Uploaded length of the download in bytes.
|
||||
BitField string `json:"bitfield"` // Hexadecimal representation of the download progress. The highest bit corresponds to the piece at index 0. Any set bits indicate loaded pieces, while unset bits indicate not yet loaded and/or missing pieces. Any overflow bits at the end are set to zero. When the download was not started yet, this key will not be included in the response.
|
||||
DownloadSpeed string `json:"downloadSpeed"` // Download speed of this download measured in bytes/sec.
|
||||
UploadSpeed string `json:"uploadSpeed"` // Upload speed of this download measured in bytes/sec.
|
||||
UploadSpeed string `json:"uploadSpeed"` // LocalUpload speed of this download measured in bytes/sec.
|
||||
InfoHash string `json:"infoHash"` // InfoHash. BitTorrent only.
|
||||
NumSeeders string `json:"numSeeders"` // The number of seeders aria2 has connected to. BitTorrent only.
|
||||
Seeder string `json:"seeder"` // true if the local endpoint is a seeder. Otherwise false. BitTorrent only.
|
||||
@@ -60,7 +60,7 @@ type PeerInfo struct {
|
||||
AmChoking string `json:"amChoking"` // true if aria2 is choking the peer. Otherwise false.
|
||||
PeerChoking string `json:"peerChoking"` // true if the peer is choking aria2. Otherwise false.
|
||||
DownloadSpeed string `json:"downloadSpeed"` // Download speed (byte/sec) that this client obtains from the peer.
|
||||
UploadSpeed string `json:"uploadSpeed"` // Upload speed(byte/sec) that this client uploads to the peer.
|
||||
UploadSpeed string `json:"uploadSpeed"` // LocalUpload speed(byte/sec) that this client uploads to the peer.
|
||||
Seeder string `json:"seeder"` // true if this peer is a seeder. Otherwise false.
|
||||
}
|
||||
|
||||
|
||||
@@ -23,6 +23,8 @@ var (
|
||||
ErrExpired = serializer.NewError(serializer.CodeSignExpired, "签名已过期", nil)
|
||||
)
|
||||
|
||||
const CrHeaderPrefix = "X-Cr-"
|
||||
|
||||
// General 通用的认证接口
|
||||
var General Auth
|
||||
|
||||
@@ -64,12 +66,12 @@ func CheckRequest(instance Auth, r *http.Request) error {
|
||||
return instance.Check(getSignContent(r), sign[0])
|
||||
}
|
||||
|
||||
// getSignContent 签名请求 path、正文、以`X-`开头的 Header. 如果 Header 中包含 `X-Policy`,
|
||||
// getSignContent 签名请求 path、正文、以`X-`开头的 Header. 如果请求 path 为从机上传 API,
|
||||
// 则不对正文签名。返回待签名/验证的字符串
|
||||
func getSignContent(r *http.Request) (rawSignString string) {
|
||||
// 读取所有body正文
|
||||
var body = []byte{}
|
||||
if _, ok := r.Header["X-Cr-Policy"]; !ok {
|
||||
if !strings.Contains(r.URL.Path, "/api/v3/slave/upload/") {
|
||||
if r.Body != nil {
|
||||
body, _ = ioutil.ReadAll(r.Body)
|
||||
_ = r.Body.Close()
|
||||
@@ -80,7 +82,7 @@ func getSignContent(r *http.Request) (rawSignString string) {
|
||||
// 决定要签名的header
|
||||
var signedHeader []string
|
||||
for k, _ := range r.Header {
|
||||
if strings.HasPrefix(k, "X-Cr-") && k != "X-Cr-Filename" {
|
||||
if strings.HasPrefix(k, CrHeaderPrefix) && k != CrHeaderPrefix+"Filename" {
|
||||
signedHeader = append(signedHeader, fmt.Sprintf("%s=%s", k, r.Header.Get(k)))
|
||||
}
|
||||
}
|
||||
|
||||
12
pkg/cache/driver.go
vendored
12
pkg/cache/driver.go
vendored
@@ -2,6 +2,7 @@ package cache
|
||||
|
||||
import (
|
||||
"github.com/cloudreve/Cloudreve/v3/pkg/conf"
|
||||
"github.com/cloudreve/Cloudreve/v3/pkg/util"
|
||||
"github.com/gin-gonic/gin"
|
||||
)
|
||||
|
||||
@@ -9,9 +10,7 @@ import (
|
||||
var Store Driver = NewMemoStore()
|
||||
|
||||
// Init 初始化缓存
|
||||
func Init() {
|
||||
//Store = NewRedisStore(10, "tcp", "127.0.0.1:6379", "", "0")
|
||||
//return
|
||||
func Init(isSlave bool) {
|
||||
if conf.RedisConfig.Server != "" && gin.Mode() != gin.TestMode {
|
||||
Store = NewRedisStore(
|
||||
10,
|
||||
@@ -21,6 +20,13 @@ func Init() {
|
||||
conf.RedisConfig.DB,
|
||||
)
|
||||
}
|
||||
|
||||
if isSlave {
|
||||
err := Store.Sets(conf.OptionOverwrite, "setting_")
|
||||
if err != nil {
|
||||
util.Log().Warning("无法覆盖数据库设置: %s", err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Driver 键值缓存存储容器
|
||||
|
||||
6
pkg/cache/driver_test.go
vendored
6
pkg/cache/driver_test.go
vendored
@@ -56,6 +56,10 @@ func TestInit(t *testing.T) {
|
||||
asserts := assert.New(t)
|
||||
|
||||
asserts.NotPanics(func() {
|
||||
Init()
|
||||
Init(false)
|
||||
})
|
||||
|
||||
asserts.NotPanics(func() {
|
||||
Init(true)
|
||||
})
|
||||
}
|
||||
|
||||
@@ -1,11 +1,15 @@
|
||||
package cluster
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
model "github.com/cloudreve/Cloudreve/v3/models"
|
||||
"github.com/cloudreve/Cloudreve/v3/pkg/aria2/common"
|
||||
"github.com/cloudreve/Cloudreve/v3/pkg/aria2/rpc"
|
||||
"github.com/cloudreve/Cloudreve/v3/pkg/auth"
|
||||
"github.com/cloudreve/Cloudreve/v3/pkg/conf"
|
||||
"github.com/cloudreve/Cloudreve/v3/pkg/request"
|
||||
"github.com/cloudreve/Cloudreve/v3/pkg/serializer"
|
||||
"github.com/cloudreve/Cloudreve/v3/pkg/util"
|
||||
@@ -40,7 +44,7 @@ func (node *SlaveNode) Init(nodeModel *model.Node) {
|
||||
var endpoint *url.URL
|
||||
if serverURL, err := url.Parse(node.Model.Server); err == nil {
|
||||
var controller *url.URL
|
||||
controller, _ = url.Parse("/api/v3/slave")
|
||||
controller, _ = url.Parse("/api/v3/slave/")
|
||||
endpoint = serverURL.ResolveReference(controller)
|
||||
}
|
||||
|
||||
@@ -408,3 +412,40 @@ func getAria2RequestBody(body *serializer.SlaveAria2Call) (io.Reader, error) {
|
||||
|
||||
return strings.NewReader(string(reqBodyEncoded)), nil
|
||||
}
|
||||
|
||||
// RemoteCallback 发送远程存储策略上传回调请求
|
||||
func RemoteCallback(url string, body serializer.UploadCallback) error {
|
||||
callbackBody, err := json.Marshal(struct {
|
||||
Data serializer.UploadCallback `json:"data"`
|
||||
}{
|
||||
Data: body,
|
||||
})
|
||||
if err != nil {
|
||||
return serializer.NewError(serializer.CodeCallbackError, "无法编码回调正文", err)
|
||||
}
|
||||
|
||||
resp := request.GeneralClient.Request(
|
||||
"POST",
|
||||
url,
|
||||
bytes.NewReader(callbackBody),
|
||||
request.WithTimeout(time.Duration(conf.SlaveConfig.CallbackTimeout)*time.Second),
|
||||
request.WithCredential(auth.General, int64(conf.SlaveConfig.SignatureTTL)),
|
||||
)
|
||||
|
||||
if resp.Err != nil {
|
||||
return serializer.NewError(serializer.CodeCallbackError, "从机无法发起回调请求", resp.Err)
|
||||
}
|
||||
|
||||
// 解析回调服务端响应
|
||||
response, err := resp.DecodeResponse()
|
||||
if err != nil {
|
||||
msg := fmt.Sprintf("从机无法解析主机返回的响应 (StatusCode=%d)", resp.Response.StatusCode)
|
||||
return serializer.NewError(serializer.CodeCallbackError, msg, err)
|
||||
}
|
||||
|
||||
if response.Code != 0 {
|
||||
return serializer.NewError(response.Code, response.Msg, errors.New(response.Error))
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -1,8 +1,12 @@
|
||||
package cluster
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
model "github.com/cloudreve/Cloudreve/v3/models"
|
||||
"github.com/cloudreve/Cloudreve/v3/pkg/cache"
|
||||
"github.com/cloudreve/Cloudreve/v3/pkg/mocks/requestmock"
|
||||
"github.com/cloudreve/Cloudreve/v3/pkg/request"
|
||||
"github.com/cloudreve/Cloudreve/v3/pkg/serializer"
|
||||
"github.com/stretchr/testify/assert"
|
||||
@@ -441,3 +445,115 @@ func TestSlaveCaller_DeleteTempFile(t *testing.T) {
|
||||
a.NoError(err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRemoteCallback(t *testing.T) {
|
||||
asserts := assert.New(t)
|
||||
|
||||
// 回调成功
|
||||
{
|
||||
clientMock := requestmock.RequestMock{}
|
||||
mockResp, _ := json.Marshal(serializer.Response{Code: 0})
|
||||
clientMock.On(
|
||||
"Request",
|
||||
"POST",
|
||||
"http://test/test/url",
|
||||
testMock.Anything,
|
||||
testMock.Anything,
|
||||
).Return(&request.Response{
|
||||
Err: nil,
|
||||
Response: &http.Response{
|
||||
StatusCode: 200,
|
||||
Body: ioutil.NopCloser(bytes.NewReader(mockResp)),
|
||||
},
|
||||
})
|
||||
request.GeneralClient = clientMock
|
||||
resp := RemoteCallback("http://test/test/url", serializer.UploadCallback{})
|
||||
asserts.NoError(resp)
|
||||
clientMock.AssertExpectations(t)
|
||||
}
|
||||
|
||||
// 服务端返回业务错误
|
||||
{
|
||||
clientMock := requestmock.RequestMock{}
|
||||
mockResp, _ := json.Marshal(serializer.Response{Code: 401})
|
||||
clientMock.On(
|
||||
"Request",
|
||||
"POST",
|
||||
"http://test/test/url",
|
||||
testMock.Anything,
|
||||
testMock.Anything,
|
||||
).Return(&request.Response{
|
||||
Err: nil,
|
||||
Response: &http.Response{
|
||||
StatusCode: 200,
|
||||
Body: ioutil.NopCloser(bytes.NewReader(mockResp)),
|
||||
},
|
||||
})
|
||||
request.GeneralClient = clientMock
|
||||
resp := RemoteCallback("http://test/test/url", serializer.UploadCallback{})
|
||||
asserts.EqualValues(401, resp.(serializer.AppError).Code)
|
||||
clientMock.AssertExpectations(t)
|
||||
}
|
||||
|
||||
// 无法解析回调响应
|
||||
{
|
||||
clientMock := requestmock.RequestMock{}
|
||||
clientMock.On(
|
||||
"Request",
|
||||
"POST",
|
||||
"http://test/test/url",
|
||||
testMock.Anything,
|
||||
testMock.Anything,
|
||||
).Return(&request.Response{
|
||||
Err: nil,
|
||||
Response: &http.Response{
|
||||
StatusCode: 200,
|
||||
Body: ioutil.NopCloser(strings.NewReader("mockResp")),
|
||||
},
|
||||
})
|
||||
request.GeneralClient = clientMock
|
||||
resp := RemoteCallback("http://test/test/url", serializer.UploadCallback{})
|
||||
asserts.Error(resp)
|
||||
clientMock.AssertExpectations(t)
|
||||
}
|
||||
|
||||
// HTTP状态码非200
|
||||
{
|
||||
clientMock := requestmock.RequestMock{}
|
||||
clientMock.On(
|
||||
"Request",
|
||||
"POST",
|
||||
"http://test/test/url",
|
||||
testMock.Anything,
|
||||
testMock.Anything,
|
||||
).Return(&request.Response{
|
||||
Err: nil,
|
||||
Response: &http.Response{
|
||||
StatusCode: 404,
|
||||
Body: ioutil.NopCloser(strings.NewReader("mockResp")),
|
||||
},
|
||||
})
|
||||
request.GeneralClient = clientMock
|
||||
resp := RemoteCallback("http://test/test/url", serializer.UploadCallback{})
|
||||
asserts.Error(resp)
|
||||
clientMock.AssertExpectations(t)
|
||||
}
|
||||
|
||||
// 无法发起回调
|
||||
{
|
||||
clientMock := requestmock.RequestMock{}
|
||||
clientMock.On(
|
||||
"Request",
|
||||
"POST",
|
||||
"http://test/test/url",
|
||||
testMock.Anything,
|
||||
testMock.Anything,
|
||||
).Return(&request.Response{
|
||||
Err: errors.New("error"),
|
||||
})
|
||||
request.GeneralClient = clientMock
|
||||
resp := RemoteCallback("http://test/test/url", serializer.UploadCallback{})
|
||||
asserts.Error(resp)
|
||||
clientMock.AssertExpectations(t)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -45,21 +45,6 @@ type slave struct {
|
||||
SignatureTTL int `validate:"omitempty,gte=1"`
|
||||
}
|
||||
|
||||
// captcha 验证码配置
|
||||
type captcha struct {
|
||||
Height int `validate:"gte=0"`
|
||||
Width int `validate:"gte=0"`
|
||||
Mode int `validate:"gte=0,lte=3"`
|
||||
ComplexOfNoiseText int `validate:"gte=0,lte=2"`
|
||||
ComplexOfNoiseDot int `validate:"gte=0,lte=2"`
|
||||
IsShowHollowLine bool
|
||||
IsShowNoiseDot bool
|
||||
IsShowNoiseText bool
|
||||
IsShowSlimeLine bool
|
||||
IsShowSineLine bool
|
||||
CaptchaLen int `validate:"gt=0"`
|
||||
}
|
||||
|
||||
// redis 配置
|
||||
type redis struct {
|
||||
Network string
|
||||
@@ -68,17 +53,6 @@ type redis struct {
|
||||
DB string
|
||||
}
|
||||
|
||||
// 缩略图 配置
|
||||
type thumb struct {
|
||||
MaxWidth uint
|
||||
MaxHeight uint
|
||||
FileSuffix string `validate:"min=1"`
|
||||
MaxTaskCount int
|
||||
EncodeMethod string `validate:"eq=jpg|eq=png"`
|
||||
EncodeQuality int `validate:"gte=1,lte=100"`
|
||||
GCAfterGen bool
|
||||
}
|
||||
|
||||
// 跨域配置
|
||||
type cors struct {
|
||||
AllowOrigins []string
|
||||
@@ -91,6 +65,7 @@ type cors struct {
|
||||
var cfg *ini.File
|
||||
|
||||
const defaultConf = `[System]
|
||||
Debug = false
|
||||
Mode = master
|
||||
Listen = :5212
|
||||
SessionSecret = {SessionSecret}
|
||||
@@ -131,9 +106,7 @@ func Init(path string) {
|
||||
"System": SystemConfig,
|
||||
"SSL": SSLConfig,
|
||||
"UnixSocket": UnixConfig,
|
||||
"Captcha": CaptchaConfig,
|
||||
"Redis": RedisConfig,
|
||||
"Thumbnail": ThumbConfig,
|
||||
"CORS": CORSConfig,
|
||||
"Slave": SlaveConfig,
|
||||
}
|
||||
@@ -144,6 +117,11 @@ func Init(path string) {
|
||||
}
|
||||
}
|
||||
|
||||
// 映射数据库配置覆盖
|
||||
for _, key := range cfg.Section("OptionOverwrite").Keys() {
|
||||
OptionOverwrite[key.Name()] = key.Value()
|
||||
}
|
||||
|
||||
// 重设log等级
|
||||
if !SystemConfig.Debug {
|
||||
util.Level = util.LevelInformational
|
||||
|
||||
@@ -56,7 +56,11 @@ User = root
|
||||
Password = root
|
||||
Host = 127.0.0.1:3306
|
||||
Name = v3
|
||||
TablePrefix = v3_`
|
||||
TablePrefix = v3_
|
||||
|
||||
[OptionOverwrite]
|
||||
key=value
|
||||
`
|
||||
err := ioutil.WriteFile("testConf.ini", []byte(testCase), 0644)
|
||||
defer func() { err = os.Remove("testConf.ini") }()
|
||||
if err != nil {
|
||||
@@ -65,6 +69,7 @@ TablePrefix = v3_`
|
||||
asserts.NotPanics(func() {
|
||||
Init("testConf.ini")
|
||||
})
|
||||
asserts.Equal(OptionOverwrite["key"], "value")
|
||||
}
|
||||
|
||||
func TestMapSection(t *testing.T) {
|
||||
|
||||
@@ -1,7 +1,5 @@
|
||||
package conf
|
||||
|
||||
import "github.com/mojocn/base64Captcha"
|
||||
|
||||
// RedisConfig Redis服务器配置
|
||||
var RedisConfig = &redis{
|
||||
Network: "tcp",
|
||||
@@ -25,21 +23,6 @@ var SystemConfig = &system{
|
||||
Listen: ":5212",
|
||||
}
|
||||
|
||||
// CaptchaConfig 验证码配置
|
||||
var CaptchaConfig = &captcha{
|
||||
Height: 60,
|
||||
Width: 240,
|
||||
Mode: 3,
|
||||
ComplexOfNoiseText: base64Captcha.CaptchaComplexLower,
|
||||
ComplexOfNoiseDot: base64Captcha.CaptchaComplexLower,
|
||||
IsShowHollowLine: false,
|
||||
IsShowNoiseDot: false,
|
||||
IsShowNoiseText: false,
|
||||
IsShowSlimeLine: false,
|
||||
IsShowSineLine: false,
|
||||
CaptchaLen: 6,
|
||||
}
|
||||
|
||||
// CORSConfig 跨域配置
|
||||
var CORSConfig = &cors{
|
||||
AllowOrigins: []string{"UNSET"},
|
||||
@@ -49,17 +32,6 @@ var CORSConfig = &cors{
|
||||
ExposeHeaders: nil,
|
||||
}
|
||||
|
||||
// ThumbConfig 缩略图配置
|
||||
var ThumbConfig = &thumb{
|
||||
MaxWidth: 400,
|
||||
MaxHeight: 300,
|
||||
FileSuffix: "._thumb",
|
||||
MaxTaskCount: -1,
|
||||
EncodeMethod: "jpg",
|
||||
GCAfterGen: false,
|
||||
EncodeQuality: 85,
|
||||
}
|
||||
|
||||
// SlaveConfig 从机配置
|
||||
var SlaveConfig = &slave{
|
||||
CallbackTimeout: 20,
|
||||
@@ -75,3 +47,5 @@ var SSLConfig = &ssl{
|
||||
var UnixConfig = &unix{
|
||||
Listen: "",
|
||||
}
|
||||
|
||||
var OptionOverwrite = map[string]interface{}{}
|
||||
|
||||
@@ -1,13 +1,13 @@
|
||||
package conf
|
||||
|
||||
// BackendVersion 当前后端版本号
|
||||
var BackendVersion = "3.4.2"
|
||||
var BackendVersion = "3.5.0"
|
||||
|
||||
// RequiredDBVersion 与当前版本匹配的数据库版本
|
||||
var RequiredDBVersion = "3.4.0"
|
||||
var RequiredDBVersion = "3.5.0"
|
||||
|
||||
// RequiredStaticVersion 与当前版本匹配的静态资源版本
|
||||
var RequiredStaticVersion = "3.4.2"
|
||||
var RequiredStaticVersion = "3.5.0"
|
||||
|
||||
// IsPro 是否为Pro版本
|
||||
var IsPro = "false"
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
package crontab
|
||||
|
||||
import (
|
||||
"context"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
@@ -8,6 +9,7 @@ import (
|
||||
|
||||
model "github.com/cloudreve/Cloudreve/v3/models"
|
||||
"github.com/cloudreve/Cloudreve/v3/pkg/cache"
|
||||
"github.com/cloudreve/Cloudreve/v3/pkg/filesystem"
|
||||
"github.com/cloudreve/Cloudreve/v3/pkg/util"
|
||||
)
|
||||
|
||||
@@ -53,3 +55,45 @@ func collectCache(store *cache.MemoStore) {
|
||||
util.Log().Debug("清理内存缓存")
|
||||
store.GarbageCollect()
|
||||
}
|
||||
|
||||
func uploadSessionCollect() {
|
||||
placeholders := model.GetUploadPlaceholderFiles(0)
|
||||
|
||||
// 将过期的上传会话按照用户分组
|
||||
userToFiles := make(map[uint][]uint)
|
||||
for _, file := range placeholders {
|
||||
_, sessionExist := cache.Get(filesystem.UploadSessionCachePrefix + *file.UploadSessionID)
|
||||
if sessionExist {
|
||||
continue
|
||||
}
|
||||
|
||||
if _, ok := userToFiles[file.UserID]; !ok {
|
||||
userToFiles[file.UserID] = make([]uint, 0)
|
||||
}
|
||||
|
||||
userToFiles[file.UserID] = append(userToFiles[file.UserID], file.ID)
|
||||
}
|
||||
|
||||
// 删除过期的会话
|
||||
for uid, filesIDs := range userToFiles {
|
||||
user, err := model.GetUserByID(uid)
|
||||
if err != nil {
|
||||
util.Log().Warning("上传会话所属用户不存在, %s", err)
|
||||
continue
|
||||
}
|
||||
|
||||
fs, err := filesystem.NewFileSystem(&user)
|
||||
if err != nil {
|
||||
util.Log().Warning("无法初始化文件系统, %s", err)
|
||||
continue
|
||||
}
|
||||
|
||||
if err = fs.Delete(context.Background(), []uint{}, filesIDs, false); err != nil {
|
||||
util.Log().Warning("无法删除上传会话, %s", err)
|
||||
}
|
||||
|
||||
fs.Recycle()
|
||||
}
|
||||
|
||||
util.Log().Info("定时任务 [cron_recycle_upload_session] 执行完毕")
|
||||
}
|
||||
|
||||
@@ -21,13 +21,18 @@ func Reload() {
|
||||
func Init() {
|
||||
util.Log().Info("初始化定时任务...")
|
||||
// 读取cron日程设置
|
||||
options := model.GetSettingByNames("cron_garbage_collect")
|
||||
options := model.GetSettingByNames(
|
||||
"cron_garbage_collect",
|
||||
"cron_recycle_upload_session",
|
||||
)
|
||||
Cron := cron.New()
|
||||
for k, v := range options {
|
||||
var handler func()
|
||||
switch k {
|
||||
case "cron_garbage_collect":
|
||||
handler = garbageCollect
|
||||
case "cron_recycle_upload_session":
|
||||
handler = uploadSessionCollect
|
||||
default:
|
||||
util.Log().Warning("未知定时任务类型 [%s],跳过", k)
|
||||
continue
|
||||
|
||||
@@ -28,17 +28,17 @@ import (
|
||||
*/
|
||||
|
||||
// Compress 创建给定目录和文件的压缩文件
|
||||
func (fs *FileSystem) Compress(ctx context.Context, folderIDs, fileIDs []uint, isArchive bool) (string, error) {
|
||||
func (fs *FileSystem) Compress(ctx context.Context, writer io.Writer, folderIDs, fileIDs []uint, isArchive bool) error {
|
||||
// 查找待压缩目录
|
||||
folders, err := model.GetFoldersByIDs(folderIDs, fs.User.ID)
|
||||
if err != nil && len(folderIDs) != 0 {
|
||||
return "", ErrDBListObjects
|
||||
return ErrDBListObjects
|
||||
}
|
||||
|
||||
// 查找待压缩文件
|
||||
files, err := model.GetFilesByIDs(fileIDs, fs.User.ID)
|
||||
if err != nil && len(fileIDs) != 0 {
|
||||
return "", ErrDBListObjects
|
||||
return ErrDBListObjects
|
||||
}
|
||||
|
||||
// 如果上下文限制了父目录,则进行检查
|
||||
@@ -46,14 +46,14 @@ func (fs *FileSystem) Compress(ctx context.Context, folderIDs, fileIDs []uint, i
|
||||
// 检查目录
|
||||
for _, folder := range folders {
|
||||
if *folder.ParentID != parent.ID {
|
||||
return "", ErrObjectNotExist
|
||||
return ErrObjectNotExist
|
||||
}
|
||||
}
|
||||
|
||||
// 检查文件
|
||||
for _, file := range files {
|
||||
if file.FolderID != parent.ID {
|
||||
return "", ErrObjectNotExist
|
||||
return ErrObjectNotExist
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -73,25 +73,8 @@ func (fs *FileSystem) Compress(ctx context.Context, folderIDs, fileIDs []uint, i
|
||||
files[i].Position = ""
|
||||
}
|
||||
|
||||
// 创建临时压缩文件
|
||||
saveFolder := "archive"
|
||||
if !isArchive {
|
||||
saveFolder = "compress"
|
||||
}
|
||||
zipFilePath := filepath.Join(
|
||||
util.RelativePath(model.GetSettingByName("temp_path")),
|
||||
saveFolder,
|
||||
fmt.Sprintf("archive_%d.zip", time.Now().UnixNano()),
|
||||
)
|
||||
zipFile, err := util.CreatNestedFile(zipFilePath)
|
||||
if err != nil {
|
||||
util.Log().Warning("%s", err)
|
||||
return "", err
|
||||
}
|
||||
defer zipFile.Close()
|
||||
|
||||
// 创建压缩文件Writer
|
||||
zipWriter := zip.NewWriter(zipFile)
|
||||
zipWriter := zip.NewWriter(writer)
|
||||
defer zipWriter.Close()
|
||||
|
||||
ctx = reqContext
|
||||
@@ -101,10 +84,9 @@ func (fs *FileSystem) Compress(ctx context.Context, folderIDs, fileIDs []uint, i
|
||||
select {
|
||||
case <-reqContext.Done():
|
||||
// 取消压缩请求
|
||||
fs.cancelCompress(ctx, zipWriter, zipFile, zipFilePath)
|
||||
return "", ErrClientCanceled
|
||||
return ErrClientCanceled
|
||||
default:
|
||||
fs.doCompress(ctx, nil, &folders[i], zipWriter, isArchive)
|
||||
fs.doCompress(reqContext, nil, &folders[i], zipWriter, isArchive)
|
||||
}
|
||||
|
||||
}
|
||||
@@ -112,22 +94,13 @@ func (fs *FileSystem) Compress(ctx context.Context, folderIDs, fileIDs []uint, i
|
||||
select {
|
||||
case <-reqContext.Done():
|
||||
// 取消压缩请求
|
||||
fs.cancelCompress(ctx, zipWriter, zipFile, zipFilePath)
|
||||
return "", ErrClientCanceled
|
||||
return ErrClientCanceled
|
||||
default:
|
||||
fs.doCompress(ctx, &files[i], nil, zipWriter, isArchive)
|
||||
fs.doCompress(reqContext, &files[i], nil, zipWriter, isArchive)
|
||||
}
|
||||
}
|
||||
|
||||
return zipFilePath, nil
|
||||
}
|
||||
|
||||
// cancelCompress 取消压缩进程
|
||||
func (fs *FileSystem) cancelCompress(ctx context.Context, zipWriter *zip.Writer, file *os.File, path string) {
|
||||
util.Log().Debug("客户端取消压缩请求")
|
||||
zipWriter.Close()
|
||||
file.Close()
|
||||
_ = os.Remove(path)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (fs *FileSystem) doCompress(ctx context.Context, file *model.File, folder *model.Folder, zipWriter *zip.Writer, isArchive bool) {
|
||||
@@ -217,6 +190,8 @@ func (fs *FileSystem) Decompress(ctx context.Context, src, dst string) error {
|
||||
return err
|
||||
}
|
||||
|
||||
defer fileStream.Close()
|
||||
|
||||
tempZipFilePath = filepath.Join(
|
||||
util.RelativePath(model.GetSettingByName("temp_path")),
|
||||
"decompress",
|
||||
@@ -238,6 +213,7 @@ func (fs *FileSystem) Decompress(ctx context.Context, src, dst string) error {
|
||||
}
|
||||
|
||||
zipFile.Close()
|
||||
fileStream.Close()
|
||||
|
||||
// 解压缩文件
|
||||
r, err := zip.OpenReader(tempZipFilePath)
|
||||
@@ -303,7 +279,12 @@ func (fs *FileSystem) Decompress(ctx context.Context, src, dst string) error {
|
||||
}
|
||||
}()
|
||||
|
||||
err = fs.UploadFromStream(ctx, fileStream, savePath, uint64(size))
|
||||
err = fs.UploadFromStream(ctx, &fsctx.FileStream{
|
||||
File: fileStream,
|
||||
Size: uint64(size),
|
||||
Name: path.Base(savePath),
|
||||
VirtualPath: path.Dir(savePath),
|
||||
}, true)
|
||||
fileStream.Close()
|
||||
if err != nil {
|
||||
util.Log().Debug("无法上传压缩包内的文件%s , %s , 跳过", rawPath, err)
|
||||
|
||||
@@ -1,8 +1,11 @@
|
||||
package filesystem
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"errors"
|
||||
"github.com/cloudreve/Cloudreve/v3/pkg/util"
|
||||
testMock "github.com/stretchr/testify/mock"
|
||||
"io"
|
||||
"os"
|
||||
"strings"
|
||||
@@ -12,11 +15,8 @@ import (
|
||||
model "github.com/cloudreve/Cloudreve/v3/models"
|
||||
"github.com/cloudreve/Cloudreve/v3/pkg/cache"
|
||||
"github.com/cloudreve/Cloudreve/v3/pkg/filesystem/fsctx"
|
||||
"github.com/cloudreve/Cloudreve/v3/pkg/request"
|
||||
"github.com/cloudreve/Cloudreve/v3/pkg/util"
|
||||
"github.com/jinzhu/gorm"
|
||||
"github.com/stretchr/testify/assert"
|
||||
testMock "github.com/stretchr/testify/mock"
|
||||
)
|
||||
|
||||
func TestFileSystem_Compress(t *testing.T) {
|
||||
@@ -58,12 +58,11 @@ func TestFileSystem_Compress(t *testing.T) {
|
||||
)
|
||||
// 查找上传策略
|
||||
asserts.NoError(cache.Set("policy_1", model.Policy{Type: "local"}, -1))
|
||||
w := &bytes.Buffer{}
|
||||
|
||||
zipFile, err := fs.Compress(ctx, []uint{1}, []uint{1}, true)
|
||||
err := fs.Compress(ctx, w, []uint{1}, []uint{1}, true)
|
||||
asserts.NoError(err)
|
||||
asserts.NotEmpty(zipFile)
|
||||
asserts.Contains(zipFile, "archive_")
|
||||
asserts.Contains(zipFile, "tests")
|
||||
asserts.NotEmpty(w.Len())
|
||||
}
|
||||
|
||||
// 上下文取消
|
||||
@@ -84,9 +83,10 @@ func TestFileSystem_Compress(t *testing.T) {
|
||||
)
|
||||
asserts.NoError(cache.Set("setting_temp_path", "tests", -1))
|
||||
|
||||
zipFile, err := fs.Compress(ctx, []uint{1}, []uint{1}, true)
|
||||
w := &bytes.Buffer{}
|
||||
err := fs.Compress(ctx, w, []uint{1}, []uint{1}, true)
|
||||
asserts.Error(err)
|
||||
asserts.Empty(zipFile)
|
||||
asserts.NotEmpty(w.Len())
|
||||
}
|
||||
|
||||
// 限制父目录
|
||||
@@ -108,10 +108,11 @@ func TestFileSystem_Compress(t *testing.T) {
|
||||
)
|
||||
asserts.NoError(cache.Set("setting_temp_path", "tests", -1))
|
||||
|
||||
zipFile, err := fs.Compress(ctx, []uint{1}, []uint{1}, true)
|
||||
w := &bytes.Buffer{}
|
||||
err := fs.Compress(ctx, w, []uint{1}, []uint{1}, true)
|
||||
asserts.Error(err)
|
||||
asserts.Equal(ErrObjectNotExist, err)
|
||||
asserts.Empty(zipFile)
|
||||
asserts.Empty(w.Len())
|
||||
}
|
||||
|
||||
}
|
||||
@@ -171,7 +172,7 @@ func TestFileSystem_Decompress(t *testing.T) {
|
||||
fs.FileTarget = []model.File{{SourceName: "1.zip", Policy: model.Policy{Type: "mock"}}}
|
||||
fs.FileTarget[0].Policy.ID = 1
|
||||
testHandler := new(FileHeaderMock)
|
||||
testHandler.On("Get", testMock.Anything, "1.zip").Return(request.NopRSCloser{}, errors.New("error"))
|
||||
testHandler.On("Get", testMock.Anything, "1.zip").Return(MockRSC{}, errors.New("error"))
|
||||
fs.Handler = testHandler
|
||||
err := fs.Decompress(ctx, "/1.zip", "/")
|
||||
asserts.NoError(mock.ExpectationsWereMet())
|
||||
@@ -185,7 +186,7 @@ func TestFileSystem_Decompress(t *testing.T) {
|
||||
fs.FileTarget = []model.File{{SourceName: "1.zip", Policy: model.Policy{Type: "mock"}}}
|
||||
fs.FileTarget[0].Policy.ID = 1
|
||||
testHandler := new(FileHeaderMock)
|
||||
testHandler.On("Get", testMock.Anything, "1.zip").Return(request.NopRSCloser{}, nil)
|
||||
testHandler.On("Get", testMock.Anything, "1.zip").Return(MockRSC{}, nil)
|
||||
fs.Handler = testHandler
|
||||
err := fs.Decompress(ctx, "/1.zip", "/")
|
||||
asserts.NoError(mock.ExpectationsWereMet())
|
||||
|
||||
31
pkg/filesystem/chunk/backoff/backoff.go
Normal file
31
pkg/filesystem/chunk/backoff/backoff.go
Normal file
@@ -0,0 +1,31 @@
|
||||
package backoff
|
||||
|
||||
import "time"
|
||||
|
||||
// Backoff used for retry sleep backoff
|
||||
type Backoff interface {
|
||||
Next() bool
|
||||
Reset()
|
||||
}
|
||||
|
||||
// ConstantBackoff implements Backoff interface with constant sleep time
|
||||
type ConstantBackoff struct {
|
||||
Sleep time.Duration
|
||||
Max int
|
||||
|
||||
tried int
|
||||
}
|
||||
|
||||
func (c *ConstantBackoff) Next() bool {
|
||||
c.tried++
|
||||
if c.tried > c.Max {
|
||||
return false
|
||||
}
|
||||
|
||||
time.Sleep(c.Sleep)
|
||||
return true
|
||||
}
|
||||
|
||||
func (c *ConstantBackoff) Reset() {
|
||||
c.tried = 0
|
||||
}
|
||||
22
pkg/filesystem/chunk/backoff/backoff_test.go
Normal file
22
pkg/filesystem/chunk/backoff/backoff_test.go
Normal file
@@ -0,0 +1,22 @@
|
||||
package backoff
|
||||
|
||||
import (
|
||||
"github.com/stretchr/testify/assert"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func TestConstantBackoff_Next(t *testing.T) {
|
||||
a := assert.New(t)
|
||||
|
||||
b := &ConstantBackoff{Sleep: time.Duration(0), Max: 3}
|
||||
a.True(b.Next())
|
||||
a.True(b.Next())
|
||||
a.True(b.Next())
|
||||
a.False(b.Next())
|
||||
b.Reset()
|
||||
a.True(b.Next())
|
||||
a.True(b.Next())
|
||||
a.True(b.Next())
|
||||
a.False(b.Next())
|
||||
}
|
||||
162
pkg/filesystem/chunk/chunk.go
Normal file
162
pkg/filesystem/chunk/chunk.go
Normal file
@@ -0,0 +1,162 @@
|
||||
package chunk
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"github.com/cloudreve/Cloudreve/v3/pkg/filesystem/chunk/backoff"
|
||||
"github.com/cloudreve/Cloudreve/v3/pkg/filesystem/fsctx"
|
||||
"github.com/cloudreve/Cloudreve/v3/pkg/util"
|
||||
"io"
|
||||
"os"
|
||||
)
|
||||
|
||||
const bufferTempPattern = "cdChunk.*.tmp"
|
||||
|
||||
// ChunkProcessFunc callback function for processing a chunk
|
||||
type ChunkProcessFunc func(c *ChunkGroup, chunk io.Reader) error
|
||||
|
||||
// ChunkGroup manage groups of chunks
|
||||
type ChunkGroup struct {
|
||||
file fsctx.FileHeader
|
||||
chunkSize uint64
|
||||
backoff backoff.Backoff
|
||||
enableRetryBuffer bool
|
||||
|
||||
fileInfo *fsctx.UploadTaskInfo
|
||||
currentIndex int
|
||||
chunkNum uint64
|
||||
bufferTemp *os.File
|
||||
}
|
||||
|
||||
func NewChunkGroup(file fsctx.FileHeader, chunkSize uint64, backoff backoff.Backoff, useBuffer bool) *ChunkGroup {
|
||||
c := &ChunkGroup{
|
||||
file: file,
|
||||
chunkSize: chunkSize,
|
||||
backoff: backoff,
|
||||
fileInfo: file.Info(),
|
||||
currentIndex: -1,
|
||||
enableRetryBuffer: useBuffer,
|
||||
}
|
||||
|
||||
if c.chunkSize == 0 {
|
||||
c.chunkSize = c.fileInfo.Size
|
||||
}
|
||||
|
||||
if c.fileInfo.Size == 0 {
|
||||
c.chunkNum = 1
|
||||
} else {
|
||||
c.chunkNum = c.fileInfo.Size / c.chunkSize
|
||||
if c.fileInfo.Size%c.chunkSize != 0 {
|
||||
c.chunkNum++
|
||||
}
|
||||
}
|
||||
|
||||
return c
|
||||
}
|
||||
|
||||
// TempAvailable returns if current chunk temp file is available to be read
|
||||
func (c *ChunkGroup) TempAvailable() bool {
|
||||
if c.bufferTemp != nil {
|
||||
state, _ := c.bufferTemp.Stat()
|
||||
return state != nil && state.Size() == c.Length()
|
||||
}
|
||||
|
||||
return false
|
||||
}
|
||||
|
||||
// Process a chunk with retry logic
|
||||
func (c *ChunkGroup) Process(processor ChunkProcessFunc) error {
|
||||
reader := io.LimitReader(c.file, int64(c.chunkSize))
|
||||
|
||||
// If useBuffer is enabled, tee the reader to a temp file
|
||||
if c.enableRetryBuffer && c.bufferTemp == nil && !c.file.Seekable() {
|
||||
c.bufferTemp, _ = os.CreateTemp("", bufferTempPattern)
|
||||
reader = io.TeeReader(reader, c.bufferTemp)
|
||||
}
|
||||
|
||||
if c.bufferTemp != nil {
|
||||
defer func() {
|
||||
if c.bufferTemp != nil {
|
||||
c.bufferTemp.Close()
|
||||
os.Remove(c.bufferTemp.Name())
|
||||
c.bufferTemp = nil
|
||||
}
|
||||
}()
|
||||
|
||||
// if temp buffer file is available, use it
|
||||
if c.TempAvailable() {
|
||||
if _, err := c.bufferTemp.Seek(0, io.SeekStart); err != nil {
|
||||
return fmt.Errorf("failed to seek temp file back to chunk start: %w", err)
|
||||
}
|
||||
|
||||
util.Log().Debug("Chunk %d will be read from temp file %q.", c.Index(), c.bufferTemp.Name())
|
||||
reader = c.bufferTemp
|
||||
}
|
||||
}
|
||||
|
||||
err := processor(c, reader)
|
||||
if err != nil {
|
||||
if err != context.Canceled && (c.file.Seekable() || c.TempAvailable()) && c.backoff.Next() {
|
||||
if c.file.Seekable() {
|
||||
if _, seekErr := c.file.Seek(c.Start(), io.SeekStart); seekErr != nil {
|
||||
return fmt.Errorf("failed to seek back to chunk start: %w, last error: %s", seekErr, err)
|
||||
}
|
||||
}
|
||||
|
||||
util.Log().Debug("Retrying chunk %d, last error: %s", c.currentIndex, err)
|
||||
return c.Process(processor)
|
||||
}
|
||||
|
||||
return err
|
||||
}
|
||||
|
||||
util.Log().Debug("Chunk %d processed", c.currentIndex)
|
||||
return nil
|
||||
}
|
||||
|
||||
// Start returns the byte index of current chunk
|
||||
func (c *ChunkGroup) Start() int64 {
|
||||
return int64(uint64(c.Index()) * c.chunkSize)
|
||||
}
|
||||
|
||||
// Total returns the total length
|
||||
func (c *ChunkGroup) Total() int64 {
|
||||
return int64(c.fileInfo.Size)
|
||||
}
|
||||
|
||||
// Num returns the total chunk number
|
||||
func (c *ChunkGroup) Num() int {
|
||||
return int(c.chunkNum)
|
||||
}
|
||||
|
||||
// RangeHeader returns header value of Content-Range
|
||||
func (c *ChunkGroup) RangeHeader() string {
|
||||
return fmt.Sprintf("bytes %d-%d/%d", c.Start(), c.Start()+c.Length()-1, c.Total())
|
||||
}
|
||||
|
||||
// Index returns current chunk index, starts from 0
|
||||
func (c *ChunkGroup) Index() int {
|
||||
return c.currentIndex
|
||||
}
|
||||
|
||||
// Next switch to next chunk, returns whether all chunks are processed
|
||||
func (c *ChunkGroup) Next() bool {
|
||||
c.currentIndex++
|
||||
c.backoff.Reset()
|
||||
return c.currentIndex < int(c.chunkNum)
|
||||
}
|
||||
|
||||
// Length returns the length of current chunk
|
||||
func (c *ChunkGroup) Length() int64 {
|
||||
contentLength := c.chunkSize
|
||||
if c.Index() == int(c.chunkNum-1) {
|
||||
contentLength = c.fileInfo.Size - c.chunkSize*(c.chunkNum-1)
|
||||
}
|
||||
|
||||
return int64(contentLength)
|
||||
}
|
||||
|
||||
// IsLast returns if current chunk is the last one
|
||||
func (c *ChunkGroup) IsLast() bool {
|
||||
return c.Index() == int(c.chunkNum-1)
|
||||
}
|
||||
250
pkg/filesystem/chunk/chunk_test.go
Normal file
250
pkg/filesystem/chunk/chunk_test.go
Normal file
@@ -0,0 +1,250 @@
|
||||
package chunk
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"github.com/cloudreve/Cloudreve/v3/pkg/filesystem/chunk/backoff"
|
||||
"github.com/cloudreve/Cloudreve/v3/pkg/filesystem/fsctx"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"io"
|
||||
"os"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestNewChunkGroup(t *testing.T) {
|
||||
a := assert.New(t)
|
||||
|
||||
testCases := []struct {
|
||||
fileSize uint64
|
||||
chunkSize uint64
|
||||
expectedInnerChunkSize uint64
|
||||
expectedChunkNum uint64
|
||||
expectedInfo [][2]int //Start, Index,Length
|
||||
}{
|
||||
{10, 0, 10, 1, [][2]int{{0, 10}}},
|
||||
{0, 0, 0, 1, [][2]int{{0, 0}}},
|
||||
{0, 10, 10, 1, [][2]int{{0, 0}}},
|
||||
{50, 10, 10, 5, [][2]int{
|
||||
{0, 10},
|
||||
{10, 10},
|
||||
{20, 10},
|
||||
{30, 10},
|
||||
{40, 10},
|
||||
}},
|
||||
{50, 50, 50, 1, [][2]int{
|
||||
{0, 50},
|
||||
}},
|
||||
|
||||
{50, 15, 15, 4, [][2]int{
|
||||
{0, 15},
|
||||
{15, 15},
|
||||
{30, 15},
|
||||
{45, 5},
|
||||
}},
|
||||
}
|
||||
|
||||
for index, testCase := range testCases {
|
||||
file := &fsctx.FileStream{Size: testCase.fileSize}
|
||||
chunkGroup := NewChunkGroup(file, testCase.chunkSize, &backoff.ConstantBackoff{}, true)
|
||||
a.EqualValues(testCase.expectedChunkNum, chunkGroup.Num(),
|
||||
"TestCase:%d,ChunkNum()", index)
|
||||
a.EqualValues(testCase.expectedInnerChunkSize, chunkGroup.chunkSize,
|
||||
"TestCase:%d,InnerChunkSize()", index)
|
||||
a.EqualValues(testCase.expectedChunkNum, chunkGroup.Num(),
|
||||
"TestCase:%d,len(Chunks)", index)
|
||||
a.EqualValues(testCase.fileSize, chunkGroup.Total())
|
||||
|
||||
for cIndex, info := range testCase.expectedInfo {
|
||||
a.True(chunkGroup.Next())
|
||||
a.EqualValues(info[1], chunkGroup.Length(),
|
||||
"TestCase:%d,Chunks[%d].Length()", index, cIndex)
|
||||
a.EqualValues(info[0], chunkGroup.Start(),
|
||||
"TestCase:%d,Chunks[%d].Start()", index, cIndex)
|
||||
|
||||
a.Equal(cIndex == len(testCase.expectedInfo)-1, chunkGroup.IsLast(),
|
||||
"TestCase:%d,Chunks[%d].IsLast()", index, cIndex)
|
||||
|
||||
a.NotEmpty(chunkGroup.RangeHeader())
|
||||
}
|
||||
a.False(chunkGroup.Next())
|
||||
}
|
||||
}
|
||||
|
||||
func TestChunkGroup_TempAvailablet(t *testing.T) {
|
||||
a := assert.New(t)
|
||||
|
||||
file := &fsctx.FileStream{Size: 1}
|
||||
c := NewChunkGroup(file, 0, &backoff.ConstantBackoff{}, true)
|
||||
a.False(c.TempAvailable())
|
||||
|
||||
f, err := os.CreateTemp("", "TestChunkGroup_TempAvailablet.*")
|
||||
defer func() {
|
||||
f.Close()
|
||||
os.Remove(f.Name())
|
||||
}()
|
||||
a.NoError(err)
|
||||
c.bufferTemp = f
|
||||
|
||||
a.False(c.TempAvailable())
|
||||
f.Write([]byte("1"))
|
||||
a.True(c.TempAvailable())
|
||||
|
||||
}
|
||||
|
||||
func TestChunkGroup_Process(t *testing.T) {
|
||||
a := assert.New(t)
|
||||
file := &fsctx.FileStream{Size: 10}
|
||||
|
||||
// success
|
||||
{
|
||||
file.File = io.NopCloser(strings.NewReader("1234567890"))
|
||||
c := NewChunkGroup(file, 5, &backoff.ConstantBackoff{}, true)
|
||||
count := 0
|
||||
a.True(c.Next())
|
||||
a.NoError(c.Process(func(c *ChunkGroup, chunk io.Reader) error {
|
||||
count++
|
||||
res, err := io.ReadAll(chunk)
|
||||
a.NoError(err)
|
||||
a.EqualValues("12345", string(res))
|
||||
return nil
|
||||
}))
|
||||
a.True(c.Next())
|
||||
a.NoError(c.Process(func(c *ChunkGroup, chunk io.Reader) error {
|
||||
count++
|
||||
res, err := io.ReadAll(chunk)
|
||||
a.NoError(err)
|
||||
a.EqualValues("67890", string(res))
|
||||
return nil
|
||||
}))
|
||||
a.False(c.Next())
|
||||
a.Equal(2, count)
|
||||
}
|
||||
|
||||
// retry, read from buffer file
|
||||
{
|
||||
file.File = io.NopCloser(strings.NewReader("1234567890"))
|
||||
c := NewChunkGroup(file, 5, &backoff.ConstantBackoff{Max: 2}, true)
|
||||
count := 0
|
||||
a.True(c.Next())
|
||||
a.NoError(c.Process(func(c *ChunkGroup, chunk io.Reader) error {
|
||||
count++
|
||||
res, err := io.ReadAll(chunk)
|
||||
a.NoError(err)
|
||||
a.EqualValues("12345", string(res))
|
||||
return nil
|
||||
}))
|
||||
a.True(c.Next())
|
||||
a.NoError(c.Process(func(c *ChunkGroup, chunk io.Reader) error {
|
||||
count++
|
||||
res, err := io.ReadAll(chunk)
|
||||
a.NoError(err)
|
||||
a.EqualValues("67890", string(res))
|
||||
if count == 2 {
|
||||
return errors.New("error")
|
||||
}
|
||||
return nil
|
||||
}))
|
||||
a.False(c.Next())
|
||||
a.Equal(3, count)
|
||||
}
|
||||
|
||||
// retry, read from seeker
|
||||
{
|
||||
f, _ := os.CreateTemp("", "TestChunkGroup_Process.*")
|
||||
f.Write([]byte("1234567890"))
|
||||
f.Seek(0, 0)
|
||||
defer func() {
|
||||
f.Close()
|
||||
os.Remove(f.Name())
|
||||
}()
|
||||
file.File = f
|
||||
file.Seeker = f
|
||||
c := NewChunkGroup(file, 5, &backoff.ConstantBackoff{Max: 2}, false)
|
||||
count := 0
|
||||
a.True(c.Next())
|
||||
a.NoError(c.Process(func(c *ChunkGroup, chunk io.Reader) error {
|
||||
count++
|
||||
res, err := io.ReadAll(chunk)
|
||||
a.NoError(err)
|
||||
a.EqualValues("12345", string(res))
|
||||
return nil
|
||||
}))
|
||||
a.True(c.Next())
|
||||
a.NoError(c.Process(func(c *ChunkGroup, chunk io.Reader) error {
|
||||
count++
|
||||
res, err := io.ReadAll(chunk)
|
||||
a.NoError(err)
|
||||
a.EqualValues("67890", string(res))
|
||||
if count == 2 {
|
||||
return errors.New("error")
|
||||
}
|
||||
return nil
|
||||
}))
|
||||
a.False(c.Next())
|
||||
a.Equal(3, count)
|
||||
}
|
||||
|
||||
// retry, seek error
|
||||
{
|
||||
f, _ := os.CreateTemp("", "TestChunkGroup_Process.*")
|
||||
f.Write([]byte("1234567890"))
|
||||
f.Seek(0, 0)
|
||||
defer func() {
|
||||
f.Close()
|
||||
os.Remove(f.Name())
|
||||
}()
|
||||
file.File = f
|
||||
file.Seeker = f
|
||||
c := NewChunkGroup(file, 5, &backoff.ConstantBackoff{Max: 2}, false)
|
||||
count := 0
|
||||
a.True(c.Next())
|
||||
a.NoError(c.Process(func(c *ChunkGroup, chunk io.Reader) error {
|
||||
count++
|
||||
res, err := io.ReadAll(chunk)
|
||||
a.NoError(err)
|
||||
a.EqualValues("12345", string(res))
|
||||
return nil
|
||||
}))
|
||||
a.True(c.Next())
|
||||
f.Close()
|
||||
a.Error(c.Process(func(c *ChunkGroup, chunk io.Reader) error {
|
||||
count++
|
||||
if count == 2 {
|
||||
return errors.New("error")
|
||||
}
|
||||
return nil
|
||||
}))
|
||||
a.False(c.Next())
|
||||
a.Equal(2, count)
|
||||
}
|
||||
|
||||
// retry, finally error
|
||||
{
|
||||
f, _ := os.CreateTemp("", "TestChunkGroup_Process.*")
|
||||
f.Write([]byte("1234567890"))
|
||||
f.Seek(0, 0)
|
||||
defer func() {
|
||||
f.Close()
|
||||
os.Remove(f.Name())
|
||||
}()
|
||||
file.File = f
|
||||
file.Seeker = f
|
||||
c := NewChunkGroup(file, 5, &backoff.ConstantBackoff{Max: 2}, false)
|
||||
count := 0
|
||||
a.True(c.Next())
|
||||
a.NoError(c.Process(func(c *ChunkGroup, chunk io.Reader) error {
|
||||
count++
|
||||
res, err := io.ReadAll(chunk)
|
||||
a.NoError(err)
|
||||
a.EqualValues("12345", string(res))
|
||||
return nil
|
||||
}))
|
||||
a.True(c.Next())
|
||||
a.Error(c.Process(func(c *ChunkGroup, chunk io.Reader) error {
|
||||
count++
|
||||
return errors.New("error")
|
||||
}))
|
||||
a.False(c.Next())
|
||||
a.Equal(4, count)
|
||||
}
|
||||
}
|
||||
@@ -183,9 +183,11 @@ func (handler Driver) Get(ctx context.Context, path string) (response.RSCloser,
|
||||
}
|
||||
|
||||
// Put 将文件流保存到指定目录
|
||||
func (handler Driver) Put(ctx context.Context, file io.ReadCloser, dst string, size uint64) error {
|
||||
func (handler Driver) Put(ctx context.Context, file fsctx.FileHeader) error {
|
||||
defer file.Close()
|
||||
|
||||
opt := &cossdk.ObjectPutOptions{}
|
||||
_, err := handler.Client.Object.Put(ctx, dst, file, opt)
|
||||
_, err := handler.Client.Object.Put(ctx, file.Info().SavePath, file, opt)
|
||||
return err
|
||||
}
|
||||
|
||||
@@ -324,21 +326,16 @@ func (handler Driver) signSourceURL(ctx context.Context, path string, ttl int64,
|
||||
}
|
||||
|
||||
// Token 获取上传策略和认证Token
|
||||
func (handler Driver) Token(ctx context.Context, TTL int64, key string) (serializer.UploadCredential, error) {
|
||||
// 读取上下文中生成的存储路径
|
||||
savePath, ok := ctx.Value(fsctx.SavePathCtx).(string)
|
||||
if !ok {
|
||||
return serializer.UploadCredential{}, errors.New("无法获取存储路径")
|
||||
}
|
||||
|
||||
func (handler Driver) Token(ctx context.Context, ttl int64, uploadSession *serializer.UploadSession, file fsctx.FileHeader) (*serializer.UploadCredential, error) {
|
||||
// 生成回调地址
|
||||
siteURL := model.GetSiteURL()
|
||||
apiBaseURI, _ := url.Parse("/api/v3/callback/cos/" + key)
|
||||
apiBaseURI, _ := url.Parse("/api/v3/callback/cos/" + uploadSession.Key)
|
||||
apiURL := siteURL.ResolveReference(apiBaseURI).String()
|
||||
|
||||
// 上传策略
|
||||
savePath := file.Info().SavePath
|
||||
startTime := time.Now()
|
||||
endTime := startTime.Add(time.Duration(TTL) * time.Second)
|
||||
endTime := startTime.Add(time.Duration(ttl) * time.Second)
|
||||
keyTime := fmt.Sprintf("%d;%d", startTime.Unix(), endTime.Unix())
|
||||
postPolicy := UploadPolicy{
|
||||
Expiration: endTime.UTC().Format(time.RFC3339),
|
||||
@@ -346,7 +343,7 @@ func (handler Driver) Token(ctx context.Context, TTL int64, key string) (seriali
|
||||
map[string]string{"bucket": handler.Policy.BucketName},
|
||||
map[string]string{"$key": savePath},
|
||||
map[string]string{"x-cos-meta-callback": apiURL},
|
||||
map[string]string{"x-cos-meta-key": key},
|
||||
map[string]string{"x-cos-meta-key": uploadSession.Key},
|
||||
map[string]string{"q-sign-algorithm": "sha1"},
|
||||
map[string]string{"q-ak": handler.Policy.AccessKey},
|
||||
map[string]string{"q-sign-time": keyTime},
|
||||
@@ -358,16 +355,22 @@ func (handler Driver) Token(ctx context.Context, TTL int64, key string) (seriali
|
||||
[]interface{}{"content-length-range", 0, handler.Policy.MaxSize})
|
||||
}
|
||||
|
||||
res, err := handler.getUploadCredential(ctx, postPolicy, keyTime)
|
||||
res, err := handler.getUploadCredential(ctx, postPolicy, keyTime, savePath)
|
||||
if err == nil {
|
||||
res.SessionID = uploadSession.Key
|
||||
res.Callback = apiURL
|
||||
res.Key = key
|
||||
res.UploadURLs = []string{handler.Policy.Server}
|
||||
}
|
||||
|
||||
return res, err
|
||||
|
||||
}
|
||||
|
||||
// 取消上传凭证
|
||||
func (handler Driver) CancelToken(ctx context.Context, uploadSession *serializer.UploadSession) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
// Meta 获取文件信息
|
||||
func (handler Driver) Meta(ctx context.Context, path string) (*MetaData, error) {
|
||||
res, err := handler.Client.Object.Head(ctx, path, &cossdk.ObjectHeadOptions{})
|
||||
@@ -381,17 +384,11 @@ func (handler Driver) Meta(ctx context.Context, path string) (*MetaData, error)
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (handler Driver) getUploadCredential(ctx context.Context, policy UploadPolicy, keyTime string) (serializer.UploadCredential, error) {
|
||||
// 读取上下文中生成的存储路径
|
||||
savePath, ok := ctx.Value(fsctx.SavePathCtx).(string)
|
||||
if !ok {
|
||||
return serializer.UploadCredential{}, errors.New("无法获取存储路径")
|
||||
}
|
||||
|
||||
func (handler Driver) getUploadCredential(ctx context.Context, policy UploadPolicy, keyTime string, savePath string) (*serializer.UploadCredential, error) {
|
||||
// 编码上传策略
|
||||
policyJSON, err := json.Marshal(policy)
|
||||
if err != nil {
|
||||
return serializer.UploadCredential{}, err
|
||||
return nil, err
|
||||
}
|
||||
policyEncoded := base64.StdEncoding.EncodeToString(policyJSON)
|
||||
|
||||
@@ -399,14 +396,14 @@ func (handler Driver) getUploadCredential(ctx context.Context, policy UploadPoli
|
||||
hmacSign := hmac.New(sha1.New, []byte(handler.Policy.SecretKey))
|
||||
_, err = io.WriteString(hmacSign, keyTime)
|
||||
if err != nil {
|
||||
return serializer.UploadCredential{}, err
|
||||
return nil, err
|
||||
}
|
||||
signKey := fmt.Sprintf("%x", hmacSign.Sum(nil))
|
||||
|
||||
sha1Sign := sha1.New()
|
||||
_, err = sha1Sign.Write(policyJSON)
|
||||
if err != nil {
|
||||
return serializer.UploadCredential{}, err
|
||||
return nil, err
|
||||
}
|
||||
stringToSign := fmt.Sprintf("%x", sha1Sign.Sum(nil))
|
||||
|
||||
@@ -414,15 +411,15 @@ func (handler Driver) getUploadCredential(ctx context.Context, policy UploadPoli
|
||||
hmacFinalSign := hmac.New(sha1.New, []byte(signKey))
|
||||
_, err = hmacFinalSign.Write([]byte(stringToSign))
|
||||
if err != nil {
|
||||
return serializer.UploadCredential{}, err
|
||||
return nil, err
|
||||
}
|
||||
signature := hmacFinalSign.Sum(nil)
|
||||
|
||||
return serializer.UploadCredential{
|
||||
Policy: policyEncoded,
|
||||
Path: savePath,
|
||||
AccessKey: handler.Policy.AccessKey,
|
||||
Token: fmt.Sprintf("%x", signature),
|
||||
KeyTime: keyTime,
|
||||
return &serializer.UploadCredential{
|
||||
Policy: policyEncoded,
|
||||
Path: savePath,
|
||||
AccessKey: handler.Policy.AccessKey,
|
||||
Credential: fmt.Sprintf("%x", signature),
|
||||
KeyTime: keyTime,
|
||||
}, nil
|
||||
}
|
||||
|
||||
@@ -2,9 +2,9 @@ package driver
|
||||
|
||||
import (
|
||||
"context"
|
||||
"github.com/cloudreve/Cloudreve/v3/pkg/filesystem/fsctx"
|
||||
"github.com/cloudreve/Cloudreve/v3/pkg/filesystem/response"
|
||||
"github.com/cloudreve/Cloudreve/v3/pkg/serializer"
|
||||
"io"
|
||||
"net/url"
|
||||
)
|
||||
|
||||
@@ -12,7 +12,7 @@ import (
|
||||
type Handler interface {
|
||||
// 上传文件, dst为文件存储路径,size 为文件大小。上下文关闭
|
||||
// 时,应取消上传并清理临时文件
|
||||
Put(ctx context.Context, file io.ReadCloser, dst string, size uint64) error
|
||||
Put(ctx context.Context, file fsctx.FileHeader) error
|
||||
|
||||
// 删除一个或多个给定路径的文件,返回删除失败的文件路径列表及错误
|
||||
Delete(ctx context.Context, files []string) ([]string, error)
|
||||
@@ -29,8 +29,11 @@ type Handler interface {
|
||||
// isDownload - 是否直接下载
|
||||
Source(ctx context.Context, path string, url url.URL, ttl int64, isDownload bool, speed int) (string, error)
|
||||
|
||||
// Token 获取有效期为ttl的上传凭证和签名,同时回调会话有效期为sessionTTL
|
||||
Token(ctx context.Context, ttl int64, callbackKey string) (serializer.UploadCredential, error)
|
||||
// Token 获取有效期为ttl的上传凭证和签名
|
||||
Token(ctx context.Context, ttl int64, uploadSession *serializer.UploadSession, file fsctx.FileHeader) (*serializer.UploadCredential, error)
|
||||
|
||||
// CancelToken 取消已经创建的有状态上传凭证
|
||||
CancelToken(ctx context.Context, uploadSession *serializer.UploadSession) error
|
||||
|
||||
// List 递归列取远程端path路径下文件、目录,不包含path本身,
|
||||
// 返回的对象路径以path作为起始根目录.
|
||||
|
||||
@@ -1,38 +0,0 @@
|
||||
package local
|
||||
|
||||
import (
|
||||
"io"
|
||||
)
|
||||
|
||||
// FileStream 用户传来的文件
|
||||
type FileStream struct {
|
||||
File io.ReadCloser
|
||||
Size uint64
|
||||
VirtualPath string
|
||||
Name string
|
||||
MIMEType string
|
||||
}
|
||||
|
||||
func (file FileStream) Read(p []byte) (n int, err error) {
|
||||
return file.File.Read(p)
|
||||
}
|
||||
|
||||
func (file FileStream) GetMIMEType() string {
|
||||
return file.MIMEType
|
||||
}
|
||||
|
||||
func (file FileStream) GetSize() uint64 {
|
||||
return file.Size
|
||||
}
|
||||
|
||||
func (file FileStream) Close() error {
|
||||
return file.File.Close()
|
||||
}
|
||||
|
||||
func (file FileStream) GetFileName() string {
|
||||
return file.Name
|
||||
}
|
||||
|
||||
func (file FileStream) GetVirtualPath() string {
|
||||
return file.VirtualPath
|
||||
}
|
||||
@@ -1,48 +0,0 @@
|
||||
package local
|
||||
|
||||
import (
|
||||
"github.com/stretchr/testify/assert"
|
||||
"io/ioutil"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestFileStream_GetFileName(t *testing.T) {
|
||||
asserts := assert.New(t)
|
||||
file := FileStream{Name: "123"}
|
||||
asserts.Equal("123", file.GetFileName())
|
||||
}
|
||||
|
||||
func TestFileStream_GetMIMEType(t *testing.T) {
|
||||
asserts := assert.New(t)
|
||||
file := FileStream{MIMEType: "123"}
|
||||
asserts.Equal("123", file.GetMIMEType())
|
||||
}
|
||||
|
||||
func TestFileStream_GetSize(t *testing.T) {
|
||||
asserts := assert.New(t)
|
||||
file := FileStream{Size: 123}
|
||||
asserts.Equal(uint64(123), file.GetSize())
|
||||
}
|
||||
|
||||
func TestFileStream_Read(t *testing.T) {
|
||||
asserts := assert.New(t)
|
||||
file := FileStream{
|
||||
File: ioutil.NopCloser(strings.NewReader("123")),
|
||||
}
|
||||
var p = make([]byte, 3)
|
||||
{
|
||||
n, err := file.Read(p)
|
||||
asserts.Equal(3, n)
|
||||
asserts.NoError(err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestFileStream_Close(t *testing.T) {
|
||||
asserts := assert.New(t)
|
||||
file := FileStream{
|
||||
File: ioutil.NopCloser(strings.NewReader("123")),
|
||||
}
|
||||
err := file.Close()
|
||||
asserts.NoError(err)
|
||||
}
|
||||
@@ -12,13 +12,16 @@ import (
|
||||
model "github.com/cloudreve/Cloudreve/v3/models"
|
||||
"github.com/cloudreve/Cloudreve/v3/pkg/auth"
|
||||
"github.com/cloudreve/Cloudreve/v3/pkg/cache"
|
||||
"github.com/cloudreve/Cloudreve/v3/pkg/conf"
|
||||
"github.com/cloudreve/Cloudreve/v3/pkg/filesystem/fsctx"
|
||||
"github.com/cloudreve/Cloudreve/v3/pkg/filesystem/response"
|
||||
"github.com/cloudreve/Cloudreve/v3/pkg/serializer"
|
||||
"github.com/cloudreve/Cloudreve/v3/pkg/util"
|
||||
)
|
||||
|
||||
const (
|
||||
Perm = 0744
|
||||
)
|
||||
|
||||
// Driver 本地策略适配器
|
||||
type Driver struct {
|
||||
Policy *model.Policy
|
||||
@@ -79,29 +82,17 @@ func (handler Driver) Get(ctx context.Context, path string) (response.RSCloser,
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// 开启一个协程,用于请求结束后关闭reader
|
||||
// go closeReader(ctx, file)
|
||||
|
||||
return file, nil
|
||||
}
|
||||
|
||||
// closeReader 用于在请求结束后关闭reader
|
||||
// TODO 让业务代码自己关闭
|
||||
func closeReader(ctx context.Context, closer io.Closer) {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
_ = closer.Close()
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
// Put 将文件流保存到指定目录
|
||||
func (handler Driver) Put(ctx context.Context, file io.ReadCloser, dst string, size uint64) error {
|
||||
func (handler Driver) Put(ctx context.Context, file fsctx.FileHeader) error {
|
||||
defer file.Close()
|
||||
dst = util.RelativePath(filepath.FromSlash(dst))
|
||||
fileInfo := file.Info()
|
||||
dst := util.RelativePath(filepath.FromSlash(fileInfo.SavePath))
|
||||
|
||||
// 如果禁止了 Overwrite,则检查是否有重名冲突
|
||||
if ctx.Value(fsctx.DisableOverwrite) != nil {
|
||||
// 如果非 Overwrite,则检查是否有重名冲突
|
||||
if fileInfo.Mode&fsctx.Overwrite != fsctx.Overwrite {
|
||||
if util.Exists(dst) {
|
||||
util.Log().Warning("物理同名文件已存在或不可用: %s", dst)
|
||||
return errors.New("物理同名文件已存在或不可用")
|
||||
@@ -111,26 +102,73 @@ func (handler Driver) Put(ctx context.Context, file io.ReadCloser, dst string, s
|
||||
// 如果目标目录不存在,创建
|
||||
basePath := filepath.Dir(dst)
|
||||
if !util.Exists(basePath) {
|
||||
err := os.MkdirAll(basePath, 0744)
|
||||
err := os.MkdirAll(basePath, Perm)
|
||||
if err != nil {
|
||||
util.Log().Warning("无法创建目录,%s", err)
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
// 创建目标文件
|
||||
out, err := os.Create(dst)
|
||||
var (
|
||||
out *os.File
|
||||
err error
|
||||
)
|
||||
|
||||
openMode := os.O_CREATE | os.O_RDWR
|
||||
if fileInfo.Mode&fsctx.Append == fsctx.Append {
|
||||
openMode |= os.O_APPEND
|
||||
} else {
|
||||
openMode |= os.O_TRUNC
|
||||
}
|
||||
|
||||
out, err = os.OpenFile(dst, openMode, Perm)
|
||||
if err != nil {
|
||||
util.Log().Warning("无法创建文件,%s", err)
|
||||
util.Log().Warning("无法打开或创建文件,%s", err)
|
||||
return err
|
||||
}
|
||||
defer out.Close()
|
||||
|
||||
if fileInfo.Mode&fsctx.Append == fsctx.Append {
|
||||
stat, err := out.Stat()
|
||||
if err != nil {
|
||||
util.Log().Warning("无法读取文件信息,%s", err)
|
||||
return err
|
||||
}
|
||||
|
||||
if uint64(stat.Size()) < fileInfo.AppendStart {
|
||||
return errors.New("未上传完成的文件分片与预期大小不一致")
|
||||
} else if uint64(stat.Size()) > fileInfo.AppendStart {
|
||||
out.Close()
|
||||
if err := handler.Truncate(ctx, dst, fileInfo.AppendStart); err != nil {
|
||||
return fmt.Errorf("覆盖分片时发生错误: %w", err)
|
||||
}
|
||||
|
||||
out, err = os.OpenFile(dst, openMode, Perm)
|
||||
defer out.Close()
|
||||
if err != nil {
|
||||
util.Log().Warning("无法打开或创建文件,%s", err)
|
||||
return err
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// 写入文件内容
|
||||
_, err = io.Copy(out, file)
|
||||
return err
|
||||
}
|
||||
|
||||
func (handler Driver) Truncate(ctx context.Context, src string, size uint64) error {
|
||||
util.Log().Warning("截断文件 [%s] 至 [%d]", src, size)
|
||||
out, err := os.OpenFile(src, os.O_WRONLY, Perm)
|
||||
if err != nil {
|
||||
util.Log().Warning("无法打开文件,%s", err)
|
||||
return err
|
||||
}
|
||||
|
||||
defer out.Close()
|
||||
return out.Truncate(int64(size))
|
||||
}
|
||||
|
||||
// Delete 删除一个或多个文件,
|
||||
// 返回未删除的文件,及遇到的最后一个错误
|
||||
func (handler Driver) Delete(ctx context.Context, files []string) ([]string, error) {
|
||||
@@ -149,7 +187,7 @@ func (handler Driver) Delete(ctx context.Context, files []string) ([]string, err
|
||||
}
|
||||
|
||||
// 尝试删除文件的缩略图(如果有)
|
||||
_ = os.Remove(util.RelativePath(value + conf.ThumbConfig.FileSuffix))
|
||||
_ = os.Remove(util.RelativePath(value + model.GetSettingByNameWithDefault("thumb_file_suffix", "._thumb")))
|
||||
}
|
||||
|
||||
return deleteFailed, retErr
|
||||
@@ -157,7 +195,7 @@ func (handler Driver) Delete(ctx context.Context, files []string) ([]string, err
|
||||
|
||||
// Thumb 获取文件缩略图
|
||||
func (handler Driver) Thumb(ctx context.Context, path string) (*response.ContentResponse, error) {
|
||||
file, err := handler.Get(ctx, path+conf.ThumbConfig.FileSuffix)
|
||||
file, err := handler.Get(ctx, path+model.GetSettingByNameWithDefault("thumb_file_suffix", "._thumb"))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@@ -227,6 +265,18 @@ func (handler Driver) Source(
|
||||
}
|
||||
|
||||
// Token 获取上传策略和认证Token,本地策略直接返回空值
|
||||
func (handler Driver) Token(ctx context.Context, ttl int64, key string) (serializer.UploadCredential, error) {
|
||||
return serializer.UploadCredential{}, nil
|
||||
func (handler Driver) Token(ctx context.Context, ttl int64, uploadSession *serializer.UploadSession, file fsctx.FileHeader) (*serializer.UploadCredential, error) {
|
||||
if util.Exists(uploadSession.SavePath) {
|
||||
return nil, errors.New("placeholder file already exist")
|
||||
}
|
||||
|
||||
return &serializer.UploadCredential{
|
||||
SessionID: uploadSession.Key,
|
||||
ChunkSize: handler.Policy.OptionsSerialized.ChunkSize,
|
||||
}, nil
|
||||
}
|
||||
|
||||
// 取消上传凭证
|
||||
func (handler Driver) CancelToken(ctx context.Context, uploadSession *serializer.UploadSession) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -4,13 +4,12 @@ import (
|
||||
"context"
|
||||
model "github.com/cloudreve/Cloudreve/v3/models"
|
||||
"github.com/cloudreve/Cloudreve/v3/pkg/auth"
|
||||
"github.com/cloudreve/Cloudreve/v3/pkg/conf"
|
||||
"github.com/cloudreve/Cloudreve/v3/pkg/filesystem/fsctx"
|
||||
"github.com/cloudreve/Cloudreve/v3/pkg/serializer"
|
||||
"github.com/cloudreve/Cloudreve/v3/pkg/util"
|
||||
"github.com/jinzhu/gorm"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"io"
|
||||
"io/ioutil"
|
||||
"net/url"
|
||||
"os"
|
||||
"strings"
|
||||
@@ -20,52 +19,74 @@ import (
|
||||
func TestHandler_Put(t *testing.T) {
|
||||
asserts := assert.New(t)
|
||||
handler := Driver{}
|
||||
ctx := context.WithValue(context.Background(), fsctx.DisableOverwrite, true)
|
||||
os.Remove(util.RelativePath("test/test/txt"))
|
||||
|
||||
defer func() {
|
||||
os.Remove(util.RelativePath("TestHandler_Put.txt"))
|
||||
os.Remove(util.RelativePath("inner/TestHandler_Put.txt"))
|
||||
}()
|
||||
|
||||
testCases := []struct {
|
||||
file io.ReadCloser
|
||||
dst string
|
||||
err bool
|
||||
file fsctx.FileHeader
|
||||
errContains string
|
||||
}{
|
||||
{
|
||||
file: ioutil.NopCloser(strings.NewReader("test input file")),
|
||||
dst: "test/test/txt",
|
||||
err: false,
|
||||
},
|
||||
{
|
||||
file: ioutil.NopCloser(strings.NewReader("test input file")),
|
||||
dst: "test/test/txt",
|
||||
err: true,
|
||||
},
|
||||
{
|
||||
file: ioutil.NopCloser(strings.NewReader("test input file")),
|
||||
dst: "/notexist:/S.TXT",
|
||||
err: true,
|
||||
},
|
||||
{&fsctx.FileStream{
|
||||
SavePath: "TestHandler_Put.txt",
|
||||
File: io.NopCloser(strings.NewReader("")),
|
||||
}, ""},
|
||||
{&fsctx.FileStream{
|
||||
SavePath: "TestHandler_Put.txt",
|
||||
File: io.NopCloser(strings.NewReader("")),
|
||||
}, "物理同名文件已存在或不可用"},
|
||||
{&fsctx.FileStream{
|
||||
SavePath: "inner/TestHandler_Put.txt",
|
||||
File: io.NopCloser(strings.NewReader("")),
|
||||
}, ""},
|
||||
{&fsctx.FileStream{
|
||||
Mode: fsctx.Append | fsctx.Overwrite,
|
||||
SavePath: "inner/TestHandler_Put.txt",
|
||||
File: io.NopCloser(strings.NewReader("123")),
|
||||
}, ""},
|
||||
{&fsctx.FileStream{
|
||||
AppendStart: 10,
|
||||
Mode: fsctx.Append | fsctx.Overwrite,
|
||||
SavePath: "inner/TestHandler_Put.txt",
|
||||
File: io.NopCloser(strings.NewReader("123")),
|
||||
}, "未上传完成的文件分片与预期大小不一致"},
|
||||
{&fsctx.FileStream{
|
||||
Mode: fsctx.Append | fsctx.Overwrite,
|
||||
SavePath: "inner/TestHandler_Put.txt",
|
||||
File: io.NopCloser(strings.NewReader("123")),
|
||||
}, ""},
|
||||
}
|
||||
|
||||
for _, testCase := range testCases {
|
||||
err := handler.Put(ctx, testCase.file, testCase.dst, 15)
|
||||
if testCase.err {
|
||||
err := handler.Put(context.Background(), testCase.file)
|
||||
if testCase.errContains != "" {
|
||||
asserts.Error(err)
|
||||
asserts.Contains(err.Error(), testCase.errContains)
|
||||
} else {
|
||||
asserts.NoError(err)
|
||||
asserts.True(util.Exists(util.RelativePath(testCase.dst)))
|
||||
asserts.True(util.Exists(util.RelativePath(testCase.file.Info().SavePath)))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestDriver_TruncateFailed(t *testing.T) {
|
||||
a := assert.New(t)
|
||||
h := Driver{}
|
||||
a.Error(h.Truncate(context.Background(), "TestDriver_TruncateFailed", 0))
|
||||
}
|
||||
|
||||
func TestHandler_Delete(t *testing.T) {
|
||||
asserts := assert.New(t)
|
||||
handler := Driver{}
|
||||
ctx := context.Background()
|
||||
filePath := util.RelativePath("test.file")
|
||||
filePath := util.RelativePath("TestHandler_Delete.file")
|
||||
|
||||
file, err := os.Create(filePath)
|
||||
asserts.NoError(err)
|
||||
_ = file.Close()
|
||||
list, err := handler.Delete(ctx, []string{"test.file"})
|
||||
list, err := handler.Delete(ctx, []string{"TestHandler_Delete.file"})
|
||||
asserts.Equal([]string{}, list)
|
||||
asserts.NoError(err)
|
||||
|
||||
@@ -73,7 +94,7 @@ func TestHandler_Delete(t *testing.T) {
|
||||
_ = file.Close()
|
||||
file, _ = os.OpenFile(filePath, os.O_RDWR, os.FileMode(0))
|
||||
asserts.NoError(err)
|
||||
list, err = handler.Delete(ctx, []string{"test.file", "test.notexist"})
|
||||
list, err = handler.Delete(ctx, []string{"TestHandler_Delete.file", "test.notexist"})
|
||||
file.Close()
|
||||
asserts.Equal([]string{}, list)
|
||||
asserts.NoError(err)
|
||||
@@ -84,7 +105,7 @@ func TestHandler_Delete(t *testing.T) {
|
||||
|
||||
file, err = os.Create(filePath)
|
||||
asserts.NoError(err)
|
||||
list, err = handler.Delete(ctx, []string{"test.file"})
|
||||
list, err = handler.Delete(ctx, []string{"TestHandler_Delete.file"})
|
||||
_ = file.Close()
|
||||
asserts.Equal([]string{}, list)
|
||||
asserts.NoError(err)
|
||||
@@ -116,7 +137,7 @@ func TestHandler_Thumb(t *testing.T) {
|
||||
asserts := assert.New(t)
|
||||
handler := Driver{}
|
||||
ctx := context.Background()
|
||||
file, err := os.Create(util.RelativePath("TestHandler_Thumb" + conf.ThumbConfig.FileSuffix))
|
||||
file, err := os.Create(util.RelativePath("TestHandler_Thumb._thumb"))
|
||||
asserts.NoError(err)
|
||||
file.Close()
|
||||
|
||||
@@ -160,6 +181,25 @@ func TestHandler_Source(t *testing.T) {
|
||||
asserts.Contains(sourceURL, "https://cloudreve.org")
|
||||
}
|
||||
|
||||
// 下载
|
||||
{
|
||||
file := model.File{
|
||||
Model: gorm.Model{
|
||||
ID: 1,
|
||||
},
|
||||
Name: "test.jpg",
|
||||
}
|
||||
ctx := context.WithValue(ctx, fsctx.FileModelCtx, file)
|
||||
baseURL, err := url.Parse("https://cloudreve.org")
|
||||
asserts.NoError(err)
|
||||
sourceURL, err := handler.Source(ctx, "", *baseURL, 0, true, 0)
|
||||
asserts.NoError(err)
|
||||
asserts.NotEmpty(sourceURL)
|
||||
asserts.Contains(sourceURL, "sign=")
|
||||
asserts.Contains(sourceURL, "download")
|
||||
asserts.Contains(sourceURL, "https://cloudreve.org")
|
||||
}
|
||||
|
||||
// 无法获取上下文
|
||||
{
|
||||
baseURL, err := url.Parse("https://cloudreve.org")
|
||||
@@ -241,10 +281,29 @@ func TestHandler_GetDownloadURL(t *testing.T) {
|
||||
|
||||
func TestHandler_Token(t *testing.T) {
|
||||
asserts := assert.New(t)
|
||||
handler := Driver{}
|
||||
handler := Driver{
|
||||
Policy: &model.Policy{},
|
||||
}
|
||||
ctx := context.Background()
|
||||
_, err := handler.Token(ctx, 10, "123")
|
||||
upSession := &serializer.UploadSession{SavePath: "TestHandler_Token"}
|
||||
_, err := handler.Token(ctx, 10, upSession, &fsctx.FileStream{})
|
||||
asserts.NoError(err)
|
||||
|
||||
file, _ := os.Create("TestHandler_Token")
|
||||
defer func() {
|
||||
file.Close()
|
||||
os.Remove("TestHandler_Token")
|
||||
}()
|
||||
|
||||
_, err = handler.Token(ctx, 10, upSession, &fsctx.FileStream{})
|
||||
asserts.Error(err)
|
||||
asserts.Contains(err.Error(), "already exist")
|
||||
}
|
||||
|
||||
func TestDriver_CancelToken(t *testing.T) {
|
||||
a := assert.New(t)
|
||||
handler := Driver{}
|
||||
a.NoError(handler.CancelToken(context.Background(), &serializer.UploadSession{}))
|
||||
}
|
||||
|
||||
func TestDriver_List(t *testing.T) {
|
||||
|
||||
@@ -1,7 +1,6 @@
|
||||
package onedrive
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
@@ -18,7 +17,10 @@ import (
|
||||
|
||||
model "github.com/cloudreve/Cloudreve/v3/models"
|
||||
"github.com/cloudreve/Cloudreve/v3/pkg/cache"
|
||||
"github.com/cloudreve/Cloudreve/v3/pkg/filesystem/chunk"
|
||||
"github.com/cloudreve/Cloudreve/v3/pkg/filesystem/chunk/backoff"
|
||||
"github.com/cloudreve/Cloudreve/v3/pkg/filesystem/fsctx"
|
||||
"github.com/cloudreve/Cloudreve/v3/pkg/mq"
|
||||
"github.com/cloudreve/Cloudreve/v3/pkg/request"
|
||||
"github.com/cloudreve/Cloudreve/v3/pkg/util"
|
||||
)
|
||||
@@ -29,7 +31,8 @@ const (
|
||||
// ChunkSize 服务端中转分片上传分片大小
|
||||
ChunkSize uint64 = 10 * 1024 * 1024
|
||||
// ListRetry 列取请求重试次数
|
||||
ListRetry = 1
|
||||
ListRetry = 1
|
||||
chunkRetrySleep = time.Second * 5
|
||||
)
|
||||
|
||||
// GetSourcePath 获取文件的绝对路径
|
||||
@@ -219,28 +222,21 @@ func (client *Client) GetUploadSessionStatus(ctx context.Context, uploadURL stri
|
||||
}
|
||||
|
||||
// UploadChunk 上传分片
|
||||
func (client *Client) UploadChunk(ctx context.Context, uploadURL string, chunk *Chunk) (*UploadSessionResponse, error) {
|
||||
func (client *Client) UploadChunk(ctx context.Context, uploadURL string, content io.Reader, current *chunk.ChunkGroup) (*UploadSessionResponse, error) {
|
||||
res, err := client.request(
|
||||
ctx, "PUT", uploadURL, bytes.NewReader(chunk.Data[0:chunk.ChunkSize]),
|
||||
request.WithContentLength(int64(chunk.ChunkSize)),
|
||||
ctx, "PUT", uploadURL, content,
|
||||
request.WithContentLength(current.Length()),
|
||||
request.WithHeader(http.Header{
|
||||
"Content-Range": {fmt.Sprintf("bytes %d-%d/%d", chunk.Offset, chunk.Offset+chunk.ChunkSize-1, chunk.Total)},
|
||||
"Content-Range": {current.RangeHeader()},
|
||||
}),
|
||||
request.WithoutHeader([]string{"Authorization", "Content-Type"}),
|
||||
request.WithTimeout(time.Duration(300)*time.Second),
|
||||
)
|
||||
if err != nil {
|
||||
// 如果重试次数小于限制,5秒后重试
|
||||
if chunk.Retried < model.GetIntSetting("onedrive_chunk_retries", 1) {
|
||||
chunk.Retried++
|
||||
util.Log().Debug("分片偏移%d上传失败[%s],5秒钟后重试", chunk.Offset, err)
|
||||
time.Sleep(time.Duration(5) * time.Second)
|
||||
return client.UploadChunk(ctx, uploadURL, chunk)
|
||||
}
|
||||
return nil, err
|
||||
return nil, fmt.Errorf("failed to upload OneDrive chunk #%d: %w", current.Index(), err)
|
||||
}
|
||||
|
||||
if chunk.IsLast() {
|
||||
if current.IsLast() {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
@@ -257,13 +253,17 @@ func (client *Client) UploadChunk(ctx context.Context, uploadURL string, chunk *
|
||||
}
|
||||
|
||||
// Upload 上传文件
|
||||
func (client *Client) Upload(ctx context.Context, dst string, size int, file io.Reader) error {
|
||||
func (client *Client) Upload(ctx context.Context, file fsctx.FileHeader) error {
|
||||
fileInfo := file.Info()
|
||||
// 决定是否覆盖文件
|
||||
overwrite := "replace"
|
||||
if ctx.Value(fsctx.DisableOverwrite) != nil {
|
||||
overwrite = "fail"
|
||||
overwrite := "fail"
|
||||
if fileInfo.Mode&fsctx.Overwrite == fsctx.Overwrite {
|
||||
overwrite = "replace"
|
||||
}
|
||||
|
||||
size := int(fileInfo.Size)
|
||||
dst := fileInfo.SavePath
|
||||
|
||||
// 小文件,使用简单上传接口上传
|
||||
if size <= int(SmallFileSize) {
|
||||
_, err := client.SimpleUpload(ctx, dst, file, int64(size), WithConflictBehavior(overwrite))
|
||||
@@ -277,46 +277,24 @@ func (client *Client) Upload(ctx context.Context, dst string, size int, file io.
|
||||
return err
|
||||
}
|
||||
|
||||
offset := 0
|
||||
chunkNum := size / int(ChunkSize)
|
||||
if size%int(ChunkSize) != 0 {
|
||||
chunkNum++
|
||||
// Initial chunk groups
|
||||
chunks := chunk.NewChunkGroup(file, client.Policy.OptionsSerialized.ChunkSize, &backoff.ConstantBackoff{
|
||||
Max: model.GetIntSetting("chunk_retries", 5),
|
||||
Sleep: chunkRetrySleep,
|
||||
}, model.IsTrueVal(model.GetSettingByName("use_temp_chunk_buffer")))
|
||||
|
||||
uploadFunc := func(current *chunk.ChunkGroup, content io.Reader) error {
|
||||
_, err := client.UploadChunk(ctx, uploadURL, content, current)
|
||||
return err
|
||||
}
|
||||
|
||||
chunkData := make([]byte, ChunkSize)
|
||||
|
||||
for i := 0; i < chunkNum; i++ {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
util.Log().Debug("OneDrive 客户端取消")
|
||||
return ErrClientCanceled
|
||||
default:
|
||||
// 分块
|
||||
chunkSize := int(ChunkSize)
|
||||
if size-offset < chunkSize {
|
||||
chunkSize = size - offset
|
||||
}
|
||||
|
||||
// 因为后面需要错误重试,这里要把分片内容读到内存中
|
||||
chunkContent := chunkData[:chunkSize]
|
||||
_, err := io.ReadFull(file, chunkContent)
|
||||
|
||||
chunk := Chunk{
|
||||
Offset: offset,
|
||||
ChunkSize: chunkSize,
|
||||
Total: size,
|
||||
Data: chunkContent,
|
||||
}
|
||||
|
||||
// 上传
|
||||
_, err = client.UploadChunk(ctx, uploadURL, &chunk)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
offset += chunkSize
|
||||
// upload chunks
|
||||
for chunks.Next() {
|
||||
if err := chunks.Process(uploadFunc); err != nil {
|
||||
return fmt.Errorf("failed to upload chunk #%d: %w", chunks.Index(), err)
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -345,16 +323,6 @@ func (client *Client) SimpleUpload(ctx context.Context, dst string, body io.Read
|
||||
request.WithTimeout(time.Duration(150)*time.Second),
|
||||
)
|
||||
if err != nil {
|
||||
retried := 0
|
||||
if v, ok := ctx.Value(fsctx.RetryCtx).(int); ok {
|
||||
retried = v
|
||||
}
|
||||
if retried < model.GetIntSetting("onedrive_chunk_retries", 1) {
|
||||
retried++
|
||||
util.Log().Debug("文件[%s]上传失败[%s],5秒钟后重试", dst, err)
|
||||
time.Sleep(time.Duration(5) * time.Second)
|
||||
return client.SimpleUpload(context.WithValue(ctx, fsctx.RetryCtx, retried), dst, body, size, opts...)
|
||||
}
|
||||
return nil, err
|
||||
}
|
||||
|
||||
@@ -483,9 +451,9 @@ func (client *Client) GetThumbURL(ctx context.Context, dst string, w, h uint) (s
|
||||
// MonitorUpload 监控客户端分片上传进度
|
||||
func (client *Client) MonitorUpload(uploadURL, callbackKey, path string, size uint64, ttl int64) {
|
||||
// 回调完成通知chan
|
||||
callbackChan := make(chan bool)
|
||||
callbackSignal.Store(callbackKey, callbackChan)
|
||||
defer callbackSignal.Delete(callbackKey)
|
||||
callbackChan := mq.GlobalMQ.Subscribe(callbackKey, 1)
|
||||
defer mq.GlobalMQ.Unsubscribe(callbackKey, callbackChan)
|
||||
|
||||
timeout := model.GetIntSetting("onedrive_monitor_timeout", 600)
|
||||
interval := model.GetIntSetting("onedrive_callback_check", 20)
|
||||
|
||||
@@ -510,16 +478,16 @@ func (client *Client) MonitorUpload(uploadURL, callbackKey, path string, size ui
|
||||
if resErr, ok := err.(*RespError); ok {
|
||||
if resErr.APIError.Code == "itemNotFound" {
|
||||
util.Log().Debug("上传会话已完成,稍后检查回调")
|
||||
time.Sleep(time.Duration(interval) * time.Second)
|
||||
util.Log().Debug("开始检查回调")
|
||||
_, ok := cache.Get("callback_" + callbackKey)
|
||||
if ok {
|
||||
select {
|
||||
case <-time.After(time.Duration(interval) * time.Second):
|
||||
util.Log().Warning("未发送回调,删除文件")
|
||||
cache.Deletes([]string{callbackKey}, "callback_")
|
||||
_, err = client.Delete(context.Background(), []string{path})
|
||||
if err != nil {
|
||||
util.Log().Warning("无法删除未回调的文件,%s", err)
|
||||
}
|
||||
case <-callbackChan:
|
||||
util.Log().Debug("客户端完成回调")
|
||||
}
|
||||
return
|
||||
}
|
||||
@@ -556,15 +524,6 @@ func (client *Client) MonitorUpload(uploadURL, callbackKey, path string, size ui
|
||||
}
|
||||
}
|
||||
|
||||
// FinishCallback 向Monitor发送回调结束信号
|
||||
func FinishCallback(key string) {
|
||||
if signal, ok := callbackSignal.Load(key); ok {
|
||||
if signalChan, ok := signal.(chan bool); ok {
|
||||
close(signalChan)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func sysError(err error) *RespError {
|
||||
return &RespError{APIError: APIError{
|
||||
Code: "system",
|
||||
|
||||
@@ -4,6 +4,11 @@ import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"github.com/cloudreve/Cloudreve/v3/pkg/filesystem/chunk"
|
||||
"github.com/cloudreve/Cloudreve/v3/pkg/filesystem/chunk/backoff"
|
||||
"github.com/cloudreve/Cloudreve/v3/pkg/filesystem/fsctx"
|
||||
"github.com/cloudreve/Cloudreve/v3/pkg/mq"
|
||||
"io"
|
||||
"io/ioutil"
|
||||
"net/http"
|
||||
"strings"
|
||||
@@ -12,7 +17,6 @@ import (
|
||||
|
||||
model "github.com/cloudreve/Cloudreve/v3/models"
|
||||
"github.com/cloudreve/Cloudreve/v3/pkg/cache"
|
||||
"github.com/cloudreve/Cloudreve/v3/pkg/filesystem/fsctx"
|
||||
"github.com/cloudreve/Cloudreve/v3/pkg/request"
|
||||
"github.com/stretchr/testify/assert"
|
||||
testMock "github.com/stretchr/testify/mock"
|
||||
@@ -307,6 +311,31 @@ func TestClient_Meta(t *testing.T) {
|
||||
asserts.NotNil(res)
|
||||
asserts.Equal("123321", res.Name)
|
||||
}
|
||||
|
||||
// 返回正常, 使用资源id
|
||||
{
|
||||
client.Credential.ExpiresIn = time.Now().Add(time.Duration(100) * time.Hour).Unix()
|
||||
clientMock := ClientMock{}
|
||||
clientMock.On(
|
||||
"Request",
|
||||
"GET",
|
||||
testMock.Anything,
|
||||
testMock.Anything,
|
||||
testMock.Anything,
|
||||
).Return(&request.Response{
|
||||
Err: nil,
|
||||
Response: &http.Response{
|
||||
StatusCode: 200,
|
||||
Body: ioutil.NopCloser(strings.NewReader(`{"name":"123321"}`)),
|
||||
},
|
||||
})
|
||||
client.Request = clientMock
|
||||
res, err := client.Meta(context.Background(), "123321", "123")
|
||||
clientMock.AssertExpectations(t)
|
||||
asserts.NoError(err)
|
||||
asserts.NotNil(res)
|
||||
asserts.Equal("123321", res.Name)
|
||||
}
|
||||
}
|
||||
|
||||
func TestClient_CreateUploadSession(t *testing.T) {
|
||||
@@ -442,9 +471,11 @@ func TestClient_UploadChunk(t *testing.T) {
|
||||
client, _ := NewClient(&model.Policy{})
|
||||
client.Credential.AccessToken = "AccessToken"
|
||||
client.Credential.ExpiresIn = time.Now().Add(time.Duration(100) * time.Hour).Unix()
|
||||
cg := chunk.NewChunkGroup(&fsctx.FileStream{Size: 15}, 10, &backoff.ConstantBackoff{}, false)
|
||||
|
||||
// 非最后分片,正常
|
||||
{
|
||||
cg.Next()
|
||||
client.Credential.ExpiresIn = time.Now().Add(time.Duration(100) * time.Hour).Unix()
|
||||
clientMock := ClientMock{}
|
||||
clientMock.On(
|
||||
@@ -453,6 +484,10 @@ func TestClient_UploadChunk(t *testing.T) {
|
||||
"http://dev.com",
|
||||
testMock.Anything,
|
||||
testMock.Anything,
|
||||
testMock.Anything,
|
||||
testMock.Anything,
|
||||
testMock.Anything,
|
||||
testMock.Anything,
|
||||
).Return(&request.Response{
|
||||
Err: nil,
|
||||
Response: &http.Response{
|
||||
@@ -461,13 +496,7 @@ func TestClient_UploadChunk(t *testing.T) {
|
||||
},
|
||||
})
|
||||
client.Request = clientMock
|
||||
res, err := client.UploadChunk(context.Background(), "http://dev.com", &Chunk{
|
||||
Offset: 0,
|
||||
ChunkSize: 10,
|
||||
Total: 100,
|
||||
Retried: 0,
|
||||
Data: []byte("12313121231312"),
|
||||
})
|
||||
res, err := client.UploadChunk(context.Background(), "http://dev.com", strings.NewReader("1234567890"), cg)
|
||||
clientMock.AssertExpectations(t)
|
||||
asserts.NoError(err)
|
||||
asserts.Equal("http://dev.com/2", res.UploadURL)
|
||||
@@ -491,13 +520,7 @@ func TestClient_UploadChunk(t *testing.T) {
|
||||
},
|
||||
})
|
||||
client.Request = clientMock
|
||||
res, err := client.UploadChunk(context.Background(), "http://dev.com", &Chunk{
|
||||
Offset: 0,
|
||||
ChunkSize: 10,
|
||||
Total: 100,
|
||||
Retried: 0,
|
||||
Data: []byte("12313112313122"),
|
||||
})
|
||||
res, err := client.UploadChunk(context.Background(), "http://dev.com", strings.NewReader("1234567890"), cg)
|
||||
clientMock.AssertExpectations(t)
|
||||
asserts.Error(err)
|
||||
asserts.Nil(res)
|
||||
@@ -505,6 +528,7 @@ func TestClient_UploadChunk(t *testing.T) {
|
||||
|
||||
// 最后分片,正常
|
||||
{
|
||||
cg.Next()
|
||||
client.Credential.ExpiresIn = time.Now().Add(time.Duration(100) * time.Hour).Unix()
|
||||
clientMock := ClientMock{}
|
||||
clientMock.On(
|
||||
@@ -521,53 +545,26 @@ func TestClient_UploadChunk(t *testing.T) {
|
||||
},
|
||||
})
|
||||
client.Request = clientMock
|
||||
res, err := client.UploadChunk(context.Background(), "http://dev.com", &Chunk{
|
||||
Offset: 95,
|
||||
ChunkSize: 5,
|
||||
Total: 100,
|
||||
Retried: 0,
|
||||
Data: []byte("1231312"),
|
||||
})
|
||||
res, err := client.UploadChunk(context.Background(), "http://dev.com", strings.NewReader("12345"), cg)
|
||||
clientMock.AssertExpectations(t)
|
||||
asserts.NoError(err)
|
||||
asserts.Nil(res)
|
||||
}
|
||||
|
||||
// 最后分片,第一次失败,重试后成功
|
||||
// 最后分片,失败
|
||||
{
|
||||
cache.Set("setting_onedrive_chunk_retries", "1", 0)
|
||||
cache.Set("setting_chunk_retries", "1", 0)
|
||||
client.Credential.ExpiresIn = 0
|
||||
go func() {
|
||||
time.Sleep(time.Duration(2) * time.Second)
|
||||
client.Credential.ExpiresIn = time.Now().Add(time.Duration(100) * time.Hour).Unix()
|
||||
}()
|
||||
clientMock := ClientMock{}
|
||||
clientMock.On(
|
||||
"Request",
|
||||
"PUT",
|
||||
"http://dev.com",
|
||||
testMock.Anything,
|
||||
testMock.Anything,
|
||||
).Return(&request.Response{
|
||||
Err: nil,
|
||||
Response: &http.Response{
|
||||
StatusCode: 200,
|
||||
Body: ioutil.NopCloser(strings.NewReader(`???`)),
|
||||
},
|
||||
})
|
||||
client.Request = clientMock
|
||||
chunk := &Chunk{
|
||||
Offset: 95,
|
||||
ChunkSize: 5,
|
||||
Total: 100,
|
||||
Retried: 0,
|
||||
Data: []byte("1231312"),
|
||||
}
|
||||
res, err := client.UploadChunk(context.Background(), "http://dev.com", chunk)
|
||||
res, err := client.UploadChunk(context.Background(), "http://dev.com", strings.NewReader("12345"), cg)
|
||||
clientMock.AssertExpectations(t)
|
||||
asserts.NoError(err)
|
||||
asserts.Error(err)
|
||||
asserts.Nil(res)
|
||||
asserts.EqualValues(1, chunk.Retried)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -576,39 +573,18 @@ func TestClient_Upload(t *testing.T) {
|
||||
client, _ := NewClient(&model.Policy{})
|
||||
client.Credential.AccessToken = "AccessToken"
|
||||
client.Credential.ExpiresIn = time.Now().Add(time.Duration(100) * time.Hour).Unix()
|
||||
ctx := context.WithValue(context.Background(), fsctx.DisableOverwrite, true)
|
||||
ctx := context.Background()
|
||||
cache.Set("setting_chunk_retries", "1", 0)
|
||||
cache.Set("setting_use_temp_chunk_buffer", "false", 0)
|
||||
|
||||
// 小文件,简单上传,失败
|
||||
{
|
||||
client.Credential.ExpiresIn = 0
|
||||
err := client.Upload(ctx, "123.jpg", 3, strings.NewReader("123"))
|
||||
asserts.Error(err)
|
||||
}
|
||||
|
||||
// 上下文取消
|
||||
{
|
||||
client.Credential.ExpiresIn = time.Now().Add(time.Duration(100) * time.Hour).Unix()
|
||||
clientMock := ClientMock{}
|
||||
clientMock.On(
|
||||
"Request",
|
||||
"POST",
|
||||
testMock.Anything,
|
||||
testMock.Anything,
|
||||
testMock.Anything,
|
||||
).Return(&request.Response{
|
||||
Err: nil,
|
||||
Response: &http.Response{
|
||||
StatusCode: 200,
|
||||
Body: ioutil.NopCloser(strings.NewReader(`{"uploadUrl":"123321"}`)),
|
||||
},
|
||||
err := client.Upload(ctx, &fsctx.FileStream{
|
||||
Size: 5,
|
||||
File: io.NopCloser(strings.NewReader("12345")),
|
||||
})
|
||||
client.Request = clientMock
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
cancel()
|
||||
err := client.Upload(ctx, "123.jpg", 15*1024*1024, strings.NewReader("123"))
|
||||
clientMock.AssertExpectations(t)
|
||||
asserts.Error(err)
|
||||
asserts.Equal(ErrClientCanceled, err)
|
||||
}
|
||||
|
||||
// 无法创建分片会话
|
||||
@@ -629,11 +605,54 @@ func TestClient_Upload(t *testing.T) {
|
||||
},
|
||||
})
|
||||
client.Request = clientMock
|
||||
err := client.Upload(context.Background(), "123.jpg", 15*1024*1024, strings.NewReader("123"))
|
||||
err := client.Upload(context.Background(), &fsctx.FileStream{
|
||||
Size: SmallFileSize + 1,
|
||||
File: io.NopCloser(strings.NewReader("12345")),
|
||||
})
|
||||
clientMock.AssertExpectations(t)
|
||||
asserts.Error(err)
|
||||
}
|
||||
|
||||
// 分片上传失败
|
||||
{
|
||||
client.Credential.ExpiresIn = time.Now().Add(time.Duration(100) * time.Hour).Unix()
|
||||
clientMock := ClientMock{}
|
||||
clientMock.On(
|
||||
"Request",
|
||||
"POST",
|
||||
testMock.Anything,
|
||||
testMock.Anything,
|
||||
testMock.Anything,
|
||||
).Return(&request.Response{
|
||||
Err: nil,
|
||||
Response: &http.Response{
|
||||
StatusCode: 200,
|
||||
Body: ioutil.NopCloser(strings.NewReader(`{"uploadUrl":"123321"}`)),
|
||||
},
|
||||
})
|
||||
clientMock.On(
|
||||
"Request",
|
||||
"PUT",
|
||||
testMock.Anything,
|
||||
testMock.Anything,
|
||||
testMock.Anything,
|
||||
).Return(&request.Response{
|
||||
Err: nil,
|
||||
Response: &http.Response{
|
||||
StatusCode: 400,
|
||||
Body: ioutil.NopCloser(strings.NewReader(`{"uploadUrl":"123321"}`)),
|
||||
},
|
||||
})
|
||||
client.Request = clientMock
|
||||
err := client.Upload(context.Background(), &fsctx.FileStream{
|
||||
Size: SmallFileSize + 1,
|
||||
File: io.NopCloser(strings.NewReader("12345")),
|
||||
})
|
||||
clientMock.AssertExpectations(t)
|
||||
asserts.Error(err)
|
||||
asserts.Contains(err.Error(), "failed to upload chunk")
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
func TestClient_SimpleUpload(t *testing.T) {
|
||||
@@ -641,9 +660,9 @@ func TestClient_SimpleUpload(t *testing.T) {
|
||||
client, _ := NewClient(&model.Policy{})
|
||||
client.Credential.AccessToken = "AccessToken"
|
||||
client.Credential.ExpiresIn = time.Now().Add(time.Duration(100) * time.Hour).Unix()
|
||||
cache.Set("setting_onedrive_chunk_retries", "1", 0)
|
||||
cache.Set("setting_chunk_retries", "1", 0)
|
||||
|
||||
// 请求失败,并重试
|
||||
// 请求失败
|
||||
{
|
||||
client.Credential.ExpiresIn = 0
|
||||
res, err := client.SimpleUpload(context.Background(), "123.jpg", strings.NewReader("123"), 3)
|
||||
@@ -651,7 +670,6 @@ func TestClient_SimpleUpload(t *testing.T) {
|
||||
asserts.Nil(res)
|
||||
}
|
||||
|
||||
cache.Set("setting_onedrive_chunk_retries", "0", 0)
|
||||
// 返回未知响应
|
||||
{
|
||||
client.Credential.ExpiresIn = time.Now().Add(time.Duration(100) * time.Hour).Unix()
|
||||
@@ -988,7 +1006,7 @@ func TestClient_MonitorUpload(t *testing.T) {
|
||||
asserts.NotPanics(func() {
|
||||
go func() {
|
||||
time.Sleep(time.Duration(1) * time.Second)
|
||||
FinishCallback("key")
|
||||
mq.GlobalMQ.Publish("key", mq.Message{})
|
||||
}()
|
||||
client.MonitorUpload("url", "key", "path", 10, 10)
|
||||
})
|
||||
|
||||
@@ -4,7 +4,6 @@ import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/url"
|
||||
"path"
|
||||
"path/filepath"
|
||||
@@ -31,6 +30,10 @@ type Driver struct {
|
||||
// NewDriver 从存储策略初始化新的Driver实例
|
||||
func NewDriver(policy *model.Policy) (driver.Handler, error) {
|
||||
client, err := NewClient(policy)
|
||||
if policy.OptionsSerialized.ChunkSize == 0 {
|
||||
policy.OptionsSerialized.ChunkSize = 50 << 20 // 50MB
|
||||
}
|
||||
|
||||
return Driver{
|
||||
Policy: policy,
|
||||
Client: client,
|
||||
@@ -121,9 +124,10 @@ func (handler Driver) Get(ctx context.Context, path string) (response.RSCloser,
|
||||
}
|
||||
|
||||
// Put 将文件流保存到指定目录
|
||||
func (handler Driver) Put(ctx context.Context, file io.ReadCloser, dst string, size uint64) error {
|
||||
func (handler Driver) Put(ctx context.Context, file fsctx.FileHeader) error {
|
||||
defer file.Close()
|
||||
return handler.Client.Upload(ctx, dst, int(size), file)
|
||||
|
||||
return handler.Client.Upload(ctx, file)
|
||||
}
|
||||
|
||||
// Delete 删除一个或多个文件,
|
||||
@@ -223,38 +227,26 @@ func (handler Driver) replaceSourceHost(origin string) (string, error) {
|
||||
}
|
||||
|
||||
// Token 获取上传会话URL
|
||||
func (handler Driver) Token(ctx context.Context, TTL int64, key string) (serializer.UploadCredential, error) {
|
||||
func (handler Driver) Token(ctx context.Context, ttl int64, uploadSession *serializer.UploadSession, file fsctx.FileHeader) (*serializer.UploadCredential, error) {
|
||||
fileInfo := file.Info()
|
||||
|
||||
// 读取上下文中生成的存储路径和文件大小
|
||||
savePath, ok := ctx.Value(fsctx.SavePathCtx).(string)
|
||||
if !ok {
|
||||
return serializer.UploadCredential{}, errors.New("无法获取存储路径")
|
||||
}
|
||||
fileSize, ok := ctx.Value(fsctx.FileSizeCtx).(uint64)
|
||||
if !ok {
|
||||
return serializer.UploadCredential{}, errors.New("无法获取文件大小")
|
||||
}
|
||||
|
||||
// 如果小于4MB,则由服务端中转
|
||||
if fileSize <= SmallFileSize {
|
||||
return serializer.UploadCredential{}, nil
|
||||
}
|
||||
|
||||
// 生成回调地址
|
||||
siteURL := model.GetSiteURL()
|
||||
apiBaseURI, _ := url.Parse("/api/v3/callback/onedrive/finish/" + key)
|
||||
apiURL := siteURL.ResolveReference(apiBaseURI)
|
||||
|
||||
uploadURL, err := handler.Client.CreateUploadSession(ctx, savePath, WithConflictBehavior("fail"))
|
||||
uploadURL, err := handler.Client.CreateUploadSession(ctx, fileInfo.SavePath, WithConflictBehavior("fail"))
|
||||
if err != nil {
|
||||
return serializer.UploadCredential{}, err
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// 监控回调及上传
|
||||
go handler.Client.MonitorUpload(uploadURL, key, savePath, fileSize, TTL)
|
||||
go handler.Client.MonitorUpload(uploadURL, uploadSession.Key, fileInfo.SavePath, fileInfo.Size, ttl)
|
||||
|
||||
return serializer.UploadCredential{
|
||||
Policy: uploadURL,
|
||||
Token: apiURL.String(),
|
||||
uploadSession.UploadURL = uploadURL
|
||||
return &serializer.UploadCredential{
|
||||
SessionID: uploadSession.Key,
|
||||
ChunkSize: handler.Policy.OptionsSerialized.ChunkSize,
|
||||
UploadURLs: []string{uploadURL},
|
||||
}, nil
|
||||
}
|
||||
|
||||
// 取消上传凭证
|
||||
func (handler Driver) CancelToken(ctx context.Context, uploadSession *serializer.UploadSession) error {
|
||||
return handler.Client.DeleteUploadSession(ctx, uploadSession.UploadURL)
|
||||
}
|
||||
|
||||
@@ -4,6 +4,9 @@ import (
|
||||
"context"
|
||||
"fmt"
|
||||
"github.com/cloudreve/Cloudreve/v3/pkg/auth"
|
||||
"github.com/cloudreve/Cloudreve/v3/pkg/mq"
|
||||
"github.com/cloudreve/Cloudreve/v3/pkg/serializer"
|
||||
"github.com/jinzhu/gorm"
|
||||
"io"
|
||||
"io/ioutil"
|
||||
"net/http"
|
||||
@@ -12,51 +15,23 @@ import (
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/DATA-DOG/go-sqlmock"
|
||||
model "github.com/cloudreve/Cloudreve/v3/models"
|
||||
"github.com/cloudreve/Cloudreve/v3/pkg/cache"
|
||||
"github.com/cloudreve/Cloudreve/v3/pkg/filesystem/fsctx"
|
||||
"github.com/cloudreve/Cloudreve/v3/pkg/request"
|
||||
"github.com/cloudreve/Cloudreve/v3/pkg/serializer"
|
||||
"github.com/stretchr/testify/assert"
|
||||
testMock "github.com/stretchr/testify/mock"
|
||||
)
|
||||
|
||||
func TestDriver_Token(t *testing.T) {
|
||||
asserts := assert.New(t)
|
||||
handler := Driver{
|
||||
Policy: &model.Policy{
|
||||
AccessKey: "ak",
|
||||
SecretKey: "sk",
|
||||
BucketName: "test",
|
||||
Server: "test.com",
|
||||
},
|
||||
}
|
||||
|
||||
// 无法获取文件路径
|
||||
{
|
||||
ctx := context.WithValue(context.Background(), fsctx.FileSizeCtx, uint64(10))
|
||||
res, err := handler.Token(ctx, 10, "key")
|
||||
asserts.Error(err)
|
||||
asserts.Equal(serializer.UploadCredential{}, res)
|
||||
}
|
||||
|
||||
// 无法获取文件大小
|
||||
{
|
||||
ctx := context.WithValue(context.Background(), fsctx.SavePathCtx, "/123")
|
||||
res, err := handler.Token(ctx, 10, "key")
|
||||
asserts.Error(err)
|
||||
asserts.Equal(serializer.UploadCredential{}, res)
|
||||
}
|
||||
|
||||
// 小文件成功
|
||||
{
|
||||
ctx := context.WithValue(context.Background(), fsctx.SavePathCtx, "/123")
|
||||
ctx = context.WithValue(ctx, fsctx.FileSizeCtx, uint64(10))
|
||||
res, err := handler.Token(ctx, 10, "key")
|
||||
asserts.NoError(err)
|
||||
asserts.Equal(serializer.UploadCredential{}, res)
|
||||
}
|
||||
h, _ := NewDriver(&model.Policy{
|
||||
AccessKey: "ak",
|
||||
SecretKey: "sk",
|
||||
BucketName: "test",
|
||||
Server: "test.com",
|
||||
})
|
||||
handler := h.(Driver)
|
||||
|
||||
// 分片上传 失败
|
||||
{
|
||||
@@ -78,11 +53,9 @@ func TestDriver_Token(t *testing.T) {
|
||||
},
|
||||
})
|
||||
handler.Client.Request = clientMock
|
||||
ctx := context.WithValue(context.Background(), fsctx.SavePathCtx, "/123")
|
||||
ctx = context.WithValue(ctx, fsctx.FileSizeCtx, uint64(20*1024*1024))
|
||||
res, err := handler.Token(ctx, 10, "key")
|
||||
res, err := handler.Token(context.Background(), 10, &serializer.UploadSession{}, &fsctx.FileStream{})
|
||||
asserts.Error(err)
|
||||
asserts.Equal(serializer.UploadCredential{}, res)
|
||||
asserts.Nil(res)
|
||||
}
|
||||
|
||||
// 分片上传 成功
|
||||
@@ -108,15 +81,13 @@ func TestDriver_Token(t *testing.T) {
|
||||
},
|
||||
})
|
||||
handler.Client.Request = clientMock
|
||||
ctx := context.WithValue(context.Background(), fsctx.SavePathCtx, "/123")
|
||||
ctx = context.WithValue(ctx, fsctx.FileSizeCtx, uint64(20*1024*1024))
|
||||
go func() {
|
||||
time.Sleep(time.Duration(1) * time.Second)
|
||||
FinishCallback("key")
|
||||
mq.GlobalMQ.Publish("TestDriver_Token", mq.Message{})
|
||||
}()
|
||||
res, err := handler.Token(ctx, 10, "key")
|
||||
res, err := handler.Token(context.Background(), 10, &serializer.UploadSession{Key: "TestDriver_Token"}, &fsctx.FileStream{})
|
||||
asserts.NoError(err)
|
||||
asserts.Equal("123321", res.Policy)
|
||||
asserts.Equal("123321", res.UploadURLs[0])
|
||||
}
|
||||
}
|
||||
|
||||
@@ -295,12 +266,8 @@ func TestDriver_Thumb(t *testing.T) {
|
||||
// 失败
|
||||
{
|
||||
ctx := context.WithValue(context.Background(), fsctx.ThumbSizeCtx, [2]uint{10, 20})
|
||||
ctx = context.WithValue(ctx, fsctx.FileModelCtx, model.File{})
|
||||
mock.ExpectBegin()
|
||||
mock.ExpectExec("UPDATE(.+)").WillReturnResult(sqlmock.NewResult(1, 1))
|
||||
mock.ExpectCommit()
|
||||
ctx = context.WithValue(ctx, fsctx.FileModelCtx, model.File{PicInfo: "1,1", Model: gorm.Model{ID: 1}})
|
||||
res, err := handler.Thumb(ctx, "123.jpg")
|
||||
asserts.NoError(mock.ExpectationsWereMet())
|
||||
asserts.Error(err)
|
||||
asserts.Empty(res.URL)
|
||||
}
|
||||
@@ -308,7 +275,6 @@ func TestDriver_Thumb(t *testing.T) {
|
||||
// 上下文错误
|
||||
{
|
||||
_, err := handler.Thumb(context.Background(), "123.jpg")
|
||||
asserts.NoError(mock.ExpectationsWereMet())
|
||||
asserts.Error(err)
|
||||
}
|
||||
}
|
||||
@@ -329,7 +295,6 @@ func TestDriver_Delete(t *testing.T) {
|
||||
// 失败
|
||||
{
|
||||
_, err := handler.Delete(context.Background(), []string{"1"})
|
||||
asserts.NoError(mock.ExpectationsWereMet())
|
||||
asserts.Error(err)
|
||||
}
|
||||
|
||||
@@ -350,7 +315,7 @@ func TestDriver_Put(t *testing.T) {
|
||||
|
||||
// 失败
|
||||
{
|
||||
err := handler.Put(context.Background(), ioutil.NopCloser(strings.NewReader("")), "dst", 0)
|
||||
err := handler.Put(context.Background(), &fsctx.FileStream{})
|
||||
asserts.Error(err)
|
||||
}
|
||||
}
|
||||
@@ -418,3 +383,55 @@ func TestDriver_Get(t *testing.T) {
|
||||
asserts.NoError(err)
|
||||
asserts.Equal("123", string(content))
|
||||
}
|
||||
|
||||
func TestDriver_replaceSourceHost(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
origin string
|
||||
cdn string
|
||||
want string
|
||||
wantErr bool
|
||||
}{
|
||||
{"TestNoReplace", "http://1dr.ms/download.aspx?123456", "", "http://1dr.ms/download.aspx?123456", false},
|
||||
{"TestReplaceCorrect", "http://1dr.ms/download.aspx?123456", "https://test.com:8080", "https://test.com:8080/download.aspx?123456", false},
|
||||
{"TestCdnFormatError", "http://1dr.ms/download.aspx?123456", string([]byte{0x7f}), "", true},
|
||||
{"TestSrcFormatError", string([]byte{0x7f}), "https://test.com:8080", "", true},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
policy := &model.Policy{}
|
||||
policy.OptionsSerialized.OdProxy = tt.cdn
|
||||
handler := Driver{
|
||||
Policy: policy,
|
||||
}
|
||||
got, err := handler.replaceSourceHost(tt.origin)
|
||||
if (err != nil) != tt.wantErr {
|
||||
t.Errorf("replaceSourceHost() error = %v, wantErr %v", err, tt.wantErr)
|
||||
return
|
||||
}
|
||||
if got != tt.want {
|
||||
t.Errorf("replaceSourceHost() got = %v, want %v", got, tt.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestDriver_CancelToken(t *testing.T) {
|
||||
asserts := assert.New(t)
|
||||
handler := Driver{
|
||||
Policy: &model.Policy{
|
||||
AccessKey: "ak",
|
||||
SecretKey: "sk",
|
||||
BucketName: "test",
|
||||
Server: "test.com",
|
||||
},
|
||||
}
|
||||
handler.Client, _ = NewClient(&model.Policy{})
|
||||
handler.Client.Credential.ExpiresIn = time.Now().Add(time.Duration(100) * time.Hour).Unix()
|
||||
|
||||
// 失败
|
||||
{
|
||||
err := handler.CancelToken(context.Background(), &serializer.UploadSession{})
|
||||
asserts.Error(err)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,38 +0,0 @@
|
||||
package onedrive
|
||||
|
||||
import (
|
||||
model "github.com/cloudreve/Cloudreve/v3/models"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestDriver_replaceSourceHost(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
origin string
|
||||
cdn string
|
||||
want string
|
||||
wantErr bool
|
||||
}{
|
||||
{"TestNoReplace", "http://1dr.ms/download.aspx?123456", "", "http://1dr.ms/download.aspx?123456", false},
|
||||
{"TestReplaceCorrect", "http://1dr.ms/download.aspx?123456", "https://test.com:8080", "https://test.com:8080/download.aspx?123456", false},
|
||||
{"TestCdnFormatError", "http://1dr.ms/download.aspx?123456", string([]byte{0x7f}), "", true},
|
||||
{"TestSrcFormatError", string([]byte{0x7f}), "https://test.com:8080", "", true},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
policy := &model.Policy{}
|
||||
policy.OptionsSerialized.OdProxy = tt.cdn
|
||||
handler := Driver{
|
||||
Policy: policy,
|
||||
}
|
||||
got, err := handler.replaceSourceHost(tt.origin)
|
||||
if (err != nil) != tt.wantErr {
|
||||
t.Errorf("replaceSourceHost() error = %v, wantErr %v", err, tt.wantErr)
|
||||
return
|
||||
}
|
||||
if got != tt.want {
|
||||
t.Errorf("replaceSourceHost() got = %v, want %v", got, tt.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -3,7 +3,6 @@ package onedrive
|
||||
import (
|
||||
"encoding/gob"
|
||||
"net/url"
|
||||
"sync"
|
||||
)
|
||||
|
||||
// RespError 接口返回错误
|
||||
@@ -99,15 +98,6 @@ type ListResponse struct {
|
||||
Context string `json:"@odata.context"`
|
||||
}
|
||||
|
||||
// Chunk 文件分片
|
||||
type Chunk struct {
|
||||
Offset int
|
||||
ChunkSize int
|
||||
Total int
|
||||
Retried int
|
||||
Data []byte
|
||||
}
|
||||
|
||||
// oauthEndpoint OAuth接口地址
|
||||
type oauthEndpoint struct {
|
||||
token url.URL
|
||||
@@ -143,10 +133,3 @@ type Site struct {
|
||||
func init() {
|
||||
gob.Register(Credential{})
|
||||
}
|
||||
|
||||
// IsLast 返回是否为最后一个分片
|
||||
func (chunk *Chunk) IsLast() bool {
|
||||
return chunk.Total-chunk.Offset == chunk.ChunkSize
|
||||
}
|
||||
|
||||
var callbackSignal sync.Map
|
||||
|
||||
@@ -1,188 +0,0 @@
|
||||
package oss
|
||||
|
||||
import (
|
||||
"io/ioutil"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/cloudreve/Cloudreve/v3/pkg/cache"
|
||||
"github.com/stretchr/testify/assert"
|
||||
)
|
||||
|
||||
func TestGetPublicKey(t *testing.T) {
|
||||
asserts := assert.New(t)
|
||||
testCases := []struct {
|
||||
Request http.Request
|
||||
ResNil bool
|
||||
Error bool
|
||||
}{
|
||||
// Header解码失败
|
||||
{
|
||||
Request: http.Request{
|
||||
Header: http.Header{
|
||||
"X-Oss-Pub-Key-Url": {"中文"},
|
||||
},
|
||||
},
|
||||
ResNil: true,
|
||||
Error: true,
|
||||
},
|
||||
// 公钥URL无效
|
||||
{
|
||||
Request: http.Request{
|
||||
Header: http.Header{
|
||||
"X-Oss-Pub-Key-Url": {"aHR0cHM6Ly9wb3JuaHViLmNvbQ=="},
|
||||
},
|
||||
},
|
||||
ResNil: true,
|
||||
Error: true,
|
||||
},
|
||||
// 请求失败
|
||||
{
|
||||
Request: http.Request{
|
||||
Header: http.Header{
|
||||
"X-Oss-Pub-Key-Url": {"aHR0cDovL2dvc3NwdWJsaWMuYWxpY2RuLmNvbS8yMzQyMzQ="},
|
||||
},
|
||||
},
|
||||
ResNil: true,
|
||||
Error: true,
|
||||
},
|
||||
// 成功
|
||||
{
|
||||
Request: http.Request{
|
||||
Header: http.Header{
|
||||
"X-Oss-Pub-Key-Url": {"aHR0cDovL2dvc3NwdWJsaWMuYWxpY2RuLmNvbS9jYWxsYmFja19wdWJfa2V5X3YxLnBlbQ=="},
|
||||
},
|
||||
},
|
||||
ResNil: false,
|
||||
Error: false,
|
||||
},
|
||||
}
|
||||
|
||||
for i, testCase := range testCases {
|
||||
asserts.NoError(cache.Deletes([]string{"oss_public_key"}, ""))
|
||||
res, err := GetPublicKey(&testCase.Request)
|
||||
if testCase.Error {
|
||||
asserts.Error(err, "Test Case #%d", i)
|
||||
} else {
|
||||
asserts.NoError(err, "Test Case #%d", i)
|
||||
}
|
||||
if testCase.ResNil {
|
||||
asserts.Empty(res, "Test Case #%d", i)
|
||||
} else {
|
||||
asserts.NotEmpty(res, "Test Case #%d", i)
|
||||
}
|
||||
}
|
||||
|
||||
// 测试缓存
|
||||
asserts.NoError(cache.Set("oss_public_key", []byte("123"), 0))
|
||||
res, err := GetPublicKey(nil)
|
||||
asserts.NoError(err)
|
||||
asserts.Equal([]byte("123"), res)
|
||||
}
|
||||
|
||||
func TestVerifyCallbackSignature(t *testing.T) {
|
||||
asserts := assert.New(t)
|
||||
testPubKey := `-----BEGIN PUBLIC KEY-----
|
||||
MFwwDQYJKoZIhvcNAQEBBQADSwAwSAJBAKs/JBGzwUB2aVht4crBx3oIPBLNsjGs
|
||||
C0fTXv+nvlmklvkcolvpvXLTjaxUHR3W9LXxQ2EHXAJfCB+6H2YF1k8CAwEAAQ==
|
||||
-----END PUBLIC KEY-----
|
||||
`
|
||||
|
||||
// 成功
|
||||
{
|
||||
asserts.NoError(cache.Set("oss_public_key", []byte(testPubKey), 0))
|
||||
r := http.Request{
|
||||
URL: &url.URL{Path: "/api/v3/callback/oss/TnXx5E5VyfJUyM1UdkdDu1rtnJ34EbmH"},
|
||||
Header: map[string][]string{
|
||||
"Authorization": {"e5LwzwTkP9AFAItT4YzvdJOHd0Y0wqTMWhsV/h5SG90JYGAmMd+8LQyj96R+9qUfJWjMt6suuUh7LaOryR87Dw=="},
|
||||
"X-Oss-Pub-Key-Url": {"aHR0cHM6Ly9nb3NzcHVibGljLmFsaWNkbi5jb20vY2FsbGJhY2tfcHViX2tleV92MS5wZW0="},
|
||||
},
|
||||
Body: ioutil.NopCloser(strings.NewReader(`{"name":"2f7b2ccf30e9270ea920f1ab8a4037a546a2f0d5.jpg","source_name":"1/1_hFRtDLgM_2f7b2ccf30e9270ea920f1ab8a4037a546a2f0d5.jpg","size":114020,"pic_info":"810,539"}`)),
|
||||
}
|
||||
asserts.NoError(VerifyCallbackSignature(&r))
|
||||
}
|
||||
|
||||
// 签名错误
|
||||
{
|
||||
asserts.NoError(cache.Set("oss_public_key", []byte(testPubKey), 0))
|
||||
r := http.Request{
|
||||
URL: &url.URL{Path: "/api/v3/callback/oss/TnXx5E5VyfJUyM1UdkdDu1rtnJ34EbmH"},
|
||||
Header: map[string][]string{
|
||||
"Authorization": {"e3LwzwTkP9AFAItT4YzvdJOHd0Y0wqTMWhsV/h5SG90JYGAmMd+8LQyj96R+9qUfJWjMt6suuUh7LaOryR87Dw=="},
|
||||
"X-Oss-Pub-Key-Url": {"aHR0cHM6Ly9nb3NzcHVibGljLmFsaWNkbi5jb20vY2FsbGJhY2tfcHViX2tleV92MS5wZW0="},
|
||||
},
|
||||
Body: ioutil.NopCloser(strings.NewReader(`{"name":"2f7b2ccf30e9270ea920f1ab8a4037a546a2f0d5.jpg","source_name":"1/1_hFRtDLgM_2f7b2ccf30e9270ea920f1ab8a4037a546a2f0d5.jpg","size":114020,"pic_info":"810,539"}`)),
|
||||
}
|
||||
asserts.Error(VerifyCallbackSignature(&r))
|
||||
}
|
||||
|
||||
// GetPubKey 失败
|
||||
{
|
||||
asserts.NoError(cache.Deletes([]string{"oss_public_key"}, ""))
|
||||
r := http.Request{
|
||||
URL: &url.URL{Path: "/api/v3/callback/oss/TnXx5E5VyfJUyM1UdkdDu1rtnJ34EbmH"},
|
||||
Header: map[string][]string{
|
||||
"Authorization": {"e5LwzwTkP9AFAItT4YzvdJOHd0Y0wqTMWhsV/h5SG90JYGAmMd+8LQyj96R+9qUfJWjMt6suuUh7LaOryR87Dw=="},
|
||||
},
|
||||
Body: ioutil.NopCloser(strings.NewReader(`{"name":"2f7b2ccf30e9270ea920f1ab8a4037a546a2f0d5.jpg","source_name":"1/1_hFRtDLgM_2f7b2ccf30e9270ea920f1ab8a4037a546a2f0d5.jpg","size":114020,"pic_info":"810,539"}`)),
|
||||
}
|
||||
asserts.Error(VerifyCallbackSignature(&r))
|
||||
}
|
||||
|
||||
// getRequestMD5 失败
|
||||
{
|
||||
asserts.NoError(cache.Set("oss_public_key", []byte(testPubKey), 0))
|
||||
r := http.Request{
|
||||
URL: &url.URL{Path: "%测试"},
|
||||
Header: map[string][]string{
|
||||
"Authorization": {"e5LwzwTkP9AFAItT4YzvdJOHd0Y0wqTMWhsV/h5SG90JYGAmMd+8LQyj96R+9qUfJWjMt6suuUh7LaOryR87Dw=="},
|
||||
"X-Oss-Pub-Key-Url": {"aHR0cHM6Ly9nb3NzcHVibGljLmFsaWNkbi5jb20vY2FsbGJhY2tfcHViX2tleV92MS5wZW0="},
|
||||
},
|
||||
Body: ioutil.NopCloser(strings.NewReader(`{"name":"2f7b2ccf30e9270ea920f1ab8a4037a546a2f0d5.jpg","source_name":"1/1_hFRtDLgM_2f7b2ccf30e9270ea920f1ab8a4037a546a2f0d5.jpg","size":114020,"pic_info":"810,539"}`)),
|
||||
}
|
||||
asserts.Error(VerifyCallbackSignature(&r))
|
||||
}
|
||||
|
||||
// 无 Authorization 头
|
||||
{
|
||||
asserts.NoError(cache.Set("oss_public_key", []byte(testPubKey), 0))
|
||||
r := http.Request{
|
||||
URL: &url.URL{Path: "/api/v3/callback/oss/TnXx5E5VyfJUyM1UdkdDu1rtnJ34EbmH"},
|
||||
Header: map[string][]string{
|
||||
"X-Oss-Pub-Key-Url": {"aHR0cHM6Ly9nb3NzcHVibGljLmFsaWNkbi5jb20vY2FsbGJhY2tfcHViX2tleV92MS5wZW0="},
|
||||
},
|
||||
Body: ioutil.NopCloser(strings.NewReader(`{"name":"2f7b2ccf30e9270ea920f1ab8a4037a546a2f0d5.jpg","source_name":"1/1_hFRtDLgM_2f7b2ccf30e9270ea920f1ab8a4037a546a2f0d5.jpg","size":114020,"pic_info":"810,539"}`)),
|
||||
}
|
||||
asserts.Error(VerifyCallbackSignature(&r))
|
||||
}
|
||||
|
||||
// pub block 不存在
|
||||
{
|
||||
asserts.NoError(cache.Set("oss_public_key", []byte(""), 0))
|
||||
r := http.Request{
|
||||
URL: &url.URL{Path: "/api/v3/callback/oss/TnXx5E5VyfJUyM1UdkdDu1rtnJ34EbmH"},
|
||||
Header: map[string][]string{
|
||||
"Authorization": {"e5LwzwTkP9AFAItT4YzvdJOHd0Y0wqTMWhsV/h5SG90JYGAmMd+8LQyj96R+9qUfJWjMt6suuUh7LaOryR87Dw=="},
|
||||
"X-Oss-Pub-Key-Url": {"aHR0cHM6Ly9nb3NzcHVibGljLmFsaWNkbi5jb20vY2FsbGJhY2tfcHViX2tleV92MS5wZW0="},
|
||||
},
|
||||
Body: ioutil.NopCloser(strings.NewReader(`{"name":"2f7b2ccf30e9270ea920f1ab8a4037a546a2f0d5.jpg","source_name":"1/1_hFRtDLgM_2f7b2ccf30e9270ea920f1ab8a4037a546a2f0d5.jpg","size":114020,"pic_info":"810,539"}`)),
|
||||
}
|
||||
asserts.Error(VerifyCallbackSignature(&r))
|
||||
}
|
||||
|
||||
// ParsePKIXPublicKey出错
|
||||
{
|
||||
asserts.NoError(cache.Set("oss_public_key", []byte("-----BEGIN PUBLIC KEY-----\n-----END PUBLIC KEY-----"), 0))
|
||||
r := http.Request{
|
||||
URL: &url.URL{Path: "/api/v3/callback/oss/TnXx5E5VyfJUyM1UdkdDu1rtnJ34EbmH"},
|
||||
Header: map[string][]string{
|
||||
"Authorization": {"e5LwzwTkP9AFAItT4YzvdJOHd0Y0wqTMWhsV/h5SG90JYGAmMd+8LQyj96R+9qUfJWjMt6suuUh7LaOryR87Dw=="},
|
||||
"X-Oss-Pub-Key-Url": {"aHR0cHM6Ly9nb3NzcHVibGljLmFsaWNkbi5jb20vY2FsbGJhY2tfcHViX2tleV92MS5wZW0="},
|
||||
},
|
||||
Body: ioutil.NopCloser(strings.NewReader(`{"name":"2f7b2ccf30e9270ea920f1ab8a4037a546a2f0d5.jpg","source_name":"1/1_hFRtDLgM_2f7b2ccf30e9270ea920f1ab8a4037a546a2f0d5.jpg","size":114020,"pic_info":"810,539"}`)),
|
||||
}
|
||||
asserts.Error(VerifyCallbackSignature(&r))
|
||||
}
|
||||
}
|
||||
@@ -2,8 +2,6 @@ package oss
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/hmac"
|
||||
"crypto/sha1"
|
||||
"encoding/base64"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
@@ -15,8 +13,10 @@ import (
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/aliyun/aliyun-oss-go-sdk/oss"
|
||||
"github.com/HFO4/aliyun-oss-go-sdk/oss"
|
||||
model "github.com/cloudreve/Cloudreve/v3/models"
|
||||
"github.com/cloudreve/Cloudreve/v3/pkg/filesystem/chunk"
|
||||
"github.com/cloudreve/Cloudreve/v3/pkg/filesystem/chunk/backoff"
|
||||
"github.com/cloudreve/Cloudreve/v3/pkg/filesystem/fsctx"
|
||||
"github.com/cloudreve/Cloudreve/v3/pkg/filesystem/response"
|
||||
"github.com/cloudreve/Cloudreve/v3/pkg/request"
|
||||
@@ -48,17 +48,29 @@ type Driver struct {
|
||||
type key int
|
||||
|
||||
const (
|
||||
chunkRetrySleep = time.Duration(5) * time.Second
|
||||
|
||||
// MultiPartUploadThreshold 服务端使用分片上传的阈值
|
||||
MultiPartUploadThreshold uint64 = 5 * (1 << 30) // 5GB
|
||||
// VersionID 文件版本标识
|
||||
VersionID key = iota
|
||||
)
|
||||
|
||||
// CORS 创建跨域策略
|
||||
func (handler *Driver) CORS() error {
|
||||
// 初始化客户端
|
||||
if err := handler.InitOSSClient(false); err != nil {
|
||||
return err
|
||||
func NewDriver(policy *model.Policy) (*Driver, error) {
|
||||
if policy.OptionsSerialized.ChunkSize == 0 {
|
||||
policy.OptionsSerialized.ChunkSize = 25 << 20 // 25 MB
|
||||
}
|
||||
|
||||
driver := &Driver{
|
||||
Policy: policy,
|
||||
HTTPClient: request.NewClient(),
|
||||
}
|
||||
|
||||
return driver, driver.InitOSSClient(false)
|
||||
}
|
||||
|
||||
// CORS 创建跨域策略
|
||||
func (handler *Driver) CORS() error {
|
||||
return handler.client.SetBucketCORS(handler.Policy.BucketName, []oss.CORSRule{
|
||||
{
|
||||
AllowedOrigin: []string{"*"},
|
||||
@@ -82,39 +94,31 @@ func (handler *Driver) InitOSSClient(forceUsePublicEndpoint bool) error {
|
||||
return errors.New("存储策略为空")
|
||||
}
|
||||
|
||||
if handler.client == nil {
|
||||
// 决定是否使用内网 Endpoint
|
||||
endpoint := handler.Policy.Server
|
||||
if handler.Policy.OptionsSerialized.ServerSideEndpoint != "" && !forceUsePublicEndpoint {
|
||||
endpoint = handler.Policy.OptionsSerialized.ServerSideEndpoint
|
||||
}
|
||||
|
||||
// 初始化客户端
|
||||
client, err := oss.New(endpoint, handler.Policy.AccessKey, handler.Policy.SecretKey)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
handler.client = client
|
||||
|
||||
// 初始化存储桶
|
||||
bucket, err := client.Bucket(handler.Policy.BucketName)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
handler.bucket = bucket
|
||||
|
||||
// 决定是否使用内网 Endpoint
|
||||
endpoint := handler.Policy.Server
|
||||
if handler.Policy.OptionsSerialized.ServerSideEndpoint != "" && !forceUsePublicEndpoint {
|
||||
endpoint = handler.Policy.OptionsSerialized.ServerSideEndpoint
|
||||
}
|
||||
|
||||
// 初始化客户端
|
||||
client, err := oss.New(endpoint, handler.Policy.AccessKey, handler.Policy.SecretKey)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
handler.client = client
|
||||
|
||||
// 初始化存储桶
|
||||
bucket, err := client.Bucket(handler.Policy.BucketName)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
handler.bucket = bucket
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// List 列出OSS上的文件
|
||||
func (handler Driver) List(ctx context.Context, base string, recursive bool) ([]response.Object, error) {
|
||||
// 初始化客户端
|
||||
if err := handler.InitOSSClient(false); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
func (handler *Driver) List(ctx context.Context, base string, recursive bool) ([]response.Object, error) {
|
||||
// 列取文件
|
||||
base = strings.TrimPrefix(base, "/")
|
||||
if base != "" {
|
||||
@@ -181,7 +185,7 @@ func (handler Driver) List(ctx context.Context, base string, recursive bool) ([]
|
||||
}
|
||||
|
||||
// Get 获取文件
|
||||
func (handler Driver) Get(ctx context.Context, path string) (response.RSCloser, error) {
|
||||
func (handler *Driver) Get(ctx context.Context, path string) (response.RSCloser, error) {
|
||||
// 通过VersionID禁止缓存
|
||||
ctx = context.WithValue(ctx, VersionID, time.Now().UnixNano())
|
||||
|
||||
@@ -224,45 +228,54 @@ func (handler Driver) Get(ctx context.Context, path string) (response.RSCloser,
|
||||
}
|
||||
|
||||
// Put 将文件流保存到指定目录
|
||||
func (handler Driver) Put(ctx context.Context, file io.ReadCloser, dst string, size uint64) error {
|
||||
func (handler *Driver) Put(ctx context.Context, file fsctx.FileHeader) error {
|
||||
defer file.Close()
|
||||
|
||||
// 初始化客户端
|
||||
if err := handler.InitOSSClient(false); err != nil {
|
||||
return err
|
||||
}
|
||||
fileInfo := file.Info()
|
||||
|
||||
// 凭证有效期
|
||||
credentialTTL := model.GetIntSetting("upload_credential_timeout", 3600)
|
||||
credentialTTL := model.GetIntSetting("upload_session_timeout", 3600)
|
||||
|
||||
// 是否允许覆盖
|
||||
overwrite := true
|
||||
if ctx.Value(fsctx.DisableOverwrite) != nil {
|
||||
overwrite = false
|
||||
}
|
||||
|
||||
overwrite := fileInfo.Mode&fsctx.Overwrite == fsctx.Overwrite
|
||||
options := []oss.Option{
|
||||
oss.Expires(time.Now().Add(time.Duration(credentialTTL) * time.Second)),
|
||||
oss.ForbidOverWrite(!overwrite),
|
||||
}
|
||||
|
||||
// 上传文件
|
||||
err := handler.bucket.PutObject(dst, file, options...)
|
||||
// 小文件直接上传
|
||||
if fileInfo.Size < MultiPartUploadThreshold {
|
||||
return handler.bucket.PutObject(fileInfo.SavePath, file, options...)
|
||||
}
|
||||
|
||||
// 超过阈值时使用分片上传
|
||||
imur, err := handler.bucket.InitiateMultipartUpload(fileInfo.SavePath, options...)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to initiate multipart upload: %w", err)
|
||||
}
|
||||
|
||||
chunks := chunk.NewChunkGroup(file, handler.Policy.OptionsSerialized.ChunkSize, &backoff.ConstantBackoff{
|
||||
Max: model.GetIntSetting("chunk_retries", 5),
|
||||
Sleep: chunkRetrySleep,
|
||||
}, model.IsTrueVal(model.GetSettingByName("use_temp_chunk_buffer")))
|
||||
|
||||
uploadFunc := func(current *chunk.ChunkGroup, content io.Reader) error {
|
||||
_, err := handler.bucket.UploadPart(imur, content, current.Length(), current.Index()+1)
|
||||
return err
|
||||
}
|
||||
|
||||
return nil
|
||||
for chunks.Next() {
|
||||
if err := chunks.Process(uploadFunc); err != nil {
|
||||
return fmt.Errorf("failed to upload chunk #%d: %w", chunks.Index(), err)
|
||||
}
|
||||
}
|
||||
|
||||
_, err = handler.bucket.CompleteMultipartUpload(imur, oss.CompleteAll("yes"), oss.ForbidOverWrite(!overwrite))
|
||||
return err
|
||||
}
|
||||
|
||||
// Delete 删除一个或多个文件,
|
||||
// 返回未删除的文件
|
||||
func (handler Driver) Delete(ctx context.Context, files []string) ([]string, error) {
|
||||
// 初始化客户端
|
||||
if err := handler.InitOSSClient(false); err != nil {
|
||||
return files, err
|
||||
}
|
||||
|
||||
func (handler *Driver) Delete(ctx context.Context, files []string) ([]string, error) {
|
||||
// 删除文件
|
||||
delRes, err := handler.bucket.DeleteObjects(files)
|
||||
|
||||
@@ -280,7 +293,7 @@ func (handler Driver) Delete(ctx context.Context, files []string) ([]string, err
|
||||
}
|
||||
|
||||
// Thumb 获取文件缩略图
|
||||
func (handler Driver) Thumb(ctx context.Context, path string) (*response.ContentResponse, error) {
|
||||
func (handler *Driver) Thumb(ctx context.Context, path string) (*response.ContentResponse, error) {
|
||||
// 初始化客户端
|
||||
if err := handler.InitOSSClient(true); err != nil {
|
||||
return nil, err
|
||||
@@ -314,7 +327,7 @@ func (handler Driver) Thumb(ctx context.Context, path string) (*response.Content
|
||||
}
|
||||
|
||||
// Source 获取外链URL
|
||||
func (handler Driver) Source(
|
||||
func (handler *Driver) Source(
|
||||
ctx context.Context,
|
||||
path string,
|
||||
baseURL url.URL,
|
||||
@@ -359,7 +372,7 @@ func (handler Driver) Source(
|
||||
return handler.signSourceURL(ctx, path, ttl, signOptions)
|
||||
}
|
||||
|
||||
func (handler Driver) signSourceURL(ctx context.Context, path string, ttl int64, options []oss.Option) (string, error) {
|
||||
func (handler *Driver) signSourceURL(ctx context.Context, path string, ttl int64, options []oss.Option) (string, error) {
|
||||
signedURL, err := handler.bucket.SignURL(path, oss.HTTPGet, ttl, options...)
|
||||
if err != nil {
|
||||
return "", err
|
||||
@@ -371,9 +384,6 @@ func (handler Driver) signSourceURL(ctx context.Context, path string, ttl int64,
|
||||
return "", err
|
||||
}
|
||||
|
||||
// 优先使用https
|
||||
finalURL.Scheme = "https"
|
||||
|
||||
// 公有空间替换掉Key及不支持的头
|
||||
if !handler.Policy.IsPrivate {
|
||||
query := finalURL.Query()
|
||||
@@ -397,16 +407,11 @@ func (handler Driver) signSourceURL(ctx context.Context, path string, ttl int64,
|
||||
}
|
||||
|
||||
// Token 获取上传策略和认证Token
|
||||
func (handler Driver) Token(ctx context.Context, TTL int64, key string) (serializer.UploadCredential, error) {
|
||||
// 读取上下文中生成的存储路径
|
||||
savePath, ok := ctx.Value(fsctx.SavePathCtx).(string)
|
||||
if !ok {
|
||||
return serializer.UploadCredential{}, errors.New("无法获取存储路径")
|
||||
}
|
||||
func (handler *Driver) Token(ctx context.Context, ttl int64, uploadSession *serializer.UploadSession, file fsctx.FileHeader) (*serializer.UploadCredential, error) {
|
||||
|
||||
// 生成回调地址
|
||||
siteURL := model.GetSiteURL()
|
||||
apiBaseURI, _ := url.Parse("/api/v3/callback/oss/" + key)
|
||||
apiBaseURI, _ := url.Parse("/api/v3/callback/oss/" + uploadSession.Key)
|
||||
apiURL := siteURL.ResolveReference(apiBaseURI)
|
||||
|
||||
// 回调策略
|
||||
@@ -415,61 +420,66 @@ func (handler Driver) Token(ctx context.Context, TTL int64, key string) (seriali
|
||||
CallbackBody: `{"name":${x:fname},"source_name":${object},"size":${size},"pic_info":"${imageInfo.width},${imageInfo.height}"}`,
|
||||
CallbackBodyType: "application/json",
|
||||
}
|
||||
|
||||
// 上传策略
|
||||
postPolicy := UploadPolicy{
|
||||
Expiration: time.Now().UTC().Add(time.Duration(TTL) * time.Second).Format(time.RFC3339),
|
||||
Conditions: []interface{}{
|
||||
map[string]string{"bucket": handler.Policy.BucketName},
|
||||
[]string{"starts-with", "$key", path.Dir(savePath)},
|
||||
},
|
||||
callbackPolicyJSON, err := json.Marshal(callbackPolicy)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to encode callback policy: %w", err)
|
||||
}
|
||||
callbackPolicyEncoded := base64.StdEncoding.EncodeToString(callbackPolicyJSON)
|
||||
|
||||
if handler.Policy.MaxSize > 0 {
|
||||
postPolicy.Conditions = append(postPolicy.Conditions,
|
||||
[]interface{}{"content-length-range", 0, handler.Policy.MaxSize})
|
||||
// 初始化分片上传
|
||||
fileInfo := file.Info()
|
||||
options := []oss.Option{
|
||||
oss.Expires(time.Now().Add(time.Duration(ttl) * time.Second)),
|
||||
oss.ForbidOverWrite(true),
|
||||
}
|
||||
|
||||
return handler.getUploadCredential(ctx, postPolicy, callbackPolicy, TTL)
|
||||
}
|
||||
|
||||
func (handler Driver) getUploadCredential(ctx context.Context, policy UploadPolicy, callback CallbackPolicy, TTL int64) (serializer.UploadCredential, error) {
|
||||
// 读取上下文中生成的存储路径
|
||||
savePath, ok := ctx.Value(fsctx.SavePathCtx).(string)
|
||||
if !ok {
|
||||
return serializer.UploadCredential{}, errors.New("无法获取存储路径")
|
||||
imur, err := handler.bucket.InitiateMultipartUpload(fileInfo.SavePath, options...)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to initialize multipart upload: %w", err)
|
||||
}
|
||||
uploadSession.UploadID = imur.UploadID
|
||||
|
||||
// 处理回调策略
|
||||
callbackPolicyEncoded := ""
|
||||
if callback.CallbackURL != "" {
|
||||
callbackPolicyJSON, err := json.Marshal(callback)
|
||||
// 为每个分片签名上传 URL
|
||||
chunks := chunk.NewChunkGroup(file, handler.Policy.OptionsSerialized.ChunkSize, &backoff.ConstantBackoff{}, false)
|
||||
urls := make([]string, chunks.Num())
|
||||
for chunks.Next() {
|
||||
err := chunks.Process(func(c *chunk.ChunkGroup, chunk io.Reader) error {
|
||||
signedURL, err := handler.bucket.SignURL(fileInfo.SavePath, oss.HTTPPut, ttl,
|
||||
oss.PartNumber(c.Index()+1),
|
||||
oss.UploadID(imur.UploadID),
|
||||
oss.ContentType("application/octet-stream"))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
urls[c.Index()] = signedURL
|
||||
return nil
|
||||
})
|
||||
if err != nil {
|
||||
return serializer.UploadCredential{}, err
|
||||
return nil, err
|
||||
}
|
||||
callbackPolicyEncoded = base64.StdEncoding.EncodeToString(callbackPolicyJSON)
|
||||
policy.Conditions = append(policy.Conditions, map[string]string{"callback": callbackPolicyEncoded})
|
||||
}
|
||||
|
||||
// 编码上传策略
|
||||
policyJSON, err := json.Marshal(policy)
|
||||
// 签名完成分片上传的URL
|
||||
completeURL, err := handler.bucket.SignURL(fileInfo.SavePath, oss.HTTPPost, ttl,
|
||||
oss.UploadID(imur.UploadID),
|
||||
oss.Expires(time.Now().Add(time.Duration(ttl)*time.Second)),
|
||||
oss.CompleteAll("yes"),
|
||||
oss.ForbidOverWrite(true),
|
||||
oss.CallbackParam(callbackPolicyEncoded))
|
||||
if err != nil {
|
||||
return serializer.UploadCredential{}, err
|
||||
return nil, err
|
||||
}
|
||||
policyEncoded := base64.StdEncoding.EncodeToString(policyJSON)
|
||||
|
||||
// 签名上传策略
|
||||
hmacSign := hmac.New(sha1.New, []byte(handler.Policy.SecretKey))
|
||||
_, err = io.WriteString(hmacSign, policyEncoded)
|
||||
if err != nil {
|
||||
return serializer.UploadCredential{}, err
|
||||
}
|
||||
signature := base64.StdEncoding.EncodeToString(hmacSign.Sum(nil))
|
||||
|
||||
return serializer.UploadCredential{
|
||||
Policy: fmt.Sprintf("%s:%s", callbackPolicyEncoded, policyEncoded),
|
||||
Path: savePath,
|
||||
AccessKey: handler.Policy.AccessKey,
|
||||
Token: signature,
|
||||
return &serializer.UploadCredential{
|
||||
SessionID: uploadSession.Key,
|
||||
ChunkSize: handler.Policy.OptionsSerialized.ChunkSize,
|
||||
UploadID: imur.UploadID,
|
||||
UploadURLs: urls,
|
||||
CompleteURL: completeURL,
|
||||
}, nil
|
||||
}
|
||||
|
||||
// 取消上传凭证
|
||||
func (handler *Driver) CancelToken(ctx context.Context, uploadSession *serializer.UploadSession) error {
|
||||
return handler.bucket.AbortMultipartUpload(oss.InitiateMultipartUploadResult{UploadID: uploadSession.UploadID, Key: uploadSession.SavePath}, nil)
|
||||
}
|
||||
|
||||
@@ -1,354 +0,0 @@
|
||||
package oss
|
||||
|
||||
import (
|
||||
"context"
|
||||
"io"
|
||||
"io/ioutil"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
model "github.com/cloudreve/Cloudreve/v3/models"
|
||||
"github.com/cloudreve/Cloudreve/v3/pkg/cache"
|
||||
"github.com/cloudreve/Cloudreve/v3/pkg/filesystem/fsctx"
|
||||
"github.com/cloudreve/Cloudreve/v3/pkg/request"
|
||||
"github.com/stretchr/testify/assert"
|
||||
testMock "github.com/stretchr/testify/mock"
|
||||
)
|
||||
|
||||
func TestDriver_InitOSSClient(t *testing.T) {
|
||||
asserts := assert.New(t)
|
||||
handler := Driver{
|
||||
Policy: &model.Policy{
|
||||
AccessKey: "ak",
|
||||
SecretKey: "sk",
|
||||
BucketName: "test",
|
||||
Server: "test.com",
|
||||
},
|
||||
}
|
||||
|
||||
// 成功
|
||||
{
|
||||
asserts.NoError(handler.InitOSSClient(false))
|
||||
}
|
||||
|
||||
// 使用内网Endpoint
|
||||
{
|
||||
handler.Policy.OptionsSerialized.ServerSideEndpoint = "endpoint2"
|
||||
asserts.NoError(handler.InitOSSClient(false))
|
||||
}
|
||||
|
||||
// 未指定存储策略
|
||||
{
|
||||
handler := Driver{}
|
||||
asserts.Error(handler.InitOSSClient(false))
|
||||
}
|
||||
}
|
||||
|
||||
func TestDriver_CORS(t *testing.T) {
|
||||
asserts := assert.New(t)
|
||||
handler := Driver{
|
||||
Policy: &model.Policy{
|
||||
AccessKey: "ak",
|
||||
SecretKey: "sk",
|
||||
BucketName: "test",
|
||||
Server: "test.com",
|
||||
},
|
||||
}
|
||||
|
||||
// 失败
|
||||
{
|
||||
asserts.NotPanics(func() {
|
||||
handler.CORS()
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestDriver_Token(t *testing.T) {
|
||||
asserts := assert.New(t)
|
||||
handler := Driver{
|
||||
Policy: &model.Policy{
|
||||
AccessKey: "ak",
|
||||
SecretKey: "sk",
|
||||
BucketName: "test",
|
||||
Server: "test.com",
|
||||
},
|
||||
}
|
||||
|
||||
// 成功
|
||||
{
|
||||
ctx := context.WithValue(context.Background(), fsctx.SavePathCtx, "/123")
|
||||
cache.Set("setting_siteURL", "http://test.cloudreve.org", 0)
|
||||
res, err := handler.Token(ctx, 10, "key")
|
||||
asserts.NoError(err)
|
||||
asserts.NotEmpty(res.Policy)
|
||||
asserts.NotEmpty(res.Token)
|
||||
asserts.Equal(handler.Policy.AccessKey, res.AccessKey)
|
||||
asserts.Equal("/123", res.Path)
|
||||
}
|
||||
|
||||
// 上下文错误
|
||||
{
|
||||
ctx := context.Background()
|
||||
_, err := handler.Token(ctx, 10, "key")
|
||||
asserts.Error(err)
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
func TestDriver_Source(t *testing.T) {
|
||||
asserts := assert.New(t)
|
||||
handler := Driver{
|
||||
Policy: &model.Policy{
|
||||
AccessKey: "ak",
|
||||
SecretKey: "sk",
|
||||
BucketName: "test",
|
||||
Server: "test.com",
|
||||
IsPrivate: true,
|
||||
},
|
||||
}
|
||||
|
||||
// 正常 非下载 无限速
|
||||
{
|
||||
res, err := handler.Source(context.Background(), "/123", url.URL{}, 10, false, 0)
|
||||
asserts.NoError(err)
|
||||
resURL, err := url.Parse(res)
|
||||
asserts.NoError(err)
|
||||
query := resURL.Query()
|
||||
asserts.NotEmpty(query.Get("Signature"))
|
||||
asserts.NotEmpty(query.Get("Expires"))
|
||||
asserts.Equal("ak", query.Get("OSSAccessKeyId"))
|
||||
}
|
||||
|
||||
// 限速 + 下载
|
||||
{
|
||||
ctx := context.WithValue(context.Background(), fsctx.FileModelCtx, model.File{Name: "123.txt"})
|
||||
res, err := handler.Source(ctx, "/123", url.URL{}, 10, true, 102401)
|
||||
asserts.NoError(err)
|
||||
resURL, err := url.Parse(res)
|
||||
asserts.NoError(err)
|
||||
query := resURL.Query()
|
||||
asserts.NotEmpty(query.Get("Signature"))
|
||||
asserts.NotEmpty(query.Get("Expires"))
|
||||
asserts.Equal("ak", query.Get("OSSAccessKeyId"))
|
||||
asserts.EqualValues("819208", query.Get("x-oss-traffic-limit"))
|
||||
asserts.NotEmpty(query.Get("response-content-disposition"))
|
||||
}
|
||||
|
||||
// 限速超出范围 + 下载
|
||||
{
|
||||
ctx := context.WithValue(context.Background(), fsctx.FileModelCtx, model.File{Name: "123.txt"})
|
||||
res, err := handler.Source(ctx, "/123", url.URL{}, 10, true, 10)
|
||||
asserts.NoError(err)
|
||||
resURL, err := url.Parse(res)
|
||||
asserts.NoError(err)
|
||||
query := resURL.Query()
|
||||
asserts.NotEmpty(query.Get("Signature"))
|
||||
asserts.NotEmpty(query.Get("Expires"))
|
||||
asserts.Equal("ak", query.Get("OSSAccessKeyId"))
|
||||
asserts.EqualValues("819200", query.Get("x-oss-traffic-limit"))
|
||||
asserts.NotEmpty(query.Get("response-content-disposition"))
|
||||
}
|
||||
|
||||
// 限速超出范围 + 下载
|
||||
{
|
||||
ctx := context.WithValue(context.Background(), fsctx.FileModelCtx, model.File{Name: "123.txt"})
|
||||
res, err := handler.Source(ctx, "/123", url.URL{}, 10, true, 838860801)
|
||||
asserts.NoError(err)
|
||||
resURL, err := url.Parse(res)
|
||||
asserts.NoError(err)
|
||||
query := resURL.Query()
|
||||
asserts.NotEmpty(query.Get("Signature"))
|
||||
asserts.NotEmpty(query.Get("Expires"))
|
||||
asserts.Equal("ak", query.Get("OSSAccessKeyId"))
|
||||
asserts.EqualValues("838860800", query.Get("x-oss-traffic-limit"))
|
||||
asserts.NotEmpty(query.Get("response-content-disposition"))
|
||||
}
|
||||
|
||||
// 公共空间
|
||||
{
|
||||
handler.Policy.IsPrivate = false
|
||||
res, err := handler.Source(context.Background(), "/123", url.URL{}, 10, false, 0)
|
||||
asserts.NoError(err)
|
||||
resURL, err := url.Parse(res)
|
||||
asserts.NoError(err)
|
||||
query := resURL.Query()
|
||||
asserts.Empty(query.Get("Signature"))
|
||||
}
|
||||
|
||||
// 正常 指定了CDN域名
|
||||
{
|
||||
handler.Policy.BaseURL = "https://cqu.edu.cn"
|
||||
res, err := handler.Source(context.Background(), "/123", url.URL{}, 10, false, 0)
|
||||
asserts.NoError(err)
|
||||
resURL, err := url.Parse(res)
|
||||
asserts.NoError(err)
|
||||
query := resURL.Query()
|
||||
asserts.Empty(query.Get("Signature"))
|
||||
asserts.Contains(resURL.String(), handler.Policy.BaseURL)
|
||||
}
|
||||
|
||||
// 强制使用公网 Endpoint
|
||||
{
|
||||
handler.Policy.BaseURL = ""
|
||||
handler.Policy.OptionsSerialized.ServerSideEndpoint = "endpoint.com"
|
||||
res, err := handler.Source(context.WithValue(context.Background(), fsctx.ForceUsePublicEndpointCtx, false), "/123", url.URL{}, 10, false, 0)
|
||||
asserts.NoError(err)
|
||||
resURL, err := url.Parse(res)
|
||||
asserts.NoError(err)
|
||||
query := resURL.Query()
|
||||
asserts.Empty(query.Get("Signature"))
|
||||
asserts.Contains(resURL.String(), "endpoint.com")
|
||||
}
|
||||
}
|
||||
|
||||
func TestDriver_Thumb(t *testing.T) {
|
||||
asserts := assert.New(t)
|
||||
handler := Driver{
|
||||
Policy: &model.Policy{
|
||||
AccessKey: "ak",
|
||||
SecretKey: "sk",
|
||||
BucketName: "test",
|
||||
Server: "test.com",
|
||||
},
|
||||
}
|
||||
|
||||
// 上下文不存在
|
||||
{
|
||||
ctx := context.Background()
|
||||
res, err := handler.Thumb(ctx, "/123.txt")
|
||||
asserts.Error(err)
|
||||
asserts.Nil(res)
|
||||
}
|
||||
|
||||
// 成功
|
||||
{
|
||||
cache.Set("setting_preview_timeout", "60", 0)
|
||||
ctx := context.WithValue(context.Background(), fsctx.ThumbSizeCtx, [2]uint{10, 20})
|
||||
res, err := handler.Thumb(ctx, "/123.jpg")
|
||||
asserts.NoError(err)
|
||||
resURL, err := url.Parse(res.URL)
|
||||
asserts.NoError(err)
|
||||
urlQuery := resURL.Query()
|
||||
asserts.Equal("image/resize,m_lfit,h_20,w_10", urlQuery.Get("x-oss-process"))
|
||||
}
|
||||
}
|
||||
|
||||
func TestDriver_Delete(t *testing.T) {
|
||||
asserts := assert.New(t)
|
||||
handler := Driver{
|
||||
Policy: &model.Policy{
|
||||
AccessKey: "ak",
|
||||
SecretKey: "sk",
|
||||
BucketName: "test",
|
||||
Server: "oss-cn-shanghai.aliyuncs.com",
|
||||
},
|
||||
}
|
||||
|
||||
// 失败
|
||||
{
|
||||
res, err := handler.Delete(context.Background(), []string{"1", "2", "3"})
|
||||
asserts.Error(err)
|
||||
asserts.Equal([]string{"1", "2", "3"}, res)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDriver_Put(t *testing.T) {
|
||||
asserts := assert.New(t)
|
||||
handler := Driver{
|
||||
Policy: &model.Policy{
|
||||
AccessKey: "ak",
|
||||
SecretKey: "sk",
|
||||
BucketName: "test",
|
||||
Server: "oss-cn-shanghai.aliyuncs.com",
|
||||
},
|
||||
}
|
||||
cache.Set("setting_upload_credential_timeout", "3600", 0)
|
||||
ctx := context.WithValue(context.Background(), fsctx.DisableOverwrite, true)
|
||||
|
||||
// 失败
|
||||
{
|
||||
err := handler.Put(ctx, ioutil.NopCloser(strings.NewReader("123")), "/123.txt", 3)
|
||||
asserts.Error(err)
|
||||
}
|
||||
}
|
||||
|
||||
type ClientMock struct {
|
||||
testMock.Mock
|
||||
}
|
||||
|
||||
func (m ClientMock) Request(method, target string, body io.Reader, opts ...request.Option) *request.Response {
|
||||
args := m.Called(method, target, body, opts)
|
||||
return args.Get(0).(*request.Response)
|
||||
}
|
||||
|
||||
func TestDriver_Get(t *testing.T) {
|
||||
asserts := assert.New(t)
|
||||
handler := Driver{
|
||||
Policy: &model.Policy{
|
||||
AccessKey: "ak",
|
||||
SecretKey: "sk",
|
||||
BucketName: "test",
|
||||
Server: "oss-cn-shanghai.aliyuncs.com",
|
||||
},
|
||||
HTTPClient: request.NewClient(),
|
||||
}
|
||||
cache.Set("setting_preview_timeout", "3600", 0)
|
||||
|
||||
// 响应失败
|
||||
{
|
||||
res, err := handler.Get(context.Background(), "123.txt")
|
||||
asserts.Error(err)
|
||||
asserts.Nil(res)
|
||||
}
|
||||
|
||||
// 响应成功
|
||||
{
|
||||
ctx := context.WithValue(context.Background(), fsctx.FileModelCtx, model.File{Size: 3})
|
||||
clientMock := ClientMock{}
|
||||
clientMock.On(
|
||||
"Request",
|
||||
"GET",
|
||||
testMock.Anything,
|
||||
testMock.Anything,
|
||||
testMock.Anything,
|
||||
).Return(&request.Response{
|
||||
Err: nil,
|
||||
Response: &http.Response{
|
||||
StatusCode: 200,
|
||||
Body: ioutil.NopCloser(strings.NewReader(`123`)),
|
||||
},
|
||||
})
|
||||
handler.HTTPClient = clientMock
|
||||
res, err := handler.Get(ctx, "123.txt")
|
||||
clientMock.AssertExpectations(t)
|
||||
asserts.NoError(err)
|
||||
n, err := res.Seek(0, io.SeekEnd)
|
||||
asserts.NoError(err)
|
||||
asserts.EqualValues(3, n)
|
||||
content, err := ioutil.ReadAll(res)
|
||||
asserts.NoError(err)
|
||||
asserts.Equal("123", string(content))
|
||||
}
|
||||
}
|
||||
|
||||
func TestDriver_List(t *testing.T) {
|
||||
asserts := assert.New(t)
|
||||
handler := Driver{
|
||||
Policy: &model.Policy{
|
||||
AccessKey: "ak",
|
||||
SecretKey: "sk",
|
||||
BucketName: "test",
|
||||
Server: "test.com",
|
||||
IsPrivate: true,
|
||||
},
|
||||
}
|
||||
|
||||
// 连接失败
|
||||
{
|
||||
res, err := handler.List(context.Background(), "/", true)
|
||||
asserts.Error(err)
|
||||
asserts.Empty(res)
|
||||
}
|
||||
}
|
||||
@@ -2,9 +2,9 @@ package qiniu
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/base64"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"path"
|
||||
@@ -17,17 +17,35 @@ import (
|
||||
"github.com/cloudreve/Cloudreve/v3/pkg/filesystem/response"
|
||||
"github.com/cloudreve/Cloudreve/v3/pkg/request"
|
||||
"github.com/cloudreve/Cloudreve/v3/pkg/serializer"
|
||||
"github.com/qiniu/api.v7/v7/auth/qbox"
|
||||
"github.com/qiniu/api.v7/v7/storage"
|
||||
"github.com/qiniu/go-sdk/v7/auth/qbox"
|
||||
"github.com/qiniu/go-sdk/v7/storage"
|
||||
)
|
||||
|
||||
// Driver 本地策略适配器
|
||||
type Driver struct {
|
||||
Policy *model.Policy
|
||||
mac *qbox.Mac
|
||||
cfg *storage.Config
|
||||
bucket *storage.BucketManager
|
||||
}
|
||||
|
||||
func NewDriver(policy *model.Policy) *Driver {
|
||||
if policy.OptionsSerialized.ChunkSize == 0 {
|
||||
policy.OptionsSerialized.ChunkSize = 25 << 20 // 25 MB
|
||||
}
|
||||
|
||||
mac := qbox.NewMac(policy.AccessKey, policy.SecretKey)
|
||||
cfg := &storage.Config{UseHTTPS: true}
|
||||
return &Driver{
|
||||
Policy: policy,
|
||||
mac: mac,
|
||||
cfg: cfg,
|
||||
bucket: storage.NewBucketManager(mac, cfg),
|
||||
}
|
||||
}
|
||||
|
||||
// List 列出给定路径下的文件
|
||||
func (handler Driver) List(ctx context.Context, base string, recursive bool) ([]response.Object, error) {
|
||||
func (handler *Driver) List(ctx context.Context, base string, recursive bool) ([]response.Object, error) {
|
||||
base = strings.TrimPrefix(base, "/")
|
||||
if base != "" {
|
||||
base += "/"
|
||||
@@ -43,14 +61,8 @@ func (handler Driver) List(ctx context.Context, base string, recursive bool) ([]
|
||||
delimiter = "/"
|
||||
}
|
||||
|
||||
mac := qbox.NewMac(handler.Policy.AccessKey, handler.Policy.SecretKey)
|
||||
cfg := storage.Config{
|
||||
UseHTTPS: true,
|
||||
}
|
||||
bucketManager := storage.NewBucketManager(mac, &cfg)
|
||||
|
||||
for {
|
||||
entries, folders, nextMarker, hashNext, err := bucketManager.ListFiles(
|
||||
entries, folders, nextMarker, hashNext, err := handler.bucket.ListFiles(
|
||||
handler.Policy.BucketName,
|
||||
base, delimiter, marker, 1000)
|
||||
if err != nil {
|
||||
@@ -100,7 +112,7 @@ func (handler Driver) List(ctx context.Context, base string, recursive bool) ([]
|
||||
}
|
||||
|
||||
// Get 获取文件
|
||||
func (handler Driver) Get(ctx context.Context, path string) (response.RSCloser, error) {
|
||||
func (handler *Driver) Get(ctx context.Context, path string) (response.RSCloser, error) {
|
||||
// 给文件名加上随机参数以强制拉取
|
||||
path = fmt.Sprintf("%s?v=%d", path, time.Now().UnixNano())
|
||||
|
||||
@@ -118,7 +130,7 @@ func (handler Driver) Get(ctx context.Context, path string) (response.RSCloser,
|
||||
}
|
||||
|
||||
// 获取文件数据流
|
||||
client := request.HTTPClient{}
|
||||
client := request.NewClient()
|
||||
resp, err := client.Request(
|
||||
"GET",
|
||||
downloadURL,
|
||||
@@ -144,19 +156,25 @@ func (handler Driver) Get(ctx context.Context, path string) (response.RSCloser,
|
||||
}
|
||||
|
||||
// Put 将文件流保存到指定目录
|
||||
func (handler Driver) Put(ctx context.Context, file io.ReadCloser, dst string, size uint64) error {
|
||||
func (handler *Driver) Put(ctx context.Context, file fsctx.FileHeader) error {
|
||||
defer file.Close()
|
||||
|
||||
// 凭证有效期
|
||||
credentialTTL := model.GetIntSetting("upload_credential_timeout", 3600)
|
||||
credentialTTL := model.GetIntSetting("upload_session_timeout", 3600)
|
||||
|
||||
// 生成上传策略
|
||||
fileInfo := file.Info()
|
||||
scope := handler.Policy.BucketName
|
||||
if fileInfo.Mode&fsctx.Overwrite == fsctx.Overwrite {
|
||||
scope = fmt.Sprintf("%s:%s", handler.Policy.BucketName, fileInfo.SavePath)
|
||||
}
|
||||
|
||||
putPolicy := storage.PutPolicy{
|
||||
// 指定为覆盖策略
|
||||
Scope: fmt.Sprintf("%s:%s", handler.Policy.BucketName, dst),
|
||||
SaveKey: dst,
|
||||
Scope: scope,
|
||||
SaveKey: fileInfo.SavePath,
|
||||
ForceSaveKey: true,
|
||||
FsizeLimit: int64(size),
|
||||
FsizeLimit: int64(fileInfo.Size),
|
||||
}
|
||||
// 是否开启了MIMEType限制
|
||||
if handler.Policy.OptionsSerialized.MimeType != "" {
|
||||
@@ -164,7 +182,7 @@ func (handler Driver) Put(ctx context.Context, file io.ReadCloser, dst string, s
|
||||
}
|
||||
|
||||
// 生成上传凭证
|
||||
token, err := handler.getUploadCredential(ctx, putPolicy, int64(credentialTTL))
|
||||
token, err := handler.getUploadCredential(ctx, putPolicy, fileInfo, int64(credentialTTL), false)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -178,7 +196,7 @@ func (handler Driver) Put(ctx context.Context, file io.ReadCloser, dst string, s
|
||||
}
|
||||
|
||||
// 开始上传
|
||||
err = formUploader.Put(ctx, &ret, token.Token, dst, file, int64(size), &putExtra)
|
||||
err = formUploader.Put(ctx, &ret, token.Credential, fileInfo.SavePath, file, int64(fileInfo.Size), &putExtra)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -188,19 +206,14 @@ func (handler Driver) Put(ctx context.Context, file io.ReadCloser, dst string, s
|
||||
|
||||
// Delete 删除一个或多个文件,
|
||||
// 返回未删除的文件
|
||||
func (handler Driver) Delete(ctx context.Context, files []string) ([]string, error) {
|
||||
func (handler *Driver) Delete(ctx context.Context, files []string) ([]string, error) {
|
||||
// TODO 大于一千个文件需要分批发送
|
||||
deleteOps := make([]string, 0, len(files))
|
||||
for _, key := range files {
|
||||
deleteOps = append(deleteOps, storage.URIDelete(handler.Policy.BucketName, key))
|
||||
}
|
||||
|
||||
mac := qbox.NewMac(handler.Policy.AccessKey, handler.Policy.SecretKey)
|
||||
cfg := storage.Config{
|
||||
UseHTTPS: true,
|
||||
}
|
||||
bucketManager := storage.NewBucketManager(mac, &cfg)
|
||||
rets, err := bucketManager.Batch(deleteOps)
|
||||
rets, err := handler.bucket.Batch(deleteOps)
|
||||
|
||||
// 处理删除结果
|
||||
if err != nil {
|
||||
@@ -217,7 +230,7 @@ func (handler Driver) Delete(ctx context.Context, files []string) ([]string, err
|
||||
}
|
||||
|
||||
// Thumb 获取文件缩略图
|
||||
func (handler Driver) Thumb(ctx context.Context, path string) (*response.ContentResponse, error) {
|
||||
func (handler *Driver) Thumb(ctx context.Context, path string) (*response.ContentResponse, error) {
|
||||
var (
|
||||
thumbSize = [2]uint{400, 300}
|
||||
ok = false
|
||||
@@ -238,7 +251,7 @@ func (handler Driver) Thumb(ctx context.Context, path string) (*response.Content
|
||||
}
|
||||
|
||||
// Source 获取外链URL
|
||||
func (handler Driver) Source(
|
||||
func (handler *Driver) Source(
|
||||
ctx context.Context,
|
||||
path string,
|
||||
baseURL url.URL,
|
||||
@@ -261,12 +274,11 @@ func (handler Driver) Source(
|
||||
return handler.signSourceURL(ctx, path, ttl), nil
|
||||
}
|
||||
|
||||
func (handler Driver) signSourceURL(ctx context.Context, path string, ttl int64) string {
|
||||
func (handler *Driver) signSourceURL(ctx context.Context, path string, ttl int64) string {
|
||||
var sourceURL string
|
||||
if handler.Policy.IsPrivate {
|
||||
mac := qbox.NewMac(handler.Policy.AccessKey, handler.Policy.SecretKey)
|
||||
deadline := time.Now().Add(time.Second * time.Duration(ttl)).Unix()
|
||||
sourceURL = storage.MakePrivateURL(mac, handler.Policy.BaseURL, path, deadline)
|
||||
sourceURL = storage.MakePrivateURL(handler.mac, handler.Policy.BaseURL, path, deadline)
|
||||
} else {
|
||||
sourceURL = storage.MakePublicURL(handler.Policy.BaseURL, path)
|
||||
}
|
||||
@@ -274,25 +286,20 @@ func (handler Driver) signSourceURL(ctx context.Context, path string, ttl int64)
|
||||
}
|
||||
|
||||
// Token 获取上传策略和认证Token
|
||||
func (handler Driver) Token(ctx context.Context, TTL int64, key string) (serializer.UploadCredential, error) {
|
||||
func (handler *Driver) Token(ctx context.Context, ttl int64, uploadSession *serializer.UploadSession, file fsctx.FileHeader) (*serializer.UploadCredential, error) {
|
||||
// 生成回调地址
|
||||
siteURL := model.GetSiteURL()
|
||||
apiBaseURI, _ := url.Parse("/api/v3/callback/qiniu/" + key)
|
||||
apiBaseURI, _ := url.Parse("/api/v3/callback/qiniu/" + uploadSession.Key)
|
||||
apiURL := siteURL.ResolveReference(apiBaseURI)
|
||||
|
||||
// 读取上下文中生成的存储路径
|
||||
savePath, ok := ctx.Value(fsctx.SavePathCtx).(string)
|
||||
if !ok {
|
||||
return serializer.UploadCredential{}, errors.New("无法获取存储路径")
|
||||
}
|
||||
|
||||
// 创建上传策略
|
||||
fileInfo := file.Info()
|
||||
putPolicy := storage.PutPolicy{
|
||||
Scope: handler.Policy.BucketName,
|
||||
CallbackURL: apiURL.String(),
|
||||
CallbackBody: `{"name":"$(fname)","source_name":"$(key)","size":$(fsize),"pic_info":"$(imageInfo.width),$(imageInfo.height)"}`,
|
||||
CallbackBody: `{"size":$(fsize),"pic_info":"$(imageInfo.width),$(imageInfo.height)"}`,
|
||||
CallbackBodyType: "application/json",
|
||||
SaveKey: savePath,
|
||||
SaveKey: fileInfo.SavePath,
|
||||
ForceSaveKey: true,
|
||||
FsizeLimit: int64(handler.Policy.MaxSize),
|
||||
}
|
||||
@@ -301,16 +308,46 @@ func (handler Driver) Token(ctx context.Context, TTL int64, key string) (seriali
|
||||
putPolicy.MimeLimit = handler.Policy.OptionsSerialized.MimeType
|
||||
}
|
||||
|
||||
return handler.getUploadCredential(ctx, putPolicy, TTL)
|
||||
credential, err := handler.getUploadCredential(ctx, putPolicy, fileInfo, ttl, true)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to init parts: %w", err)
|
||||
}
|
||||
|
||||
credential.SessionID = uploadSession.Key
|
||||
credential.ChunkSize = handler.Policy.OptionsSerialized.ChunkSize
|
||||
|
||||
uploadSession.UploadURL = credential.UploadURLs[0]
|
||||
uploadSession.Credential = credential.Credential
|
||||
|
||||
return credential, nil
|
||||
}
|
||||
|
||||
// getUploadCredential 签名上传策略
|
||||
func (handler Driver) getUploadCredential(ctx context.Context, policy storage.PutPolicy, TTL int64) (serializer.UploadCredential, error) {
|
||||
// getUploadCredential 签名上传策略并创建上传会话
|
||||
func (handler *Driver) getUploadCredential(ctx context.Context, policy storage.PutPolicy, file *fsctx.UploadTaskInfo, TTL int64, resume bool) (*serializer.UploadCredential, error) {
|
||||
// 上传凭证
|
||||
policy.Expires = uint64(TTL)
|
||||
mac := qbox.NewMac(handler.Policy.AccessKey, handler.Policy.SecretKey)
|
||||
upToken := policy.UploadToken(mac)
|
||||
upToken := policy.UploadToken(handler.mac)
|
||||
|
||||
return serializer.UploadCredential{
|
||||
Token: upToken,
|
||||
}, nil
|
||||
// 初始化分片上传
|
||||
resumeUploader := storage.NewResumeUploaderV2(handler.cfg)
|
||||
upHost, err := resumeUploader.UpHost(handler.Policy.AccessKey, handler.Policy.BucketName)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
ret := &storage.InitPartsRet{}
|
||||
if resume {
|
||||
err = resumeUploader.InitParts(ctx, upToken, upHost, handler.Policy.BucketName, file.SavePath, true, ret)
|
||||
}
|
||||
|
||||
return &serializer.UploadCredential{
|
||||
UploadURLs: []string{upHost + "/buckets/" + handler.Policy.BucketName + "/objects/" + base64.URLEncoding.EncodeToString([]byte(file.SavePath)) + "/uploads/" + ret.UploadID},
|
||||
Credential: upToken,
|
||||
}, err
|
||||
}
|
||||
|
||||
// 取消上传凭证
|
||||
func (handler Driver) CancelToken(ctx context.Context, uploadSession *serializer.UploadSession) error {
|
||||
resumeUploader := storage.NewResumeUploaderV2(handler.cfg)
|
||||
return resumeUploader.Client.CallWith(ctx, nil, "DELETE", uploadSession.UploadURL, http.Header{"Authorization": {"UpToken " + uploadSession.Credential}}, nil, 0)
|
||||
}
|
||||
|
||||
195
pkg/filesystem/driver/remote/client.go
Normal file
195
pkg/filesystem/driver/remote/client.go
Normal file
@@ -0,0 +1,195 @@
|
||||
package remote
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
model "github.com/cloudreve/Cloudreve/v3/models"
|
||||
"github.com/cloudreve/Cloudreve/v3/pkg/auth"
|
||||
"github.com/cloudreve/Cloudreve/v3/pkg/filesystem/chunk"
|
||||
"github.com/cloudreve/Cloudreve/v3/pkg/filesystem/chunk/backoff"
|
||||
"github.com/cloudreve/Cloudreve/v3/pkg/filesystem/fsctx"
|
||||
"github.com/cloudreve/Cloudreve/v3/pkg/request"
|
||||
"github.com/cloudreve/Cloudreve/v3/pkg/serializer"
|
||||
"github.com/cloudreve/Cloudreve/v3/pkg/util"
|
||||
"github.com/gofrs/uuid"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"path"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
const (
|
||||
basePath = "/api/v3/slave/"
|
||||
OverwriteHeader = auth.CrHeaderPrefix + "Overwrite"
|
||||
chunkRetrySleep = time.Duration(5) * time.Second
|
||||
)
|
||||
|
||||
// Client to operate uploading to remote slave server
|
||||
type Client interface {
|
||||
// CreateUploadSession creates remote upload session
|
||||
CreateUploadSession(ctx context.Context, session *serializer.UploadSession, ttl int64, overwrite bool) error
|
||||
// GetUploadURL signs an url for uploading file
|
||||
GetUploadURL(ttl int64, sessionID string) (string, string, error)
|
||||
// Upload uploads file to remote server
|
||||
Upload(ctx context.Context, file fsctx.FileHeader) error
|
||||
// DeleteUploadSession deletes remote upload session
|
||||
DeleteUploadSession(ctx context.Context, sessionID string) error
|
||||
}
|
||||
|
||||
// NewClient creates new Client from given policy
|
||||
func NewClient(policy *model.Policy) (Client, error) {
|
||||
authInstance := auth.HMACAuth{[]byte(policy.SecretKey)}
|
||||
serverURL, err := url.Parse(policy.Server)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
base, _ := url.Parse(basePath)
|
||||
signTTL := model.GetIntSetting("slave_api_timeout", 60)
|
||||
|
||||
return &remoteClient{
|
||||
policy: policy,
|
||||
authInstance: authInstance,
|
||||
httpClient: request.NewClient(
|
||||
request.WithEndpoint(serverURL.ResolveReference(base).String()),
|
||||
request.WithCredential(authInstance, int64(signTTL)),
|
||||
request.WithMasterMeta(),
|
||||
request.WithSlaveMeta(policy.AccessKey),
|
||||
),
|
||||
}, nil
|
||||
}
|
||||
|
||||
type remoteClient struct {
|
||||
policy *model.Policy
|
||||
authInstance auth.Auth
|
||||
httpClient request.Client
|
||||
}
|
||||
|
||||
func (c *remoteClient) Upload(ctx context.Context, file fsctx.FileHeader) error {
|
||||
ttl := model.GetIntSetting("upload_session_timeout", 86400)
|
||||
fileInfo := file.Info()
|
||||
session := &serializer.UploadSession{
|
||||
Key: uuid.Must(uuid.NewV4()).String(),
|
||||
VirtualPath: fileInfo.VirtualPath,
|
||||
Name: fileInfo.FileName,
|
||||
Size: fileInfo.Size,
|
||||
SavePath: fileInfo.SavePath,
|
||||
LastModified: fileInfo.LastModified,
|
||||
Policy: *c.policy,
|
||||
}
|
||||
|
||||
// Create upload session
|
||||
overwrite := fileInfo.Mode&fsctx.Overwrite == fsctx.Overwrite
|
||||
if err := c.CreateUploadSession(ctx, session, int64(ttl), overwrite); err != nil {
|
||||
return fmt.Errorf("failed to create upload session: %w", err)
|
||||
}
|
||||
|
||||
// Initial chunk groups
|
||||
chunks := chunk.NewChunkGroup(file, c.policy.OptionsSerialized.ChunkSize, &backoff.ConstantBackoff{
|
||||
Max: model.GetIntSetting("chunk_retries", 5),
|
||||
Sleep: chunkRetrySleep,
|
||||
}, model.IsTrueVal(model.GetSettingByName("use_temp_chunk_buffer")))
|
||||
|
||||
uploadFunc := func(current *chunk.ChunkGroup, content io.Reader) error {
|
||||
return c.uploadChunk(ctx, session.Key, current.Index(), content, overwrite, current.Length())
|
||||
}
|
||||
|
||||
// upload chunks
|
||||
for chunks.Next() {
|
||||
if err := chunks.Process(uploadFunc); err != nil {
|
||||
if err := c.DeleteUploadSession(ctx, session.Key); err != nil {
|
||||
util.Log().Warning("failed to delete upload session: %s", err)
|
||||
}
|
||||
|
||||
return fmt.Errorf("failed to upload chunk #%d: %w", chunks.Index(), err)
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (c *remoteClient) DeleteUploadSession(ctx context.Context, sessionID string) error {
|
||||
resp, err := c.httpClient.Request(
|
||||
"DELETE",
|
||||
"upload/"+sessionID,
|
||||
nil,
|
||||
request.WithContext(ctx),
|
||||
).CheckHTTPResponse(200).DecodeResponse()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if resp.Code != 0 {
|
||||
return serializer.NewErrorFromResponse(resp)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (c *remoteClient) CreateUploadSession(ctx context.Context, session *serializer.UploadSession, ttl int64, overwrite bool) error {
|
||||
reqBodyEncoded, err := json.Marshal(map[string]interface{}{
|
||||
"session": session,
|
||||
"ttl": ttl,
|
||||
"overwrite": overwrite,
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
bodyReader := strings.NewReader(string(reqBodyEncoded))
|
||||
resp, err := c.httpClient.Request(
|
||||
"PUT",
|
||||
"upload",
|
||||
bodyReader,
|
||||
request.WithContext(ctx),
|
||||
).CheckHTTPResponse(200).DecodeResponse()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if resp.Code != 0 {
|
||||
return serializer.NewErrorFromResponse(resp)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (c *remoteClient) GetUploadURL(ttl int64, sessionID string) (string, string, error) {
|
||||
base, err := url.Parse(c.policy.Server)
|
||||
if err != nil {
|
||||
return "", "", err
|
||||
}
|
||||
|
||||
base.Path = path.Join(base.Path, basePath, "upload", sessionID)
|
||||
req, err := http.NewRequest("POST", base.String(), nil)
|
||||
if err != nil {
|
||||
return "", "", err
|
||||
}
|
||||
|
||||
req = auth.SignRequest(c.authInstance, req, ttl)
|
||||
return req.URL.String(), req.Header["Authorization"][0], nil
|
||||
}
|
||||
|
||||
func (c *remoteClient) uploadChunk(ctx context.Context, sessionID string, index int, chunk io.Reader, overwrite bool, size int64) error {
|
||||
resp, err := c.httpClient.Request(
|
||||
"POST",
|
||||
fmt.Sprintf("upload/%s?chunk=%d", sessionID, index),
|
||||
chunk,
|
||||
request.WithContext(ctx),
|
||||
request.WithTimeout(time.Duration(0)),
|
||||
request.WithContentLength(size),
|
||||
request.WithHeader(map[string][]string{OverwriteHeader: {fmt.Sprintf("%t", overwrite)}}),
|
||||
).CheckHTTPResponse(200).DecodeResponse()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if resp.Code != 0 {
|
||||
return serializer.NewErrorFromResponse(resp)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
262
pkg/filesystem/driver/remote/client_test.go
Normal file
262
pkg/filesystem/driver/remote/client_test.go
Normal file
@@ -0,0 +1,262 @@
|
||||
package remote
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
model "github.com/cloudreve/Cloudreve/v3/models"
|
||||
"github.com/cloudreve/Cloudreve/v3/pkg/cache"
|
||||
"github.com/cloudreve/Cloudreve/v3/pkg/filesystem/fsctx"
|
||||
"github.com/cloudreve/Cloudreve/v3/pkg/mocks/requestmock"
|
||||
"github.com/cloudreve/Cloudreve/v3/pkg/request"
|
||||
"github.com/stretchr/testify/assert"
|
||||
testMock "github.com/stretchr/testify/mock"
|
||||
"io/ioutil"
|
||||
"net/http"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestNewClient(t *testing.T) {
|
||||
a := assert.New(t)
|
||||
policy := &model.Policy{}
|
||||
|
||||
// 无法解析服务端url
|
||||
{
|
||||
policy.Server = string([]byte{0x7f})
|
||||
c, err := NewClient(policy)
|
||||
a.Error(err)
|
||||
a.Nil(c)
|
||||
}
|
||||
|
||||
// 成功
|
||||
{
|
||||
policy.Server = ""
|
||||
c, err := NewClient(policy)
|
||||
a.NoError(err)
|
||||
a.NotNil(c)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRemoteClient_Upload(t *testing.T) {
|
||||
a := assert.New(t)
|
||||
c, _ := NewClient(&model.Policy{})
|
||||
|
||||
// 无法创建上传会话
|
||||
{
|
||||
clientMock := requestmock.RequestMock{}
|
||||
c.(*remoteClient).httpClient = &clientMock
|
||||
clientMock.On(
|
||||
"Request",
|
||||
"PUT",
|
||||
"upload",
|
||||
testMock.Anything,
|
||||
testMock.Anything,
|
||||
).Return(&request.Response{
|
||||
Err: errors.New("error"),
|
||||
})
|
||||
err := c.Upload(context.Background(), &fsctx.FileStream{})
|
||||
a.Error(err)
|
||||
a.Contains(err.Error(), "error")
|
||||
clientMock.AssertExpectations(t)
|
||||
}
|
||||
|
||||
// 分片上传失败,成功删除上传会话
|
||||
{
|
||||
cache.Set("setting_chunk_retries", "1", 0)
|
||||
clientMock := requestmock.RequestMock{}
|
||||
c.(*remoteClient).httpClient = &clientMock
|
||||
clientMock.On(
|
||||
"Request",
|
||||
"PUT",
|
||||
"upload",
|
||||
testMock.Anything,
|
||||
testMock.Anything,
|
||||
).Return(&request.Response{
|
||||
Err: nil,
|
||||
Response: &http.Response{
|
||||
StatusCode: 200,
|
||||
Body: ioutil.NopCloser(strings.NewReader(`{"code":0}`)),
|
||||
},
|
||||
})
|
||||
clientMock.On(
|
||||
"Request",
|
||||
"POST",
|
||||
testMock.Anything,
|
||||
testMock.Anything,
|
||||
testMock.Anything,
|
||||
).Return(&request.Response{
|
||||
Err: errors.New("error"),
|
||||
})
|
||||
clientMock.On(
|
||||
"Request",
|
||||
"DELETE",
|
||||
testMock.Anything,
|
||||
testMock.Anything,
|
||||
testMock.Anything,
|
||||
).Return(&request.Response{
|
||||
Err: nil,
|
||||
Response: &http.Response{
|
||||
StatusCode: 200,
|
||||
Body: ioutil.NopCloser(strings.NewReader(`{"code":0}`)),
|
||||
},
|
||||
})
|
||||
err := c.Upload(context.Background(), &fsctx.FileStream{})
|
||||
a.Error(err)
|
||||
a.Contains(err.Error(), "error")
|
||||
clientMock.AssertExpectations(t)
|
||||
}
|
||||
|
||||
// 分片上传失败,无法删除上传会话
|
||||
{
|
||||
cache.Set("setting_chunk_retries", "1", 0)
|
||||
clientMock := requestmock.RequestMock{}
|
||||
c.(*remoteClient).httpClient = &clientMock
|
||||
clientMock.On(
|
||||
"Request",
|
||||
"PUT",
|
||||
"upload",
|
||||
testMock.Anything,
|
||||
testMock.Anything,
|
||||
).Return(&request.Response{
|
||||
Err: nil,
|
||||
Response: &http.Response{
|
||||
StatusCode: 200,
|
||||
Body: ioutil.NopCloser(strings.NewReader(`{"code":0}`)),
|
||||
},
|
||||
})
|
||||
clientMock.On(
|
||||
"Request",
|
||||
"POST",
|
||||
testMock.Anything,
|
||||
testMock.Anything,
|
||||
testMock.Anything,
|
||||
).Return(&request.Response{
|
||||
Err: errors.New("error"),
|
||||
})
|
||||
clientMock.On(
|
||||
"Request",
|
||||
"DELETE",
|
||||
testMock.Anything,
|
||||
testMock.Anything,
|
||||
testMock.Anything,
|
||||
).Return(&request.Response{
|
||||
Err: errors.New("error2"),
|
||||
Response: &http.Response{
|
||||
StatusCode: 200,
|
||||
Body: ioutil.NopCloser(strings.NewReader(`{"code":0}`)),
|
||||
},
|
||||
})
|
||||
err := c.Upload(context.Background(), &fsctx.FileStream{})
|
||||
a.Error(err)
|
||||
a.Contains(err.Error(), "error")
|
||||
clientMock.AssertExpectations(t)
|
||||
}
|
||||
|
||||
// 成功
|
||||
{
|
||||
cache.Set("setting_chunk_retries", "1", 0)
|
||||
clientMock := requestmock.RequestMock{}
|
||||
c.(*remoteClient).httpClient = &clientMock
|
||||
clientMock.On(
|
||||
"Request",
|
||||
"PUT",
|
||||
"upload",
|
||||
testMock.Anything,
|
||||
testMock.Anything,
|
||||
).Return(&request.Response{
|
||||
Err: nil,
|
||||
Response: &http.Response{
|
||||
StatusCode: 200,
|
||||
Body: ioutil.NopCloser(strings.NewReader(`{"code":0}`)),
|
||||
},
|
||||
})
|
||||
clientMock.On(
|
||||
"Request",
|
||||
"POST",
|
||||
testMock.Anything,
|
||||
testMock.Anything,
|
||||
testMock.Anything,
|
||||
).Return(&request.Response{
|
||||
Response: &http.Response{
|
||||
StatusCode: 200,
|
||||
Body: ioutil.NopCloser(strings.NewReader(`{"code":0}`)),
|
||||
},
|
||||
})
|
||||
err := c.Upload(context.Background(), &fsctx.FileStream{})
|
||||
a.NoError(err)
|
||||
clientMock.AssertExpectations(t)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRemoteClient_CreateUploadSessionFailed(t *testing.T) {
|
||||
a := assert.New(t)
|
||||
c, _ := NewClient(&model.Policy{})
|
||||
|
||||
clientMock := requestmock.RequestMock{}
|
||||
c.(*remoteClient).httpClient = &clientMock
|
||||
clientMock.On(
|
||||
"Request",
|
||||
"PUT",
|
||||
"upload",
|
||||
testMock.Anything,
|
||||
testMock.Anything,
|
||||
).Return(&request.Response{
|
||||
Err: nil,
|
||||
Response: &http.Response{
|
||||
StatusCode: 200,
|
||||
Body: ioutil.NopCloser(strings.NewReader(`{"code":500,"msg":"error"}`)),
|
||||
},
|
||||
})
|
||||
err := c.Upload(context.Background(), &fsctx.FileStream{})
|
||||
a.Error(err)
|
||||
a.Contains(err.Error(), "error")
|
||||
clientMock.AssertExpectations(t)
|
||||
}
|
||||
|
||||
func TestRemoteClient_UploadChunkFailed(t *testing.T) {
|
||||
a := assert.New(t)
|
||||
c, _ := NewClient(&model.Policy{})
|
||||
|
||||
clientMock := requestmock.RequestMock{}
|
||||
c.(*remoteClient).httpClient = &clientMock
|
||||
clientMock.On(
|
||||
"Request",
|
||||
"POST",
|
||||
testMock.Anything,
|
||||
testMock.Anything,
|
||||
testMock.Anything,
|
||||
).Return(&request.Response{
|
||||
Err: nil,
|
||||
Response: &http.Response{
|
||||
StatusCode: 200,
|
||||
Body: ioutil.NopCloser(strings.NewReader(`{"code":500,"msg":"error"}`)),
|
||||
},
|
||||
})
|
||||
err := c.(*remoteClient).uploadChunk(context.Background(), "", 0, strings.NewReader(""), false, 0)
|
||||
a.Error(err)
|
||||
a.Contains(err.Error(), "error")
|
||||
clientMock.AssertExpectations(t)
|
||||
}
|
||||
|
||||
func TestRemoteClient_GetUploadURL(t *testing.T) {
|
||||
a := assert.New(t)
|
||||
c, _ := NewClient(&model.Policy{})
|
||||
|
||||
// url 解析失败
|
||||
{
|
||||
c.(*remoteClient).policy.Server = string([]byte{0x7f})
|
||||
res, sign, err := c.GetUploadURL(0, "")
|
||||
a.Error(err)
|
||||
a.Empty(res)
|
||||
a.Empty(sign)
|
||||
}
|
||||
|
||||
// 成功
|
||||
{
|
||||
c.(*remoteClient).policy.Server = ""
|
||||
res, sign, err := c.GetUploadURL(0, "")
|
||||
a.NoError(err)
|
||||
a.NotEmpty(res)
|
||||
a.NotEmpty(sign)
|
||||
}
|
||||
}
|
||||
@@ -6,8 +6,6 @@ import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"path"
|
||||
"strings"
|
||||
@@ -26,10 +24,28 @@ type Driver struct {
|
||||
Client request.Client
|
||||
Policy *model.Policy
|
||||
AuthInstance auth.Auth
|
||||
|
||||
uploadClient Client
|
||||
}
|
||||
|
||||
// NewDriver initializes a new Driver from policy
|
||||
// TODO: refactor all method into upload client
|
||||
func NewDriver(policy *model.Policy) (*Driver, error) {
|
||||
client, err := NewClient(policy)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return &Driver{
|
||||
Policy: policy,
|
||||
Client: request.NewClient(),
|
||||
AuthInstance: auth.HMACAuth{[]byte(policy.SecretKey)},
|
||||
uploadClient: client,
|
||||
}, nil
|
||||
}
|
||||
|
||||
// List 列取文件
|
||||
func (handler Driver) List(ctx context.Context, path string, recursive bool) ([]response.Object, error) {
|
||||
func (handler *Driver) List(ctx context.Context, path string, recursive bool) ([]response.Object, error) {
|
||||
var res []response.Object
|
||||
|
||||
reqBody := serializer.ListRequest{
|
||||
@@ -71,7 +87,7 @@ func (handler Driver) List(ctx context.Context, path string, recursive bool) ([]
|
||||
}
|
||||
|
||||
// getAPIUrl 获取接口请求地址
|
||||
func (handler Driver) getAPIUrl(scope string, routes ...string) string {
|
||||
func (handler *Driver) getAPIUrl(scope string, routes ...string) string {
|
||||
serverURL, err := url.Parse(handler.Policy.Server)
|
||||
if err != nil {
|
||||
return ""
|
||||
@@ -97,7 +113,7 @@ func (handler Driver) getAPIUrl(scope string, routes ...string) string {
|
||||
}
|
||||
|
||||
// Get 获取文件内容
|
||||
func (handler Driver) Get(ctx context.Context, path string) (response.RSCloser, error) {
|
||||
func (handler *Driver) Get(ctx context.Context, path string) (response.RSCloser, error) {
|
||||
// 尝试获取速度限制
|
||||
speedLimit := 0
|
||||
if user, ok := ctx.Value(fsctx.UserCtx).(model.User); ok {
|
||||
@@ -134,62 +150,15 @@ func (handler Driver) Get(ctx context.Context, path string) (response.RSCloser,
|
||||
}
|
||||
|
||||
// Put 将文件流保存到指定目录
|
||||
func (handler Driver) Put(ctx context.Context, file io.ReadCloser, dst string, size uint64) error {
|
||||
func (handler *Driver) Put(ctx context.Context, file fsctx.FileHeader) error {
|
||||
defer file.Close()
|
||||
|
||||
// 凭证有效期
|
||||
credentialTTL := model.GetIntSetting("upload_credential_timeout", 3600)
|
||||
|
||||
// 生成上传策略
|
||||
policy := serializer.UploadPolicy{
|
||||
SavePath: path.Dir(dst),
|
||||
FileName: path.Base(dst),
|
||||
AutoRename: false,
|
||||
MaxSize: size,
|
||||
}
|
||||
credential, err := handler.getUploadCredential(ctx, policy, int64(credentialTTL))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// 对文件名进行URLEncode
|
||||
fileName := url.QueryEscape(path.Base(dst))
|
||||
|
||||
// 决定是否要禁用文件覆盖
|
||||
overwrite := "true"
|
||||
if ctx.Value(fsctx.DisableOverwrite) != nil {
|
||||
overwrite = "false"
|
||||
}
|
||||
|
||||
// 上传文件
|
||||
resp, err := handler.Client.Request(
|
||||
"POST",
|
||||
handler.Policy.GetUploadURL(),
|
||||
file,
|
||||
request.WithHeader(map[string][]string{
|
||||
"X-Cr-Policy": {credential.Policy},
|
||||
"X-Cr-FileName": {fileName},
|
||||
"X-Cr-Overwrite": {overwrite},
|
||||
}),
|
||||
request.WithContentLength(int64(size)),
|
||||
request.WithTimeout(time.Duration(0)),
|
||||
request.WithMasterMeta(),
|
||||
request.WithSlaveMeta(handler.Policy.AccessKey),
|
||||
request.WithCredential(handler.AuthInstance, int64(credentialTTL)),
|
||||
).CheckHTTPResponse(200).DecodeResponse()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if resp.Code != 0 {
|
||||
return errors.New(resp.Msg)
|
||||
}
|
||||
|
||||
return nil
|
||||
return handler.uploadClient.Upload(ctx, file)
|
||||
}
|
||||
|
||||
// Delete 删除一个或多个文件,
|
||||
// 返回未删除的文件,及遇到的最后一个错误
|
||||
func (handler Driver) Delete(ctx context.Context, files []string) ([]string, error) {
|
||||
func (handler *Driver) Delete(ctx context.Context, files []string) ([]string, error) {
|
||||
// 封装接口请求正文
|
||||
reqBody := serializer.RemoteDeleteRequest{
|
||||
Files: files,
|
||||
@@ -235,7 +204,7 @@ func (handler Driver) Delete(ctx context.Context, files []string) ([]string, err
|
||||
}
|
||||
|
||||
// Thumb 获取文件缩略图
|
||||
func (handler Driver) Thumb(ctx context.Context, path string) (*response.ContentResponse, error) {
|
||||
func (handler *Driver) Thumb(ctx context.Context, path string) (*response.ContentResponse, error) {
|
||||
sourcePath := base64.RawURLEncoding.EncodeToString([]byte(path))
|
||||
thumbURL := handler.getAPIUrl("thumb") + "/" + sourcePath
|
||||
ttl := model.GetIntSetting("preview_timeout", 60)
|
||||
@@ -251,7 +220,7 @@ func (handler Driver) Thumb(ctx context.Context, path string) (*response.Content
|
||||
}
|
||||
|
||||
// Source 获取外链URL
|
||||
func (handler Driver) Source(
|
||||
func (handler *Driver) Source(
|
||||
ctx context.Context,
|
||||
path string,
|
||||
baseURL url.URL,
|
||||
@@ -305,43 +274,32 @@ func (handler Driver) Source(
|
||||
}
|
||||
|
||||
// Token 获取上传策略和认证Token
|
||||
func (handler Driver) Token(ctx context.Context, TTL int64, key string) (serializer.UploadCredential, error) {
|
||||
// 生成回调地址
|
||||
func (handler *Driver) Token(ctx context.Context, ttl int64, uploadSession *serializer.UploadSession, file fsctx.FileHeader) (*serializer.UploadCredential, error) {
|
||||
siteURL := model.GetSiteURL()
|
||||
apiBaseURI, _ := url.Parse("/api/v3/callback/remote/" + key)
|
||||
apiBaseURI, _ := url.Parse(path.Join("/api/v3/callback/remote", uploadSession.Key, uploadSession.CallbackSecret))
|
||||
apiURL := siteURL.ResolveReference(apiBaseURI)
|
||||
|
||||
// 生成上传策略
|
||||
policy := serializer.UploadPolicy{
|
||||
SavePath: handler.Policy.DirNameRule,
|
||||
FileName: handler.Policy.FileNameRule,
|
||||
AutoRename: handler.Policy.AutoRename,
|
||||
MaxSize: handler.Policy.MaxSize,
|
||||
AllowedExtension: handler.Policy.OptionsSerialized.FileType,
|
||||
CallbackURL: apiURL.String(),
|
||||
// 在从机端创建上传会话
|
||||
uploadSession.Callback = apiURL.String()
|
||||
if err := handler.uploadClient.CreateUploadSession(ctx, uploadSession, ttl, false); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return handler.getUploadCredential(ctx, policy, TTL)
|
||||
}
|
||||
|
||||
func (handler Driver) getUploadCredential(ctx context.Context, policy serializer.UploadPolicy, TTL int64) (serializer.UploadCredential, error) {
|
||||
policyEncoded, err := policy.EncodeUploadPolicy()
|
||||
// 获取上传地址
|
||||
uploadURL, sign, err := handler.uploadClient.GetUploadURL(ttl, uploadSession.Key)
|
||||
if err != nil {
|
||||
return serializer.UploadCredential{}, err
|
||||
return nil, fmt.Errorf("failed to sign upload url: %w", err)
|
||||
}
|
||||
|
||||
// 签名上传策略
|
||||
uploadRequest, _ := http.NewRequest("POST", "/api/v3/slave/upload", nil)
|
||||
uploadRequest.Header = map[string][]string{
|
||||
"X-Cr-Policy": {policyEncoded},
|
||||
"X-Cr-Overwrite": {"false"},
|
||||
}
|
||||
auth.SignRequest(handler.AuthInstance, uploadRequest, TTL)
|
||||
|
||||
if credential, ok := uploadRequest.Header["Authorization"]; ok && len(credential) == 1 {
|
||||
return serializer.UploadCredential{
|
||||
Token: credential[0],
|
||||
Policy: policyEncoded,
|
||||
}, nil
|
||||
}
|
||||
return serializer.UploadCredential{}, errors.New("无法签名上传策略")
|
||||
return &serializer.UploadCredential{
|
||||
SessionID: uploadSession.Key,
|
||||
ChunkSize: handler.Policy.OptionsSerialized.ChunkSize,
|
||||
UploadURLs: []string{uploadURL},
|
||||
Credential: sign,
|
||||
}, nil
|
||||
}
|
||||
|
||||
// 取消上传凭证
|
||||
func (handler *Driver) CancelToken(ctx context.Context, uploadSession *serializer.UploadSession) error {
|
||||
return handler.uploadClient.DeleteUploadSession(ctx, uploadSession.Key)
|
||||
}
|
||||
|
||||
@@ -2,6 +2,9 @@ package remote
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"github.com/cloudreve/Cloudreve/v3/pkg/mocks/remoteclientmock"
|
||||
"github.com/cloudreve/Cloudreve/v3/pkg/serializer"
|
||||
"io"
|
||||
"io/ioutil"
|
||||
"net/http"
|
||||
@@ -14,45 +17,26 @@ import (
|
||||
"github.com/cloudreve/Cloudreve/v3/pkg/cache"
|
||||
"github.com/cloudreve/Cloudreve/v3/pkg/filesystem/fsctx"
|
||||
"github.com/cloudreve/Cloudreve/v3/pkg/request"
|
||||
"github.com/cloudreve/Cloudreve/v3/pkg/serializer"
|
||||
"github.com/stretchr/testify/assert"
|
||||
testMock "github.com/stretchr/testify/mock"
|
||||
)
|
||||
|
||||
func TestHandler_Token(t *testing.T) {
|
||||
asserts := assert.New(t)
|
||||
handler := Driver{
|
||||
Policy: &model.Policy{
|
||||
MaxSize: 10,
|
||||
AutoRename: true,
|
||||
DirNameRule: "dir",
|
||||
FileNameRule: "file",
|
||||
OptionsSerialized: model.PolicyOption{
|
||||
FileType: []string{"txt"},
|
||||
},
|
||||
Server: "http://test.com",
|
||||
},
|
||||
AuthInstance: auth.HMACAuth{},
|
||||
func TestNewDriver(t *testing.T) {
|
||||
a := assert.New(t)
|
||||
|
||||
// remoteClient 初始化失败
|
||||
{
|
||||
d, err := NewDriver(&model.Policy{Server: string([]byte{0x7f})})
|
||||
a.Error(err)
|
||||
a.Nil(d)
|
||||
}
|
||||
ctx := context.WithValue(context.Background(), fsctx.DisableOverwrite, true)
|
||||
auth.General = auth.HMACAuth{SecretKey: []byte("test")}
|
||||
|
||||
// 成功
|
||||
{
|
||||
cache.Set("setting_siteURL", "http://test.cloudreve.org", 0)
|
||||
credential, err := handler.Token(ctx, 10, "123")
|
||||
asserts.NoError(err)
|
||||
policy, err := serializer.DecodeUploadPolicy(credential.Policy)
|
||||
asserts.NoError(err)
|
||||
asserts.Equal("http://test.cloudreve.org/api/v3/callback/remote/123", policy.CallbackURL)
|
||||
asserts.Equal(uint64(10), policy.MaxSize)
|
||||
asserts.Equal(true, policy.AutoRename)
|
||||
asserts.Equal("dir", policy.SavePath)
|
||||
asserts.Equal("file", policy.FileName)
|
||||
asserts.Equal("file", policy.FileName)
|
||||
asserts.Equal([]string{"txt"}, policy.AllowedExtension)
|
||||
d, err := NewDriver(&model.Policy{})
|
||||
a.NoError(err)
|
||||
a.NotNil(d)
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
func TestHandler_Source(t *testing.T) {
|
||||
@@ -369,87 +353,17 @@ func TestHandler_Get(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestHandler_Put(t *testing.T) {
|
||||
asserts := assert.New(t)
|
||||
handler := Driver{
|
||||
Policy: &model.Policy{
|
||||
Type: "remote",
|
||||
SecretKey: "test",
|
||||
Server: "http://test.com",
|
||||
},
|
||||
AuthInstance: auth.HMACAuth{},
|
||||
}
|
||||
ctx := context.Background()
|
||||
asserts.NoError(cache.Set("setting_upload_credential_timeout", "3600", 0))
|
||||
|
||||
// 成功
|
||||
{
|
||||
ctx = context.WithValue(ctx, fsctx.UserCtx, model.User{})
|
||||
clientMock := ClientMock{}
|
||||
clientMock.On(
|
||||
"Request",
|
||||
"POST",
|
||||
"http://test.com/api/v3/slave/upload",
|
||||
testMock.Anything,
|
||||
testMock.Anything,
|
||||
).Return(&request.Response{
|
||||
Err: nil,
|
||||
Response: &http.Response{
|
||||
StatusCode: 200,
|
||||
Body: ioutil.NopCloser(strings.NewReader(`{"code":0}`)),
|
||||
},
|
||||
})
|
||||
handler.Client = clientMock
|
||||
err := handler.Put(ctx, ioutil.NopCloser(strings.NewReader("test input file")), "/", 15)
|
||||
clientMock.AssertExpectations(t)
|
||||
asserts.NoError(err)
|
||||
}
|
||||
|
||||
// 请求失败
|
||||
{
|
||||
ctx = context.WithValue(ctx, fsctx.UserCtx, model.User{})
|
||||
clientMock := ClientMock{}
|
||||
clientMock.On(
|
||||
"Request",
|
||||
"POST",
|
||||
"http://test.com/api/v3/slave/upload",
|
||||
testMock.Anything,
|
||||
testMock.Anything,
|
||||
).Return(&request.Response{
|
||||
Err: nil,
|
||||
Response: &http.Response{
|
||||
StatusCode: 404,
|
||||
Body: ioutil.NopCloser(strings.NewReader(`{"code":0}`)),
|
||||
},
|
||||
})
|
||||
handler.Client = clientMock
|
||||
err := handler.Put(ctx, ioutil.NopCloser(strings.NewReader("test input file")), "/", 15)
|
||||
clientMock.AssertExpectations(t)
|
||||
asserts.Error(err)
|
||||
}
|
||||
|
||||
// 返回错误
|
||||
{
|
||||
ctx = context.WithValue(ctx, fsctx.UserCtx, model.User{})
|
||||
clientMock := ClientMock{}
|
||||
clientMock.On(
|
||||
"Request",
|
||||
"POST",
|
||||
"http://test.com/api/v3/slave/upload",
|
||||
testMock.Anything,
|
||||
testMock.Anything,
|
||||
).Return(&request.Response{
|
||||
Err: nil,
|
||||
Response: &http.Response{
|
||||
StatusCode: 200,
|
||||
Body: ioutil.NopCloser(strings.NewReader(`{"code":1}`)),
|
||||
},
|
||||
})
|
||||
handler.Client = clientMock
|
||||
err := handler.Put(ctx, ioutil.NopCloser(strings.NewReader("test input file")), "/", 15)
|
||||
clientMock.AssertExpectations(t)
|
||||
asserts.Error(err)
|
||||
}
|
||||
|
||||
a := assert.New(t)
|
||||
handler, _ := NewDriver(&model.Policy{
|
||||
Type: "remote",
|
||||
SecretKey: "test",
|
||||
Server: "http://test.com",
|
||||
})
|
||||
clientMock := &remoteclientmock.RemoteClientMock{}
|
||||
handler.uploadClient = clientMock
|
||||
clientMock.On("Upload", testMock.Anything, testMock.Anything).Return(errors.New("error"))
|
||||
a.Error(handler.Put(context.Background(), &fsctx.FileStream{}))
|
||||
clientMock.AssertExpectations(t)
|
||||
}
|
||||
|
||||
func TestHandler_Thumb(t *testing.T) {
|
||||
@@ -468,3 +382,60 @@ func TestHandler_Thumb(t *testing.T) {
|
||||
asserts.NoError(err)
|
||||
asserts.True(resp.Redirect)
|
||||
}
|
||||
|
||||
func TestHandler_Token(t *testing.T) {
|
||||
a := assert.New(t)
|
||||
handler, _ := NewDriver(&model.Policy{})
|
||||
|
||||
// 无法创建上传会话
|
||||
{
|
||||
clientMock := &remoteclientmock.RemoteClientMock{}
|
||||
handler.uploadClient = clientMock
|
||||
clientMock.On("CreateUploadSession", testMock.Anything, testMock.Anything, int64(10), false).Return(errors.New("error"))
|
||||
res, err := handler.Token(context.Background(), 10, &serializer.UploadSession{}, &fsctx.FileStream{})
|
||||
a.Error(err)
|
||||
a.Contains(err.Error(), "error")
|
||||
a.Nil(res)
|
||||
clientMock.AssertExpectations(t)
|
||||
}
|
||||
|
||||
// 无法创建上传地址
|
||||
{
|
||||
clientMock := &remoteclientmock.RemoteClientMock{}
|
||||
handler.uploadClient = clientMock
|
||||
clientMock.On("CreateUploadSession", testMock.Anything, testMock.Anything, int64(10), false).Return(nil)
|
||||
clientMock.On("GetUploadURL", int64(10), "").Return("", "", errors.New("error"))
|
||||
res, err := handler.Token(context.Background(), 10, &serializer.UploadSession{}, &fsctx.FileStream{})
|
||||
a.Error(err)
|
||||
a.Contains(err.Error(), "error")
|
||||
a.Nil(res)
|
||||
clientMock.AssertExpectations(t)
|
||||
}
|
||||
|
||||
// 成功
|
||||
{
|
||||
clientMock := &remoteclientmock.RemoteClientMock{}
|
||||
handler.uploadClient = clientMock
|
||||
clientMock.On("CreateUploadSession", testMock.Anything, testMock.Anything, int64(10), false).Return(nil)
|
||||
clientMock.On("GetUploadURL", int64(10), "").Return("1", "2", nil)
|
||||
res, err := handler.Token(context.Background(), 10, &serializer.UploadSession{}, &fsctx.FileStream{})
|
||||
a.NoError(err)
|
||||
a.NotNil(res)
|
||||
a.Equal("1", res.UploadURLs[0])
|
||||
a.Equal("2", res.Credential)
|
||||
clientMock.AssertExpectations(t)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDriver_CancelToken(t *testing.T) {
|
||||
a := assert.New(t)
|
||||
handler, _ := NewDriver(&model.Policy{})
|
||||
|
||||
clientMock := &remoteclientmock.RemoteClientMock{}
|
||||
handler.uploadClient = clientMock
|
||||
clientMock.On("DeleteUploadSession", testMock.Anything, "key").Return(errors.New("error"))
|
||||
err := handler.CancelToken(context.Background(), &serializer.UploadSession{Key: "key"})
|
||||
a.Error(err)
|
||||
a.Contains(err.Error(), "error")
|
||||
clientMock.AssertExpectations(t)
|
||||
}
|
||||
|
||||
@@ -2,12 +2,10 @@ package s3
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/hmac"
|
||||
"crypto/sha256"
|
||||
"encoding/base64"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"github.com/cloudreve/Cloudreve/v3/pkg/filesystem/chunk"
|
||||
"github.com/cloudreve/Cloudreve/v3/pkg/filesystem/chunk/backoff"
|
||||
"github.com/cloudreve/Cloudreve/v3/pkg/util"
|
||||
"io"
|
||||
"net/http"
|
||||
@@ -48,6 +46,18 @@ type MetaData struct {
|
||||
Etag string
|
||||
}
|
||||
|
||||
func NewDriver(policy *model.Policy) (*Driver, error) {
|
||||
if policy.OptionsSerialized.ChunkSize == 0 {
|
||||
policy.OptionsSerialized.ChunkSize = 25 << 20 // 25 MB
|
||||
}
|
||||
|
||||
driver := &Driver{
|
||||
Policy: policy,
|
||||
}
|
||||
|
||||
return driver, driver.InitS3Client()
|
||||
}
|
||||
|
||||
// InitS3Client 初始化S3会话
|
||||
func (handler *Driver) InitS3Client() error {
|
||||
if handler.Policy == nil {
|
||||
@@ -73,13 +83,7 @@ func (handler *Driver) InitS3Client() error {
|
||||
}
|
||||
|
||||
// List 列出给定路径下的文件
|
||||
func (handler Driver) List(ctx context.Context, base string, recursive bool) ([]response.Object, error) {
|
||||
|
||||
// 初始化客户端
|
||||
if err := handler.InitS3Client(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
func (handler *Driver) List(ctx context.Context, base string, recursive bool) ([]response.Object, error) {
|
||||
// 初始化列目录参数
|
||||
base = strings.TrimPrefix(base, "/")
|
||||
if base != "" {
|
||||
@@ -156,8 +160,7 @@ func (handler Driver) List(ctx context.Context, base string, recursive bool) ([]
|
||||
}
|
||||
|
||||
// Get 获取文件
|
||||
func (handler Driver) Get(ctx context.Context, path string) (response.RSCloser, error) {
|
||||
|
||||
func (handler *Driver) Get(ctx context.Context, path string) (response.RSCloser, error) {
|
||||
// 获取文件源地址
|
||||
downloadURL, err := handler.Source(
|
||||
ctx,
|
||||
@@ -198,19 +201,23 @@ func (handler Driver) Get(ctx context.Context, path string) (response.RSCloser,
|
||||
}
|
||||
|
||||
// Put 将文件流保存到指定目录
|
||||
func (handler Driver) Put(ctx context.Context, file io.ReadCloser, dst string, size uint64) error {
|
||||
func (handler *Driver) Put(ctx context.Context, file fsctx.FileHeader) error {
|
||||
defer file.Close()
|
||||
|
||||
// 初始化客户端
|
||||
if err := handler.InitS3Client(); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
uploader := s3manager.NewUploader(handler.sess)
|
||||
uploader := s3manager.NewUploader(handler.sess, func(u *s3manager.Uploader) {
|
||||
u.PartSize = int64(handler.Policy.OptionsSerialized.ChunkSize)
|
||||
})
|
||||
|
||||
dst := file.Info().SavePath
|
||||
_, err := uploader.Upload(&s3manager.UploadInput{
|
||||
Bucket: &handler.Policy.BucketName,
|
||||
Key: &dst,
|
||||
Body: file,
|
||||
Body: io.LimitReader(file, int64(file.Info().Size)),
|
||||
})
|
||||
|
||||
if err != nil {
|
||||
@@ -222,13 +229,7 @@ func (handler Driver) Put(ctx context.Context, file io.ReadCloser, dst string, s
|
||||
|
||||
// Delete 删除一个或多个文件,
|
||||
// 返回未删除的文件,及遇到的最后一个错误
|
||||
func (handler Driver) Delete(ctx context.Context, files []string) ([]string, error) {
|
||||
|
||||
// 初始化客户端
|
||||
if err := handler.InitS3Client(); err != nil {
|
||||
return files, err
|
||||
}
|
||||
|
||||
func (handler *Driver) Delete(ctx context.Context, files []string) ([]string, error) {
|
||||
failed := make([]string, 0, len(files))
|
||||
deleted := make([]string, 0, len(files))
|
||||
|
||||
@@ -262,12 +263,12 @@ func (handler Driver) Delete(ctx context.Context, files []string) ([]string, err
|
||||
}
|
||||
|
||||
// Thumb 获取文件缩略图
|
||||
func (handler Driver) Thumb(ctx context.Context, path string) (*response.ContentResponse, error) {
|
||||
func (handler *Driver) Thumb(ctx context.Context, path string) (*response.ContentResponse, error) {
|
||||
return nil, errors.New("未实现")
|
||||
}
|
||||
|
||||
// Source 获取外链URL
|
||||
func (handler Driver) Source(
|
||||
func (handler *Driver) Source(
|
||||
ctx context.Context,
|
||||
path string,
|
||||
baseURL url.URL,
|
||||
@@ -324,48 +325,75 @@ func (handler Driver) Source(
|
||||
}
|
||||
|
||||
// Token 获取上传策略和认证Token
|
||||
func (handler Driver) Token(ctx context.Context, TTL int64, key string) (serializer.UploadCredential, error) {
|
||||
|
||||
// 读取上下文中生成的存储路径和文件大小
|
||||
savePath, ok := ctx.Value(fsctx.SavePathCtx).(string)
|
||||
if !ok {
|
||||
return serializer.UploadCredential{}, errors.New("无法获取存储路径")
|
||||
func (handler *Driver) Token(ctx context.Context, ttl int64, uploadSession *serializer.UploadSession, file fsctx.FileHeader) (*serializer.UploadCredential, error) {
|
||||
// 检查文件是否存在
|
||||
fileInfo := file.Info()
|
||||
if _, err := handler.Meta(ctx, fileInfo.SavePath); err == nil {
|
||||
return nil, fmt.Errorf("file already exist")
|
||||
}
|
||||
|
||||
// 生成回调地址
|
||||
siteURL := model.GetSiteURL()
|
||||
apiBaseURI, _ := url.Parse("/api/v3/callback/s3/" + key)
|
||||
apiURL := siteURL.ResolveReference(apiBaseURI)
|
||||
|
||||
// 上传策略
|
||||
putPolicy := UploadPolicy{
|
||||
Expiration: time.Now().UTC().Add(time.Duration(TTL) * time.Second).Format(time.RFC3339),
|
||||
Conditions: []interface{}{
|
||||
map[string]string{"bucket": handler.Policy.BucketName},
|
||||
[]string{"starts-with", "$key", savePath},
|
||||
[]string{"starts-with", "$success_action_redirect", apiURL.String()},
|
||||
[]string{"starts-with", "$name", ""},
|
||||
[]string{"starts-with", "$Content-Type", ""},
|
||||
map[string]string{"x-amz-algorithm": "AWS4-HMAC-SHA256"},
|
||||
},
|
||||
// 创建分片上传
|
||||
expires := time.Now().Add(time.Duration(ttl) * time.Second)
|
||||
res, err := handler.svc.CreateMultipartUpload(&s3.CreateMultipartUploadInput{
|
||||
Bucket: &handler.Policy.BucketName,
|
||||
Key: &fileInfo.SavePath,
|
||||
Expires: &expires,
|
||||
})
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to create multipart upload: %w", err)
|
||||
}
|
||||
|
||||
if handler.Policy.MaxSize > 0 {
|
||||
putPolicy.Conditions = append(putPolicy.Conditions,
|
||||
[]interface{}{"content-length-range", 0, handler.Policy.MaxSize})
|
||||
uploadSession.UploadID = *res.UploadId
|
||||
|
||||
// 为每个分片签名上传 URL
|
||||
chunks := chunk.NewChunkGroup(file, handler.Policy.OptionsSerialized.ChunkSize, &backoff.ConstantBackoff{}, false)
|
||||
urls := make([]string, chunks.Num())
|
||||
for chunks.Next() {
|
||||
err := chunks.Process(func(c *chunk.ChunkGroup, chunk io.Reader) error {
|
||||
signedReq, _ := handler.svc.UploadPartRequest(&s3.UploadPartInput{
|
||||
Bucket: &handler.Policy.BucketName,
|
||||
Key: &fileInfo.SavePath,
|
||||
PartNumber: aws.Int64(int64(c.Index() + 1)),
|
||||
UploadId: res.UploadId,
|
||||
})
|
||||
|
||||
signedURL, err := signedReq.Presign(time.Duration(ttl) * time.Second)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
urls[c.Index()] = signedURL
|
||||
return nil
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
|
||||
// 生成上传凭证
|
||||
return handler.getUploadCredential(ctx, putPolicy, apiURL)
|
||||
}
|
||||
// 签名完成分片上传的请求URL
|
||||
signedReq, _ := handler.svc.CompleteMultipartUploadRequest(&s3.CompleteMultipartUploadInput{
|
||||
Bucket: &handler.Policy.BucketName,
|
||||
Key: &fileInfo.SavePath,
|
||||
UploadId: res.UploadId,
|
||||
})
|
||||
|
||||
// Meta 获取文件信息
|
||||
func (handler Driver) Meta(ctx context.Context, path string) (*MetaData, error) {
|
||||
// 初始化客户端
|
||||
if err := handler.InitS3Client(); err != nil {
|
||||
signedURL, err := signedReq.Presign(time.Duration(ttl) * time.Second)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// 生成上传凭证
|
||||
return &serializer.UploadCredential{
|
||||
SessionID: uploadSession.Key,
|
||||
ChunkSize: handler.Policy.OptionsSerialized.ChunkSize,
|
||||
UploadID: *res.UploadId,
|
||||
UploadURLs: urls,
|
||||
CompleteURL: signedURL,
|
||||
}, nil
|
||||
}
|
||||
|
||||
// Meta 获取文件信息
|
||||
func (handler *Driver) Meta(ctx context.Context, path string) (*MetaData, error) {
|
||||
res, err := handler.svc.GetObject(
|
||||
&s3.GetObjectInput{
|
||||
Bucket: &handler.Policy.BucketName,
|
||||
@@ -383,58 +411,8 @@ func (handler Driver) Meta(ctx context.Context, path string) (*MetaData, error)
|
||||
|
||||
}
|
||||
|
||||
func (handler Driver) getUploadCredential(ctx context.Context, policy UploadPolicy, callback *url.URL) (serializer.UploadCredential, error) {
|
||||
|
||||
// 读取上下文中生成的存储路径和文件大小
|
||||
savePath, ok := ctx.Value(fsctx.SavePathCtx).(string)
|
||||
if !ok {
|
||||
return serializer.UploadCredential{}, errors.New("无法获取存储路径")
|
||||
}
|
||||
|
||||
longDate := time.Now().UTC().Format("20060102T150405Z")
|
||||
shortDate := time.Now().UTC().Format("20060102")
|
||||
|
||||
credential := handler.Policy.AccessKey + "/" + shortDate + "/" + handler.Policy.OptionsSerialized.Region + "/s3/aws4_request"
|
||||
policy.Conditions = append(policy.Conditions, map[string]string{"x-amz-credential": credential})
|
||||
policy.Conditions = append(policy.Conditions, map[string]string{"x-amz-date": longDate})
|
||||
|
||||
// 编码上传策略
|
||||
policyJSON, err := json.Marshal(policy)
|
||||
if err != nil {
|
||||
return serializer.UploadCredential{}, err
|
||||
}
|
||||
policyEncoded := base64.StdEncoding.EncodeToString(policyJSON)
|
||||
|
||||
//签名
|
||||
signature := getHMAC([]byte("AWS4"+handler.Policy.SecretKey), []byte(shortDate))
|
||||
signature = getHMAC(signature, []byte(handler.Policy.OptionsSerialized.Region))
|
||||
signature = getHMAC(signature, []byte("s3"))
|
||||
signature = getHMAC(signature, []byte("aws4_request"))
|
||||
signature = getHMAC(signature, []byte(policyEncoded))
|
||||
|
||||
return serializer.UploadCredential{
|
||||
Policy: policyEncoded,
|
||||
Callback: callback.String(),
|
||||
Token: hex.EncodeToString(signature),
|
||||
AccessKey: credential,
|
||||
Path: savePath,
|
||||
KeyTime: longDate,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func getHMAC(key []byte, data []byte) []byte {
|
||||
hash := hmac.New(sha256.New, key)
|
||||
hash.Write(data)
|
||||
return hash.Sum(nil)
|
||||
}
|
||||
|
||||
// CORS 创建跨域策略
|
||||
func (handler Driver) CORS() error {
|
||||
// 初始化客户端
|
||||
if err := handler.InitS3Client(); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
func (handler *Driver) CORS() error {
|
||||
rule := s3.CORSRule{
|
||||
AllowedMethods: aws.StringSlice([]string{
|
||||
"GET",
|
||||
@@ -445,6 +423,7 @@ func (handler Driver) CORS() error {
|
||||
}),
|
||||
AllowedOrigins: aws.StringSlice([]string{"*"}),
|
||||
AllowedHeaders: aws.StringSlice([]string{"*"}),
|
||||
ExposeHeaders: aws.StringSlice([]string{"ETag"}),
|
||||
MaxAgeSeconds: aws.Int64(3600),
|
||||
}
|
||||
|
||||
@@ -457,3 +436,13 @@ func (handler Driver) CORS() error {
|
||||
|
||||
return err
|
||||
}
|
||||
|
||||
// 取消上传凭证
|
||||
func (handler *Driver) CancelToken(ctx context.Context, uploadSession *serializer.UploadSession) error {
|
||||
_, err := handler.svc.AbortMultipartUpload(&s3.AbortMultipartUploadInput{
|
||||
UploadId: &uploadSession.UploadID,
|
||||
Bucket: &handler.Policy.BucketName,
|
||||
Key: &uploadSession.SavePath,
|
||||
})
|
||||
return err
|
||||
}
|
||||
|
||||
@@ -5,9 +5,9 @@ import (
|
||||
model "github.com/cloudreve/Cloudreve/v3/models"
|
||||
"github.com/cloudreve/Cloudreve/v3/pkg/cluster"
|
||||
"github.com/cloudreve/Cloudreve/v3/pkg/filesystem/driver"
|
||||
"github.com/cloudreve/Cloudreve/v3/pkg/filesystem/fsctx"
|
||||
"github.com/cloudreve/Cloudreve/v3/pkg/filesystem/response"
|
||||
"github.com/cloudreve/Cloudreve/v3/pkg/serializer"
|
||||
"io"
|
||||
"net/url"
|
||||
)
|
||||
|
||||
@@ -27,8 +27,8 @@ func NewDriver(master cluster.Node, handler driver.Handler, policy *model.Policy
|
||||
}
|
||||
}
|
||||
|
||||
func (d *Driver) Put(ctx context.Context, file io.ReadCloser, dst string, size uint64) error {
|
||||
return d.handler.Put(ctx, file, dst, size)
|
||||
func (d *Driver) Put(ctx context.Context, file fsctx.FileHeader) error {
|
||||
return d.handler.Put(ctx, file)
|
||||
}
|
||||
|
||||
func (d *Driver) Delete(ctx context.Context, files []string) ([]string, error) {
|
||||
@@ -47,10 +47,15 @@ func (d *Driver) Source(ctx context.Context, path string, url url.URL, ttl int64
|
||||
return "", ErrNotImplemented
|
||||
}
|
||||
|
||||
func (d *Driver) Token(ctx context.Context, ttl int64, callbackKey string) (serializer.UploadCredential, error) {
|
||||
return serializer.UploadCredential{}, ErrNotImplemented
|
||||
func (d *Driver) Token(ctx context.Context, ttl int64, uploadSession *serializer.UploadSession, file fsctx.FileHeader) (*serializer.UploadCredential, error) {
|
||||
return nil, ErrNotImplemented
|
||||
}
|
||||
|
||||
func (d *Driver) List(ctx context.Context, path string, recursive bool) ([]response.Object, error) {
|
||||
return nil, ErrNotImplemented
|
||||
}
|
||||
|
||||
// 取消上传凭证
|
||||
func (handler Driver) CancelToken(ctx context.Context, uploadSession *serializer.UploadSession) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -13,7 +13,6 @@ import (
|
||||
"github.com/cloudreve/Cloudreve/v3/pkg/mq"
|
||||
"github.com/cloudreve/Cloudreve/v3/pkg/request"
|
||||
"github.com/cloudreve/Cloudreve/v3/pkg/serializer"
|
||||
"io"
|
||||
"net/url"
|
||||
"time"
|
||||
)
|
||||
@@ -31,7 +30,7 @@ func NewDriver(node cluster.Node, handler driver.Handler, policy *model.Policy)
|
||||
var endpoint *url.URL
|
||||
if serverURL, err := url.Parse(node.DBModel().Server); err == nil {
|
||||
var controller *url.URL
|
||||
controller, _ = url.Parse("/api/v3/slave")
|
||||
controller, _ = url.Parse("/api/v3/slave/")
|
||||
endpoint = serverURL.ResolveReference(controller)
|
||||
}
|
||||
|
||||
@@ -50,15 +49,13 @@ func NewDriver(node cluster.Node, handler driver.Handler, policy *model.Policy)
|
||||
}
|
||||
|
||||
// Put 将ctx中指定的从机物理文件由从机上传到目标存储策略
|
||||
func (d *Driver) Put(ctx context.Context, file io.ReadCloser, dst string, size uint64) error {
|
||||
src, ok := ctx.Value(fsctx.SlaveSrcPath).(string)
|
||||
if !ok {
|
||||
return ErrSlaveSrcPathNotExist
|
||||
}
|
||||
func (d *Driver) Put(ctx context.Context, file fsctx.FileHeader) error {
|
||||
defer file.Close()
|
||||
|
||||
fileInfo := file.Info()
|
||||
req := serializer.SlaveTransferReq{
|
||||
Src: src,
|
||||
Dst: dst,
|
||||
Src: fileInfo.Src,
|
||||
Dst: fileInfo.SavePath,
|
||||
Policy: d.policy,
|
||||
}
|
||||
|
||||
@@ -112,10 +109,15 @@ func (d *Driver) Source(ctx context.Context, path string, url url.URL, ttl int64
|
||||
return "", ErrNotImplemented
|
||||
}
|
||||
|
||||
func (d *Driver) Token(ctx context.Context, ttl int64, callbackKey string) (serializer.UploadCredential, error) {
|
||||
return serializer.UploadCredential{}, ErrNotImplemented
|
||||
func (d *Driver) Token(ctx context.Context, ttl int64, uploadSession *serializer.UploadSession, file fsctx.FileHeader) (*serializer.UploadCredential, error) {
|
||||
return nil, ErrNotImplemented
|
||||
}
|
||||
|
||||
func (d *Driver) List(ctx context.Context, path string, recursive bool) ([]response.Object, error) {
|
||||
return nil, ErrNotImplemented
|
||||
}
|
||||
|
||||
// 取消上传凭证
|
||||
func (handler Driver) CancelToken(ctx context.Context, uploadSession *serializer.UploadSession) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -1,55 +0,0 @@
|
||||
package template
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"io"
|
||||
"net/url"
|
||||
|
||||
model "github.com/cloudreve/Cloudreve/v3/models"
|
||||
"github.com/cloudreve/Cloudreve/v3/pkg/filesystem/response"
|
||||
"github.com/cloudreve/Cloudreve/v3/pkg/serializer"
|
||||
)
|
||||
|
||||
// Driver 适配器模板
|
||||
type Driver struct {
|
||||
Policy *model.Policy
|
||||
}
|
||||
|
||||
// Get 获取文件
|
||||
func (handler Driver) Get(ctx context.Context, path string) (response.RSCloser, error) {
|
||||
return nil, errors.New("未实现")
|
||||
}
|
||||
|
||||
// Put 将文件流保存到指定目录
|
||||
func (handler Driver) Put(ctx context.Context, file io.ReadCloser, dst string, size uint64) error {
|
||||
return errors.New("未实现")
|
||||
}
|
||||
|
||||
// Delete 删除一个或多个文件,
|
||||
// 返回未删除的文件,及遇到的最后一个错误
|
||||
func (handler Driver) Delete(ctx context.Context, files []string) ([]string, error) {
|
||||
return []string{}, errors.New("未实现")
|
||||
}
|
||||
|
||||
// Thumb 获取文件缩略图
|
||||
func (handler Driver) Thumb(ctx context.Context, path string) (*response.ContentResponse, error) {
|
||||
return nil, errors.New("未实现")
|
||||
}
|
||||
|
||||
// Source 获取外链URL
|
||||
func (handler Driver) Source(
|
||||
ctx context.Context,
|
||||
path string,
|
||||
baseURL url.URL,
|
||||
ttl int64,
|
||||
isDownload bool,
|
||||
speed int,
|
||||
) (string, error) {
|
||||
return "", errors.New("未实现")
|
||||
}
|
||||
|
||||
// Token 获取上传策略和认证Token
|
||||
func (handler Driver) Token(ctx context.Context, TTL int64, key string) (serializer.UploadCredential, error) {
|
||||
return serializer.UploadCredential{}, errors.New("未实现")
|
||||
}
|
||||
@@ -9,7 +9,6 @@ import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"path"
|
||||
@@ -119,7 +118,7 @@ func (handler Driver) Get(ctx context.Context, path string) (response.RSCloser,
|
||||
}
|
||||
|
||||
// 获取文件数据流
|
||||
client := request.HTTPClient{}
|
||||
client := request.NewClient()
|
||||
resp, err := client.Request(
|
||||
"GET",
|
||||
downloadURL,
|
||||
@@ -146,7 +145,7 @@ func (handler Driver) Get(ctx context.Context, path string) (response.RSCloser,
|
||||
}
|
||||
|
||||
// Put 将文件流保存到指定目录
|
||||
func (handler Driver) Put(ctx context.Context, file io.ReadCloser, dst string, size uint64) error {
|
||||
func (handler Driver) Put(ctx context.Context, file fsctx.FileHeader) error {
|
||||
defer file.Close()
|
||||
|
||||
up := upyun.NewUpYun(&upyun.UpYunConfig{
|
||||
@@ -155,7 +154,7 @@ func (handler Driver) Put(ctx context.Context, file io.ReadCloser, dst string, s
|
||||
Password: handler.Policy.SecretKey,
|
||||
})
|
||||
err := up.Put(&upyun.PutObjectConfig{
|
||||
Path: dst,
|
||||
Path: file.Info().SavePath,
|
||||
Reader: file,
|
||||
})
|
||||
|
||||
@@ -311,45 +310,29 @@ func (handler Driver) signURL(ctx context.Context, path *url.URL, TTL int64) (st
|
||||
}
|
||||
|
||||
// Token 获取上传策略和认证Token
|
||||
func (handler Driver) Token(ctx context.Context, TTL int64, key string) (serializer.UploadCredential, error) {
|
||||
// 读取上下文中生成的存储路径和文件大小
|
||||
savePath, ok := ctx.Value(fsctx.SavePathCtx).(string)
|
||||
if !ok {
|
||||
return serializer.UploadCredential{}, errors.New("无法获取存储路径")
|
||||
}
|
||||
fileSize, ok := ctx.Value(fsctx.FileSizeCtx).(uint64)
|
||||
if !ok {
|
||||
return serializer.UploadCredential{}, errors.New("无法获取文件大小")
|
||||
}
|
||||
|
||||
// 检查文件大小
|
||||
|
||||
func (handler Driver) Token(ctx context.Context, ttl int64, uploadSession *serializer.UploadSession, file fsctx.FileHeader) (*serializer.UploadCredential, error) {
|
||||
// 生成回调地址
|
||||
siteURL := model.GetSiteURL()
|
||||
apiBaseURI, _ := url.Parse("/api/v3/callback/upyun/" + key)
|
||||
apiBaseURI, _ := url.Parse("/api/v3/callback/upyun/" + uploadSession.Key)
|
||||
apiURL := siteURL.ResolveReference(apiBaseURI)
|
||||
|
||||
// 上传策略
|
||||
fileInfo := file.Info()
|
||||
putPolicy := UploadPolicy{
|
||||
Bucket: handler.Policy.BucketName,
|
||||
// TODO escape
|
||||
SaveKey: savePath,
|
||||
Expiration: time.Now().Add(time.Duration(TTL) * time.Second).Unix(),
|
||||
SaveKey: fileInfo.SavePath,
|
||||
Expiration: time.Now().Add(time.Duration(ttl) * time.Second).Unix(),
|
||||
CallbackURL: apiURL.String(),
|
||||
ContentLength: fileSize,
|
||||
ContentLengthRange: fmt.Sprintf("0,%d", fileSize),
|
||||
ContentLength: fileInfo.Size,
|
||||
ContentLengthRange: fmt.Sprintf("0,%d", fileInfo.Size),
|
||||
AllowFileType: strings.Join(handler.Policy.OptionsSerialized.FileType, ","),
|
||||
}
|
||||
|
||||
// 生成上传凭证
|
||||
return handler.getUploadCredential(ctx, putPolicy)
|
||||
}
|
||||
|
||||
func (handler Driver) getUploadCredential(ctx context.Context, policy UploadPolicy) (serializer.UploadCredential, error) {
|
||||
// 生成上传策略
|
||||
policyJSON, err := json.Marshal(policy)
|
||||
policyJSON, err := json.Marshal(putPolicy)
|
||||
if err != nil {
|
||||
return serializer.UploadCredential{}, err
|
||||
return nil, err
|
||||
}
|
||||
policyEncoded := base64.StdEncoding.EncodeToString(policyJSON)
|
||||
|
||||
@@ -357,12 +340,19 @@ func (handler Driver) getUploadCredential(ctx context.Context, policy UploadPoli
|
||||
elements := []string{"POST", "/" + handler.Policy.BucketName, policyEncoded}
|
||||
signStr := handler.Sign(ctx, elements)
|
||||
|
||||
return serializer.UploadCredential{
|
||||
Policy: policyEncoded,
|
||||
Token: signStr,
|
||||
return &serializer.UploadCredential{
|
||||
SessionID: uploadSession.Key,
|
||||
Policy: policyEncoded,
|
||||
Credential: signStr,
|
||||
UploadURLs: []string{"https://v0.api.upyun.com/" + handler.Policy.BucketName},
|
||||
}, nil
|
||||
}
|
||||
|
||||
// 取消上传凭证
|
||||
func (handler Driver) CancelToken(ctx context.Context, uploadSession *serializer.UploadSession) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
// Sign 计算又拍云的签名头
|
||||
func (handler Driver) Sign(ctx context.Context, elements []string) string {
|
||||
password := fmt.Sprintf("%x", md5.Sum([]byte(handler.Policy.SecretKey)))
|
||||
|
||||
@@ -7,19 +7,20 @@ import (
|
||||
)
|
||||
|
||||
var (
|
||||
ErrUnknownPolicyType = errors.New("未知存储策略类型")
|
||||
ErrFileSizeTooBig = errors.New("单个文件尺寸太大")
|
||||
ErrFileExtensionNotAllowed = errors.New("不允许上传此类型的文件")
|
||||
ErrInsufficientCapacity = errors.New("容量空间不足")
|
||||
ErrIllegalObjectName = errors.New("目标名称非法")
|
||||
ErrClientCanceled = errors.New("客户端取消操作")
|
||||
ErrRootProtected = errors.New("无法对根目录进行操作")
|
||||
ErrInsertFileRecord = serializer.NewError(serializer.CodeDBError, "无法插入文件记录", nil)
|
||||
ErrFileExisted = serializer.NewError(serializer.CodeObjectExist, "同名文件或目录已存在", nil)
|
||||
ErrFolderExisted = serializer.NewError(serializer.CodeObjectExist, "同名目录已存在", nil)
|
||||
ErrPathNotExist = serializer.NewError(404, "路径不存在", nil)
|
||||
ErrObjectNotExist = serializer.NewError(404, "文件不存在", nil)
|
||||
ErrIO = serializer.NewError(serializer.CodeIOFailed, "无法读取文件数据", nil)
|
||||
ErrDBListObjects = serializer.NewError(serializer.CodeDBError, "无法列取对象记录", nil)
|
||||
ErrDBDeleteObjects = serializer.NewError(serializer.CodeDBError, "无法删除对象记录", nil)
|
||||
ErrUnknownPolicyType = errors.New("未知存储策略类型")
|
||||
ErrFileSizeTooBig = errors.New("单个文件尺寸太大")
|
||||
ErrFileExtensionNotAllowed = errors.New("不允许上传此类型的文件")
|
||||
ErrInsufficientCapacity = errors.New("容量空间不足")
|
||||
ErrIllegalObjectName = errors.New("目标名称非法")
|
||||
ErrClientCanceled = errors.New("客户端取消操作")
|
||||
ErrRootProtected = errors.New("无法对根目录进行操作")
|
||||
ErrInsertFileRecord = serializer.NewError(serializer.CodeDBError, "无法插入文件记录", nil)
|
||||
ErrFileExisted = serializer.NewError(serializer.CodeObjectExist, "同名文件或目录已存在", nil)
|
||||
ErrFileUploadSessionExisted = serializer.NewError(serializer.CodeObjectExist, "当前目录下已经有同名文件正在上传中,请尝试清空上传会话", nil)
|
||||
ErrFolderExisted = serializer.NewError(serializer.CodeObjectExist, "同名目录已存在", nil)
|
||||
ErrPathNotExist = serializer.NewError(404, "路径不存在", nil)
|
||||
ErrObjectNotExist = serializer.NewError(404, "文件不存在", nil)
|
||||
ErrIO = serializer.NewError(serializer.CodeIOFailed, "无法读取文件数据", nil)
|
||||
ErrDBListObjects = serializer.NewError(serializer.CodeDBError, "无法列取对象记录", nil)
|
||||
ErrDBDeleteObjects = serializer.NewError(serializer.CodeDBError, "无法删除对象记录", nil)
|
||||
)
|
||||
|
||||
@@ -5,6 +5,7 @@ import (
|
||||
"io"
|
||||
|
||||
model "github.com/cloudreve/Cloudreve/v3/models"
|
||||
"github.com/cloudreve/Cloudreve/v3/pkg/cache"
|
||||
"github.com/cloudreve/Cloudreve/v3/pkg/conf"
|
||||
"github.com/cloudreve/Cloudreve/v3/pkg/filesystem/fsctx"
|
||||
"github.com/cloudreve/Cloudreve/v3/pkg/filesystem/response"
|
||||
@@ -43,41 +44,39 @@ func (fs *FileSystem) withSpeedLimit(rs response.RSCloser) response.RSCloser {
|
||||
}
|
||||
|
||||
// AddFile 新增文件记录
|
||||
func (fs *FileSystem) AddFile(ctx context.Context, parent *model.Folder) (*model.File, error) {
|
||||
func (fs *FileSystem) AddFile(ctx context.Context, parent *model.Folder, file fsctx.FileHeader) (*model.File, error) {
|
||||
// 添加文件记录前的钩子
|
||||
err := fs.Trigger(ctx, "BeforeAddFile")
|
||||
err := fs.Trigger(ctx, "BeforeAddFile", file)
|
||||
if err != nil {
|
||||
if err := fs.Trigger(ctx, "BeforeAddFileFailed"); err != nil {
|
||||
util.Log().Debug("BeforeAddFileFailed 钩子执行失败,%s", err)
|
||||
}
|
||||
return nil, err
|
||||
}
|
||||
|
||||
file := ctx.Value(fsctx.FileHeaderCtx).(FileHeader)
|
||||
filePath := ctx.Value(fsctx.SavePathCtx).(string)
|
||||
|
||||
uploadInfo := file.Info()
|
||||
newFile := model.File{
|
||||
Name: file.GetFileName(),
|
||||
SourceName: filePath,
|
||||
UserID: fs.User.ID,
|
||||
Size: file.GetSize(),
|
||||
FolderID: parent.ID,
|
||||
PolicyID: fs.User.Policy.ID,
|
||||
Name: uploadInfo.FileName,
|
||||
SourceName: uploadInfo.SavePath,
|
||||
UserID: fs.User.ID,
|
||||
Size: uploadInfo.Size,
|
||||
FolderID: parent.ID,
|
||||
PolicyID: fs.Policy.ID,
|
||||
MetadataSerialized: uploadInfo.Metadata,
|
||||
UploadSessionID: uploadInfo.UploadSessionID,
|
||||
}
|
||||
|
||||
if fs.User.Policy.IsThumbExist(file.GetFileName()) {
|
||||
if fs.Policy.IsThumbExist(uploadInfo.FileName) {
|
||||
newFile.PicInfo = "1,1"
|
||||
}
|
||||
|
||||
_, err = newFile.Create()
|
||||
err = newFile.Create()
|
||||
|
||||
if err != nil {
|
||||
if err := fs.Trigger(ctx, "AfterValidateFailed"); err != nil {
|
||||
if err := fs.Trigger(ctx, "AfterValidateFailed", file); err != nil {
|
||||
util.Log().Debug("AfterValidateFailed 钩子执行失败,%s", err)
|
||||
}
|
||||
return nil, ErrFileExisted.WithError(err)
|
||||
}
|
||||
|
||||
fs.User.Storage += newFile.Size
|
||||
return &newFile, nil
|
||||
}
|
||||
|
||||
@@ -153,14 +152,7 @@ func (fs *FileSystem) GetDownloadContent(ctx context.Context, id uint) (response
|
||||
|
||||
// GetContent 获取文件内容,path为虚拟路径
|
||||
func (fs *FileSystem) GetContent(ctx context.Context, id uint) (response.RSCloser, error) {
|
||||
// 触发`下载前`钩子
|
||||
err := fs.Trigger(ctx, "BeforeFileDownload")
|
||||
if err != nil {
|
||||
util.Log().Debug("BeforeFileDownload 钩子执行失败,%s", err)
|
||||
return nil, err
|
||||
}
|
||||
|
||||
err = fs.resetFileIDIfNotExist(ctx, id)
|
||||
err := fs.resetFileIDIfNotExist(ctx, id)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@@ -184,23 +176,41 @@ func (fs *FileSystem) deleteGroupedFile(ctx context.Context, files map[uint][]*m
|
||||
|
||||
for policyID, toBeDeletedFiles := range files {
|
||||
// 列举出需要物理删除的文件的物理路径
|
||||
sourceNames := make([]string, 0, len(toBeDeletedFiles))
|
||||
sourceNamesAll := make([]string, 0, len(toBeDeletedFiles))
|
||||
uploadSessions := make([]*serializer.UploadSession, 0, len(toBeDeletedFiles))
|
||||
|
||||
for i := 0; i < len(toBeDeletedFiles); i++ {
|
||||
sourceNames = append(sourceNames, toBeDeletedFiles[i].SourceName)
|
||||
sourceNamesAll = append(sourceNamesAll, toBeDeletedFiles[i].SourceName)
|
||||
|
||||
if toBeDeletedFiles[i].UploadSessionID != nil {
|
||||
if session, ok := cache.Get(UploadSessionCachePrefix + *toBeDeletedFiles[i].UploadSessionID); ok {
|
||||
uploadSession := session.(serializer.UploadSession)
|
||||
uploadSessions = append(uploadSessions, &uploadSession)
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
// 切换上传策略
|
||||
fs.Policy = toBeDeletedFiles[0].GetPolicy()
|
||||
err := fs.DispatchHandler()
|
||||
if err != nil {
|
||||
failed[policyID] = sourceNames
|
||||
failed[policyID] = sourceNamesAll
|
||||
continue
|
||||
}
|
||||
|
||||
// 执行删除
|
||||
failedFile, _ := fs.Handler.Delete(ctx, sourceNames)
|
||||
failed[policyID] = failedFile
|
||||
// 取消上传会话
|
||||
for _, upSession := range uploadSessions {
|
||||
if err := fs.Handler.CancelToken(ctx, upSession); err != nil {
|
||||
util.Log().Warning("无法取消 [%s] 的上传会话: %s", upSession.Name, err)
|
||||
}
|
||||
|
||||
cache.Deletes([]string{upSession.Key}, UploadSessionCachePrefix)
|
||||
}
|
||||
|
||||
// 执行删除
|
||||
failedFile, _ := fs.Handler.Delete(ctx, sourceNamesAll)
|
||||
failed[policyID] = failedFile
|
||||
}
|
||||
|
||||
return failed
|
||||
@@ -215,7 +225,7 @@ func (fs *FileSystem) GroupFileByPolicy(ctx context.Context, files []model.File)
|
||||
// 如果已存在分组,直接追加
|
||||
policyGroup[files[key].PolicyID] = append(file, &files[key])
|
||||
} else {
|
||||
// 分布不存在,创建
|
||||
// 分组不存在,创建
|
||||
policyGroup[files[key].PolicyID] = make([]*model.File, 0)
|
||||
policyGroup[files[key].PolicyID] = append(policyGroup[files[key].PolicyID], &files[key])
|
||||
}
|
||||
@@ -350,7 +360,7 @@ func (fs *FileSystem) resetPolicyToFirstFile(ctx context.Context) error {
|
||||
}
|
||||
|
||||
// Search 搜索文件
|
||||
func (fs *FileSystem) Search(ctx context.Context, keywords ...interface{}) ([]Object, error) {
|
||||
func (fs *FileSystem) Search(ctx context.Context, keywords ...interface{}) ([]serializer.Object, error) {
|
||||
files, _ := model.GetFilesByKeywords(fs.User.ID, keywords...)
|
||||
fs.SetTargetFile(&files)
|
||||
|
||||
|
||||
@@ -2,6 +2,7 @@ package filesystem
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"os"
|
||||
"testing"
|
||||
|
||||
@@ -9,7 +10,6 @@ import (
|
||||
model "github.com/cloudreve/Cloudreve/v3/models"
|
||||
"github.com/cloudreve/Cloudreve/v3/pkg/auth"
|
||||
"github.com/cloudreve/Cloudreve/v3/pkg/cache"
|
||||
"github.com/cloudreve/Cloudreve/v3/pkg/filesystem/driver/local"
|
||||
"github.com/cloudreve/Cloudreve/v3/pkg/filesystem/fsctx"
|
||||
"github.com/cloudreve/Cloudreve/v3/pkg/serializer"
|
||||
"github.com/cloudreve/Cloudreve/v3/pkg/util"
|
||||
@@ -19,9 +19,10 @@ import (
|
||||
|
||||
func TestFileSystem_AddFile(t *testing.T) {
|
||||
asserts := assert.New(t)
|
||||
file := local.FileStream{
|
||||
Size: 5,
|
||||
Name: "1.png",
|
||||
file := fsctx.FileStream{
|
||||
Size: 5,
|
||||
Name: "1.png",
|
||||
SavePath: "/Uploads/1_sad.png",
|
||||
}
|
||||
folder := model.Folder{
|
||||
Model: gorm.Model{
|
||||
@@ -40,24 +41,55 @@ func TestFileSystem_AddFile(t *testing.T) {
|
||||
},
|
||||
},
|
||||
},
|
||||
Policy: &model.Policy{Type: "cos"},
|
||||
}
|
||||
ctx := context.WithValue(context.Background(), fsctx.FileHeaderCtx, file)
|
||||
ctx = context.WithValue(ctx, fsctx.SavePathCtx, "/Uploads/1_sad.png")
|
||||
|
||||
_, err := fs.AddFile(ctx, &folder)
|
||||
_, err := fs.AddFile(context.Background(), &folder, &file)
|
||||
|
||||
asserts.Error(err)
|
||||
|
||||
mock.ExpectBegin()
|
||||
mock.ExpectExec("INSERT(.+)").WillReturnResult(sqlmock.NewResult(1, 1))
|
||||
mock.ExpectExec("UPDATE(.+)storage(.+)").WillReturnResult(sqlmock.NewResult(1, 1))
|
||||
mock.ExpectCommit()
|
||||
|
||||
f, err := fs.AddFile(ctx, &folder)
|
||||
f, err := fs.AddFile(context.Background(), &folder, &file)
|
||||
|
||||
asserts.NoError(err)
|
||||
asserts.NoError(mock.ExpectationsWereMet())
|
||||
asserts.Equal("/Uploads/1_sad.png", f.SourceName)
|
||||
asserts.NotEmpty(f.PicInfo)
|
||||
|
||||
// 前置钩子执行失败
|
||||
{
|
||||
hookExecuted := false
|
||||
fs.Use("BeforeAddFile", func(ctx context.Context, fs *FileSystem, file fsctx.FileHeader) error {
|
||||
hookExecuted = true
|
||||
return errors.New("error")
|
||||
})
|
||||
f, err := fs.AddFile(context.Background(), &folder, &file)
|
||||
asserts.Error(err)
|
||||
asserts.Nil(f)
|
||||
asserts.True(hookExecuted)
|
||||
}
|
||||
|
||||
// 后置钩子执行失败
|
||||
{
|
||||
hookExecuted := false
|
||||
mock.ExpectBegin()
|
||||
mock.ExpectExec("INSERT(.+)").WillReturnError(errors.New("error"))
|
||||
mock.ExpectRollback()
|
||||
fs.Hooks = map[string][]Hook{}
|
||||
fs.Use("AfterValidateFailed", func(ctx context.Context, fs *FileSystem, file fsctx.FileHeader) error {
|
||||
hookExecuted = true
|
||||
return errors.New("error")
|
||||
})
|
||||
f, err := fs.AddFile(context.Background(), &folder, &file)
|
||||
asserts.Error(err)
|
||||
asserts.Nil(f)
|
||||
asserts.True(hookExecuted)
|
||||
asserts.NoError(mock.ExpectationsWereMet())
|
||||
}
|
||||
}
|
||||
|
||||
func TestFileSystem_GetContent(t *testing.T) {
|
||||
@@ -264,6 +296,22 @@ func TestFileSystem_deleteGroupedFile(t *testing.T) {
|
||||
3: {},
|
||||
}, failed)
|
||||
}
|
||||
// 包含上传会话文件
|
||||
{
|
||||
sessionID := "session"
|
||||
cache.Set(UploadSessionCachePrefix+sessionID, serializer.UploadSession{Key: sessionID}, 0)
|
||||
files[1].Policy.Type = "local"
|
||||
files[3].Policy.Type = "local"
|
||||
files[0].UploadSessionID = &sessionID
|
||||
failed := fs.deleteGroupedFile(ctx, fs.GroupFileByPolicy(ctx, files))
|
||||
asserts.Equal(map[uint][]string{
|
||||
1: {},
|
||||
2: {},
|
||||
3: {},
|
||||
}, failed)
|
||||
_, ok := cache.Get(UploadSessionCachePrefix + sessionID)
|
||||
asserts.False(ok)
|
||||
}
|
||||
}
|
||||
|
||||
func TestFileSystem_GetSource(t *testing.T) {
|
||||
|
||||
@@ -3,18 +3,10 @@ package filesystem
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"github.com/cloudreve/Cloudreve/v3/pkg/cluster"
|
||||
"github.com/cloudreve/Cloudreve/v3/pkg/filesystem/driver"
|
||||
"github.com/cloudreve/Cloudreve/v3/pkg/filesystem/driver/shadow/masterinslave"
|
||||
"github.com/cloudreve/Cloudreve/v3/pkg/filesystem/driver/shadow/slaveinmaster"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"sync"
|
||||
|
||||
model "github.com/cloudreve/Cloudreve/v3/models"
|
||||
"github.com/cloudreve/Cloudreve/v3/pkg/auth"
|
||||
"github.com/cloudreve/Cloudreve/v3/pkg/cluster"
|
||||
"github.com/cloudreve/Cloudreve/v3/pkg/conf"
|
||||
"github.com/cloudreve/Cloudreve/v3/pkg/filesystem/driver"
|
||||
"github.com/cloudreve/Cloudreve/v3/pkg/filesystem/driver/cos"
|
||||
"github.com/cloudreve/Cloudreve/v3/pkg/filesystem/driver/local"
|
||||
"github.com/cloudreve/Cloudreve/v3/pkg/filesystem/driver/onedrive"
|
||||
@@ -22,11 +14,16 @@ import (
|
||||
"github.com/cloudreve/Cloudreve/v3/pkg/filesystem/driver/qiniu"
|
||||
"github.com/cloudreve/Cloudreve/v3/pkg/filesystem/driver/remote"
|
||||
"github.com/cloudreve/Cloudreve/v3/pkg/filesystem/driver/s3"
|
||||
"github.com/cloudreve/Cloudreve/v3/pkg/filesystem/driver/shadow/masterinslave"
|
||||
"github.com/cloudreve/Cloudreve/v3/pkg/filesystem/driver/shadow/slaveinmaster"
|
||||
"github.com/cloudreve/Cloudreve/v3/pkg/filesystem/driver/upyun"
|
||||
"github.com/cloudreve/Cloudreve/v3/pkg/request"
|
||||
"github.com/cloudreve/Cloudreve/v3/pkg/serializer"
|
||||
"github.com/gin-gonic/gin"
|
||||
cossdk "github.com/tencentyun/cos-go-sdk-v5"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"sync"
|
||||
)
|
||||
|
||||
// FSPool 文件系统资源池
|
||||
@@ -36,16 +33,6 @@ var FSPool = sync.Pool{
|
||||
},
|
||||
}
|
||||
|
||||
// FileHeader 上传来的文件数据处理器
|
||||
type FileHeader interface {
|
||||
io.Reader
|
||||
io.Closer
|
||||
GetSize() uint64
|
||||
GetMIMEType() string
|
||||
GetFileName() string
|
||||
GetVirtualPath() string
|
||||
}
|
||||
|
||||
// FileSystem 管理文件的文件系统
|
||||
type FileSystem struct {
|
||||
// 文件系统所有者
|
||||
@@ -104,6 +91,8 @@ func (fs *FileSystem) reset() {
|
||||
func NewFileSystem(user *model.User) (*FileSystem, error) {
|
||||
fs := getEmptyFS()
|
||||
fs.User = user
|
||||
fs.Policy = &fs.User.Policy
|
||||
|
||||
// 分配存储策略适配器
|
||||
err := fs.DispatchHandler()
|
||||
|
||||
@@ -132,17 +121,11 @@ func NewAnonymousFileSystem() (*FileSystem, error) {
|
||||
|
||||
// DispatchHandler 根据存储策略分配文件适配器
|
||||
func (fs *FileSystem) DispatchHandler() error {
|
||||
var policyType string
|
||||
var currentPolicy *model.Policy
|
||||
|
||||
if fs.Policy == nil {
|
||||
// 如果没有具体指定,就是用用户当前存储策略
|
||||
policyType = fs.User.Policy.Type
|
||||
currentPolicy = &fs.User.Policy
|
||||
} else {
|
||||
policyType = fs.Policy.Type
|
||||
currentPolicy = fs.Policy
|
||||
return errors.New("未设置存储策略")
|
||||
}
|
||||
policyType := fs.Policy.Type
|
||||
currentPolicy := fs.Policy
|
||||
|
||||
switch policyType {
|
||||
case "mock", "anonymous":
|
||||
@@ -153,23 +136,19 @@ func (fs *FileSystem) DispatchHandler() error {
|
||||
}
|
||||
return nil
|
||||
case "remote":
|
||||
fs.Handler = remote.Driver{
|
||||
Policy: currentPolicy,
|
||||
Client: request.NewClient(),
|
||||
AuthInstance: auth.HMACAuth{[]byte(currentPolicy.SecretKey)},
|
||||
handler, err := remote.NewDriver(currentPolicy)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
|
||||
fs.Handler = handler
|
||||
case "qiniu":
|
||||
fs.Handler = qiniu.Driver{
|
||||
Policy: currentPolicy,
|
||||
}
|
||||
fs.Handler = qiniu.NewDriver(currentPolicy)
|
||||
return nil
|
||||
case "oss":
|
||||
fs.Handler = oss.Driver{
|
||||
Policy: currentPolicy,
|
||||
HTTPClient: request.NewClient(),
|
||||
}
|
||||
return nil
|
||||
handler, err := oss.NewDriver(currentPolicy)
|
||||
fs.Handler = handler
|
||||
return err
|
||||
case "upyun":
|
||||
fs.Handler = upyun.Driver{
|
||||
Policy: currentPolicy,
|
||||
@@ -194,13 +173,14 @@ func (fs *FileSystem) DispatchHandler() error {
|
||||
}
|
||||
return nil
|
||||
case "s3":
|
||||
fs.Handler = s3.Driver{
|
||||
Policy: currentPolicy,
|
||||
}
|
||||
return nil
|
||||
handler, err := s3.NewDriver(currentPolicy)
|
||||
fs.Handler = handler
|
||||
return err
|
||||
default:
|
||||
return ErrUnknownPolicyType
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// NewFileSystemFromContext 从gin.Context创建文件系统
|
||||
@@ -221,19 +201,14 @@ func NewFileSystemFromCallback(c *gin.Context) (*FileSystem, error) {
|
||||
}
|
||||
|
||||
// 获取回调会话
|
||||
callbackSessionRaw, ok := c.Get("callbackSession")
|
||||
callbackSessionRaw, ok := c.Get(UploadSessionCtx)
|
||||
if !ok {
|
||||
return nil, errors.New("找不到回调会话")
|
||||
}
|
||||
callbackSession := callbackSessionRaw.(*serializer.UploadSession)
|
||||
|
||||
// 重新指向上传策略
|
||||
policy, err := model.GetPolicyByID(callbackSession.PolicyID)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
fs.Policy = &policy
|
||||
fs.User.Policy = policy
|
||||
fs.Policy = &callbackSession.Policy
|
||||
err = fs.DispatchHandler()
|
||||
|
||||
return fs, err
|
||||
@@ -241,7 +216,7 @@ func NewFileSystemFromCallback(c *gin.Context) (*FileSystem, error) {
|
||||
|
||||
// SwitchToSlaveHandler 将负责上传的 Handler 切换为从机节点
|
||||
func (fs *FileSystem) SwitchToSlaveHandler(node cluster.Node) {
|
||||
fs.Handler = slaveinmaster.NewDriver(node, fs.Handler, &fs.User.Policy)
|
||||
fs.Handler = slaveinmaster.NewDriver(node, fs.Handler, fs.Policy)
|
||||
}
|
||||
|
||||
// SwitchToShadowHandler 将负责上传的 Handler 切换为从机节点转存使用的影子处理器
|
||||
|
||||
@@ -2,16 +2,16 @@ package filesystem
|
||||
|
||||
import (
|
||||
"github.com/cloudreve/Cloudreve/v3/pkg/cluster"
|
||||
"github.com/cloudreve/Cloudreve/v3/pkg/conf"
|
||||
"github.com/cloudreve/Cloudreve/v3/pkg/filesystem/driver/shadow/masterinslave"
|
||||
"github.com/cloudreve/Cloudreve/v3/pkg/filesystem/driver/shadow/slaveinmaster"
|
||||
"github.com/cloudreve/Cloudreve/v3/pkg/serializer"
|
||||
"net/http/httptest"
|
||||
|
||||
"github.com/DATA-DOG/go-sqlmock"
|
||||
model "github.com/cloudreve/Cloudreve/v3/models"
|
||||
"github.com/cloudreve/Cloudreve/v3/pkg/cache"
|
||||
"github.com/cloudreve/Cloudreve/v3/pkg/filesystem/driver/local"
|
||||
"github.com/cloudreve/Cloudreve/v3/pkg/filesystem/driver/remote"
|
||||
"github.com/cloudreve/Cloudreve/v3/pkg/serializer"
|
||||
"github.com/gin-gonic/gin"
|
||||
"github.com/stretchr/testify/assert"
|
||||
|
||||
@@ -36,7 +36,7 @@ func TestNewFileSystem(t *testing.T) {
|
||||
fs, err = NewFileSystem(&user)
|
||||
asserts.NoError(err)
|
||||
asserts.NotNil(fs.Handler)
|
||||
asserts.IsType(remote.Driver{}, fs.Handler)
|
||||
asserts.IsType(&remote.Driver{}, fs.Handler)
|
||||
|
||||
user.Policy.Type = "unknown"
|
||||
fs, err = NewFileSystem(&user)
|
||||
@@ -64,9 +64,10 @@ func TestNewFileSystemFromContext(t *testing.T) {
|
||||
func TestDispatchHandler(t *testing.T) {
|
||||
asserts := assert.New(t)
|
||||
fs := &FileSystem{
|
||||
User: &model.User{Policy: model.Policy{
|
||||
User: &model.User{},
|
||||
Policy: &model.Policy{
|
||||
Type: "local",
|
||||
}},
|
||||
},
|
||||
}
|
||||
|
||||
// 未指定,使用用户默认
|
||||
@@ -95,7 +96,7 @@ func TestDispatchHandler(t *testing.T) {
|
||||
err = fs.DispatchHandler()
|
||||
asserts.NoError(err)
|
||||
|
||||
fs.Policy = &model.Policy{Type: "oss"}
|
||||
fs.Policy = &model.Policy{Type: "oss", Server: "https://s.com", BucketName: "1234"}
|
||||
err = fs.DispatchHandler()
|
||||
asserts.NoError(err)
|
||||
|
||||
@@ -140,23 +141,6 @@ func TestNewFileSystemFromCallback(t *testing.T) {
|
||||
asserts.Error(err)
|
||||
}
|
||||
|
||||
// 找不到上传策略
|
||||
{
|
||||
c, _ := gin.CreateTestContext(httptest.NewRecorder())
|
||||
c.Set("user", &model.User{
|
||||
Policy: model.Policy{
|
||||
Type: "local",
|
||||
},
|
||||
})
|
||||
c.Set("callbackSession", &serializer.UploadSession{PolicyID: 138})
|
||||
cache.Deletes([]string{"138"}, "policy_")
|
||||
mock.ExpectQuery("SELECT(.+)").WillReturnRows(sqlmock.NewRows([]string{"id"}))
|
||||
fs, err := NewFileSystemFromCallback(c)
|
||||
asserts.NoError(mock.ExpectationsWereMet())
|
||||
asserts.Nil(fs)
|
||||
asserts.Error(err)
|
||||
}
|
||||
|
||||
// 成功
|
||||
{
|
||||
c, _ := gin.CreateTestContext(httptest.NewRecorder())
|
||||
@@ -165,11 +149,8 @@ func TestNewFileSystemFromCallback(t *testing.T) {
|
||||
Type: "local",
|
||||
},
|
||||
})
|
||||
c.Set("callbackSession", &serializer.UploadSession{PolicyID: 138})
|
||||
cache.Deletes([]string{"138"}, "policy_")
|
||||
mock.ExpectQuery("SELECT(.+)").WillReturnRows(sqlmock.NewRows([]string{"id", "type", "options"}).AddRow(138, "local", "{}"))
|
||||
c.Set(UploadSessionCtx, &serializer.UploadSession{Policy: model.Policy{Type: "local"}})
|
||||
fs, err := NewFileSystemFromCallback(c)
|
||||
asserts.NoError(mock.ExpectationsWereMet())
|
||||
asserts.NotNil(fs)
|
||||
asserts.NoError(err)
|
||||
}
|
||||
@@ -234,6 +215,16 @@ func TestNewAnonymousFileSystem(t *testing.T) {
|
||||
asserts.Error(err)
|
||||
asserts.Nil(fs)
|
||||
}
|
||||
|
||||
// 从机
|
||||
{
|
||||
conf.SystemConfig.Mode = "slave"
|
||||
fs, err := NewAnonymousFileSystem()
|
||||
asserts.NoError(mock.ExpectationsWereMet())
|
||||
asserts.NoError(err)
|
||||
asserts.NotNil(fs)
|
||||
asserts.NotNil(fs.Handler)
|
||||
}
|
||||
}
|
||||
|
||||
func TestFileSystem_Recycle(t *testing.T) {
|
||||
|
||||
@@ -5,10 +5,6 @@ type key int
|
||||
const (
|
||||
// GinCtx Gin的上下文
|
||||
GinCtx key = iota
|
||||
// SavePathCtx 文件物理路径
|
||||
SavePathCtx
|
||||
// FileHeaderCtx 上传的文件
|
||||
FileHeaderCtx
|
||||
// PathCtx 文件或目录的虚拟路径
|
||||
PathCtx
|
||||
// FileModelCtx 文件数据库模型
|
||||
@@ -37,10 +33,6 @@ const (
|
||||
ForceUsePublicEndpointCtx
|
||||
// CancelFuncCtx Context 取消函數
|
||||
CancelFuncCtx
|
||||
// ValidateCapacityOnceCtx 限定归还容量的操作只執行一次
|
||||
ValidateCapacityOnceCtx
|
||||
// 禁止上传时同名覆盖操作
|
||||
DisableOverwrite
|
||||
// 文件在从机节点中的路径
|
||||
SlaveSrcPath
|
||||
)
|
||||
|
||||
113
pkg/filesystem/fsctx/stream.go
Normal file
113
pkg/filesystem/fsctx/stream.go
Normal file
@@ -0,0 +1,113 @@
|
||||
package fsctx
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"io"
|
||||
"time"
|
||||
)
|
||||
|
||||
type WriteMode int
|
||||
|
||||
const (
|
||||
Overwrite WriteMode = 0x00001
|
||||
// Append 只适用于本地策略
|
||||
Append WriteMode = 0x00002
|
||||
Nop WriteMode = 0x00004
|
||||
)
|
||||
|
||||
type UploadTaskInfo struct {
|
||||
Size uint64
|
||||
MIMEType string
|
||||
FileName string
|
||||
VirtualPath string
|
||||
Mode WriteMode
|
||||
Metadata map[string]string
|
||||
LastModified *time.Time
|
||||
SavePath string
|
||||
UploadSessionID *string
|
||||
AppendStart uint64
|
||||
Model interface{}
|
||||
Src string
|
||||
}
|
||||
|
||||
// FileHeader 上传来的文件数据处理器
|
||||
type FileHeader interface {
|
||||
io.Reader
|
||||
io.Closer
|
||||
io.Seeker
|
||||
Info() *UploadTaskInfo
|
||||
SetSize(uint64)
|
||||
SetModel(fileModel interface{})
|
||||
Seekable() bool
|
||||
}
|
||||
|
||||
// FileStream 用户传来的文件
|
||||
type FileStream struct {
|
||||
Mode WriteMode
|
||||
LastModified *time.Time
|
||||
Metadata map[string]string
|
||||
File io.ReadCloser
|
||||
Seeker io.Seeker
|
||||
Size uint64
|
||||
VirtualPath string
|
||||
Name string
|
||||
MIMEType string
|
||||
SavePath string
|
||||
UploadSessionID *string
|
||||
AppendStart uint64
|
||||
Model interface{}
|
||||
Src string
|
||||
}
|
||||
|
||||
func (file *FileStream) Read(p []byte) (n int, err error) {
|
||||
if file.File != nil {
|
||||
return file.File.Read(p)
|
||||
}
|
||||
|
||||
return 0, io.EOF
|
||||
}
|
||||
|
||||
func (file *FileStream) Close() error {
|
||||
if file.File != nil {
|
||||
return file.File.Close()
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (file *FileStream) Seek(offset int64, whence int) (int64, error) {
|
||||
if file.Seekable() {
|
||||
return file.Seeker.Seek(offset, whence)
|
||||
}
|
||||
|
||||
return 0, errors.New("no seeker")
|
||||
}
|
||||
|
||||
func (file *FileStream) Seekable() bool {
|
||||
return file.Seeker != nil
|
||||
}
|
||||
|
||||
func (file *FileStream) Info() *UploadTaskInfo {
|
||||
return &UploadTaskInfo{
|
||||
Size: file.Size,
|
||||
MIMEType: file.MIMEType,
|
||||
FileName: file.Name,
|
||||
VirtualPath: file.VirtualPath,
|
||||
Mode: file.Mode,
|
||||
Metadata: file.Metadata,
|
||||
LastModified: file.LastModified,
|
||||
SavePath: file.SavePath,
|
||||
UploadSessionID: file.UploadSessionID,
|
||||
AppendStart: file.AppendStart,
|
||||
Model: file.Model,
|
||||
Src: file.Src,
|
||||
}
|
||||
}
|
||||
|
||||
func (file *FileStream) SetSize(size uint64) {
|
||||
file.Size = size
|
||||
}
|
||||
|
||||
func (file *FileStream) SetModel(fileModel interface{}) {
|
||||
file.Model = fileModel
|
||||
}
|
||||
78
pkg/filesystem/fsctx/stream_test.go
Normal file
78
pkg/filesystem/fsctx/stream_test.go
Normal file
@@ -0,0 +1,78 @@
|
||||
package fsctx
|
||||
|
||||
import (
|
||||
model "github.com/cloudreve/Cloudreve/v3/models"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"io"
|
||||
"io/ioutil"
|
||||
"os"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestFileStream_Read(t *testing.T) {
|
||||
asserts := assert.New(t)
|
||||
file := FileStream{
|
||||
File: ioutil.NopCloser(strings.NewReader("123")),
|
||||
}
|
||||
var p = make([]byte, 3)
|
||||
{
|
||||
n, err := file.Read(p)
|
||||
asserts.Equal(3, n)
|
||||
asserts.NoError(err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestFileStream_Close(t *testing.T) {
|
||||
asserts := assert.New(t)
|
||||
{
|
||||
file := FileStream{
|
||||
File: ioutil.NopCloser(strings.NewReader("123")),
|
||||
}
|
||||
err := file.Close()
|
||||
asserts.NoError(err)
|
||||
}
|
||||
|
||||
{
|
||||
file := FileStream{}
|
||||
err := file.Close()
|
||||
asserts.NoError(err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestFileStream_Seek(t *testing.T) {
|
||||
asserts := assert.New(t)
|
||||
f, _ := os.CreateTemp("", "*")
|
||||
defer func() {
|
||||
f.Close()
|
||||
os.Remove(f.Name())
|
||||
}()
|
||||
{
|
||||
file := FileStream{
|
||||
File: f,
|
||||
Seeker: f,
|
||||
}
|
||||
res, err := file.Seek(0, io.SeekStart)
|
||||
asserts.NoError(err)
|
||||
asserts.EqualValues(0, res)
|
||||
}
|
||||
|
||||
{
|
||||
file := FileStream{}
|
||||
res, err := file.Seek(0, io.SeekStart)
|
||||
asserts.Error(err)
|
||||
asserts.EqualValues(0, res)
|
||||
}
|
||||
}
|
||||
|
||||
func TestFileStream_Info(t *testing.T) {
|
||||
a := assert.New(t)
|
||||
file := FileStream{}
|
||||
a.NotNil(file.Info())
|
||||
|
||||
file.SetSize(10)
|
||||
a.EqualValues(10, file.Info().Size)
|
||||
|
||||
file.SetModel(&model.File{})
|
||||
a.NotNil(file.Info().Model)
|
||||
}
|
||||
@@ -2,21 +2,19 @@ package filesystem
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"io/ioutil"
|
||||
"strings"
|
||||
"sync"
|
||||
|
||||
model "github.com/cloudreve/Cloudreve/v3/models"
|
||||
"github.com/cloudreve/Cloudreve/v3/pkg/conf"
|
||||
"github.com/cloudreve/Cloudreve/v3/pkg/cache"
|
||||
"github.com/cloudreve/Cloudreve/v3/pkg/cluster"
|
||||
"github.com/cloudreve/Cloudreve/v3/pkg/filesystem/driver/local"
|
||||
"github.com/cloudreve/Cloudreve/v3/pkg/filesystem/fsctx"
|
||||
"github.com/cloudreve/Cloudreve/v3/pkg/request"
|
||||
"github.com/cloudreve/Cloudreve/v3/pkg/serializer"
|
||||
"github.com/cloudreve/Cloudreve/v3/pkg/util"
|
||||
"io/ioutil"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// Hook 钩子函数
|
||||
type Hook func(ctx context.Context, fs *FileSystem) error
|
||||
type Hook func(ctx context.Context, fs *FileSystem, file fsctx.FileHeader) error
|
||||
|
||||
// Use 注入钩子
|
||||
func (fs *FileSystem) Use(name string, hook Hook) {
|
||||
@@ -41,10 +39,10 @@ func (fs *FileSystem) CleanHooks(name string) {
|
||||
|
||||
// Trigger 触发钩子,遇到第一个错误时
|
||||
// 返回错误,后续钩子不会继续执行
|
||||
func (fs *FileSystem) Trigger(ctx context.Context, name string) error {
|
||||
func (fs *FileSystem) Trigger(ctx context.Context, name string, file fsctx.FileHeader) error {
|
||||
if hooks, ok := fs.Hooks[name]; ok {
|
||||
for _, hook := range hooks {
|
||||
err := hook(ctx, fs)
|
||||
err := hook(ctx, fs, file)
|
||||
if err != nil {
|
||||
util.Log().Warning("钩子执行失败:%s", err)
|
||||
return err
|
||||
@@ -54,56 +52,22 @@ func (fs *FileSystem) Trigger(ctx context.Context, name string) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
// HookIsFileExist 检查虚拟路径文件是否存在
|
||||
func HookIsFileExist(ctx context.Context, fs *FileSystem) error {
|
||||
filePath := ctx.Value(fsctx.PathCtx).(string)
|
||||
if ok, _ := fs.IsFileExist(filePath); ok {
|
||||
return nil
|
||||
}
|
||||
return ErrObjectNotExist
|
||||
}
|
||||
|
||||
// HookSlaveUploadValidate Slave模式下对文件上传的一系列验证
|
||||
func HookSlaveUploadValidate(ctx context.Context, fs *FileSystem) error {
|
||||
file := ctx.Value(fsctx.FileHeaderCtx).(FileHeader)
|
||||
policy := ctx.Value(fsctx.UploadPolicyCtx).(serializer.UploadPolicy)
|
||||
|
||||
// 验证单文件尺寸
|
||||
if policy.MaxSize > 0 {
|
||||
if file.GetSize() > policy.MaxSize {
|
||||
return ErrFileSizeTooBig
|
||||
}
|
||||
}
|
||||
|
||||
// 验证文件名
|
||||
if !fs.ValidateLegalName(ctx, file.GetFileName()) {
|
||||
return ErrIllegalObjectName
|
||||
}
|
||||
|
||||
// 验证扩展名
|
||||
if len(policy.AllowedExtension) > 0 && !IsInExtensionList(policy.AllowedExtension, file.GetFileName()) {
|
||||
return ErrFileExtensionNotAllowed
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// HookValidateFile 一系列对文件检验的集合
|
||||
func HookValidateFile(ctx context.Context, fs *FileSystem) error {
|
||||
file := ctx.Value(fsctx.FileHeaderCtx).(FileHeader)
|
||||
func HookValidateFile(ctx context.Context, fs *FileSystem, file fsctx.FileHeader) error {
|
||||
fileInfo := file.Info()
|
||||
|
||||
// 验证单文件尺寸
|
||||
if !fs.ValidateFileSize(ctx, file.GetSize()) {
|
||||
if !fs.ValidateFileSize(ctx, fileInfo.Size) {
|
||||
return ErrFileSizeTooBig
|
||||
}
|
||||
|
||||
// 验证文件名
|
||||
if !fs.ValidateLegalName(ctx, file.GetFileName()) {
|
||||
if !fs.ValidateLegalName(ctx, fileInfo.FileName) {
|
||||
return ErrIllegalObjectName
|
||||
}
|
||||
|
||||
// 验证扩展名
|
||||
if !fs.ValidateExtension(ctx, file.GetFileName()) {
|
||||
if !fs.ValidateExtension(ctx, fileInfo.FileName) {
|
||||
return ErrFileExtensionNotAllowed
|
||||
}
|
||||
|
||||
@@ -112,58 +76,41 @@ func HookValidateFile(ctx context.Context, fs *FileSystem) error {
|
||||
}
|
||||
|
||||
// HookResetPolicy 重设存储策略为上下文已有文件
|
||||
func HookResetPolicy(ctx context.Context, fs *FileSystem) error {
|
||||
func HookResetPolicy(ctx context.Context, fs *FileSystem, file fsctx.FileHeader) error {
|
||||
originFile, ok := ctx.Value(fsctx.FileModelCtx).(model.File)
|
||||
if !ok {
|
||||
return ErrObjectNotExist
|
||||
}
|
||||
|
||||
fs.Policy = originFile.GetPolicy()
|
||||
fs.User.Policy = *fs.Policy
|
||||
return fs.DispatchHandler()
|
||||
}
|
||||
|
||||
// HookValidateCapacity 验证并扣除用户容量,包含数据库操作
|
||||
func HookValidateCapacity(ctx context.Context, fs *FileSystem) error {
|
||||
file := ctx.Value(fsctx.FileHeaderCtx).(FileHeader)
|
||||
// HookValidateCapacity 验证用户容量
|
||||
func HookValidateCapacity(ctx context.Context, fs *FileSystem, file fsctx.FileHeader) error {
|
||||
// 验证并扣除容量
|
||||
if !fs.ValidateCapacity(ctx, file.GetSize()) {
|
||||
if fs.User.GetRemainingCapacity() < file.Info().Size {
|
||||
return ErrInsufficientCapacity
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// HookValidateCapacityWithoutIncrease 验证用户容量,不扣除
|
||||
func HookValidateCapacityWithoutIncrease(ctx context.Context, fs *FileSystem) error {
|
||||
file := ctx.Value(fsctx.FileHeaderCtx).(FileHeader)
|
||||
// 验证并扣除容量
|
||||
if fs.User.GetRemainingCapacity() < file.GetSize() {
|
||||
return ErrInsufficientCapacity
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// HookChangeCapacity 根据原有文件和新文件的大小更新用户容量
|
||||
func HookChangeCapacity(ctx context.Context, fs *FileSystem) error {
|
||||
newFile := ctx.Value(fsctx.FileHeaderCtx).(FileHeader)
|
||||
// HookValidateCapacityDiff 根据原有文件和新文件的大小验证用户容量
|
||||
func HookValidateCapacityDiff(ctx context.Context, fs *FileSystem, newFile fsctx.FileHeader) error {
|
||||
originFile := ctx.Value(fsctx.FileModelCtx).(model.File)
|
||||
newFileSize := newFile.Info().Size
|
||||
|
||||
if newFile.GetSize() > originFile.Size {
|
||||
if !fs.ValidateCapacity(ctx, newFile.GetSize()-originFile.Size) {
|
||||
return ErrInsufficientCapacity
|
||||
}
|
||||
return nil
|
||||
if newFileSize > originFile.Size {
|
||||
return HookValidateCapacity(ctx, fs, newFile)
|
||||
}
|
||||
|
||||
fs.User.DeductionStorage(originFile.Size - newFile.GetSize())
|
||||
return nil
|
||||
}
|
||||
|
||||
// HookDeleteTempFile 删除已保存的临时文件
|
||||
func HookDeleteTempFile(ctx context.Context, fs *FileSystem) error {
|
||||
filePath := ctx.Value(fsctx.SavePathCtx).(string)
|
||||
func HookDeleteTempFile(ctx context.Context, fs *FileSystem, file fsctx.FileHeader) error {
|
||||
// 删除临时文件
|
||||
_, err := fs.Handler.Delete(ctx, []string{filePath})
|
||||
_, err := fs.Handler.Delete(ctx, []string{file.Info().SavePath})
|
||||
if err != nil {
|
||||
util.Log().Warning("无法清理上传临时文件,%s", err)
|
||||
}
|
||||
@@ -172,14 +119,18 @@ func HookDeleteTempFile(ctx context.Context, fs *FileSystem) error {
|
||||
}
|
||||
|
||||
// HookCleanFileContent 清空文件内容
|
||||
func HookCleanFileContent(ctx context.Context, fs *FileSystem) error {
|
||||
filePath := ctx.Value(fsctx.SavePathCtx).(string)
|
||||
func HookCleanFileContent(ctx context.Context, fs *FileSystem, file fsctx.FileHeader) error {
|
||||
// 清空内容
|
||||
return fs.Handler.Put(ctx, ioutil.NopCloser(strings.NewReader("")), filePath, 0)
|
||||
return fs.Handler.Put(ctx, &fsctx.FileStream{
|
||||
File: ioutil.NopCloser(strings.NewReader("")),
|
||||
SavePath: file.Info().SavePath,
|
||||
Size: 0,
|
||||
Mode: fsctx.Overwrite,
|
||||
})
|
||||
}
|
||||
|
||||
// HookClearFileSize 将原始文件的尺寸设为0
|
||||
func HookClearFileSize(ctx context.Context, fs *FileSystem) error {
|
||||
func HookClearFileSize(ctx context.Context, fs *FileSystem, file fsctx.FileHeader) error {
|
||||
originFile, ok := ctx.Value(fsctx.FileModelCtx).(model.File)
|
||||
if !ok {
|
||||
return ErrObjectNotExist
|
||||
@@ -188,7 +139,7 @@ func HookClearFileSize(ctx context.Context, fs *FileSystem) error {
|
||||
}
|
||||
|
||||
// HookCancelContext 取消上下文
|
||||
func HookCancelContext(ctx context.Context, fs *FileSystem) error {
|
||||
func HookCancelContext(ctx context.Context, fs *FileSystem, file fsctx.FileHeader) error {
|
||||
cancelFunc, ok := ctx.Value(fsctx.CancelFuncCtx).(context.CancelFunc)
|
||||
if ok {
|
||||
cancelFunc()
|
||||
@@ -196,29 +147,8 @@ func HookCancelContext(ctx context.Context, fs *FileSystem) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
// HookGiveBackCapacity 归还用户容量
|
||||
func HookGiveBackCapacity(ctx context.Context, fs *FileSystem) error {
|
||||
file := ctx.Value(fsctx.FileHeaderCtx).(FileHeader)
|
||||
once, ok := ctx.Value(fsctx.ValidateCapacityOnceCtx).(*sync.Once)
|
||||
if !ok {
|
||||
once = &sync.Once{}
|
||||
}
|
||||
|
||||
// 归还用户容量
|
||||
res := true
|
||||
once.Do(func() {
|
||||
res = fs.User.DeductionStorage(file.GetSize())
|
||||
})
|
||||
|
||||
if !res {
|
||||
return errors.New("无法继续降低用户已用存储")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// HookUpdateSourceName 更新文件SourceName
|
||||
// TODO:测试
|
||||
func HookUpdateSourceName(ctx context.Context, fs *FileSystem) error {
|
||||
func HookUpdateSourceName(ctx context.Context, fs *FileSystem, file fsctx.FileHeader) error {
|
||||
originFile, ok := ctx.Value(fsctx.FileModelCtx).(model.File)
|
||||
if !ok {
|
||||
return ErrObjectNotExist
|
||||
@@ -227,103 +157,141 @@ func HookUpdateSourceName(ctx context.Context, fs *FileSystem) error {
|
||||
}
|
||||
|
||||
// GenericAfterUpdate 文件内容更新后
|
||||
func GenericAfterUpdate(ctx context.Context, fs *FileSystem) error {
|
||||
func GenericAfterUpdate(ctx context.Context, fs *FileSystem, newFile fsctx.FileHeader) error {
|
||||
// 更新文件尺寸
|
||||
originFile, ok := ctx.Value(fsctx.FileModelCtx).(model.File)
|
||||
if !ok {
|
||||
return ErrObjectNotExist
|
||||
}
|
||||
|
||||
fs.SetTargetFile(&[]model.File{originFile})
|
||||
newFile.SetModel(&originFile)
|
||||
|
||||
newFile, ok := ctx.Value(fsctx.FileHeaderCtx).(FileHeader)
|
||||
if !ok {
|
||||
return ErrObjectNotExist
|
||||
}
|
||||
err := originFile.UpdateSize(newFile.GetSize())
|
||||
err := originFile.UpdateSize(newFile.Info().Size)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// 尝试清空原有缩略图并重新生成
|
||||
if originFile.GetPolicy().IsThumbGenerateNeeded() {
|
||||
fs.recycleLock.Lock()
|
||||
go func() {
|
||||
defer fs.recycleLock.Unlock()
|
||||
if originFile.PicInfo != "" {
|
||||
_, _ = fs.Handler.Delete(ctx, []string{originFile.SourceName + conf.ThumbConfig.FileSuffix})
|
||||
fs.GenerateThumbnail(ctx, &originFile)
|
||||
}
|
||||
}()
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// SlaveAfterUpload Slave模式下上传完成钩子
|
||||
func SlaveAfterUpload(ctx context.Context, fs *FileSystem) error {
|
||||
fileHeader := ctx.Value(fsctx.FileHeaderCtx).(FileHeader)
|
||||
policy := ctx.Value(fsctx.UploadPolicyCtx).(serializer.UploadPolicy)
|
||||
func SlaveAfterUpload(session *serializer.UploadSession) Hook {
|
||||
return func(ctx context.Context, fs *FileSystem, fileHeader fsctx.FileHeader) error {
|
||||
fileInfo := fileHeader.Info()
|
||||
|
||||
// 构造一个model.File,用于生成缩略图
|
||||
file := model.File{
|
||||
Name: fileHeader.GetFileName(),
|
||||
SourceName: ctx.Value(fsctx.SavePathCtx).(string),
|
||||
}
|
||||
fs.GenerateThumbnail(ctx, &file)
|
||||
// 构造一个model.File,用于生成缩略图
|
||||
file := model.File{
|
||||
Name: fileInfo.FileName,
|
||||
SourceName: fileInfo.SavePath,
|
||||
}
|
||||
fs.GenerateThumbnail(ctx, &file)
|
||||
|
||||
if policy.CallbackURL == "" {
|
||||
return nil
|
||||
}
|
||||
if session.Callback == "" {
|
||||
return nil
|
||||
}
|
||||
|
||||
// 发送回调请求
|
||||
callbackBody := serializer.UploadCallback{
|
||||
Name: file.Name,
|
||||
SourceName: file.SourceName,
|
||||
PicInfo: file.PicInfo,
|
||||
Size: fileHeader.GetSize(),
|
||||
// 发送回调请求
|
||||
callbackBody := serializer.UploadCallback{
|
||||
PicInfo: file.PicInfo,
|
||||
}
|
||||
|
||||
return cluster.RemoteCallback(session.Callback, callbackBody)
|
||||
}
|
||||
return request.RemoteCallback(policy.CallbackURL, callbackBody)
|
||||
}
|
||||
|
||||
// GenericAfterUpload 文件上传完成后,包含数据库操作
|
||||
func GenericAfterUpload(ctx context.Context, fs *FileSystem) error {
|
||||
// 文件存放的虚拟路径
|
||||
virtualPath := ctx.Value(fsctx.FileHeaderCtx).(FileHeader).GetVirtualPath()
|
||||
func GenericAfterUpload(ctx context.Context, fs *FileSystem, fileHeader fsctx.FileHeader) error {
|
||||
fileInfo := fileHeader.Info()
|
||||
|
||||
// 检查路径是否存在,不存在就创建
|
||||
isExist, folder := fs.IsPathExist(virtualPath)
|
||||
if !isExist {
|
||||
newFolder, err := fs.CreateDirectory(ctx, virtualPath)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
folder = newFolder
|
||||
// 创建或查找根目录
|
||||
folder, err := fs.CreateDirectory(ctx, fileInfo.VirtualPath)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// 检查文件是否存在
|
||||
if ok, _ := fs.IsChildFileExist(
|
||||
if ok, file := fs.IsChildFileExist(
|
||||
folder,
|
||||
ctx.Value(fsctx.FileHeaderCtx).(FileHeader).GetFileName(),
|
||||
fileInfo.FileName,
|
||||
); ok {
|
||||
if file.UploadSessionID != nil {
|
||||
return ErrFileUploadSessionExisted
|
||||
}
|
||||
|
||||
return ErrFileExisted
|
||||
}
|
||||
|
||||
// 向数据库中插入记录
|
||||
file, err := fs.AddFile(ctx, folder)
|
||||
file, err := fs.AddFile(ctx, folder, fileHeader)
|
||||
if err != nil {
|
||||
return ErrInsertFileRecord
|
||||
}
|
||||
fs.SetTargetFile(&[]model.File{*file})
|
||||
|
||||
// 异步尝试生成缩略图
|
||||
if fs.User.Policy.IsThumbGenerateNeeded() {
|
||||
fs.recycleLock.Lock()
|
||||
go func() {
|
||||
defer fs.recycleLock.Unlock()
|
||||
fs.GenerateThumbnail(ctx, file)
|
||||
}()
|
||||
}
|
||||
fileHeader.SetModel(file)
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// HookGenerateThumb 生成缩略图
|
||||
func HookGenerateThumb(ctx context.Context, fs *FileSystem, fileHeader fsctx.FileHeader) error {
|
||||
// 异步尝试生成缩略图
|
||||
fileMode := fileHeader.Info().Model.(*model.File)
|
||||
if fs.Policy.IsThumbGenerateNeeded() {
|
||||
fs.recycleLock.Lock()
|
||||
go func() {
|
||||
defer fs.recycleLock.Unlock()
|
||||
_, _ = fs.Handler.Delete(ctx, []string{fileMode.SourceName + model.GetSettingByNameWithDefault("thumb_file_suffix", "._thumb")})
|
||||
fs.GenerateThumbnail(ctx, fileMode)
|
||||
}()
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// HookClearFileHeaderSize 将FileHeader大小设定为0
|
||||
func HookClearFileHeaderSize(ctx context.Context, fs *FileSystem, fileHeader fsctx.FileHeader) error {
|
||||
fileHeader.SetSize(0)
|
||||
return nil
|
||||
}
|
||||
|
||||
// HookTruncateFileTo 将物理文件截断至 size
|
||||
func HookTruncateFileTo(size uint64) Hook {
|
||||
return func(ctx context.Context, fs *FileSystem, fileHeader fsctx.FileHeader) error {
|
||||
if handler, ok := fs.Handler.(local.Driver); ok {
|
||||
return handler.Truncate(ctx, fileHeader.Info().SavePath, size)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
// HookChunkUploadFinished 单个分片上传结束后
|
||||
func HookChunkUploaded(ctx context.Context, fs *FileSystem, fileHeader fsctx.FileHeader) error {
|
||||
fileInfo := fileHeader.Info()
|
||||
|
||||
// 更新文件大小
|
||||
return fileInfo.Model.(*model.File).UpdateSize(fileInfo.AppendStart + fileInfo.Size)
|
||||
}
|
||||
|
||||
// HookChunkUploadFailed 单个分片上传失败后
|
||||
func HookChunkUploadFailed(ctx context.Context, fs *FileSystem, fileHeader fsctx.FileHeader) error {
|
||||
fileInfo := fileHeader.Info()
|
||||
|
||||
// 更新文件大小
|
||||
return fileInfo.Model.(*model.File).UpdateSize(fileInfo.AppendStart)
|
||||
}
|
||||
|
||||
// HookPopPlaceholderToFile 将占位文件提升为正式文件
|
||||
func HookPopPlaceholderToFile(picInfo string) Hook {
|
||||
return func(ctx context.Context, fs *FileSystem, fileHeader fsctx.FileHeader) error {
|
||||
fileInfo := fileHeader.Info()
|
||||
fileModel := fileInfo.Model.(*model.File)
|
||||
return fileModel.PopChunkToFile(fileInfo.LastModified, picInfo)
|
||||
}
|
||||
}
|
||||
|
||||
// HookChunkUploadFinished 分片上传结束后处理文件
|
||||
func HookDeleteUploadSession(id string) Hook {
|
||||
return func(ctx context.Context, fs *FileSystem, fileHeader fsctx.FileHeader) error {
|
||||
cache.Deletes([]string{id}, UploadSessionCachePrefix)
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
@@ -3,22 +3,20 @@ package filesystem
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"io"
|
||||
"io/ioutil"
|
||||
"net/http"
|
||||
"os"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
|
||||
"github.com/DATA-DOG/go-sqlmock"
|
||||
model "github.com/cloudreve/Cloudreve/v3/models"
|
||||
"github.com/cloudreve/Cloudreve/v3/pkg/cache"
|
||||
"github.com/cloudreve/Cloudreve/v3/pkg/conf"
|
||||
"github.com/cloudreve/Cloudreve/v3/pkg/filesystem/driver/local"
|
||||
"github.com/cloudreve/Cloudreve/v3/pkg/filesystem/fsctx"
|
||||
"github.com/cloudreve/Cloudreve/v3/pkg/mocks/requestmock"
|
||||
"github.com/cloudreve/Cloudreve/v3/pkg/request"
|
||||
"github.com/cloudreve/Cloudreve/v3/pkg/serializer"
|
||||
"io/ioutil"
|
||||
"net/http"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
model "github.com/cloudreve/Cloudreve/v3/models"
|
||||
"github.com/jinzhu/gorm"
|
||||
"github.com/stretchr/testify/assert"
|
||||
testMock "github.com/stretchr/testify/mock"
|
||||
@@ -26,78 +24,72 @@ import (
|
||||
|
||||
func TestGenericBeforeUpload(t *testing.T) {
|
||||
asserts := assert.New(t)
|
||||
file := local.FileStream{
|
||||
file := &fsctx.FileStream{
|
||||
Size: 5,
|
||||
Name: "1.txt",
|
||||
}
|
||||
ctx := context.Background()
|
||||
cache.Set("pack_size_0", uint64(0), 0)
|
||||
ctx := context.WithValue(context.Background(), fsctx.FileHeaderCtx, file)
|
||||
fs := FileSystem{
|
||||
User: &model.User{
|
||||
Storage: 0,
|
||||
Group: model.Group{
|
||||
MaxStorage: 11,
|
||||
},
|
||||
Policy: model.Policy{
|
||||
MaxSize: 4,
|
||||
OptionsSerialized: model.PolicyOption{
|
||||
FileType: []string{"txt"},
|
||||
},
|
||||
},
|
||||
Policy: &model.Policy{
|
||||
MaxSize: 4,
|
||||
OptionsSerialized: model.PolicyOption{
|
||||
FileType: []string{"txt"},
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
asserts.Error(HookValidateFile(ctx, &fs))
|
||||
asserts.Error(HookValidateFile(ctx, &fs, file))
|
||||
|
||||
file.Size = 1
|
||||
file.Name = "1"
|
||||
ctx = context.WithValue(context.Background(), fsctx.FileHeaderCtx, file)
|
||||
asserts.Error(HookValidateFile(ctx, &fs))
|
||||
asserts.Error(HookValidateFile(ctx, &fs, file))
|
||||
|
||||
file.Name = "1.txt"
|
||||
ctx = context.WithValue(context.Background(), fsctx.FileHeaderCtx, file)
|
||||
asserts.NoError(HookValidateFile(ctx, &fs))
|
||||
asserts.NoError(HookValidateFile(ctx, &fs, file))
|
||||
|
||||
file.Name = "1.t/xt"
|
||||
ctx = context.WithValue(context.Background(), fsctx.FileHeaderCtx, file)
|
||||
asserts.Error(HookValidateFile(ctx, &fs))
|
||||
asserts.Error(HookValidateFile(ctx, &fs, file))
|
||||
}
|
||||
|
||||
func TestGenericAfterUploadCanceled(t *testing.T) {
|
||||
asserts := assert.New(t)
|
||||
f, err := os.Create("TestGenericAfterUploadCanceled")
|
||||
asserts.NoError(err)
|
||||
f.Close()
|
||||
file := local.FileStream{
|
||||
Size: 5,
|
||||
Name: "TestGenericAfterUploadCanceled",
|
||||
file := &fsctx.FileStream{
|
||||
Size: 5,
|
||||
Name: "TestGenericAfterUploadCanceled",
|
||||
SavePath: "TestGenericAfterUploadCanceled",
|
||||
}
|
||||
ctx := context.WithValue(context.Background(), fsctx.SavePathCtx, "TestGenericAfterUploadCanceled")
|
||||
ctx = context.WithValue(ctx, fsctx.FileHeaderCtx, file)
|
||||
ctx := context.Background()
|
||||
fs := FileSystem{
|
||||
User: &model.User{Storage: 5},
|
||||
Handler: local.Driver{},
|
||||
User: &model.User{},
|
||||
}
|
||||
|
||||
// 成功
|
||||
err = HookDeleteTempFile(ctx, &fs)
|
||||
asserts.NoError(err)
|
||||
err = HookGiveBackCapacity(ctx, &fs)
|
||||
asserts.NoError(err)
|
||||
asserts.Equal(uint64(0), fs.User.Storage)
|
||||
{
|
||||
mockHandler := &FileHeaderMock{}
|
||||
fs.Handler = mockHandler
|
||||
mockHandler.On("Delete", testMock.Anything, testMock.Anything).Return([]string{}, nil)
|
||||
err := HookDeleteTempFile(ctx, &fs, file)
|
||||
asserts.NoError(err)
|
||||
mockHandler.AssertExpectations(t)
|
||||
}
|
||||
|
||||
f, err = os.Create("TestGenericAfterUploadCanceled")
|
||||
asserts.NoError(err)
|
||||
f.Close()
|
||||
// 失败
|
||||
{
|
||||
mockHandler := &FileHeaderMock{}
|
||||
fs.Handler = mockHandler
|
||||
mockHandler.On("Delete", testMock.Anything, testMock.Anything).Return([]string{}, errors.New(""))
|
||||
err := HookDeleteTempFile(ctx, &fs, file)
|
||||
asserts.NoError(err)
|
||||
mockHandler.AssertExpectations(t)
|
||||
}
|
||||
|
||||
// 容量不能再降低
|
||||
err = HookGiveBackCapacity(ctx, &fs)
|
||||
asserts.Error(err)
|
||||
|
||||
//文件不存在
|
||||
fs.User.Storage = 5
|
||||
err = HookDeleteTempFile(ctx, &fs)
|
||||
asserts.NoError(err)
|
||||
}
|
||||
|
||||
func TestGenericAfterUpload(t *testing.T) {
|
||||
@@ -108,69 +100,98 @@ func TestGenericAfterUpload(t *testing.T) {
|
||||
ID: 1,
|
||||
},
|
||||
},
|
||||
Policy: &model.Policy{},
|
||||
}
|
||||
|
||||
ctx := context.WithValue(context.Background(), fsctx.FileHeaderCtx, local.FileStream{
|
||||
ctx := context.Background()
|
||||
file := &fsctx.FileStream{
|
||||
VirtualPath: "/我的文件",
|
||||
Name: "test.txt",
|
||||
})
|
||||
ctx = context.WithValue(ctx, fsctx.SavePathCtx, "")
|
||||
}
|
||||
|
||||
// 正常
|
||||
mock.ExpectQuery("SELECT(.+)").
|
||||
WithArgs(1).
|
||||
WillReturnRows(sqlmock.NewRows([]string{"id", "owner_id"}).AddRow(1, 1))
|
||||
mock.ExpectQuery("SELECT(.+)files").
|
||||
WithArgs(1, "我的文件").
|
||||
WillReturnRows(sqlmock.NewRows([]string{"id", "owner_id"}))
|
||||
// 1
|
||||
mock.ExpectBegin()
|
||||
mock.ExpectQuery("SELECT(.+)").
|
||||
WithArgs(1, 1, "我的文件").
|
||||
WithArgs("我的文件", 1, 1).
|
||||
WillReturnRows(sqlmock.NewRows([]string{"id", "owner_id"}).AddRow(2, 1))
|
||||
mock.ExpectCommit()
|
||||
mock.ExpectQuery("SELECT(.+)files(.+)").WillReturnError(errors.New("not found"))
|
||||
mock.ExpectBegin()
|
||||
mock.ExpectExec("INSERT(.+)files(.+)").WillReturnResult(sqlmock.NewResult(1, 1))
|
||||
mock.ExpectExec("UPDATE(.+)storage(.+)").WillReturnResult(sqlmock.NewResult(1, 1))
|
||||
mock.ExpectCommit()
|
||||
|
||||
err := GenericAfterUpload(ctx, &fs)
|
||||
err := GenericAfterUpload(ctx, &fs, file)
|
||||
asserts.NoError(err)
|
||||
asserts.NoError(mock.ExpectationsWereMet())
|
||||
|
||||
// 路径不存在
|
||||
mock.ExpectQuery("SELECT(.+)folders(.+)").WillReturnRows(
|
||||
mock.NewRows([]string{"name"}),
|
||||
)
|
||||
err = GenericAfterUpload(ctx, &fs)
|
||||
asserts.Equal(ErrRootProtected, err)
|
||||
asserts.NoError(mock.ExpectationsWereMet())
|
||||
|
||||
// 文件已存在
|
||||
mock.ExpectQuery("SELECT(.+)").
|
||||
WithArgs(1).
|
||||
WillReturnRows(sqlmock.NewRows([]string{"id", "owner_id"}).AddRow(1, 1))
|
||||
mock.ExpectQuery("SELECT(.+)files").
|
||||
WithArgs(1, "我的文件").
|
||||
WillReturnRows(sqlmock.NewRows([]string{"id", "owner_id"}))
|
||||
// 1
|
||||
mock.ExpectBegin()
|
||||
mock.ExpectQuery("SELECT(.+)").
|
||||
WithArgs(1, 1, "我的文件").
|
||||
WithArgs("我的文件", 1, 1).
|
||||
WillReturnRows(sqlmock.NewRows([]string{"id", "owner_id"}).AddRow(2, 1))
|
||||
mock.ExpectCommit()
|
||||
mock.ExpectQuery("SELECT(.+)files(.+)").WillReturnRows(
|
||||
mock.NewRows([]string{"name"}).AddRow("test.txt"),
|
||||
)
|
||||
err = GenericAfterUpload(ctx, &fs)
|
||||
err = GenericAfterUpload(ctx, &fs, file)
|
||||
asserts.Equal(ErrFileExisted, err)
|
||||
asserts.NoError(mock.ExpectationsWereMet())
|
||||
|
||||
// 文件已存在, 且为上传占位符
|
||||
mock.ExpectQuery("SELECT(.+)").
|
||||
WithArgs(1).
|
||||
WillReturnRows(sqlmock.NewRows([]string{"id", "owner_id"}).AddRow(1, 1))
|
||||
mock.ExpectQuery("SELECT(.+)files").
|
||||
WithArgs(1, "我的文件").
|
||||
WillReturnRows(sqlmock.NewRows([]string{"id", "owner_id"}))
|
||||
// 1
|
||||
mock.ExpectBegin()
|
||||
mock.ExpectQuery("SELECT(.+)").
|
||||
WithArgs("我的文件", 1, 1).
|
||||
WillReturnRows(sqlmock.NewRows([]string{"id", "owner_id"}).AddRow(2, 1))
|
||||
mock.ExpectCommit()
|
||||
mock.ExpectQuery("SELECT(.+)files(.+)").WillReturnRows(
|
||||
mock.NewRows([]string{"name", "upload_session_id"}).AddRow("test.txt", "1"),
|
||||
)
|
||||
err = GenericAfterUpload(ctx, &fs, file)
|
||||
asserts.Equal(ErrFileUploadSessionExisted, err)
|
||||
asserts.NoError(mock.ExpectationsWereMet())
|
||||
|
||||
// 插入失败
|
||||
mock.ExpectQuery("SELECT(.+)").
|
||||
WithArgs(1).
|
||||
WillReturnRows(sqlmock.NewRows([]string{"id", "owner_id"}).AddRow(1, 1))
|
||||
mock.ExpectQuery("SELECT(.+)files").
|
||||
WithArgs(1, "我的文件").
|
||||
WillReturnRows(sqlmock.NewRows([]string{"id", "owner_id"}))
|
||||
// 1
|
||||
mock.ExpectBegin()
|
||||
mock.ExpectQuery("SELECT(.+)").
|
||||
WithArgs(1, 1, "我的文件").
|
||||
WithArgs("我的文件", 1, 1).
|
||||
WillReturnRows(sqlmock.NewRows([]string{"id", "owner_id"}).AddRow(2, 1))
|
||||
mock.ExpectCommit()
|
||||
|
||||
mock.ExpectQuery("SELECT(.+)files(.+)").WillReturnError(errors.New("not found"))
|
||||
mock.ExpectBegin()
|
||||
mock.ExpectExec("INSERT(.+)files(.+)").WillReturnError(errors.New("error"))
|
||||
mock.ExpectRollback()
|
||||
|
||||
err = GenericAfterUpload(ctx, &fs)
|
||||
err = GenericAfterUpload(ctx, &fs, file)
|
||||
asserts.Equal(ErrInsertFileRecord, err)
|
||||
asserts.NoError(mock.ExpectationsWereMet())
|
||||
|
||||
@@ -180,7 +201,7 @@ func TestFileSystem_Use(t *testing.T) {
|
||||
asserts := assert.New(t)
|
||||
fs := FileSystem{}
|
||||
|
||||
hook := func(ctx context.Context, fs *FileSystem) error {
|
||||
hook := func(ctx context.Context, fs *FileSystem, fileHeader fsctx.FileHeader) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -215,56 +236,34 @@ func TestFileSystem_Trigger(t *testing.T) {
|
||||
}
|
||||
ctx := context.Background()
|
||||
|
||||
hook := func(ctx context.Context, fs *FileSystem) error {
|
||||
hook := func(ctx context.Context, fs *FileSystem, fileHeader fsctx.FileHeader) error {
|
||||
fs.User.Storage++
|
||||
return nil
|
||||
}
|
||||
|
||||
// 一个
|
||||
fs.Use("BeforeUpload", hook)
|
||||
err := fs.Trigger(ctx, "BeforeUpload")
|
||||
err := fs.Trigger(ctx, "BeforeUpload", nil)
|
||||
asserts.NoError(err)
|
||||
asserts.Equal(uint64(1), fs.User.Storage)
|
||||
|
||||
// 多个
|
||||
fs.Use("BeforeUpload", hook)
|
||||
fs.Use("BeforeUpload", hook)
|
||||
err = fs.Trigger(ctx, "BeforeUpload")
|
||||
err = fs.Trigger(ctx, "BeforeUpload", nil)
|
||||
asserts.NoError(err)
|
||||
asserts.Equal(uint64(4), fs.User.Storage)
|
||||
}
|
||||
|
||||
func TestHookIsFileExist(t *testing.T) {
|
||||
asserts := assert.New(t)
|
||||
fs := &FileSystem{User: &model.User{
|
||||
Model: gorm.Model{
|
||||
ID: 1,
|
||||
},
|
||||
}}
|
||||
ctx := context.WithValue(context.Background(), fsctx.PathCtx, "/test.txt")
|
||||
{
|
||||
mock.ExpectQuery("SELECT(.+)").
|
||||
WithArgs(1).
|
||||
WillReturnRows(sqlmock.NewRows([]string{"id", "owner_id"}).AddRow(1, 1))
|
||||
mock.ExpectQuery("SELECT(.+)").WithArgs(uint(1), "test.txt").WillReturnRows(
|
||||
sqlmock.NewRows([]string{"Name"}).AddRow("s"),
|
||||
)
|
||||
err := HookIsFileExist(ctx, fs)
|
||||
asserts.NoError(mock.ExpectationsWereMet())
|
||||
asserts.NoError(err)
|
||||
}
|
||||
{
|
||||
mock.ExpectQuery("SELECT(.+)").
|
||||
WithArgs(1).
|
||||
WillReturnRows(sqlmock.NewRows([]string{"id", "owner_id"}).AddRow(1, 1))
|
||||
mock.ExpectQuery("SELECT(.+)").WithArgs(uint(1), "test.txt").WillReturnRows(
|
||||
sqlmock.NewRows([]string{"Name"}),
|
||||
)
|
||||
err := HookIsFileExist(ctx, fs)
|
||||
asserts.NoError(mock.ExpectationsWereMet())
|
||||
asserts.Error(err)
|
||||
}
|
||||
|
||||
// 多个,有失败
|
||||
fs.Use("BeforeUpload", func(ctx context.Context, fs *FileSystem, file fsctx.FileHeader) error {
|
||||
return errors.New("error")
|
||||
})
|
||||
fs.Use("BeforeUpload", func(ctx context.Context, fs *FileSystem, file fsctx.FileHeader) error {
|
||||
asserts.Fail("following hooks executed")
|
||||
return nil
|
||||
})
|
||||
err = fs.Trigger(ctx, "BeforeUpload", nil)
|
||||
asserts.Error(err)
|
||||
}
|
||||
|
||||
func TestHookValidateCapacity(t *testing.T) {
|
||||
@@ -277,17 +276,41 @@ func TestHookValidateCapacity(t *testing.T) {
|
||||
MaxStorage: 11,
|
||||
},
|
||||
}}
|
||||
ctx := context.WithValue(context.Background(), fsctx.FileHeaderCtx, local.FileStream{Size: 10})
|
||||
ctx := context.Background()
|
||||
file := &fsctx.FileStream{Size: 11}
|
||||
{
|
||||
err := HookValidateCapacity(ctx, fs)
|
||||
err := HookValidateCapacity(ctx, fs, file)
|
||||
asserts.NoError(err)
|
||||
}
|
||||
{
|
||||
err := HookValidateCapacity(ctx, fs)
|
||||
file.Size = 12
|
||||
err := HookValidateCapacity(ctx, fs, file)
|
||||
asserts.Error(err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestHookValidateCapacityDiff(t *testing.T) {
|
||||
a := assert.New(t)
|
||||
fs := &FileSystem{User: &model.User{
|
||||
Group: model.Group{
|
||||
MaxStorage: 11,
|
||||
},
|
||||
}}
|
||||
file := model.File{Size: 10}
|
||||
ctx := context.WithValue(context.Background(), fsctx.FileModelCtx, file)
|
||||
|
||||
// 无需操作
|
||||
{
|
||||
a.NoError(HookValidateCapacityDiff(ctx, fs, &fsctx.FileStream{Size: 10}))
|
||||
}
|
||||
|
||||
// 需要验证
|
||||
{
|
||||
a.Error(HookValidateCapacityDiff(ctx, fs, &fsctx.FileStream{Size: 12}))
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
func TestHookResetPolicy(t *testing.T) {
|
||||
asserts := assert.New(t)
|
||||
fs := &FileSystem{User: &model.User{
|
||||
@@ -301,7 +324,7 @@ func TestHookResetPolicy(t *testing.T) {
|
||||
mock.ExpectQuery("SELECT(.+)policies(.+)").
|
||||
WillReturnRows(sqlmock.NewRows([]string{"id", "type"}).AddRow(2, "local"))
|
||||
ctx := context.WithValue(context.Background(), fsctx.FileModelCtx, file)
|
||||
err := HookResetPolicy(ctx, fs)
|
||||
err := HookResetPolicy(ctx, fs, nil)
|
||||
asserts.NoError(mock.ExpectationsWereMet())
|
||||
asserts.NoError(err)
|
||||
}
|
||||
@@ -310,76 +333,22 @@ func TestHookResetPolicy(t *testing.T) {
|
||||
{
|
||||
cache.Deletes([]string{"2"}, "policy_")
|
||||
ctx := context.Background()
|
||||
err := HookResetPolicy(ctx, fs)
|
||||
err := HookResetPolicy(ctx, fs, nil)
|
||||
asserts.Error(err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestHookChangeCapacity(t *testing.T) {
|
||||
asserts := assert.New(t)
|
||||
cache.Set("pack_size_1", uint64(0), 0)
|
||||
|
||||
// 容量增加 失败
|
||||
{
|
||||
fs := &FileSystem{User: &model.User{
|
||||
Model: gorm.Model{ID: 1},
|
||||
}}
|
||||
|
||||
newFile := local.FileStream{Size: 10}
|
||||
oldFile := model.File{Size: 9}
|
||||
ctx := context.WithValue(context.Background(), fsctx.FileModelCtx, oldFile)
|
||||
ctx = context.WithValue(ctx, fsctx.FileHeaderCtx, newFile)
|
||||
err := HookChangeCapacity(ctx, fs)
|
||||
asserts.Equal(ErrInsufficientCapacity, err)
|
||||
}
|
||||
|
||||
// 容量增加 成功
|
||||
{
|
||||
fs := &FileSystem{User: &model.User{
|
||||
Model: gorm.Model{ID: 1},
|
||||
Group: model.Group{MaxStorage: 1},
|
||||
}}
|
||||
|
||||
newFile := local.FileStream{Size: 10}
|
||||
oldFile := model.File{Size: 9}
|
||||
ctx := context.WithValue(context.Background(), fsctx.FileModelCtx, oldFile)
|
||||
ctx = context.WithValue(ctx, fsctx.FileHeaderCtx, newFile)
|
||||
mock.ExpectBegin()
|
||||
mock.ExpectExec("UPDATE(.+)").WithArgs(1, sqlmock.AnyArg(), 1).WillReturnResult(sqlmock.NewResult(1, 1))
|
||||
err := HookChangeCapacity(ctx, fs)
|
||||
asserts.NoError(mock.ExpectationsWereMet())
|
||||
asserts.NoError(err)
|
||||
asserts.Equal(uint64(1), fs.User.Storage)
|
||||
}
|
||||
|
||||
// 容量减少
|
||||
{
|
||||
fs := &FileSystem{User: &model.User{
|
||||
Model: gorm.Model{ID: 1},
|
||||
Storage: 1,
|
||||
}}
|
||||
|
||||
newFile := local.FileStream{Size: 9}
|
||||
oldFile := model.File{Size: 10}
|
||||
ctx := context.WithValue(context.Background(), fsctx.FileModelCtx, oldFile)
|
||||
ctx = context.WithValue(ctx, fsctx.FileHeaderCtx, newFile)
|
||||
err := HookChangeCapacity(ctx, fs)
|
||||
asserts.NoError(err)
|
||||
asserts.Equal(uint64(0), fs.User.Storage)
|
||||
}
|
||||
}
|
||||
|
||||
func TestHookCleanFileContent(t *testing.T) {
|
||||
asserts := assert.New(t)
|
||||
fs := &FileSystem{User: &model.User{
|
||||
Model: gorm.Model{ID: 1},
|
||||
}}
|
||||
|
||||
ctx := context.WithValue(context.Background(), fsctx.SavePathCtx, "123/123")
|
||||
file := &fsctx.FileStream{SavePath: "123/123"}
|
||||
handlerMock := FileHeaderMock{}
|
||||
handlerMock.On("Put", testMock.Anything, testMock.Anything, "123/123").Return(errors.New("error"))
|
||||
handlerMock.On("Put", testMock.Anything, testMock.Anything).Return(errors.New("error"))
|
||||
fs.Handler = handlerMock
|
||||
err := HookCleanFileContent(ctx, fs)
|
||||
err := HookCleanFileContent(context.Background(), fs, file)
|
||||
asserts.Error(err)
|
||||
handlerMock.AssertExpectations(t)
|
||||
}
|
||||
@@ -395,14 +364,17 @@ func TestHookClearFileSize(t *testing.T) {
|
||||
ctx := context.WithValue(
|
||||
context.Background(),
|
||||
fsctx.FileModelCtx,
|
||||
model.File{Model: gorm.Model{ID: 1}},
|
||||
model.File{Model: gorm.Model{ID: 1}, Size: 10},
|
||||
)
|
||||
mock.ExpectBegin()
|
||||
mock.ExpectExec("UPDATE(.+)").
|
||||
WithArgs(0, sqlmock.AnyArg(), 1).
|
||||
mock.ExpectExec("UPDATE(.+)files(.+)").
|
||||
WithArgs(0, sqlmock.AnyArg(), 1, 10).
|
||||
WillReturnResult(sqlmock.NewResult(1, 1))
|
||||
mock.ExpectExec("UPDATE(.+)users(.+)").
|
||||
WithArgs(10, sqlmock.AnyArg()).
|
||||
WillReturnResult(sqlmock.NewResult(1, 1))
|
||||
mock.ExpectCommit()
|
||||
err := HookClearFileSize(ctx, fs)
|
||||
err := HookClearFileSize(ctx, fs, nil)
|
||||
asserts.NoError(mock.ExpectationsWereMet())
|
||||
asserts.NoError(err)
|
||||
}
|
||||
@@ -410,7 +382,7 @@ func TestHookClearFileSize(t *testing.T) {
|
||||
// 上下文对象不存在
|
||||
{
|
||||
ctx := context.Background()
|
||||
err := HookClearFileSize(ctx, fs)
|
||||
err := HookClearFileSize(ctx, fs, nil)
|
||||
asserts.Error(err)
|
||||
}
|
||||
|
||||
@@ -432,7 +404,7 @@ func TestHookUpdateSourceName(t *testing.T) {
|
||||
mock.ExpectBegin()
|
||||
mock.ExpectExec("UPDATE(.+)").WithArgs("new.txt", sqlmock.AnyArg(), 1).WillReturnResult(sqlmock.NewResult(1, 1))
|
||||
mock.ExpectCommit()
|
||||
err := HookUpdateSourceName(ctx, fs)
|
||||
err := HookUpdateSourceName(ctx, fs, nil)
|
||||
asserts.NoError(mock.ExpectationsWereMet())
|
||||
asserts.NoError(err)
|
||||
}
|
||||
@@ -440,7 +412,7 @@ func TestHookUpdateSourceName(t *testing.T) {
|
||||
// 上下文错误
|
||||
{
|
||||
ctx := context.Background()
|
||||
err := HookUpdateSourceName(ctx, fs)
|
||||
err := HookUpdateSourceName(ctx, fs, nil)
|
||||
asserts.Error(err)
|
||||
}
|
||||
}
|
||||
@@ -457,41 +429,32 @@ func TestGenericAfterUpdate(t *testing.T) {
|
||||
Model: gorm.Model{ID: 1},
|
||||
PicInfo: "1,1",
|
||||
}
|
||||
newFile := local.FileStream{Size: 10}
|
||||
newFile := &fsctx.FileStream{Size: 10}
|
||||
ctx := context.WithValue(context.Background(), fsctx.FileModelCtx, originFile)
|
||||
ctx = context.WithValue(ctx, fsctx.FileHeaderCtx, newFile)
|
||||
|
||||
handlerMock := FileHeaderMock{}
|
||||
handlerMock.On("Delete", testMock.Anything, []string{"._thumb"}).Return([]string{}, nil)
|
||||
fs.Handler = handlerMock
|
||||
mock.ExpectBegin()
|
||||
mock.ExpectExec("UPDATE(.+)").
|
||||
WithArgs(10, sqlmock.AnyArg(), 1).
|
||||
mock.ExpectExec("UPDATE(.+)files(.+)").
|
||||
WithArgs(10, sqlmock.AnyArg(), 1, 0).
|
||||
WillReturnResult(sqlmock.NewResult(1, 1))
|
||||
mock.ExpectExec("UPDATE(.+)users(.+)").
|
||||
WithArgs(10, sqlmock.AnyArg()).
|
||||
WillReturnResult(sqlmock.NewResult(1, 1))
|
||||
mock.ExpectCommit()
|
||||
|
||||
err := GenericAfterUpdate(ctx, fs)
|
||||
err := GenericAfterUpdate(ctx, fs, newFile)
|
||||
|
||||
asserts.NoError(mock.ExpectationsWereMet())
|
||||
asserts.NoError(err)
|
||||
}
|
||||
|
||||
// 新文件上下文不存在
|
||||
{
|
||||
originFile := model.File{
|
||||
Model: gorm.Model{ID: 1},
|
||||
PicInfo: "1,1",
|
||||
}
|
||||
ctx := context.WithValue(context.Background(), fsctx.FileModelCtx, originFile)
|
||||
err := GenericAfterUpdate(ctx, fs)
|
||||
asserts.Error(err)
|
||||
}
|
||||
|
||||
// 原始文件上下文不存在
|
||||
{
|
||||
newFile := local.FileStream{Size: 10}
|
||||
ctx := context.WithValue(context.Background(), fsctx.FileHeaderCtx, newFile)
|
||||
err := GenericAfterUpdate(ctx, fs)
|
||||
newFile := &fsctx.FileStream{Size: 10}
|
||||
ctx := context.Background()
|
||||
err := GenericAfterUpdate(ctx, fs, newFile)
|
||||
asserts.Error(err)
|
||||
}
|
||||
|
||||
@@ -502,91 +465,41 @@ func TestGenericAfterUpdate(t *testing.T) {
|
||||
Model: gorm.Model{ID: 1},
|
||||
PicInfo: "1,1",
|
||||
}
|
||||
newFile := local.FileStream{Size: 10}
|
||||
newFile := &fsctx.FileStream{Size: 10}
|
||||
ctx := context.WithValue(context.Background(), fsctx.FileModelCtx, originFile)
|
||||
ctx = context.WithValue(ctx, fsctx.FileHeaderCtx, newFile)
|
||||
|
||||
mock.ExpectBegin()
|
||||
mock.ExpectExec("UPDATE(.+)").
|
||||
WithArgs(10, sqlmock.AnyArg(), 1).
|
||||
WithArgs(10, sqlmock.AnyArg(), 1, 0).
|
||||
WillReturnError(errors.New("error"))
|
||||
mock.ExpectRollback()
|
||||
|
||||
err := GenericAfterUpdate(ctx, fs)
|
||||
err := GenericAfterUpdate(ctx, fs, newFile)
|
||||
|
||||
asserts.NoError(mock.ExpectationsWereMet())
|
||||
asserts.Error(err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestHookSlaveUploadValidate(t *testing.T) {
|
||||
asserts := assert.New(t)
|
||||
conf.SystemConfig.Mode = "slave"
|
||||
fs, err := NewAnonymousFileSystem()
|
||||
conf.SystemConfig.Mode = "master"
|
||||
asserts.NoError(err)
|
||||
|
||||
// 正常
|
||||
{
|
||||
policy := serializer.UploadPolicy{
|
||||
SavePath: "",
|
||||
MaxSize: 10,
|
||||
AllowedExtension: nil,
|
||||
}
|
||||
file := local.FileStream{Name: "1.txt", Size: 10}
|
||||
ctx := context.WithValue(context.Background(), fsctx.UploadPolicyCtx, policy)
|
||||
ctx = context.WithValue(ctx, fsctx.FileHeaderCtx, file)
|
||||
asserts.NoError(HookSlaveUploadValidate(ctx, fs))
|
||||
func TestHookGenerateThumb(t *testing.T) {
|
||||
a := assert.New(t)
|
||||
mockHandler := &FileHeaderMock{}
|
||||
fs := &FileSystem{
|
||||
User: &model.User{
|
||||
Model: gorm.Model{ID: 1},
|
||||
},
|
||||
Handler: mockHandler,
|
||||
Policy: &model.Policy{Type: "local"},
|
||||
}
|
||||
|
||||
// 尺寸太大
|
||||
{
|
||||
policy := serializer.UploadPolicy{
|
||||
SavePath: "",
|
||||
MaxSize: 10,
|
||||
AllowedExtension: nil,
|
||||
}
|
||||
file := local.FileStream{Name: "1.txt", Size: 11}
|
||||
ctx := context.WithValue(context.Background(), fsctx.UploadPolicyCtx, policy)
|
||||
ctx = context.WithValue(ctx, fsctx.FileHeaderCtx, file)
|
||||
asserts.Equal(ErrFileSizeTooBig, HookSlaveUploadValidate(ctx, fs))
|
||||
}
|
||||
|
||||
// 文件名非法
|
||||
{
|
||||
policy := serializer.UploadPolicy{
|
||||
SavePath: "",
|
||||
MaxSize: 10,
|
||||
AllowedExtension: nil,
|
||||
}
|
||||
file := local.FileStream{Name: "/1.txt", Size: 10}
|
||||
ctx := context.WithValue(context.Background(), fsctx.UploadPolicyCtx, policy)
|
||||
ctx = context.WithValue(ctx, fsctx.FileHeaderCtx, file)
|
||||
asserts.Equal(ErrIllegalObjectName, HookSlaveUploadValidate(ctx, fs))
|
||||
}
|
||||
|
||||
// 扩展名非法
|
||||
{
|
||||
policy := serializer.UploadPolicy{
|
||||
SavePath: "",
|
||||
MaxSize: 10,
|
||||
AllowedExtension: []string{"jpg"},
|
||||
}
|
||||
file := local.FileStream{Name: "1.txt", Size: 10}
|
||||
ctx := context.WithValue(context.Background(), fsctx.UploadPolicyCtx, policy)
|
||||
ctx = context.WithValue(ctx, fsctx.FileHeaderCtx, file)
|
||||
asserts.Equal(ErrFileExtensionNotAllowed, HookSlaveUploadValidate(ctx, fs))
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
type ClientMock struct {
|
||||
testMock.Mock
|
||||
}
|
||||
|
||||
func (m ClientMock) Request(method, target string, body io.Reader, opts ...request.Option) *request.Response {
|
||||
args := m.Called(method, target, body, opts)
|
||||
return args.Get(0).(*request.Response)
|
||||
mockHandler.On("Delete", testMock.Anything, []string{"1.txt._thumb"}).Return([]string{}, nil)
|
||||
a.NoError(HookGenerateThumb(context.Background(), fs, &fsctx.FileStream{
|
||||
Model: &model.File{
|
||||
SourceName: "1.txt",
|
||||
},
|
||||
}))
|
||||
fs.Recycle()
|
||||
mockHandler.AssertExpectations(t)
|
||||
}
|
||||
|
||||
func TestSlaveAfterUpload(t *testing.T) {
|
||||
@@ -598,7 +511,7 @@ func TestSlaveAfterUpload(t *testing.T) {
|
||||
|
||||
// 成功
|
||||
{
|
||||
clientMock := ClientMock{}
|
||||
clientMock := requestmock.RequestMock{}
|
||||
clientMock.On(
|
||||
"Request",
|
||||
"POST",
|
||||
@@ -613,19 +526,28 @@ func TestSlaveAfterUpload(t *testing.T) {
|
||||
},
|
||||
})
|
||||
request.GeneralClient = clientMock
|
||||
ctx := context.WithValue(context.Background(), fsctx.FileHeaderCtx, local.FileStream{
|
||||
file := &fsctx.FileStream{
|
||||
Size: 10,
|
||||
VirtualPath: "/my",
|
||||
Name: "test.txt",
|
||||
})
|
||||
ctx = context.WithValue(ctx, fsctx.UploadPolicyCtx, serializer.UploadPolicy{
|
||||
CallbackURL: "http://test/callbakc",
|
||||
})
|
||||
ctx = context.WithValue(ctx, fsctx.SavePathCtx, "/not_exist")
|
||||
err := SlaveAfterUpload(ctx, fs)
|
||||
SavePath: "/not_exist",
|
||||
}
|
||||
err := SlaveAfterUpload(&serializer.UploadSession{Callback: "http://test/callbakc"})(context.Background(), fs, file)
|
||||
clientMock.AssertExpectations(t)
|
||||
asserts.NoError(err)
|
||||
}
|
||||
|
||||
// 跳过回调
|
||||
{
|
||||
file := &fsctx.FileStream{
|
||||
Size: 10,
|
||||
VirtualPath: "/my",
|
||||
Name: "test.txt",
|
||||
SavePath: "/not_exist",
|
||||
}
|
||||
err := SlaveAfterUpload(&serializer.UploadSession{})(context.Background(), fs, file)
|
||||
asserts.NoError(err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestFileSystem_CleanHooks(t *testing.T) {
|
||||
@@ -663,7 +585,7 @@ func TestHookCancelContext(t *testing.T) {
|
||||
|
||||
// empty ctx
|
||||
{
|
||||
asserts.NoError(HookCancelContext(ctx, fs))
|
||||
asserts.NoError(HookCancelContext(ctx, fs, nil))
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
t.Errorf("Channel should not be closed")
|
||||
@@ -675,62 +597,99 @@ func TestHookCancelContext(t *testing.T) {
|
||||
// with cancel ctx
|
||||
{
|
||||
ctx = context.WithValue(ctx, fsctx.CancelFuncCtx, cancel)
|
||||
asserts.NoError(HookCancelContext(ctx, fs))
|
||||
asserts.NoError(HookCancelContext(ctx, fs, nil))
|
||||
_, ok := <-ctx.Done()
|
||||
asserts.False(ok)
|
||||
}
|
||||
}
|
||||
|
||||
func TestHookGiveBackCapacity(t *testing.T) {
|
||||
asserts := assert.New(t)
|
||||
fs := &FileSystem{
|
||||
User: &model.User{
|
||||
Model: gorm.Model{ID: 1},
|
||||
Storage: 10,
|
||||
},
|
||||
}
|
||||
ctx := context.WithValue(context.Background(), fsctx.FileHeaderCtx, local.FileStream{Size: 1})
|
||||
|
||||
// without once limit
|
||||
{
|
||||
asserts.NoError(HookGiveBackCapacity(ctx, fs))
|
||||
asserts.EqualValues(9, fs.User.Storage)
|
||||
asserts.NoError(HookGiveBackCapacity(ctx, fs))
|
||||
asserts.EqualValues(8, fs.User.Storage)
|
||||
}
|
||||
|
||||
// with once limit
|
||||
{
|
||||
ctx = context.WithValue(ctx, fsctx.ValidateCapacityOnceCtx, &sync.Once{})
|
||||
asserts.NoError(HookGiveBackCapacity(ctx, fs))
|
||||
asserts.EqualValues(7, fs.User.Storage)
|
||||
asserts.NoError(HookGiveBackCapacity(ctx, fs))
|
||||
asserts.EqualValues(7, fs.User.Storage)
|
||||
}
|
||||
}
|
||||
|
||||
func TestHookValidateCapacityWithoutIncrease(t *testing.T) {
|
||||
func TestHookClearFileHeaderSize(t *testing.T) {
|
||||
a := assert.New(t)
|
||||
fs := &FileSystem{
|
||||
User: &model.User{
|
||||
Model: gorm.Model{ID: 1},
|
||||
Storage: 10,
|
||||
Group: model.Group{},
|
||||
fs := &FileSystem{}
|
||||
file := &fsctx.FileStream{Size: 10}
|
||||
a.NoError(HookClearFileHeaderSize(context.Background(), fs, file))
|
||||
a.EqualValues(0, file.Size)
|
||||
}
|
||||
|
||||
func TestHookTruncateFileTo(t *testing.T) {
|
||||
a := assert.New(t)
|
||||
fs := &FileSystem{}
|
||||
file := &fsctx.FileStream{}
|
||||
a.NoError(HookTruncateFileTo(0)(context.Background(), fs, file))
|
||||
|
||||
fs.Handler = local.Driver{}
|
||||
a.Error(HookTruncateFileTo(0)(context.Background(), fs, file))
|
||||
}
|
||||
|
||||
func TestHookChunkUploaded(t *testing.T) {
|
||||
a := assert.New(t)
|
||||
fs := &FileSystem{}
|
||||
file := &fsctx.FileStream{
|
||||
AppendStart: 10,
|
||||
Size: 10,
|
||||
Model: &model.File{
|
||||
Model: gorm.Model{ID: 1},
|
||||
},
|
||||
}
|
||||
ctx := context.WithValue(context.Background(), fsctx.FileHeaderCtx, local.FileStream{Size: 1})
|
||||
|
||||
// not enough
|
||||
{
|
||||
fs.User.Group.MaxStorage = 10
|
||||
a.Error(HookValidateCapacityWithoutIncrease(ctx, fs))
|
||||
a.EqualValues(10, fs.User.Storage)
|
||||
}
|
||||
|
||||
// enough
|
||||
{
|
||||
fs.User.Group.MaxStorage = 11
|
||||
a.NoError(HookValidateCapacityWithoutIncrease(ctx, fs))
|
||||
a.EqualValues(10, fs.User.Storage)
|
||||
}
|
||||
mock.ExpectBegin()
|
||||
mock.ExpectExec("UPDATE(.+)files(.+)").WithArgs(20, sqlmock.AnyArg(), 1, 0).WillReturnResult(sqlmock.NewResult(1, 1))
|
||||
mock.ExpectExec("UPDATE(.+)users(.+)").
|
||||
WithArgs(20, sqlmock.AnyArg()).
|
||||
WillReturnResult(sqlmock.NewResult(1, 1))
|
||||
mock.ExpectCommit()
|
||||
a.NoError(HookChunkUploaded(context.Background(), fs, file))
|
||||
a.NoError(mock.ExpectationsWereMet())
|
||||
}
|
||||
|
||||
func TestHookChunkUploadFailed(t *testing.T) {
|
||||
a := assert.New(t)
|
||||
fs := &FileSystem{}
|
||||
file := &fsctx.FileStream{
|
||||
AppendStart: 10,
|
||||
Size: 10,
|
||||
Model: &model.File{
|
||||
Model: gorm.Model{ID: 1},
|
||||
},
|
||||
}
|
||||
|
||||
mock.ExpectBegin()
|
||||
mock.ExpectExec("UPDATE(.+)files(.+)").WithArgs(10, sqlmock.AnyArg(), 1, 0).WillReturnResult(sqlmock.NewResult(1, 1))
|
||||
mock.ExpectExec("UPDATE(.+)users(.+)").
|
||||
WithArgs(10, sqlmock.AnyArg()).
|
||||
WillReturnResult(sqlmock.NewResult(1, 1))
|
||||
mock.ExpectCommit()
|
||||
a.NoError(HookChunkUploadFailed(context.Background(), fs, file))
|
||||
a.NoError(mock.ExpectationsWereMet())
|
||||
}
|
||||
|
||||
func TestHookPopPlaceholderToFile(t *testing.T) {
|
||||
a := assert.New(t)
|
||||
fs := &FileSystem{}
|
||||
file := &fsctx.FileStream{
|
||||
Model: &model.File{
|
||||
Model: gorm.Model{ID: 1},
|
||||
},
|
||||
}
|
||||
|
||||
mock.ExpectBegin()
|
||||
mock.ExpectExec("UPDATE(.+)files(.+)").WillReturnResult(sqlmock.NewResult(1, 1))
|
||||
mock.ExpectCommit()
|
||||
a.NoError(HookPopPlaceholderToFile("1,1")(context.Background(), fs, file))
|
||||
a.NoError(mock.ExpectationsWereMet())
|
||||
}
|
||||
|
||||
func TestHookDeleteUploadSession(t *testing.T) {
|
||||
a := assert.New(t)
|
||||
fs := &FileSystem{}
|
||||
file := &fsctx.FileStream{
|
||||
Model: &model.File{
|
||||
Model: gorm.Model{ID: 1},
|
||||
},
|
||||
}
|
||||
|
||||
cache.Set(UploadSessionCachePrefix+"TestHookDeleteUploadSession", "", 0)
|
||||
a.NoError(HookDeleteUploadSession("TestHookDeleteUploadSession")(context.Background(), fs, file))
|
||||
_, ok := cache.Get(UploadSessionCachePrefix + "TestHookDeleteUploadSession")
|
||||
a.False(ok)
|
||||
}
|
||||
|
||||
@@ -3,7 +3,6 @@ package filesystem
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"strconv"
|
||||
"sync"
|
||||
|
||||
"runtime"
|
||||
@@ -65,7 +64,7 @@ type Pool struct {
|
||||
// Init 初始化任务池
|
||||
func getThumbWorker() *Pool {
|
||||
once.Do(func() {
|
||||
maxWorker := conf.ThumbConfig.MaxTaskCount
|
||||
maxWorker := model.GetIntSetting("thumb_max_task_count", -1)
|
||||
if maxWorker <= 0 {
|
||||
maxWorker = runtime.GOMAXPROCS(0)
|
||||
}
|
||||
@@ -118,9 +117,9 @@ func (fs *FileSystem) GenerateThumbnail(ctx context.Context, file *model.File) {
|
||||
// 生成缩略图
|
||||
image.GetThumb(fs.GenerateThumbnailSize(w, h))
|
||||
// 保存到文件
|
||||
err = image.Save(util.RelativePath(file.SourceName + conf.ThumbConfig.FileSuffix))
|
||||
err = image.Save(util.RelativePath(file.SourceName + model.GetSettingByNameWithDefault("thumb_file_suffix", "._thumb")))
|
||||
image = nil
|
||||
if conf.ThumbConfig.GCAfterGen {
|
||||
if model.IsTrueVal(model.GetSettingByName("thumb_gc_after_gen")) {
|
||||
util.Log().Debug("GenerateThumbnail runtime.GC")
|
||||
runtime.GC()
|
||||
}
|
||||
@@ -139,17 +138,11 @@ func (fs *FileSystem) GenerateThumbnail(ctx context.Context, file *model.File) {
|
||||
|
||||
// 失败时删除缩略图文件
|
||||
if err != nil {
|
||||
_, _ = fs.Handler.Delete(newCtx, []string{file.SourceName + conf.ThumbConfig.FileSuffix})
|
||||
_, _ = fs.Handler.Delete(newCtx, []string{file.SourceName + model.GetSettingByNameWithDefault("thumb_file_suffix", "._thumb")})
|
||||
}
|
||||
}
|
||||
|
||||
// GenerateThumbnailSize 获取要生成的缩略图的尺寸
|
||||
func (fs *FileSystem) GenerateThumbnailSize(w, h int) (uint, uint) {
|
||||
if conf.SystemConfig.Mode == "master" {
|
||||
options := model.GetSettingByNames("thumb_width", "thumb_height")
|
||||
w, _ := strconv.ParseUint(options["thumb_width"], 10, 32)
|
||||
h, _ := strconv.ParseUint(options["thumb_height"], 10, 32)
|
||||
return uint(w), uint(h)
|
||||
}
|
||||
return conf.ThumbConfig.MaxWidth, conf.ThumbConfig.MaxHeight
|
||||
return uint(model.GetIntSetting("thumb_width", 400)), uint(model.GetIntSetting("thumb_width", 300))
|
||||
}
|
||||
|
||||
@@ -2,13 +2,15 @@ package filesystem
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
|
||||
"errors"
|
||||
model "github.com/cloudreve/Cloudreve/v3/models"
|
||||
"github.com/cloudreve/Cloudreve/v3/pkg/cache"
|
||||
"github.com/cloudreve/Cloudreve/v3/pkg/filesystem/response"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/cloudreve/Cloudreve/v3/pkg/request"
|
||||
testMock "github.com/stretchr/testify/mock"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
)
|
||||
|
||||
func TestFileSystem_GetThumb(t *testing.T) {
|
||||
@@ -47,3 +49,22 @@ func TestFileSystem_ThumbWorker(t *testing.T) {
|
||||
getThumbWorker().releaseWorker()
|
||||
})
|
||||
}
|
||||
|
||||
func TestFileSystem_GenerateThumbnail(t *testing.T) {
|
||||
fs := &FileSystem{User: &model.User{}}
|
||||
|
||||
// 无法生成缩略图
|
||||
{
|
||||
fs.SetTargetFile(&[]model.File{{}})
|
||||
fs.GenerateThumbnail(context.Background(), &model.File{})
|
||||
}
|
||||
|
||||
// 无法获取文件数据
|
||||
{
|
||||
testHandller := new(FileHeaderMock)
|
||||
testHandller.On("Get", testMock.Anything, "").Return(request.NopRSCloser{}, errors.New("error"))
|
||||
fs.Handler = testHandller
|
||||
fs.GenerateThumbnail(context.Background(), &model.File{Name: "test.png"})
|
||||
testHandller.AssertExpectations(t)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -5,7 +5,6 @@ import (
|
||||
"fmt"
|
||||
"path"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
model "github.com/cloudreve/Cloudreve/v3/models"
|
||||
"github.com/cloudreve/Cloudreve/v3/pkg/filesystem/fsctx"
|
||||
@@ -19,18 +18,6 @@ import (
|
||||
=================
|
||||
*/
|
||||
|
||||
// Object 文件或者目录
|
||||
type Object struct {
|
||||
ID string `json:"id"`
|
||||
Name string `json:"name"`
|
||||
Path string `json:"path"`
|
||||
Pic string `json:"pic"`
|
||||
Size uint64 `json:"size"`
|
||||
Type string `json:"type"`
|
||||
Date time.Time `json:"date"`
|
||||
Key string `json:"key,omitempty"`
|
||||
}
|
||||
|
||||
// Rename 重命名对象
|
||||
func (fs *FileSystem) Rename(ctx context.Context, dir, file []uint, new string) (err error) {
|
||||
// 验证新名字
|
||||
@@ -135,15 +122,12 @@ func (fs *FileSystem) Move(ctx context.Context, dirs, files []uint, src, dst str
|
||||
|
||||
// Delete 递归删除对象, force 为 true 时强制删除文件记录,忽略物理删除是否成功
|
||||
func (fs *FileSystem) Delete(ctx context.Context, dirs, files []uint, force bool) error {
|
||||
// 已删除的总容量,map用于去重
|
||||
var deletedStorage = make(map[uint]uint64)
|
||||
var totalStorage = make(map[uint]uint64)
|
||||
// 已删除的文件ID
|
||||
var deletedFileIDs = make([]uint, 0, len(fs.FileTarget))
|
||||
var deletedFiles = make([]*model.File, 0, len(fs.FileTarget))
|
||||
// 删除失败的文件的父目录ID
|
||||
|
||||
// 所有文件的ID
|
||||
var allFileIDs = make([]uint, 0, len(fs.FileTarget))
|
||||
var allFiles = make([]*model.File, 0, len(fs.FileTarget))
|
||||
|
||||
// 列出要删除的目录
|
||||
if len(dirs) > 0 {
|
||||
@@ -177,38 +161,35 @@ func (fs *FileSystem) Delete(ctx context.Context, dirs, files []uint, force bool
|
||||
for i := 0; i < len(fs.FileTarget); i++ {
|
||||
if !util.ContainsString(failed[fs.FileTarget[i].PolicyID], fs.FileTarget[i].SourceName) {
|
||||
// 已成功删除的文件
|
||||
deletedFileIDs = append(deletedFileIDs, fs.FileTarget[i].ID)
|
||||
deletedStorage[fs.FileTarget[i].ID] = fs.FileTarget[i].Size
|
||||
deletedFiles = append(deletedFiles, &fs.FileTarget[i])
|
||||
}
|
||||
|
||||
// 全部文件
|
||||
totalStorage[fs.FileTarget[i].ID] = fs.FileTarget[i].Size
|
||||
allFileIDs = append(allFileIDs, fs.FileTarget[i].ID)
|
||||
allFiles = append(allFiles, &fs.FileTarget[i])
|
||||
}
|
||||
|
||||
// 如果强制删除,则将全部文件视为删除成功
|
||||
if force {
|
||||
deletedFileIDs = allFileIDs
|
||||
deletedStorage = totalStorage
|
||||
deletedFiles = allFiles
|
||||
}
|
||||
|
||||
// 删除文件记录
|
||||
err = model.DeleteFileByIDs(deletedFileIDs)
|
||||
err = model.DeleteFiles(deletedFiles, fs.User.ID)
|
||||
if err != nil {
|
||||
return ErrDBDeleteObjects.WithError(err)
|
||||
}
|
||||
|
||||
// 删除文件记录对应的分享记录
|
||||
// TODO 先取消分享再删除文件
|
||||
deletedFileIDs := make([]uint, len(deletedFiles))
|
||||
for k, file := range deletedFiles {
|
||||
deletedFileIDs[k] = file.ID
|
||||
}
|
||||
|
||||
model.DeleteShareBySourceIDs(deletedFileIDs, false)
|
||||
|
||||
// 归还容量
|
||||
var total uint64
|
||||
for _, value := range deletedStorage {
|
||||
total += value
|
||||
}
|
||||
fs.User.DeductionStorage(total)
|
||||
|
||||
// 如果文件全部删除成功,继续删除目录
|
||||
if len(deletedFileIDs) == len(allFileIDs) {
|
||||
if len(deletedFiles) == len(allFiles) {
|
||||
var allFolderIDs = make([]uint, 0, len(fs.DirTarget))
|
||||
for _, value := range fs.DirTarget {
|
||||
allFolderIDs = append(allFolderIDs, value.ID)
|
||||
@@ -222,7 +203,7 @@ func (fs *FileSystem) Delete(ctx context.Context, dirs, files []uint, force bool
|
||||
model.DeleteShareBySourceIDs(allFolderIDs, true)
|
||||
}
|
||||
|
||||
if notDeleted := len(fs.FileTarget) - len(deletedFileIDs); notDeleted > 0 {
|
||||
if notDeleted := len(fs.FileTarget) - len(deletedFiles); notDeleted > 0 {
|
||||
return serializer.NewError(
|
||||
serializer.CodeNotFullySuccess,
|
||||
fmt.Sprintf("有 %d 个文件未能成功删除", notDeleted),
|
||||
@@ -240,6 +221,15 @@ func (fs *FileSystem) ListDeleteDirs(ctx context.Context, ids []uint) error {
|
||||
if err != nil {
|
||||
return ErrDBListObjects.WithError(err)
|
||||
}
|
||||
|
||||
// 忽略根目录
|
||||
for i := 0; i < len(folders); i++ {
|
||||
if folders[i].ParentID == nil {
|
||||
folders = append(folders[:i], folders[i+1:]...)
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
fs.SetTargetDir(&folders)
|
||||
|
||||
// 检索目录下的子文件
|
||||
@@ -266,7 +256,7 @@ func (fs *FileSystem) ListDeleteFiles(ctx context.Context, ids []uint) error {
|
||||
// pathProcessor为最终对象路径的处理钩子。
|
||||
// 有些情况下(如在分享页面列对象)时,
|
||||
// 路径需要截取掉被分享目录路径之前的部分。
|
||||
func (fs *FileSystem) List(ctx context.Context, dirPath string, pathProcessor func(string) string) ([]Object, error) {
|
||||
func (fs *FileSystem) List(ctx context.Context, dirPath string, pathProcessor func(string) string) ([]serializer.Object, error) {
|
||||
// 获取父目录
|
||||
isExist, folder := fs.IsPathExist(dirPath)
|
||||
if !isExist {
|
||||
@@ -289,7 +279,7 @@ func (fs *FileSystem) List(ctx context.Context, dirPath string, pathProcessor fu
|
||||
|
||||
// ListPhysical 列出存储策略中的外部目录
|
||||
// TODO:测试
|
||||
func (fs *FileSystem) ListPhysical(ctx context.Context, dirPath string) ([]Object, error) {
|
||||
func (fs *FileSystem) ListPhysical(ctx context.Context, dirPath string) ([]serializer.Object, error) {
|
||||
if err := fs.DispatchHandler(); fs.Policy == nil || err != nil {
|
||||
return nil, ErrUnknownPolicyType
|
||||
}
|
||||
@@ -319,7 +309,7 @@ func (fs *FileSystem) ListPhysical(ctx context.Context, dirPath string) ([]Objec
|
||||
return fs.listObjects(ctx, dirPath, nil, folders, nil), nil
|
||||
}
|
||||
|
||||
func (fs *FileSystem) listObjects(ctx context.Context, parent string, files []model.File, folders []model.Folder, pathProcessor func(string) string) []Object {
|
||||
func (fs *FileSystem) listObjects(ctx context.Context, parent string, files []model.File, folders []model.Folder, pathProcessor func(string) string) []serializer.Object {
|
||||
// 分享文件的ID
|
||||
shareKey := ""
|
||||
if key, ok := ctx.Value(fsctx.ShareKeyCtx).(string); ok {
|
||||
@@ -327,7 +317,7 @@ func (fs *FileSystem) listObjects(ctx context.Context, parent string, files []mo
|
||||
}
|
||||
|
||||
// 汇总处理结果
|
||||
objects := make([]Object, 0, len(files)+len(folders))
|
||||
objects := make([]serializer.Object, 0, len(files)+len(folders))
|
||||
|
||||
// 所有对象的父目录
|
||||
var processedPath string
|
||||
@@ -343,14 +333,15 @@ func (fs *FileSystem) listObjects(ctx context.Context, parent string, files []mo
|
||||
}
|
||||
}
|
||||
|
||||
objects = append(objects, Object{
|
||||
ID: hashid.HashID(subFolder.ID, hashid.FolderID),
|
||||
Name: subFolder.Name,
|
||||
Path: processedPath,
|
||||
Pic: "",
|
||||
Size: 0,
|
||||
Type: "dir",
|
||||
Date: subFolder.CreatedAt,
|
||||
objects = append(objects, serializer.Object{
|
||||
ID: hashid.HashID(subFolder.ID, hashid.FolderID),
|
||||
Name: subFolder.Name,
|
||||
Path: processedPath,
|
||||
Pic: "",
|
||||
Size: 0,
|
||||
Type: "dir",
|
||||
Date: subFolder.UpdatedAt,
|
||||
CreateDate: subFolder.CreatedAt,
|
||||
})
|
||||
}
|
||||
|
||||
@@ -363,30 +354,39 @@ func (fs *FileSystem) listObjects(ctx context.Context, parent string, files []mo
|
||||
}
|
||||
}
|
||||
|
||||
newFile := Object{
|
||||
ID: hashid.HashID(file.ID, hashid.FileID),
|
||||
Name: file.Name,
|
||||
Path: processedPath,
|
||||
Pic: file.PicInfo,
|
||||
Size: file.Size,
|
||||
Type: "file",
|
||||
Date: file.CreatedAt,
|
||||
if file.UploadSessionID == nil {
|
||||
newFile := serializer.Object{
|
||||
ID: hashid.HashID(file.ID, hashid.FileID),
|
||||
Name: file.Name,
|
||||
Path: processedPath,
|
||||
Pic: file.PicInfo,
|
||||
Size: file.Size,
|
||||
Type: "file",
|
||||
Date: file.UpdatedAt,
|
||||
SourceEnabled: file.GetPolicy().IsOriginLinkEnable,
|
||||
CreateDate: file.CreatedAt,
|
||||
}
|
||||
if shareKey != "" {
|
||||
newFile.Key = shareKey
|
||||
}
|
||||
objects = append(objects, newFile)
|
||||
}
|
||||
if shareKey != "" {
|
||||
newFile.Key = shareKey
|
||||
}
|
||||
objects = append(objects, newFile)
|
||||
}
|
||||
|
||||
return objects
|
||||
}
|
||||
|
||||
// CreateDirectory 根据给定的完整创建目录,支持递归创建
|
||||
// CreateDirectory 根据给定的完整创建目录,支持递归创建。如果目录已存在,则直接
|
||||
// 返回已存在的目录。
|
||||
func (fs *FileSystem) CreateDirectory(ctx context.Context, fullPath string) (*model.Folder, error) {
|
||||
if fullPath == "/" || fullPath == "." || fullPath == "" {
|
||||
if fullPath == "." || fullPath == "" {
|
||||
return nil, ErrRootProtected
|
||||
}
|
||||
|
||||
if fullPath == "/" {
|
||||
return fs.User.Root()
|
||||
}
|
||||
|
||||
// 获取要创建目录的父路径和目录名
|
||||
fullPath = path.Clean(fullPath)
|
||||
base := path.Dir(fullPath)
|
||||
@@ -403,10 +403,6 @@ func (fs *FileSystem) CreateDirectory(ctx context.Context, fullPath string) (*mo
|
||||
// 父目录是否存在
|
||||
isExist, parent := fs.IsPathExist(base)
|
||||
if !isExist {
|
||||
// 递归创建父目录
|
||||
if _, ok := ctx.Value(fsctx.IgnoreDirectoryConflictCtx).(bool); !ok {
|
||||
ctx = context.WithValue(ctx, fsctx.IgnoreDirectoryConflictCtx, true)
|
||||
}
|
||||
newParent, err := fs.CreateDirectory(ctx, base)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
@@ -428,11 +424,9 @@ func (fs *FileSystem) CreateDirectory(ctx context.Context, fullPath string) (*mo
|
||||
_, err := newFolder.Create()
|
||||
|
||||
if err != nil {
|
||||
if _, ok := ctx.Value(fsctx.IgnoreDirectoryConflictCtx).(bool); !ok {
|
||||
return nil, ErrFolderExisted
|
||||
}
|
||||
|
||||
return nil, fmt.Errorf("failed to create folder: %w", err)
|
||||
}
|
||||
|
||||
return &newFolder, nil
|
||||
}
|
||||
|
||||
|
||||
@@ -3,6 +3,8 @@ package filesystem
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"github.com/cloudreve/Cloudreve/v3/pkg/filesystem/response"
|
||||
testMock "github.com/stretchr/testify/mock"
|
||||
"os"
|
||||
"testing"
|
||||
|
||||
@@ -11,12 +13,10 @@ import (
|
||||
"github.com/cloudreve/Cloudreve/v3/pkg/cache"
|
||||
"github.com/cloudreve/Cloudreve/v3/pkg/conf"
|
||||
"github.com/cloudreve/Cloudreve/v3/pkg/filesystem/fsctx"
|
||||
"github.com/cloudreve/Cloudreve/v3/pkg/filesystem/response"
|
||||
"github.com/cloudreve/Cloudreve/v3/pkg/serializer"
|
||||
"github.com/cloudreve/Cloudreve/v3/pkg/util"
|
||||
"github.com/jinzhu/gorm"
|
||||
"github.com/stretchr/testify/assert"
|
||||
testMock "github.com/stretchr/testify/mock"
|
||||
)
|
||||
|
||||
func TestFileSystem_ListPhysical(t *testing.T) {
|
||||
@@ -214,7 +214,7 @@ func TestFileSystem_CreateDirectory(t *testing.T) {
|
||||
asserts.Equal(ErrFileExisted, err)
|
||||
asserts.NoError(mock.ExpectationsWereMet())
|
||||
|
||||
// 存在同名目录
|
||||
// 存在同名目录,直接返回
|
||||
mock.ExpectQuery("SELECT(.+)").
|
||||
WithArgs(1).
|
||||
WillReturnRows(sqlmock.NewRows([]string{"id", "owner_id"}).AddRow(1, 1))
|
||||
@@ -225,13 +225,37 @@ func TestFileSystem_CreateDirectory(t *testing.T) {
|
||||
|
||||
mock.ExpectQuery("SELECT(.+)files").WillReturnRows(sqlmock.NewRows([]string{"id", "name"}))
|
||||
mock.ExpectBegin()
|
||||
mock.ExpectExec("INSERT(.+)").WillReturnError(errors.New("s"))
|
||||
mock.ExpectRollback()
|
||||
_, err = fs.CreateDirectory(ctx, "/ad/ab")
|
||||
asserts.Error(err)
|
||||
// ab
|
||||
mock.ExpectQuery("SELECT(.+)").
|
||||
WithArgs("ab", 2, 1).
|
||||
WillReturnRows(sqlmock.NewRows([]string{"id", "owner_id"}).AddRow(3, 1))
|
||||
mock.ExpectCommit()
|
||||
res, err := fs.CreateDirectory(ctx, "/ad/ab")
|
||||
asserts.NoError(err)
|
||||
asserts.EqualValues(3, res.ID)
|
||||
asserts.NoError(mock.ExpectationsWereMet())
|
||||
|
||||
// 成功创建
|
||||
mock.ExpectQuery("SELECT(.+)").
|
||||
WithArgs(1).
|
||||
WillReturnRows(sqlmock.NewRows([]string{"id", "owner_id"}).AddRow(1, 1))
|
||||
// ad
|
||||
mock.ExpectQuery("SELECT(.+)").
|
||||
WithArgs(1, 1, "ad").
|
||||
WillReturnRows(sqlmock.NewRows([]string{"id", "owner_id"}).AddRow(2, 1))
|
||||
|
||||
mock.ExpectQuery("SELECT(.+)files").WillReturnRows(sqlmock.NewRows([]string{"id", "name"}))
|
||||
mock.ExpectBegin()
|
||||
mock.ExpectQuery("SELECT(.+)").
|
||||
WithArgs("ab", 2, 1).
|
||||
WillReturnRows(sqlmock.NewRows([]string{"id", "owner_id"}))
|
||||
mock.ExpectExec("INSERT(.+)").WillReturnResult(sqlmock.NewResult(1, 1))
|
||||
mock.ExpectCommit()
|
||||
_, err = fs.CreateDirectory(ctx, "/ad/ab")
|
||||
asserts.NoError(err)
|
||||
asserts.NoError(mock.ExpectationsWereMet())
|
||||
|
||||
// 成功创建, 递归创建父目录
|
||||
// 根目录
|
||||
mock.ExpectQuery("SELECT(.+)").
|
||||
WithArgs(1).
|
||||
@@ -239,20 +263,62 @@ func TestFileSystem_CreateDirectory(t *testing.T) {
|
||||
// ad
|
||||
mock.ExpectQuery("SELECT(.+)").
|
||||
WithArgs(1, 1, "ad").
|
||||
WillReturnRows(sqlmock.NewRows([]string{"id", "owner_id"}).AddRow(2, 1))
|
||||
|
||||
WillReturnRows(sqlmock.NewRows([]string{"id", "owner_id"}))
|
||||
// 根目录
|
||||
mock.ExpectQuery("SELECT(.+)").
|
||||
WithArgs(1).
|
||||
WillReturnRows(sqlmock.NewRows([]string{"id", "owner_id"}).AddRow(1, 1))
|
||||
mock.ExpectQuery("SELECT(.+)files").WillReturnRows(sqlmock.NewRows([]string{"id", "name"}))
|
||||
// 创建ad
|
||||
mock.ExpectBegin()
|
||||
mock.ExpectQuery("SELECT(.+)").
|
||||
WithArgs("ad", 1, 1).
|
||||
WillReturnRows(sqlmock.NewRows([]string{"id", "owner_id"}))
|
||||
mock.ExpectExec("INSERT(.+)").WillReturnResult(sqlmock.NewResult(2, 1))
|
||||
mock.ExpectCommit()
|
||||
mock.ExpectQuery("SELECT(.+)files").WillReturnRows(sqlmock.NewRows([]string{"id", "name"}))
|
||||
// 创建ab
|
||||
mock.ExpectBegin()
|
||||
mock.ExpectQuery("SELECT(.+)").
|
||||
WithArgs("ab", 2, 1).
|
||||
WillReturnRows(sqlmock.NewRows([]string{"id", "owner_id"}))
|
||||
mock.ExpectExec("INSERT(.+)").WillReturnResult(sqlmock.NewResult(1, 1))
|
||||
mock.ExpectCommit()
|
||||
_, err = fs.CreateDirectory(ctx, "/ad/ab")
|
||||
asserts.NoError(err)
|
||||
asserts.NoError(mock.ExpectationsWereMet())
|
||||
|
||||
// 父目录不存在
|
||||
mock.ExpectQuery("SELECT(.+)folders").WillReturnRows(sqlmock.NewRows([]string{"id", "name"}))
|
||||
_, err = fs.CreateDirectory(ctx, "/ad")
|
||||
asserts.Equal(ErrRootProtected, err)
|
||||
// 底层创建失败
|
||||
// 成功创建
|
||||
mock.ExpectQuery("SELECT(.+)").
|
||||
WithArgs(1).
|
||||
WillReturnRows(sqlmock.NewRows([]string{"id", "owner_id"}).AddRow(1, 1))
|
||||
// ad
|
||||
mock.ExpectQuery("SELECT(.+)").
|
||||
WithArgs(1, 1, "ad").
|
||||
WillReturnRows(sqlmock.NewRows([]string{"id", "owner_id"}))
|
||||
// 根目录
|
||||
mock.ExpectQuery("SELECT(.+)").
|
||||
WithArgs(1).
|
||||
WillReturnRows(sqlmock.NewRows([]string{"id", "owner_id"}).AddRow(1, 1))
|
||||
mock.ExpectQuery("SELECT(.+)files").WillReturnRows(sqlmock.NewRows([]string{"id", "name"}))
|
||||
// 创建ad
|
||||
mock.ExpectBegin()
|
||||
mock.ExpectQuery("SELECT(.+)").
|
||||
WithArgs("ad", 1, 1).
|
||||
WillReturnRows(sqlmock.NewRows([]string{"id", "owner_id"}))
|
||||
mock.ExpectExec("INSERT(.+)").WillReturnResult(sqlmock.NewResult(2, 1)).WillReturnError(errors.New("error"))
|
||||
mock.ExpectRollback()
|
||||
_, err = fs.CreateDirectory(ctx, "/ad/ab")
|
||||
asserts.Error(err)
|
||||
asserts.NoError(mock.ExpectationsWereMet())
|
||||
|
||||
// 直接创建根目录
|
||||
mock.ExpectQuery("SELECT(.+)").
|
||||
WithArgs(1).
|
||||
WillReturnRows(sqlmock.NewRows([]string{"id", "owner_id"}).AddRow(1, 1))
|
||||
_, err = fs.CreateDirectory(ctx, "/")
|
||||
asserts.NoError(err)
|
||||
asserts.NoError(mock.ExpectationsWereMet())
|
||||
}
|
||||
|
||||
@@ -296,12 +362,12 @@ func TestFileSystem_ListDeleteDirs(t *testing.T) {
|
||||
{
|
||||
mock.ExpectQuery("SELECT(.+)").
|
||||
WillReturnRows(
|
||||
sqlmock.NewRows([]string{"id"}).
|
||||
AddRow(1).
|
||||
AddRow(2).
|
||||
AddRow(3),
|
||||
sqlmock.NewRows([]string{"id", "parent_id"}).
|
||||
AddRow(1, 0).
|
||||
AddRow(2, 0).
|
||||
AddRow(3, 0),
|
||||
)
|
||||
mock.ExpectQuery("SELECT(.+)").
|
||||
mock.ExpectQuery("SELECT(.+)files(.+)").
|
||||
WithArgs(1, 2, 3).
|
||||
WillReturnRows(
|
||||
sqlmock.NewRows([]string{"id", "name"}).
|
||||
@@ -316,21 +382,47 @@ func TestFileSystem_ListDeleteDirs(t *testing.T) {
|
||||
asserts.NoError(mock.ExpectationsWereMet())
|
||||
}
|
||||
|
||||
// 成功,忽略根目录
|
||||
{
|
||||
mock.ExpectQuery("SELECT(.+)").
|
||||
WillReturnRows(
|
||||
sqlmock.NewRows([]string{"id", "parent_id"}).
|
||||
AddRow(1, 0).
|
||||
AddRow(2, nil).
|
||||
AddRow(3, 0),
|
||||
)
|
||||
mock.ExpectQuery("SELECT(.+)files(.+)").
|
||||
WithArgs(1, 3).
|
||||
WillReturnRows(
|
||||
sqlmock.NewRows([]string{"id", "name"}).
|
||||
AddRow(4, "1.txt").
|
||||
AddRow(5, "2.txt").
|
||||
AddRow(6, "3.txt"),
|
||||
)
|
||||
fs.CleanTargets()
|
||||
err := fs.ListDeleteDirs(context.Background(), []uint{1})
|
||||
asserts.NoError(err)
|
||||
asserts.Len(fs.FileTarget, 3)
|
||||
asserts.Len(fs.DirTarget, 2)
|
||||
asserts.NoError(mock.ExpectationsWereMet())
|
||||
}
|
||||
|
||||
// 检索文件发生错误
|
||||
{
|
||||
mock.ExpectQuery("SELECT(.+)").
|
||||
WillReturnRows(
|
||||
sqlmock.NewRows([]string{"id"}).
|
||||
AddRow(1).
|
||||
AddRow(2).
|
||||
AddRow(3),
|
||||
sqlmock.NewRows([]string{"id", "parent_id"}).
|
||||
AddRow(1, 0).
|
||||
AddRow(2, 0).
|
||||
AddRow(3, 0),
|
||||
)
|
||||
mock.ExpectQuery("SELECT(.+)").
|
||||
WithArgs(1, 2, 3).
|
||||
WillReturnError(errors.New("error"))
|
||||
fs.CleanTargets()
|
||||
err := fs.ListDeleteDirs(context.Background(), []uint{1})
|
||||
asserts.Error(err)
|
||||
asserts.Len(fs.DirTarget, 6)
|
||||
asserts.Len(fs.DirTarget, 3)
|
||||
asserts.NoError(mock.ExpectationsWereMet())
|
||||
}
|
||||
// 检索目录发生错误
|
||||
@@ -349,7 +441,7 @@ func TestFileSystem_Delete(t *testing.T) {
|
||||
cache.Set("pack_size_1", uint64(0), 0)
|
||||
fs := &FileSystem{User: &model.User{
|
||||
Model: gorm.Model{
|
||||
ID: 1,
|
||||
ID: 0,
|
||||
},
|
||||
Storage: 3,
|
||||
Group: model.Group{MaxStorage: 3},
|
||||
@@ -361,10 +453,10 @@ func TestFileSystem_Delete(t *testing.T) {
|
||||
fs.CleanTargets()
|
||||
mock.ExpectQuery("SELECT(.+)").
|
||||
WillReturnRows(
|
||||
sqlmock.NewRows([]string{"id"}).
|
||||
AddRow(1).
|
||||
AddRow(2).
|
||||
AddRow(3),
|
||||
sqlmock.NewRows([]string{"id", "parent_id"}).
|
||||
AddRow(1, 0).
|
||||
AddRow(2, 0).
|
||||
AddRow(3, 0),
|
||||
)
|
||||
mock.ExpectQuery("SELECT(.+)").
|
||||
WithArgs(1, 2, 3).
|
||||
@@ -380,18 +472,16 @@ func TestFileSystem_Delete(t *testing.T) {
|
||||
// 删除文件记录
|
||||
mock.ExpectBegin()
|
||||
mock.ExpectExec("DELETE(.+)").
|
||||
WillReturnResult(sqlmock.NewResult(0, 3))
|
||||
WillReturnResult(sqlmock.NewResult(0, 1))
|
||||
mock.ExpectExec("DELETE(.+)").
|
||||
WillReturnResult(sqlmock.NewResult(0, 1))
|
||||
mock.ExpectExec("UPDATE(.+)users(.+)storage(.+)").WillReturnResult(sqlmock.NewResult(1, 1))
|
||||
mock.ExpectCommit()
|
||||
// 删除对应分享
|
||||
mock.ExpectBegin()
|
||||
mock.ExpectExec("UPDATE(.+)shares").
|
||||
WillReturnResult(sqlmock.NewResult(0, 3))
|
||||
mock.ExpectCommit()
|
||||
// 归还容量
|
||||
mock.ExpectBegin()
|
||||
mock.ExpectExec("UPDATE(.+)").
|
||||
WillReturnResult(sqlmock.NewResult(0, 3))
|
||||
mock.ExpectCommit()
|
||||
// 删除目录
|
||||
mock.ExpectBegin()
|
||||
mock.ExpectExec("DELETE(.+)").
|
||||
@@ -407,7 +497,6 @@ func TestFileSystem_Delete(t *testing.T) {
|
||||
fs.DirTarget = []model.Folder{}
|
||||
err := fs.Delete(ctx, []uint{1}, []uint{1}, true)
|
||||
asserts.NoError(err)
|
||||
asserts.Equal(uint64(0), fs.User.Storage)
|
||||
}
|
||||
//全部成功
|
||||
{
|
||||
@@ -419,10 +508,10 @@ func TestFileSystem_Delete(t *testing.T) {
|
||||
asserts.NoError(err)
|
||||
mock.ExpectQuery("SELECT(.+)").
|
||||
WillReturnRows(
|
||||
sqlmock.NewRows([]string{"id"}).
|
||||
AddRow(1).
|
||||
AddRow(2).
|
||||
AddRow(3),
|
||||
sqlmock.NewRows([]string{"id", "parent_id"}).
|
||||
AddRow(1, 0).
|
||||
AddRow(2, 0).
|
||||
AddRow(3, 0),
|
||||
)
|
||||
mock.ExpectQuery("SELECT(.+)").
|
||||
WithArgs(1, 2, 3).
|
||||
@@ -438,18 +527,16 @@ func TestFileSystem_Delete(t *testing.T) {
|
||||
// 删除文件记录
|
||||
mock.ExpectBegin()
|
||||
mock.ExpectExec("DELETE(.+)").
|
||||
WillReturnResult(sqlmock.NewResult(0, 3))
|
||||
WillReturnResult(sqlmock.NewResult(0, 1))
|
||||
mock.ExpectExec("DELETE(.+)").
|
||||
WillReturnResult(sqlmock.NewResult(0, 1))
|
||||
mock.ExpectExec("UPDATE(.+)users(.+)storage(.+)").WillReturnResult(sqlmock.NewResult(1, 1))
|
||||
mock.ExpectCommit()
|
||||
// 删除对应分享
|
||||
mock.ExpectBegin()
|
||||
mock.ExpectExec("UPDATE(.+)shares").
|
||||
WillReturnResult(sqlmock.NewResult(0, 3))
|
||||
mock.ExpectCommit()
|
||||
// 归还容量
|
||||
mock.ExpectBegin()
|
||||
mock.ExpectExec("UPDATE(.+)").
|
||||
WillReturnResult(sqlmock.NewResult(0, 3))
|
||||
mock.ExpectCommit()
|
||||
// 删除目录
|
||||
mock.ExpectBegin()
|
||||
mock.ExpectExec("DELETE(.+)").
|
||||
@@ -465,7 +552,6 @@ func TestFileSystem_Delete(t *testing.T) {
|
||||
fs.DirTarget = []model.Folder{}
|
||||
err = fs.Delete(ctx, []uint{1}, []uint{1}, false)
|
||||
asserts.NoError(err)
|
||||
asserts.Equal(uint64(0), fs.User.Storage)
|
||||
}
|
||||
|
||||
}
|
||||
@@ -573,7 +659,9 @@ func TestFileSystem_Rename(t *testing.T) {
|
||||
Model: gorm.Model{
|
||||
ID: 1,
|
||||
},
|
||||
}}
|
||||
},
|
||||
Policy: &model.Policy{},
|
||||
}
|
||||
ctx := context.Background()
|
||||
|
||||
// 重命名文件 成功
|
||||
@@ -683,7 +771,7 @@ func TestFileSystem_Rename(t *testing.T) {
|
||||
|
||||
// 新名字是文件,扩展名不合法
|
||||
{
|
||||
fs.User.Policy.OptionsSerialized.FileType = []string{"txt"}
|
||||
fs.Policy.OptionsSerialized.FileType = []string{"txt"}
|
||||
err := fs.Rename(ctx, []uint{}, []uint{10}, "1.jpg")
|
||||
asserts.Error(err)
|
||||
asserts.Equal(ErrIllegalObjectName, err)
|
||||
@@ -691,7 +779,7 @@ func TestFileSystem_Rename(t *testing.T) {
|
||||
|
||||
// 新名字是目录,不应该检测扩展名
|
||||
{
|
||||
fs.User.Policy.OptionsSerialized.FileType = []string{"txt"}
|
||||
fs.Policy.OptionsSerialized.FileType = []string{"txt"}
|
||||
mock.ExpectQuery("SELECT(.+)folders(.+)").
|
||||
WithArgs(10, 1).
|
||||
WillReturnRows(sqlmock.NewRows([]string{"id", "name"}))
|
||||
|
||||
@@ -2,18 +2,18 @@ package filesystem
|
||||
|
||||
import (
|
||||
"context"
|
||||
"io"
|
||||
"os"
|
||||
"path"
|
||||
"time"
|
||||
|
||||
model "github.com/cloudreve/Cloudreve/v3/models"
|
||||
"github.com/cloudreve/Cloudreve/v3/pkg/cache"
|
||||
"github.com/cloudreve/Cloudreve/v3/pkg/filesystem/driver/local"
|
||||
"github.com/cloudreve/Cloudreve/v3/pkg/filesystem/fsctx"
|
||||
"github.com/cloudreve/Cloudreve/v3/pkg/request"
|
||||
"github.com/cloudreve/Cloudreve/v3/pkg/serializer"
|
||||
"github.com/cloudreve/Cloudreve/v3/pkg/util"
|
||||
"github.com/gin-gonic/gin"
|
||||
"github.com/gofrs/uuid"
|
||||
)
|
||||
|
||||
/* ================
|
||||
@@ -21,12 +21,17 @@ import (
|
||||
================
|
||||
*/
|
||||
|
||||
// Upload 上传文件
|
||||
func (fs *FileSystem) Upload(ctx context.Context, file FileHeader) (err error) {
|
||||
ctx = context.WithValue(ctx, fsctx.FileHeaderCtx, file)
|
||||
const (
|
||||
UploadSessionMetaKey = "upload_session"
|
||||
UploadSessionCtx = "uploadSession"
|
||||
UserCtx = "user"
|
||||
UploadSessionCachePrefix = "callback_"
|
||||
)
|
||||
|
||||
// Upload 上传文件
|
||||
func (fs *FileSystem) Upload(ctx context.Context, file *fsctx.FileStream) (err error) {
|
||||
// 上传前的钩子
|
||||
err = fs.Trigger(ctx, "BeforeUpload")
|
||||
err = fs.Trigger(ctx, "BeforeUpload", file)
|
||||
if err != nil {
|
||||
request.BlackHole(file)
|
||||
return err
|
||||
@@ -34,30 +39,34 @@ func (fs *FileSystem) Upload(ctx context.Context, file FileHeader) (err error) {
|
||||
|
||||
// 生成文件名和路径,
|
||||
var savePath string
|
||||
// 如果是更新操作就从上下文中获取
|
||||
if originFile, ok := ctx.Value(fsctx.FileModelCtx).(model.File); ok {
|
||||
savePath = originFile.SourceName
|
||||
} else {
|
||||
savePath = fs.GenerateSavePath(ctx, file)
|
||||
if file.SavePath == "" {
|
||||
// 如果是更新操作就从上下文中获取
|
||||
if originFile, ok := ctx.Value(fsctx.FileModelCtx).(model.File); ok {
|
||||
savePath = originFile.SourceName
|
||||
} else {
|
||||
savePath = fs.GenerateSavePath(ctx, file)
|
||||
}
|
||||
file.SavePath = savePath
|
||||
}
|
||||
ctx = context.WithValue(ctx, fsctx.SavePathCtx, savePath)
|
||||
|
||||
// 处理客户端未完成上传时,关闭连接
|
||||
go fs.CancelUpload(ctx, savePath, file)
|
||||
|
||||
// 保存文件
|
||||
err = fs.Handler.Put(ctx, file, savePath, file.GetSize())
|
||||
if err != nil {
|
||||
fs.Trigger(ctx, "AfterUploadFailed")
|
||||
return err
|
||||
if file.Mode&fsctx.Nop != fsctx.Nop {
|
||||
// 处理客户端未完成上传时,关闭连接
|
||||
go fs.CancelUpload(ctx, savePath, file)
|
||||
|
||||
err = fs.Handler.Put(ctx, file)
|
||||
if err != nil {
|
||||
fs.Trigger(ctx, "AfterUploadFailed", file)
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
// 上传完成后的钩子
|
||||
err = fs.Trigger(ctx, "AfterUpload")
|
||||
err = fs.Trigger(ctx, "AfterUpload", file)
|
||||
|
||||
if err != nil {
|
||||
// 上传完成后续处理失败
|
||||
followUpErr := fs.Trigger(ctx, "AfterValidateFailed")
|
||||
followUpErr := fs.Trigger(ctx, "AfterValidateFailed", file)
|
||||
// 失败后再失败...
|
||||
if followUpErr != nil {
|
||||
util.Log().Debug("AfterValidateFailed 钩子执行失败,%s", followUpErr)
|
||||
@@ -66,56 +75,28 @@ func (fs *FileSystem) Upload(ctx context.Context, file FileHeader) (err error) {
|
||||
return err
|
||||
}
|
||||
|
||||
util.Log().Info(
|
||||
"新文件PUT:%s , 大小:%d, 上传者:%s",
|
||||
file.GetFileName(),
|
||||
file.GetSize(),
|
||||
fs.User.Nick,
|
||||
)
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// GenerateSavePath 生成要存放文件的路径
|
||||
// TODO 完善测试
|
||||
func (fs *FileSystem) GenerateSavePath(ctx context.Context, file FileHeader) string {
|
||||
if fs.User.Model.ID != 0 {
|
||||
return path.Join(
|
||||
fs.User.Policy.GeneratePath(
|
||||
fs.User.Model.ID,
|
||||
file.GetVirtualPath(),
|
||||
),
|
||||
fs.User.Policy.GenerateFileName(
|
||||
fs.User.Model.ID,
|
||||
file.GetFileName(),
|
||||
),
|
||||
)
|
||||
}
|
||||
|
||||
// 匿名文件系统尝试根据上下文中的上传策略生成路径
|
||||
var anonymousPolicy model.Policy
|
||||
if policy, ok := ctx.Value(fsctx.UploadPolicyCtx).(serializer.UploadPolicy); ok {
|
||||
anonymousPolicy = model.Policy{
|
||||
Type: "remote",
|
||||
AutoRename: policy.AutoRename,
|
||||
DirNameRule: policy.SavePath,
|
||||
FileNameRule: policy.FileName,
|
||||
}
|
||||
}
|
||||
func (fs *FileSystem) GenerateSavePath(ctx context.Context, file fsctx.FileHeader) string {
|
||||
fileInfo := file.Info()
|
||||
return path.Join(
|
||||
anonymousPolicy.GeneratePath(
|
||||
0,
|
||||
"",
|
||||
fs.Policy.GeneratePath(
|
||||
fs.User.Model.ID,
|
||||
fileInfo.VirtualPath,
|
||||
),
|
||||
anonymousPolicy.GenerateFileName(
|
||||
0,
|
||||
file.GetFileName(),
|
||||
fs.Policy.GenerateFileName(
|
||||
fs.User.Model.ID,
|
||||
fileInfo.FileName,
|
||||
),
|
||||
)
|
||||
|
||||
}
|
||||
|
||||
// CancelUpload 监测客户端取消上传
|
||||
func (fs *FileSystem) CancelUpload(ctx context.Context, path string, file FileHeader) {
|
||||
func (fs *FileSystem) CancelUpload(ctx context.Context, path string, file fsctx.FileHeader) {
|
||||
var reqContext context.Context
|
||||
if ginCtx, ok := ctx.Value(fsctx.GinCtx).(*gin.Context); ok {
|
||||
reqContext = ginCtx.Request.Context()
|
||||
@@ -136,8 +117,7 @@ func (fs *FileSystem) CancelUpload(ctx context.Context, path string, file FileHe
|
||||
if fs.Hooks["AfterUploadCanceled"] == nil {
|
||||
return
|
||||
}
|
||||
ctx = context.WithValue(ctx, fsctx.SavePathCtx, path)
|
||||
err := fs.Trigger(ctx, "AfterUploadCanceled")
|
||||
err := fs.Trigger(ctx, "AfterUploadCanceled", file)
|
||||
if err != nil {
|
||||
util.Log().Debug("执行 AfterUploadCanceled 钩子出错,%s", err)
|
||||
}
|
||||
@@ -146,67 +126,81 @@ func (fs *FileSystem) CancelUpload(ctx context.Context, path string, file FileHe
|
||||
}
|
||||
}
|
||||
|
||||
// GetUploadToken 生成新的上传凭证
|
||||
func (fs *FileSystem) GetUploadToken(ctx context.Context, path string, size uint64, name string) (*serializer.UploadCredential, error) {
|
||||
// CreateUploadSession 创建上传会话
|
||||
func (fs *FileSystem) CreateUploadSession(ctx context.Context, file *fsctx.FileStream) (*serializer.UploadCredential, error) {
|
||||
// 获取相关有效期设置
|
||||
credentialTTL := model.GetIntSetting("upload_credential_timeout", 3600)
|
||||
callBackSessionTTL := model.GetIntSetting("upload_session_timeout", 86400)
|
||||
|
||||
var err error
|
||||
callbackKey := uuid.Must(uuid.NewV4()).String()
|
||||
fileSize := file.Size
|
||||
|
||||
// 检查文件大小
|
||||
if fs.User.Policy.MaxSize != 0 {
|
||||
if size > fs.User.Policy.MaxSize {
|
||||
return nil, ErrFileSizeTooBig
|
||||
}
|
||||
// 创建占位的文件,同时校验文件信息
|
||||
file.Mode = fsctx.Nop
|
||||
if callbackKey != "" {
|
||||
file.UploadSessionID = &callbackKey
|
||||
}
|
||||
|
||||
// 是否需要预先生成存储路径
|
||||
var savePath string
|
||||
if fs.User.Policy.IsPathGenerateNeeded() {
|
||||
savePath = fs.GenerateSavePath(ctx, local.FileStream{Name: name, VirtualPath: path})
|
||||
ctx = context.WithValue(ctx, fsctx.SavePathCtx, savePath)
|
||||
fs.Use("BeforeUpload", HookValidateFile)
|
||||
fs.Use("BeforeUpload", HookValidateCapacity)
|
||||
|
||||
// 验证文件规格
|
||||
if err := fs.Upload(ctx, file); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
uploadSession := &serializer.UploadSession{
|
||||
Key: callbackKey,
|
||||
UID: fs.User.ID,
|
||||
Policy: *fs.Policy,
|
||||
VirtualPath: file.VirtualPath,
|
||||
Name: file.Name,
|
||||
Size: fileSize,
|
||||
SavePath: file.SavePath,
|
||||
LastModified: file.LastModified,
|
||||
CallbackSecret: util.RandStringRunes(32),
|
||||
}
|
||||
ctx = context.WithValue(ctx, fsctx.FileSizeCtx, size)
|
||||
|
||||
// 获取上传凭证
|
||||
callbackKey := util.RandStringRunes(32)
|
||||
credential, err := fs.Handler.Token(ctx, int64(credentialTTL), callbackKey)
|
||||
credential, err := fs.Handler.Token(ctx, int64(callBackSessionTTL), uploadSession, file)
|
||||
if err != nil {
|
||||
return nil, serializer.NewError(serializer.CodeEncryptError, "无法获取上传凭证", err)
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// 创建占位符
|
||||
if !fs.Policy.IsUploadPlaceholderWithSize() {
|
||||
fs.Use("AfterUpload", HookClearFileHeaderSize)
|
||||
}
|
||||
fs.Use("AfterUpload", GenericAfterUpload)
|
||||
ctx = context.WithValue(ctx, fsctx.IgnoreDirectoryConflictCtx, true)
|
||||
if err := fs.Upload(ctx, file); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// 创建回调会话
|
||||
err = cache.Set(
|
||||
"callback_"+callbackKey,
|
||||
serializer.UploadSession{
|
||||
Key: callbackKey,
|
||||
UID: fs.User.ID,
|
||||
PolicyID: fs.User.GetPolicyID(0),
|
||||
VirtualPath: path,
|
||||
Name: name,
|
||||
Size: size,
|
||||
SavePath: savePath,
|
||||
},
|
||||
UploadSessionCachePrefix+callbackKey,
|
||||
*uploadSession,
|
||||
callBackSessionTTL,
|
||||
)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return &credential, nil
|
||||
// 补全上传凭证其他信息
|
||||
credential.Expires = time.Now().Add(time.Duration(callBackSessionTTL) * time.Second).Unix()
|
||||
|
||||
return credential, nil
|
||||
}
|
||||
|
||||
// UploadFromStream 从文件流上传文件
|
||||
func (fs *FileSystem) UploadFromStream(ctx context.Context, src io.ReadCloser, dst string, size uint64) error {
|
||||
// 构建文件头
|
||||
fileName := path.Base(dst)
|
||||
filePath := path.Dir(dst)
|
||||
fileData := local.FileStream{
|
||||
File: src,
|
||||
Size: size,
|
||||
Name: fileName,
|
||||
VirtualPath: filePath,
|
||||
func (fs *FileSystem) UploadFromStream(ctx context.Context, file *fsctx.FileStream, resetPolicy bool) error {
|
||||
if resetPolicy {
|
||||
// 重设存储策略
|
||||
fs.Policy = &fs.User.Policy
|
||||
err := fs.DispatchHandler()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
// 给文件系统分配钩子
|
||||
@@ -215,29 +209,18 @@ func (fs *FileSystem) UploadFromStream(ctx context.Context, src io.ReadCloser, d
|
||||
fs.Use("BeforeUpload", HookValidateFile)
|
||||
fs.Use("BeforeUpload", HookValidateCapacity)
|
||||
fs.Use("AfterUploadCanceled", HookDeleteTempFile)
|
||||
fs.Use("AfterUploadCanceled", HookGiveBackCapacity)
|
||||
fs.Use("AfterUpload", GenericAfterUpload)
|
||||
fs.Use("AfterUpload", HookGenerateThumb)
|
||||
fs.Use("AfterValidateFailed", HookDeleteTempFile)
|
||||
fs.Use("AfterValidateFailed", HookGiveBackCapacity)
|
||||
fs.Use("AfterUploadFailed", HookGiveBackCapacity)
|
||||
}
|
||||
fs.Lock.Unlock()
|
||||
|
||||
// 开始上传
|
||||
return fs.Upload(ctx, fileData)
|
||||
return fs.Upload(ctx, file)
|
||||
}
|
||||
|
||||
// UploadFromPath 将本机已有文件上传到用户的文件系统
|
||||
func (fs *FileSystem) UploadFromPath(ctx context.Context, src, dst string, resetPolicy bool) error {
|
||||
// 重设存储策略
|
||||
if resetPolicy {
|
||||
fs.Policy = &fs.User.Policy
|
||||
err := fs.DispatchHandler()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
func (fs *FileSystem) UploadFromPath(ctx context.Context, src, dst string, mode fsctx.WriteMode) error {
|
||||
file, err := os.Open(util.RelativePath(src))
|
||||
if err != nil {
|
||||
return err
|
||||
@@ -252,5 +235,12 @@ func (fs *FileSystem) UploadFromPath(ctx context.Context, src, dst string, reset
|
||||
size := fi.Size()
|
||||
|
||||
// 开始上传
|
||||
return fs.UploadFromStream(ctx, file, dst, uint64(size))
|
||||
return fs.UploadFromStream(ctx, &fsctx.FileStream{
|
||||
File: file,
|
||||
Seeker: file,
|
||||
Size: uint64(size),
|
||||
Name: path.Base(dst),
|
||||
VirtualPath: path.Dir(dst),
|
||||
Mode: mode,
|
||||
}, true)
|
||||
}
|
||||
|
||||
@@ -3,17 +3,9 @@ package filesystem
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"io"
|
||||
"io/ioutil"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"net/url"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/DATA-DOG/go-sqlmock"
|
||||
model "github.com/cloudreve/Cloudreve/v3/models"
|
||||
"github.com/cloudreve/Cloudreve/v3/pkg/cache"
|
||||
"github.com/cloudreve/Cloudreve/v3/pkg/filesystem/driver/local"
|
||||
"github.com/cloudreve/Cloudreve/v3/pkg/filesystem/fsctx"
|
||||
"github.com/cloudreve/Cloudreve/v3/pkg/filesystem/response"
|
||||
"github.com/cloudreve/Cloudreve/v3/pkg/serializer"
|
||||
@@ -21,12 +13,33 @@ import (
|
||||
"github.com/jinzhu/gorm"
|
||||
"github.com/stretchr/testify/assert"
|
||||
testMock "github.com/stretchr/testify/mock"
|
||||
"io/ioutil"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"net/url"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
type FileHeaderMock struct {
|
||||
testMock.Mock
|
||||
}
|
||||
|
||||
func (m FileHeaderMock) Put(ctx context.Context, file fsctx.FileHeader) error {
|
||||
args := m.Called(ctx, file)
|
||||
return args.Error(0)
|
||||
}
|
||||
|
||||
func (m FileHeaderMock) Token(ctx context.Context, ttl int64, uploadSession *serializer.UploadSession, file fsctx.FileHeader) (*serializer.UploadCredential, error) {
|
||||
args := m.Called(ctx, ttl, uploadSession, file)
|
||||
return args.Get(0).(*serializer.UploadCredential), args.Error(1)
|
||||
}
|
||||
|
||||
func (m FileHeaderMock) CancelToken(ctx context.Context, uploadSession *serializer.UploadSession) error {
|
||||
args := m.Called(ctx, uploadSession)
|
||||
return args.Error(0)
|
||||
}
|
||||
|
||||
func (m FileHeaderMock) List(ctx context.Context, path string, recursive bool) ([]response.Object, error) {
|
||||
args := m.Called(ctx, path, recursive)
|
||||
return args.Get(0).([]response.Object), args.Error(1)
|
||||
@@ -37,11 +50,6 @@ func (m FileHeaderMock) Get(ctx context.Context, path string) (response.RSCloser
|
||||
return args.Get(0).(response.RSCloser), args.Error(1)
|
||||
}
|
||||
|
||||
func (m FileHeaderMock) Put(ctx context.Context, file io.ReadCloser, dst string, size uint64) error {
|
||||
args := m.Called(ctx, file, dst)
|
||||
return args.Error(0)
|
||||
}
|
||||
|
||||
func (m FileHeaderMock) Delete(ctx context.Context, files []string) ([]string, error) {
|
||||
args := m.Called(ctx, files)
|
||||
return args.Get(0).([]string), args.Error(1)
|
||||
@@ -57,11 +65,6 @@ func (m FileHeaderMock) Source(ctx context.Context, path string, url url.URL, ex
|
||||
return args.Get(0).(string), args.Error(1)
|
||||
}
|
||||
|
||||
func (m FileHeaderMock) Token(ctx context.Context, expires int64, key string) (serializer.UploadCredential, error) {
|
||||
args := m.Called(ctx, expires, key)
|
||||
return args.Get(0).(serializer.UploadCredential), args.Error(1)
|
||||
}
|
||||
|
||||
func TestFileSystem_Upload(t *testing.T) {
|
||||
asserts := assert.New(t)
|
||||
|
||||
@@ -74,10 +77,10 @@ func TestFileSystem_Upload(t *testing.T) {
|
||||
Model: gorm.Model{
|
||||
ID: 1,
|
||||
},
|
||||
Policy: model.Policy{
|
||||
AutoRename: false,
|
||||
DirNameRule: "{path}",
|
||||
},
|
||||
},
|
||||
Policy: &model.Policy{
|
||||
AutoRename: false,
|
||||
DirNameRule: "{path}",
|
||||
},
|
||||
}
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
@@ -85,7 +88,7 @@ func TestFileSystem_Upload(t *testing.T) {
|
||||
c.Request, _ = http.NewRequest("POST", "/", nil)
|
||||
ctx = context.WithValue(ctx, fsctx.GinCtx, c)
|
||||
cancel()
|
||||
file := local.FileStream{
|
||||
file := &fsctx.FileStream{
|
||||
Size: 5,
|
||||
VirtualPath: "/",
|
||||
Name: "1.txt",
|
||||
@@ -95,17 +98,17 @@ func TestFileSystem_Upload(t *testing.T) {
|
||||
|
||||
// 正常,上下文已指定源文件
|
||||
testHandler = new(FileHeaderMock)
|
||||
testHandler.On("Put", testMock.Anything, testMock.Anything, "123/123.txt").Return(nil)
|
||||
testHandler.On("Put", testMock.Anything, testMock.Anything).Return(nil)
|
||||
fs = &FileSystem{
|
||||
Handler: testHandler,
|
||||
User: &model.User{
|
||||
Model: gorm.Model{
|
||||
ID: 1,
|
||||
},
|
||||
Policy: model.Policy{
|
||||
AutoRename: false,
|
||||
DirNameRule: "{path}",
|
||||
},
|
||||
},
|
||||
Policy: &model.Policy{
|
||||
AutoRename: false,
|
||||
DirNameRule: "{path}",
|
||||
},
|
||||
}
|
||||
ctx, cancel = context.WithCancel(context.Background())
|
||||
@@ -114,7 +117,7 @@ func TestFileSystem_Upload(t *testing.T) {
|
||||
ctx = context.WithValue(ctx, fsctx.GinCtx, c)
|
||||
ctx = context.WithValue(ctx, fsctx.FileModelCtx, model.File{SourceName: "123/123.txt"})
|
||||
cancel()
|
||||
file = local.FileStream{
|
||||
file = &fsctx.FileStream{
|
||||
Size: 5,
|
||||
VirtualPath: "/",
|
||||
Name: "1.txt",
|
||||
@@ -124,7 +127,7 @@ func TestFileSystem_Upload(t *testing.T) {
|
||||
asserts.NoError(err)
|
||||
|
||||
// BeforeUpload 返回错误
|
||||
fs.Use("BeforeUpload", func(ctx context.Context, fs *FileSystem) error {
|
||||
fs.Use("BeforeUpload", func(ctx context.Context, fs *FileSystem, file fsctx.FileHeader) error {
|
||||
return errors.New("error")
|
||||
})
|
||||
err = fs.Upload(ctx, file)
|
||||
@@ -134,7 +137,7 @@ func TestFileSystem_Upload(t *testing.T) {
|
||||
|
||||
// 上传文件失败
|
||||
testHandler2 := new(FileHeaderMock)
|
||||
testHandler2.On("Put", testMock.Anything, testMock.Anything, testMock.Anything).Return(errors.New("error"))
|
||||
testHandler2.On("Put", testMock.Anything, testMock.Anything).Return(errors.New("error"))
|
||||
fs.Handler = testHandler2
|
||||
err = fs.Upload(ctx, file)
|
||||
asserts.Error(err)
|
||||
@@ -142,12 +145,12 @@ func TestFileSystem_Upload(t *testing.T) {
|
||||
|
||||
// AfterUpload失败
|
||||
testHandler3 := new(FileHeaderMock)
|
||||
testHandler3.On("Put", testMock.Anything, testMock.Anything, testMock.Anything).Return(nil)
|
||||
testHandler3.On("Put", testMock.Anything, testMock.Anything).Return(nil)
|
||||
fs.Handler = testHandler3
|
||||
fs.Use("AfterUpload", func(ctx context.Context, fs *FileSystem) error {
|
||||
fs.Use("AfterUpload", func(ctx context.Context, fs *FileSystem, file fsctx.FileHeader) error {
|
||||
return errors.New("error")
|
||||
})
|
||||
fs.Use("AfterValidateFailed", func(ctx context.Context, fs *FileSystem) error {
|
||||
fs.Use("AfterValidateFailed", func(ctx context.Context, fs *FileSystem, file fsctx.FileHeader) error {
|
||||
return errors.New("error")
|
||||
})
|
||||
err = fs.Upload(ctx, file)
|
||||
@@ -156,43 +159,39 @@ func TestFileSystem_Upload(t *testing.T) {
|
||||
|
||||
}
|
||||
|
||||
func TestFileSystem_GenerateSavePath_Anonymous(t *testing.T) {
|
||||
asserts := assert.New(t)
|
||||
fs := FileSystem{User: &model.User{}}
|
||||
ctx := context.WithValue(
|
||||
context.Background(),
|
||||
fsctx.UploadPolicyCtx,
|
||||
serializer.UploadPolicy{
|
||||
SavePath: "{randomkey16}",
|
||||
AutoRename: false,
|
||||
},
|
||||
)
|
||||
|
||||
savePath := fs.GenerateSavePath(ctx, local.FileStream{
|
||||
Name: "test.test",
|
||||
})
|
||||
asserts.Len(savePath, 26)
|
||||
asserts.Contains(savePath, "test.test")
|
||||
}
|
||||
|
||||
func TestFileSystem_GetUploadToken(t *testing.T) {
|
||||
asserts := assert.New(t)
|
||||
fs := FileSystem{User: &model.User{Model: gorm.Model{ID: 1}}}
|
||||
fs := FileSystem{
|
||||
User: &model.User{Model: gorm.Model{ID: 1}},
|
||||
Policy: &model.Policy{},
|
||||
}
|
||||
ctx := context.Background()
|
||||
|
||||
// 成功
|
||||
{
|
||||
cache.SetSettings(map[string]string{
|
||||
"upload_credential_timeout": "10",
|
||||
"upload_session_timeout": "10",
|
||||
"upload_session_timeout": "10",
|
||||
}, "setting_")
|
||||
testHandler := new(FileHeaderMock)
|
||||
testHandler.On("Token", testMock.Anything, int64(10), testMock.Anything).Return(serializer.UploadCredential{Token: "test"}, nil)
|
||||
testHandler.On("Token", testMock.Anything, int64(10), testMock.Anything, testMock.Anything).Return(&serializer.UploadCredential{Credential: "test"}, nil)
|
||||
fs.Handler = testHandler
|
||||
res, err := fs.GetUploadToken(ctx, "/", 10, "123")
|
||||
mock.ExpectQuery("SELECT(.+)").
|
||||
WithArgs(1).
|
||||
WillReturnRows(sqlmock.NewRows([]string{"id", "owner_id"}).AddRow(1, 1))
|
||||
mock.ExpectQuery("SELECT(.+)files(.+)").WillReturnError(errors.New("not found"))
|
||||
mock.ExpectBegin()
|
||||
mock.ExpectExec("INSERT(.+)files(.+)").WillReturnResult(sqlmock.NewResult(1, 1))
|
||||
mock.ExpectExec("UPDATE(.+)storage(.+)").WillReturnResult(sqlmock.NewResult(1, 1))
|
||||
mock.ExpectCommit()
|
||||
res, err := fs.CreateUploadSession(ctx, &fsctx.FileStream{
|
||||
Size: 0,
|
||||
Name: "file",
|
||||
VirtualPath: "/",
|
||||
})
|
||||
asserts.NoError(mock.ExpectationsWereMet())
|
||||
testHandler.AssertExpectations(t)
|
||||
asserts.NoError(err)
|
||||
asserts.Equal("test", res.Token)
|
||||
asserts.Equal("test", res.Credential)
|
||||
}
|
||||
|
||||
// 无法获取上传凭证
|
||||
@@ -202,9 +201,22 @@ func TestFileSystem_GetUploadToken(t *testing.T) {
|
||||
"upload_session_timeout": "10",
|
||||
}, "setting_")
|
||||
testHandler := new(FileHeaderMock)
|
||||
testHandler.On("Token", testMock.Anything, int64(10), testMock.Anything).Return(serializer.UploadCredential{}, errors.New("error"))
|
||||
testHandler.On("Token", testMock.Anything, int64(10), testMock.Anything, testMock.Anything).Return(&serializer.UploadCredential{}, errors.New("error"))
|
||||
fs.Handler = testHandler
|
||||
_, err := fs.GetUploadToken(ctx, "/", 10, "123")
|
||||
mock.ExpectQuery("SELECT(.+)").
|
||||
WithArgs(1).
|
||||
WillReturnRows(sqlmock.NewRows([]string{"id", "owner_id"}).AddRow(1, 1))
|
||||
mock.ExpectQuery("SELECT(.+)files(.+)").WillReturnError(errors.New("not found"))
|
||||
mock.ExpectBegin()
|
||||
mock.ExpectExec("INSERT(.+)files(.+)").WillReturnResult(sqlmock.NewResult(1, 1))
|
||||
mock.ExpectExec("UPDATE(.+)storage(.+)").WillReturnResult(sqlmock.NewResult(1, 1))
|
||||
mock.ExpectCommit()
|
||||
_, err := fs.CreateUploadSession(ctx, &fsctx.FileStream{
|
||||
Size: 0,
|
||||
Name: "file",
|
||||
VirtualPath: "/",
|
||||
})
|
||||
asserts.NoError(mock.ExpectationsWereMet())
|
||||
testHandler.AssertExpectations(t)
|
||||
asserts.Error(err)
|
||||
}
|
||||
@@ -212,27 +224,41 @@ func TestFileSystem_GetUploadToken(t *testing.T) {
|
||||
|
||||
func TestFileSystem_UploadFromStream(t *testing.T) {
|
||||
asserts := assert.New(t)
|
||||
fs := FileSystem{User: &model.User{Model: gorm.Model{ID: 1}}}
|
||||
fs := FileSystem{
|
||||
User: &model.User{
|
||||
Model: gorm.Model{ID: 1},
|
||||
Policy: model.Policy{Type: "mock"},
|
||||
},
|
||||
Policy: &model.Policy{Type: "mock"},
|
||||
}
|
||||
ctx := context.Background()
|
||||
|
||||
err := fs.UploadFromStream(ctx, ioutil.NopCloser(strings.NewReader("123")), "/1.txt", 1)
|
||||
err := fs.UploadFromStream(ctx, &fsctx.FileStream{
|
||||
File: ioutil.NopCloser(strings.NewReader("123")),
|
||||
}, true)
|
||||
asserts.Error(err)
|
||||
}
|
||||
|
||||
func TestFileSystem_UploadFromPath(t *testing.T) {
|
||||
asserts := assert.New(t)
|
||||
fs := FileSystem{User: &model.User{Policy: model.Policy{Type: "mock"}, Model: gorm.Model{ID: 1}}}
|
||||
fs := FileSystem{
|
||||
User: &model.User{
|
||||
Model: gorm.Model{ID: 1},
|
||||
Policy: model.Policy{Type: "mock"},
|
||||
},
|
||||
Policy: &model.Policy{Type: "mock"},
|
||||
}
|
||||
ctx := context.Background()
|
||||
|
||||
// 文件不存在
|
||||
{
|
||||
err := fs.UploadFromPath(ctx, "test/not_exist", "/", true)
|
||||
err := fs.UploadFromPath(ctx, "test/not_exist", "/", fsctx.Overwrite)
|
||||
asserts.Error(err)
|
||||
}
|
||||
|
||||
// 文存在,上传失败
|
||||
{
|
||||
err := fs.UploadFromPath(ctx, "tests/test.zip", "/", true)
|
||||
err := fs.UploadFromPath(ctx, "tests/test.zip", "/", fsctx.Overwrite)
|
||||
asserts.Error(err)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -45,10 +45,10 @@ func (fs *FileSystem) ValidateLegalName(ctx context.Context, name string) bool {
|
||||
|
||||
// ValidateFileSize 验证上传的文件大小是否超出限制
|
||||
func (fs *FileSystem) ValidateFileSize(ctx context.Context, size uint64) bool {
|
||||
if fs.User.Policy.MaxSize == 0 {
|
||||
if fs.Policy.MaxSize == 0 {
|
||||
return true
|
||||
}
|
||||
return size <= fs.User.Policy.MaxSize
|
||||
return size <= fs.Policy.MaxSize
|
||||
}
|
||||
|
||||
// ValidateCapacity 验证并扣除用户容量
|
||||
@@ -59,11 +59,11 @@ func (fs *FileSystem) ValidateCapacity(ctx context.Context, size uint64) bool {
|
||||
// ValidateExtension 验证文件扩展名
|
||||
func (fs *FileSystem) ValidateExtension(ctx context.Context, fileName string) bool {
|
||||
// 不需要验证
|
||||
if len(fs.User.Policy.OptionsSerialized.FileType) == 0 {
|
||||
if len(fs.Policy.OptionsSerialized.FileType) == 0 {
|
||||
return true
|
||||
}
|
||||
|
||||
return IsInExtensionList(fs.User.Policy.OptionsSerialized.FileType, fileName)
|
||||
return IsInExtensionList(fs.Policy.OptionsSerialized.FileType, fileName)
|
||||
}
|
||||
|
||||
// IsInExtensionList 返回文件的扩展名是否在给定的列表范围内
|
||||
|
||||
@@ -68,10 +68,9 @@ func TestFileSystem_ValidateFileSize(t *testing.T) {
|
||||
asserts := assert.New(t)
|
||||
ctx := context.Background()
|
||||
fs := FileSystem{
|
||||
User: &model.User{
|
||||
Policy: model.Policy{
|
||||
MaxSize: 10,
|
||||
},
|
||||
User: &model.User{},
|
||||
Policy: &model.Policy{
|
||||
MaxSize: 10,
|
||||
},
|
||||
}
|
||||
|
||||
@@ -80,7 +79,7 @@ func TestFileSystem_ValidateFileSize(t *testing.T) {
|
||||
asserts.False(fs.ValidateFileSize(ctx, 11))
|
||||
|
||||
// 无限制
|
||||
fs.User.Policy.MaxSize = 0
|
||||
fs.Policy.MaxSize = 0
|
||||
asserts.True(fs.ValidateFileSize(ctx, 11))
|
||||
}
|
||||
|
||||
@@ -88,11 +87,10 @@ func TestFileSystem_ValidateExtension(t *testing.T) {
|
||||
asserts := assert.New(t)
|
||||
ctx := context.Background()
|
||||
fs := FileSystem{
|
||||
User: &model.User{
|
||||
Policy: model.Policy{
|
||||
OptionsSerialized: model.PolicyOption{
|
||||
FileType: nil,
|
||||
},
|
||||
User: &model.User{},
|
||||
Policy: &model.Policy{
|
||||
OptionsSerialized: model.PolicyOption{
|
||||
FileType: nil,
|
||||
},
|
||||
},
|
||||
}
|
||||
@@ -100,11 +98,11 @@ func TestFileSystem_ValidateExtension(t *testing.T) {
|
||||
asserts.True(fs.ValidateExtension(ctx, "1"))
|
||||
asserts.True(fs.ValidateExtension(ctx, "1.txt"))
|
||||
|
||||
fs.User.Policy.OptionsSerialized.FileType = []string{}
|
||||
fs.Policy.OptionsSerialized.FileType = []string{}
|
||||
asserts.True(fs.ValidateExtension(ctx, "1"))
|
||||
asserts.True(fs.ValidateExtension(ctx, "1.txt"))
|
||||
|
||||
fs.User.Policy.OptionsSerialized.FileType = []string{"txt", "jpg"}
|
||||
fs.Policy.OptionsSerialized.FileType = []string{"txt", "jpg"}
|
||||
asserts.False(fs.ValidateExtension(ctx, "1"))
|
||||
asserts.False(fs.ValidateExtension(ctx, "1.jpg.png"))
|
||||
asserts.True(fs.ValidateExtension(ctx, "1.txt"))
|
||||
|
||||
@@ -7,11 +7,9 @@ import (
|
||||
"github.com/cloudreve/Cloudreve/v3/pkg/auth"
|
||||
"github.com/cloudreve/Cloudreve/v3/pkg/balancer"
|
||||
"github.com/cloudreve/Cloudreve/v3/pkg/cluster"
|
||||
"github.com/cloudreve/Cloudreve/v3/pkg/request"
|
||||
"github.com/cloudreve/Cloudreve/v3/pkg/serializer"
|
||||
"github.com/cloudreve/Cloudreve/v3/pkg/task"
|
||||
testMock "github.com/stretchr/testify/mock"
|
||||
"io"
|
||||
)
|
||||
|
||||
type NodePoolMock struct {
|
||||
@@ -151,11 +149,3 @@ func (t TaskPoolMock) Add(num int) {
|
||||
func (t TaskPoolMock) Submit(job task.Job) {
|
||||
t.Called(job)
|
||||
}
|
||||
|
||||
type RequestMock struct {
|
||||
testMock.Mock
|
||||
}
|
||||
|
||||
func (r RequestMock) Request(method, target string, body io.Reader, opts ...request.Option) *request.Response {
|
||||
return r.Called(method, target, body, opts).Get(0).(*request.Response)
|
||||
}
|
||||
|
||||
32
pkg/mocks/remoteclientmock/mock.go
Normal file
32
pkg/mocks/remoteclientmock/mock.go
Normal file
@@ -0,0 +1,32 @@
|
||||
package remoteclientmock
|
||||
|
||||
import (
|
||||
"context"
|
||||
"github.com/cloudreve/Cloudreve/v3/pkg/filesystem/fsctx"
|
||||
"github.com/cloudreve/Cloudreve/v3/pkg/serializer"
|
||||
"github.com/stretchr/testify/mock"
|
||||
)
|
||||
|
||||
type RemoteClientMock struct {
|
||||
mock.Mock
|
||||
}
|
||||
|
||||
func (r *RemoteClientMock) CreateUploadSession(ctx context.Context, session *serializer.UploadSession, ttl int64, overwrite bool) error {
|
||||
return r.Called(ctx, session, ttl, overwrite).Error(0)
|
||||
}
|
||||
|
||||
func (r *RemoteClientMock) GetUploadURL(ttl int64, sessionID string) (string, string, error) {
|
||||
args := r.Called(ttl, sessionID)
|
||||
|
||||
return args.String(0), args.String(1), args.Error(2)
|
||||
}
|
||||
|
||||
func (r *RemoteClientMock) Upload(ctx context.Context, file fsctx.FileHeader) error {
|
||||
args := r.Called(ctx, file)
|
||||
return args.Error(0)
|
||||
}
|
||||
|
||||
func (r *RemoteClientMock) DeleteUploadSession(ctx context.Context, sessionID string) error {
|
||||
args := r.Called(ctx, sessionID)
|
||||
return args.Error(0)
|
||||
}
|
||||
15
pkg/mocks/requestmock/request.go
Normal file
15
pkg/mocks/requestmock/request.go
Normal file
@@ -0,0 +1,15 @@
|
||||
package requestmock
|
||||
|
||||
import (
|
||||
"github.com/cloudreve/Cloudreve/v3/pkg/request"
|
||||
"github.com/stretchr/testify/mock"
|
||||
"io"
|
||||
)
|
||||
|
||||
type RequestMock struct {
|
||||
mock.Mock
|
||||
}
|
||||
|
||||
func (r RequestMock) Request(method, target string, body io.Reader, opts ...request.Option) *request.Response {
|
||||
return r.Called(method, target, body, opts).Get(0).(*request.Response)
|
||||
}
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user