Compare commits

...

84 Commits
3.5.3 ... 3.6.2

Author SHA1 Message Date
HFO4
42a31f2fd1 fix: timeout while fetching yarn pkgs in building docker image action 2022-12-19 18:22:18 +08:00
HFO4
ca80051a89 release: 3.6.2 2022-12-19 17:53:11 +08:00
HFO4
bc0c374f00 feat(mobile): only allow request from mobile client to copy session 2022-12-19 17:35:39 +08:00
HFO4
e4c87483d6 feat(session): generate temp URL to copy/refresh user session 2022-12-19 17:34:57 +08:00
HFO4
1227f35d3c doc: change readme link 2022-12-19 17:33:15 +08:00
HFO4
08fa6964a9 doc: change readme link 2022-12-16 21:13:17 +08:00
HFO4
9eafe07f4e doc: add English README 2022-12-16 21:12:09 +08:00
HFO4
73d0f2db9b release: 3.6.1 2022-12-16 17:37:21 +08:00
HFO4
82b4e29a80 enhance: escalate ProxyHeader as a global config 2022-12-16 16:58:06 +08:00
HFO4
9860ebbca9 feat(doc preview): add magic variable for file name 2022-12-16 16:55:47 +08:00
HFO4
435a03dd34 fix: nil reference while trying to shut down DB in slave mode (#1416) 2022-12-16 16:55:28 +08:00
HFO4
4e8ab75211 feat(s3): support setting for force using path style endpoint (#1559) 2022-12-16 16:54:58 +08:00
HFO4
6ceb255512 dep: git mod tidy 2022-12-16 14:01:59 +08:00
AHdark
74e1bd6a43 Added same-site policy for session options (#1381)
* Feat: added same-site policy for session options

* Feat: configurations in conf package to control the `SameSite` mode and `Secure` value of the session.

Co-authored-by: AaronLiu <abslant@126.com>
2022-12-16 13:59:26 +08:00
topjohncian
fd59d1b5ca Enhance(dashboard): optimize get policies request (#1539) 2022-12-16 13:55:52 +08:00
Code
2bb28a9845 fix(s3): use HEAD method to get file info (#1521)
建议更换成更好的 HeadObject 方法因为 HeadObject 方法并不会返回文件 Body 因此不需要 defer res.Body.Close()
2022-12-16 13:54:12 +08:00
vvisionnn
5f4f6bd91a refactor: build docker image using build.sh (#1562) 2022-12-15 22:23:34 +08:00
Nya Candy
053e4352b4 fix: Dockerfile (#1561) 2022-12-14 22:31:36 +08:00
HFO4
08e4d2257a release: 3.6.0 2022-12-14 20:17:14 +08:00
HFO4
f02b6f0286 feat(net): customize socket file permission 2022-12-14 15:28:45 +08:00
HFO4
50a3917a65 feat(cache): set max-age for public accessible static resources 2022-12-14 15:28:19 +08:00
HFO4
8c5ba89f7d feat: mobile app promotion page 2022-12-12 20:35:48 +08:00
HFO4
4519dc025b update(version): 3.6.0-beta1 2022-11-23 20:44:09 +08:00
HFO4
92cbc9f312 i18n: logs in database script 2022-11-23 18:31:43 +08:00
HFO4
756769335f feat(dashboard): edit and remove 2FA secret for users 2022-11-23 17:55:23 +08:00
HFO4
6b63195d28 enhance(session): increase default cookie ttl to 60 days 2022-11-21 19:09:54 +08:00
HFO4
db6681f448 fix(avatar): add default cache max age for avatar response 2022-11-21 19:09:37 +08:00
HFO4
4b85541d73 fix(security): CVE-2022-32167 2022-11-21 19:08:51 +08:00
HFO4
f8ed4b4a5a feat(remote download): show download node in list page 2022-10-30 10:45:25 +08:00
HFO4
7dda81368d test(source link): add unit test 2022-10-30 09:41:14 +08:00
HFO4
1c25232b06 feat(source link): record downloads for redirected source link 2022-10-29 11:08:16 +08:00
HFO4
8d7ecedf47 feat(source link): create perm source link with shorter url 2022-10-29 11:06:07 +08:00
HFO4
1f836a4b8b feat(task): not fail immediately after failed upload in transfer tasks 2022-10-17 19:26:09 +08:00
HFO4
c17cf1946a fix(static): add placeholder empty zip file for go embed 2022-10-15 16:40:00 +08:00
HFO4
392c824a33 feat(OneDrive): support Retry-After throttling control from Graph API (#280) 2022-10-15 16:35:02 +08:00
HFO4
8494bd6eb9 fix(request): deep copy shared header object in request options 2022-10-15 16:16:17 +08:00
HFO4
c7dc143d30 Merge remote-tracking branch 'origin/master' 2022-10-15 10:16:08 +08:00
HFO4
8b30593822 fix: cannot delete mass files (>=333) in SQLite (#622) 2022-10-15 10:12:23 +08:00
HFO4
56fa01ed61 fix: failed UT 2022-10-15 09:55:44 +08:00
HFO4
560097145b fix: metadata mismatch if file name contains % while uploading to OneDrive/SharePoint 2022-10-15 09:20:25 +08:00
topjohncian
8cec65b0a7 Fix: cannot finish callback when uploading an office file using sharepoint.cn (#1503) 2022-10-15 09:06:12 +08:00
WeidiDeng
f89653cea7 feat(static): release static files into memory while startup (#1471)
* 初始化时解压zip文件读取内存中

* update go.mod

* 更新各种go版本
2022-10-15 09:05:05 +08:00
HFO4
6b0b44f6d0 test: fix failed UT 2022-10-15 09:02:28 +08:00
HFO4
63b536e5db Merge remote-tracking branch 'origin/master' 2022-10-08 19:24:57 +08:00
HFO4
19a2f69a19 test: fix failed UT 2022-10-08 19:24:38 +08:00
SuperHgO
2271fcfdef fix: s3(minio) file remove operation hangout (#1491) 2022-10-08 19:21:38 +08:00
HFO4
16b5fc3f60 fix: failed test due to missing error message in filesystem 2022-10-08 19:20:41 +08:00
HFO4
f431eb0cbd fix: metadata mismatch if file name contains % while uploading to OneDrive/SharePoint (#1301) 2022-10-08 19:09:51 +08:00
HFO4
644a326580 i18n: logs in rest pkgs 2022-10-08 18:51:52 +08:00
HFO4
f2c53dda31 Merge remote-tracking branch 'origin/master' 2022-09-29 17:43:06 +08:00
HFO4
28c2ffe72e i18n: logs in filesystem 2022-09-29 17:42:23 +08:00
HFO4
196729bae8 i18n: logs in conf/crontab/email/fs.driver 2022-09-29 17:42:05 +08:00
HFO4
9bb4a5263c i18n: logs in aria2/auth/cache/cluster/serializer 2022-09-29 17:40:56 +08:00
HFO4
7366ff534e i18n: logs in models 2022-09-29 17:40:22 +08:00
HFO4
db23f4061d i18n: logs in bootstrapper and response code in middleware 2022-09-29 17:39:48 +08:00
HFO4
16d17ac1e6 i18n: user setting route 2022-09-29 17:38:52 +08:00
HFO4
9464ee2103 i18n: user route 2022-09-29 17:37:05 +08:00
topjohncian
88e10aeaa2 Fix: unexpected querying all files when deleting an empty folder (#1469) 2022-09-29 09:26:30 +08:00
XYenon
b1685d2863 feat: seeding status for aria2 download tasks (#1422)
* feat: add aria2 seeding

* fix: move RecycleTaskType to the bottom

* refactor: refactor recycle aria2 temp file
2022-09-29 09:24:58 +08:00
WeidiDeng
846438e3af graceful 关闭服务器 (#1416) 2022-08-22 19:49:19 +08:00
HFO4
96daed26b4 i18n: objects / share / slave / tag operations 2022-07-20 20:03:41 +08:00
HFO4
906e9857bc i18n: file operation 2022-07-20 20:01:34 +08:00
HFO4
08104646ba i18n: error codes for aria2 / callback/ directory operation 2022-07-20 19:59:13 +08:00
HFO4
a1880672b1 i18n: error codes for dashboard operations 2022-07-18 20:03:57 +08:00
小白-白
9869671633 fix: incorrect progress count (#1379)
* fix: incorrect progress count

文件中转 已完成文件计数 应在文件成功上传后+1 #1367

* fix failed ut

Co-authored-by: HFO4 <912394456@qq.com>
2022-07-12 19:30:41 +08:00
AHdark
c99b36f788 chore: better way to remove frontend map files (#1380)
* Feat: better way to remove frontend map files

* Feat: Docker use `GENERATE_SOURCEMAP` in the frontend build section to avoid generating map files.
2022-07-12 19:20:14 +08:00
topjohncian
25d56fad6e Fix: admin summary cannot be cached in redis (#1329) 2022-06-14 14:42:02 +08:00
HFO4
f083d52e17 feat: tps limit for OneDrive policy 2022-06-09 16:11:36 +08:00
HFO4
4859ea6ee5 feat: update user storage in calibrating no matter if the actual storage match presisted 2022-06-09 16:11:10 +08:00
HFO4
21d2b817f4 Merge remote-tracking branch 'origin/master' 2022-05-25 20:02:43 +08:00
HFO4
04b0b87082 enhance: remove icp footer 2022-05-25 20:02:13 +08:00
HFO4
2a3759c315 i18n: reading dashboard announcements from custom tag 2022-05-25 20:01:41 +08:00
WeidiDeng
36b310133c fix: IP address is empty in unix socket mode (#1314) 2022-05-24 11:01:00 +08:00
WeidiDeng
3fa1249678 fix: use file extension to search for content-type (#1313) 2022-05-24 10:57:20 +08:00
vvisionnn
fb56b27062 fix: delete socket file before run (fixed #1262) (#1279)
* fix: delete socket file before run (fixed #1262)

* refactor: remove useless logs
2022-05-09 19:24:40 +08:00
HFO4
e705dedc22 Merge remote-tracking branch 'origin/master' 2022-05-09 19:06:21 +08:00
HFO4
7bd5a8e3cd dep: update webautn client for better compatibility 2022-05-09 14:51:11 +08:00
XYenon
5bd711afc6 fix: catch s3 presign err (#1277) 2022-05-05 14:24:35 +08:00
AaronLiu
eef6c40441 Merge pull request #1259 from xb2016/master
Update how to build
2022-05-05 13:48:26 +08:00
HFO4
a78407d878 i18n: tag management 2022-05-02 10:29:33 +08:00
HFO4
46c6ee9be7 i18n: add error codes related to sign up 2022-05-02 10:27:51 +08:00
HFO4
c9eefcb946 i18n: captcha, reset password 2022-04-30 16:51:24 +08:00
HFO4
4fe79859a9 enhance: generate error message for parameter error
i18n: use explicit error code for login controlelr
2022-04-30 16:50:59 +08:00
小白-白
4d4a31c250 Update how to build 2022-04-30 14:21:47 +08:00
160 changed files with 4155 additions and 1090 deletions

2
.github/FUNDING.yml vendored
View File

@@ -1 +1 @@
custom: ["https://cloudreve.org/buy.php"]
custom: ["https://cloudreve.org/pricing"]

View File

@@ -7,10 +7,10 @@ jobs:
name: Build
runs-on: ubuntu-18.04
steps:
- name: Set up Go 1.17
- name: Set up Go 1.18
uses: actions/setup-go@v2
with:
go-version: "1.17"
go-version: "1.18"
id: go
- name: Check out code into the Go module directory

View File

@@ -12,10 +12,10 @@ jobs:
name: Test
runs-on: ubuntu-18.04
steps:
- name: Set up Go 1.17
- name: Set up Go 1.18
uses: actions/setup-go@v2
with:
go-version: "1.17"
go-version: "1.18"
id: go
- name: Check out code into the Go module directory

View File

@@ -1,6 +1,6 @@
language: go
go:
- 1.17.x
- 1.18.x
node_js: "12.16.3"
git:
depth: 1

View File

@@ -1,42 +1,49 @@
FROM golang:1.17-alpine as cloudreve_builder
# the frontend builder
# cloudreve need node.js 16* to build frontend,
# separate build step and custom image tag will resolve this
FROM node:16-alpine as cloudreve_frontend_builder
RUN apk update \
&& apk add --no-cache wget curl git yarn zip bash \
&& git clone --recurse-submodules https://github.com/cloudreve/Cloudreve.git /cloudreve_frontend
# build frontend assets using build script, make sure all the steps just follow the regular release
WORKDIR /cloudreve_frontend
ENV GENERATE_SOURCEMAP false
RUN chmod +x ./build.sh && ./build.sh -a
# the backend builder
# cloudreve backend needs golang 1.18* to build
FROM golang:1.18-alpine as cloudreve_backend_builder
# install dependencies and build tools
RUN apk update && apk add --no-cache wget curl git yarn build-base gcc abuild binutils binutils-doc gcc-doc zip
RUN apk update \
# install dependencies and build tools
&& apk add --no-cache wget curl git build-base gcc abuild binutils binutils-doc gcc-doc zip bash \
&& git clone --recurse-submodules https://github.com/cloudreve/Cloudreve.git /cloudreve_backend
WORKDIR /cloudreve_builder
RUN git clone --recurse-submodules https://github.com/cloudreve/Cloudreve.git
# build frontend
WORKDIR /cloudreve_builder/Cloudreve/assets
RUN yarn install --network-timeout 1000000
RUN yarn run build && find . -name "*.map" -type f -delete
# build backend
WORKDIR /cloudreve_builder/Cloudreve
RUN zip -r - assets/build >assets.zip
RUN tag_name=$(git describe --tags) \
&& export COMMIT_SHA=$(git rev-parse --short HEAD) \
&& go build -a -o cloudreve -ldflags " -X 'github.com/HFO4/cloudreve/pkg/conf.BackendVersion=$tag_name' -X 'github.com/HFO4/cloudreve/pkg/conf.LastCommit=$COMMIT_SHA'"
WORKDIR /cloudreve_backend
COPY --from=cloudreve_frontend_builder /cloudreve_frontend/assets.zip ./
RUN chmod +x ./build.sh && ./build.sh -c
# build final image
# TODO: merge the frontend build and backend build into a single one image
# the final published image
FROM alpine:latest
WORKDIR /cloudreve
COPY --from=cloudreve_backend_builder /cloudreve_backend/cloudreve ./cloudreve
RUN apk update && apk add --no-cache tzdata
# we using the `Asia/Shanghai` timezone by default, you can do modification at your will
RUN cp /usr/share/zoneinfo/Asia/Shanghai /etc/localtime \
&& echo "Asia/Shanghai" > /etc/timezone
COPY --from=cloudreve_builder /cloudreve_builder/Cloudreve/cloudreve ./
# prepare permissions and aria2 dir
RUN chmod +x ./cloudreve && mkdir -p /data/aria2 && chmod -R 766 /data/aria2
RUN apk update \
&& apk add --no-cache tzdata \
&& cp /usr/share/zoneinfo/Asia/Shanghai /etc/localtime \
&& echo "Asia/Shanghai" > /etc/timezone \
&& chmod +x ./cloudreve \
&& mkdir -p /data/aria2 \
&& chmod -R 766 /data/aria2
EXPOSE 5212
VOLUME ["/cloudreve/uploads", "/cloudreve/avatar", "/data"]
ENTRYPOINT ["./cloudreve"]
ENTRYPOINT ["./cloudreve"]

108
README.md
View File

@@ -1,3 +1,5 @@
[中文版本](https://github.com/cloudreve/Cloudreve/blob/master/README_zh-CN.md)
<h1 align="center">
<br>
<a href="https://cloudreve.org/" alt="logo" ><img src="https://raw.githubusercontent.com/cloudreve/frontend/master/public/static/img/logo192.png" width="150"/></a>
@@ -5,12 +7,11 @@
Cloudreve
<br>
</h1>
<h4 align="center">支持多家云存储驱动的公有云文件系统.</h4>
<h4 align="center">Self-hosted file management system with muilt-cloud support.</h4>
<p align="center">
<a href="https://github.com/cloudreve/Cloudreve/actions/workflows/test.yml">
<img src="https://img.shields.io/github/workflow/status/cloudreve/Cloudreve/Test?style=flat-square"
<img src="https://img.shields.io/github/actions/workflow/status/cloudreve/Cloudreve/test.yml?branch=master&style=flat-square"
alt="GitHub Test Workflow">
</a>
<a href="https://codecov.io/gh/cloudreve/Cloudreve"><img src="https://img.shields.io/codecov/c/github/cloudreve/Cloudreve?style=flat-square"></a>
@@ -24,103 +25,106 @@
<img src="https://img.shields.io/docker/image-size/cloudreve/cloudreve?style=flat-square"/>
</a>
</p>
<p align="center">
<a href="https://demo.cloudreve.org">演示站</a> •
<a href="https://forum.cloudreve.org/">讨论社区</a> •
<a href="https://docs.cloudreve.org/">文档</a> •
<a href="https://github.com/cloudreve/Cloudreve/releases">下载</a> •
<a href="https://t.me/cloudreve_official">Telegram 群组</a> •
<a href="#scroll-许可证">许可证</a>
<a href="https://cloudreve.org">Homepage</a> •
<a href="https://demo.cloudreve.org">Demo</a> •
<a href="https://forum.cloudreve.org/">Discussion</a> •
<a href="https://docs.cloudreve.org/v/en/">Documents</a> •
<a href="https://github.com/cloudreve/Cloudreve/releases">Download</a> •
<a href="https://t.me/cloudreve_official">Telegram Group</a>
<a href="#scroll-License">License</a>
</p>
![Screenshot](https://raw.githubusercontent.com/cloudreve/docs/master/images/homepage.png)
## :sparkles: 特性
## :sparkles: Features
* :cloud: 支持本机、从机、七牛、阿里云 OSS、腾讯云 COS、又拍云、OneDrive (包括世纪互联版) 作为存储端
* :outbox_tray: 上传/下载 支持客户端直传,支持下载限速
* 💾 可对接 Aria2 离线下载,可使用多个从机机点分担下载任务
* 📚 在线 压缩/解压缩、多文件打包下载
* 💻 覆盖全部存储策略的 WebDAV 协议支持
* :zap: 拖拽上传、目录上传、流式上传处理
* :card_file_box: 文件拖拽管理
* :family_woman_girl_boy: 多用户、用户组
* :link: 创建文件、目录的分享链接,可设定自动过期
* :eye_speech_bubble: 视频、图像、音频、文本、Office 文档在线预览
* :art: 自定义配色、黑暗模式、PWA 应用、全站单页应用
* :rocket: All-In-One 打包,开箱即用
* :cloud: Support storing files into Local storage, Remote storage, Qiniu, Aliyun OSS, Tencent COS, Upyun, OneDrive, S3 compatible API.
* :outbox_tray: Upload/Download in directly transmission with speed limiting support.
* 💾 Integrate with Aria2 to download files offline, use multiple download nodes to share the load.
* 📚 Compress/Extract files, download files in batch.
* 💻 WebDAV support covering all storage providers.
* :zap:Drag&Drop to upload files or folders, with streaming upload processing.
* :card_file_box: Drag & Drop to manage your files.
* :family_woman_girl_boy: Multi-users with multi-groups.
* :link: Create share links for files and folders with expiration date.
* :eye_speech_bubble: Preview videos, images, audios, texts, Office documents, ePub files online.
* :art: Customize theme colors, dark mode, PWA application, SPA, i18n.
* :rocket: All-In-One packing, with all features out-of-the-box.
* 🌈 ... ...
## :hammer_and_wrench: 部署
## :hammer_and_wrench: Deploy
下载适用于您目标机器操作系统、CPU架构的主程序直接运行即可。
Download the main binary for your target machine OS, CPU architecture and run it directly.
```shell
# 解压程序包
# Extract Cloudreve binary
tar -zxvf cloudreve_VERSION_OS_ARCH.tar.gz
# 赋予执行权限
# Grant execute permission
chmod +x ./cloudreve
# 启动 Cloudreve
# Start Cloudreve
./cloudreve
```
以上为最简单的部署示例,您可以参考 [文档 - 起步](https://docs.cloudreve.org/) 进行更为完善的部署。
The above is a minimum deploy example, you can refer to [Getting started](https://docs.cloudreve.org/v/en/getting-started/install) for a completed deployment.
## :gear: 构建
## :gear: Build
自行构建前需要拥有 `Go >= 1.17``yarn`等必要依赖。
You need to have `Go >= 1.18`, `node.js`, `yarn`, `zip` and other necessary dependencies before you can build it yourself.
#### 克隆代码
#### Clone the code
```shell
git clone --recurse-submodules https://github.com/cloudreve/Cloudreve.git
```
#### 构建静态资源
#### Build static resources
```shell
# 进入前端子模块
# Enter frontend sub-module
cd assets
# 安装依赖
# Install dependencies
yarn install
# 开始构建
# Start building
yarn run build
# Delete unused map files
cd build
find . -name "*.map" -type f -delete
# Return to main folder to pack static files
cd ../../
zip -r - assets/build >assets.zip
```
#### 编译项目
#### Compile
```shell
# 获得当前版本号、Commit
# Obtain version number, commit SHA
export COMMIT_SHA=$(git rev-parse --short HEAD)
export VERSION=$(git describe --tags)
# 开始编译
# Compile
go build -a -o cloudreve -ldflags "-s -w -X 'github.com/cloudreve/Cloudreve/v3/pkg/conf.BackendVersion=$VERSION' -X 'github.com/cloudreve/Cloudreve/v3/pkg/conf.LastCommit=$COMMIT_SHA'"
```
你也可以使用项目根目录下的`build.sh`快速开始构建:
You can also start a quick build using `build.sh` in the project root directory:
```shell
./build.sh [-a] [-c] [-b] [-r]
a - 构建静态资源
c - 编译二进制文件
b - 构建前端 + 编译二进制文件
r - 交叉编译,构建用于release的版本
a - Build assets
c - Build binary backend
b - Build both assets and backend
r - Cross-compilation for final release
```
## :alembic: 技术栈
## :alembic: Stacks
* [Go ](https://golang.org/) + [Gin](https://github.com/gin-gonic/gin)
* [Go](https://golang.org/) + [Gin](https://github.com/gin-gonic/gin)
* [React](https://github.com/facebook/react) + [Redux](https://github.com/reduxjs/redux) + [Material-UI](https://github.com/mui-org/material-ui)
## :scroll: 许可证
## :scroll: License
GPL V3
---
> GitHub [@HFO4](https://github.com/HFO4) &nbsp;&middot;&nbsp;
> Twitter [@abslant00](https://twitter.com/abslant00)

130
README_zh-CN.md Normal file
View File

@@ -0,0 +1,130 @@
[English Version](https://github.com/cloudreve/Cloudreve/blob/master/README.md)
<h1 align="center">
<br>
<a href="https://cloudreve.org/" alt="logo" ><img src="https://raw.githubusercontent.com/cloudreve/frontend/master/public/static/img/logo192.png" width="150"/></a>
<br>
Cloudreve
<br>
</h1>
<h4 align="center">支持多家云存储驱动的公有云文件系统.</h4>
<p align="center">
<a href="https://github.com/cloudreve/Cloudreve/actions/workflows/test.yml">
<img src="https://img.shields.io/github/actions/workflow/status/cloudreve/Cloudreve/test.yml?branch=master&style=flat-square"
alt="GitHub Test Workflow">
</a>
<a href="https://codecov.io/gh/cloudreve/Cloudreve"><img src="https://img.shields.io/codecov/c/github/cloudreve/Cloudreve?style=flat-square"></a>
<a href="https://goreportcard.com/report/github.com/cloudreve/Cloudreve">
<img src="https://goreportcard.com/badge/github.com/cloudreve/Cloudreve?style=flat-square">
</a>
<a href="https://github.com/cloudreve/Cloudreve/releases">
<img src="https://img.shields.io/github/v/release/cloudreve/Cloudreve?include_prereleases&style=flat-square" />
</a>
<a href="https://hub.docker.com/r/cloudreve/cloudreve">
<img src="https://img.shields.io/docker/image-size/cloudreve/cloudreve?style=flat-square"/>
</a>
</p>
<p align="center">
<a href="https://cloudreve.org">主页</a> •
<a href="https://demo.cloudreve.org">演示站</a> •
<a href="https://forum.cloudreve.org/">讨论社区</a> •
<a href="https://docs.cloudreve.org/">文档</a> •
<a href="https://github.com/cloudreve/Cloudreve/releases">下载</a> •
<a href="https://t.me/cloudreve_official">Telegram 群组</a> •
<a href="#scroll-许可证">许可证</a>
</p>
![Screenshot](https://raw.githubusercontent.com/cloudreve/docs/master/images/homepage.png)
## :sparkles: 特性
* :cloud: 支持本机、从机、七牛、阿里云 OSS、腾讯云 COS、又拍云、OneDrive (包括世纪互联版) 、S3兼容协议 作为存储端
* :outbox_tray: 上传/下载 支持客户端直传,支持下载限速
* 💾 可对接 Aria2 离线下载,可使用多个从机节点分担下载任务
* 📚 在线 压缩/解压缩、多文件打包下载
* 💻 覆盖全部存储策略的 WebDAV 协议支持
* :zap: 拖拽上传、目录上传、流式上传处理
* :card_file_box: 文件拖拽管理
* :family_woman_girl_boy: 多用户、用户组、多存储策略
* :link: 创建文件、目录的分享链接,可设定自动过期
* :eye_speech_bubble: 视频、图像、音频、文本、Office 文档、 ePub 在线预览
* :art: 自定义配色、黑暗模式、PWA 应用、全站单页应用、国际化支持
* :rocket: All-In-One 打包,开箱即用
* 🌈 ... ...
## :hammer_and_wrench: 部署
下载适用于您目标机器操作系统、CPU架构的主程序直接运行即可。
```shell
# 解压程序包
tar -zxvf cloudreve_VERSION_OS_ARCH.tar.gz
# 赋予执行权限
chmod +x ./cloudreve
# 启动 Cloudreve
./cloudreve
```
以上为最简单的部署示例,您可以参考 [文档 - 起步](https://docs.cloudreve.org/) 进行更为完善的部署。
## :gear: 构建
自行构建前需要拥有 `Go >= 1.18``node.js``yarn``zip` 等必要依赖。
#### 克隆代码
```shell
git clone --recurse-submodules https://github.com/cloudreve/Cloudreve.git
```
#### 构建静态资源
```shell
# 进入前端子模块
cd assets
# 安装依赖
yarn install
# 开始构建
yarn run build
# 构建完成后删除映射文件
cd build
find . -name "*.map" -type f -delete
# 返回项目主目录打包静态资源
cd ../../
zip -r - assets/build >assets.zip
```
#### 编译项目
```shell
# 获得当前版本号、Commit
export COMMIT_SHA=$(git rev-parse --short HEAD)
export VERSION=$(git describe --tags)
# 开始编译
go build -a -o cloudreve -ldflags "-s -w -X 'github.com/cloudreve/Cloudreve/v3/pkg/conf.BackendVersion=$VERSION' -X 'github.com/cloudreve/Cloudreve/v3/pkg/conf.LastCommit=$COMMIT_SHA'"
```
你也可以使用项目根目录下的 `build.sh` 快速开始构建:
```shell
./build.sh [-a] [-c] [-b] [-r]
a - 构建静态资源
c - 编译二进制文件
b - 构建前端 + 编译二进制文件
r - 交叉编译构建用于release的版本
```
## :alembic: 技术栈
* [Go](https://golang.org/) + [Gin](https://github.com/gin-gonic/gin)
* [React](https://github.com/facebook/react) + [Redux](https://github.com/reduxjs/redux) + [Material-UI](https://github.com/mui-org/material-ui)
## :scroll: 许可证
GPL V3

2
assets

Submodule assets updated: 2d20892994...01343d7656

Binary file not shown.

432
bootstrap/embed.go Normal file
View File

@@ -0,0 +1,432 @@
// Copyright 2020 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
// Package embed provides access to files embedded in the running Go program.
//
// Go source files that import "embed" can use the //go:embed directive
// to initialize a variable of type string, []byte, or FS with the contents of
// files read from the package directory or subdirectories at compile time.
//
// For example, here are three ways to embed a file named hello.txt
// and then print its contents at run time.
//
// Embedding one file into a string:
//
// import _ "embed"
//
// //go:embed hello.txt
// var s string
// print(s)
//
// Embedding one file into a slice of bytes:
//
// import _ "embed"
//
// //go:embed hello.txt
// var b []byte
// print(string(b))
//
// Embedded one or more files into a file system:
//
// import "embed"
//
// //go:embed hello.txt
// var f embed.FS
// data, _ := f.ReadFile("hello.txt")
// print(string(data))
//
// # Directives
//
// A //go:embed directive above a variable declaration specifies which files to embed,
// using one or more path.Match patterns.
//
// The directive must immediately precede a line containing the declaration of a single variable.
// Only blank lines and // line comments are permitted between the directive and the declaration.
//
// The type of the variable must be a string type, or a slice of a byte type,
// or FS (or an alias of FS).
//
// For example:
//
// package server
//
// import "embed"
//
// // content holds our static web server content.
// //go:embed image/* template/*
// //go:embed html/index.html
// var content embed.FS
//
// The Go build system will recognize the directives and arrange for the declared variable
// (in the example above, content) to be populated with the matching files from the file system.
//
// The //go:embed directive accepts multiple space-separated patterns for
// brevity, but it can also be repeated, to avoid very long lines when there are
// many patterns. The patterns are interpreted relative to the package directory
// containing the source file. The path separator is a forward slash, even on
// Windows systems. Patterns may not contain . or .. or empty path elements,
// nor may they begin or end with a slash. To match everything in the current
// directory, use * instead of .. To allow for naming files with spaces in
// their names, patterns can be written as Go double-quoted or back-quoted
// string literals.
//
// If a pattern names a directory, all files in the subtree rooted at that directory are
// embedded (recursively), except that files with names beginning with . or _
// are excluded. So the variable in the above example is almost equivalent to:
//
// // content is our static web server content.
// //go:embed image template html/index.html
// var content embed.FS
//
// The difference is that image/* embeds image/.tempfile while image does not.
// Neither embeds image/dir/.tempfile.
//
// If a pattern begins with the prefix all:, then the rule for walking directories is changed
// to include those files beginning with . or _. For example, all:image embeds
// both image/.tempfile and image/dir/.tempfile.
//
// The //go:embed directive can be used with both exported and unexported variables,
// depending on whether the package wants to make the data available to other packages.
// It can only be used with variables at package scope, not with local variables.
//
// Patterns must not match files outside the package's module, such as .git/* or symbolic links.
// Patterns must not match files whose names include the special punctuation characters " * < > ? ` ' | / \ and :.
// Matches for empty directories are ignored. After that, each pattern in a //go:embed line
// must match at least one file or non-empty directory.
//
// If any patterns are invalid or have invalid matches, the build will fail.
//
// # Strings and Bytes
//
// The //go:embed line for a variable of type string or []byte can have only a single pattern,
// and that pattern can match only a single file. The string or []byte is initialized with
// the contents of that file.
//
// The //go:embed directive requires importing "embed", even when using a string or []byte.
// In source files that don't refer to embed.FS, use a blank import (import _ "embed").
//
// # File Systems
//
// For embedding a single file, a variable of type string or []byte is often best.
// The FS type enables embedding a tree of files, such as a directory of static
// web server content, as in the example above.
//
// FS implements the io/fs package's FS interface, so it can be used with any package that
// understands file systems, including net/http, text/template, and html/template.
//
// For example, given the content variable in the example above, we can write:
//
// http.Handle("/static/", http.StripPrefix("/static/", http.FileServer(http.FS(content))))
//
// template.ParseFS(content, "*.tmpl")
//
// # Tools
//
// To support tools that analyze Go packages, the patterns found in //go:embed lines
// are available in “go list” output. See the EmbedPatterns, TestEmbedPatterns,
// and XTestEmbedPatterns fields in the “go help list” output.
package bootstrap
import (
"errors"
"io"
"io/fs"
"time"
)
// An FS is a read-only collection of files, usually initialized with a //go:embed directive.
// When declared without a //go:embed directive, an FS is an empty file system.
//
// An FS is a read-only value, so it is safe to use from multiple goroutines
// simultaneously and also safe to assign values of type FS to each other.
//
// FS implements fs.FS, so it can be used with any package that understands
// file system interfaces, including net/http, text/template, and html/template.
//
// See the package documentation for more details about initializing an FS.
type FS struct {
// The compiler knows the layout of this struct.
// See cmd/compile/internal/staticdata's WriteEmbed.
//
// The files list is sorted by name but not by simple string comparison.
// Instead, each file's name takes the form "dir/elem" or "dir/elem/".
// The optional trailing slash indicates that the file is itself a directory.
// The files list is sorted first by dir (if dir is missing, it is taken to be ".")
// and then by base, so this list of files:
//
// p
// q/
// q/r
// q/s/
// q/s/t
// q/s/u
// q/v
// w
//
// is actually sorted as:
//
// p # dir=. elem=p
// q/ # dir=. elem=q
// w/ # dir=. elem=w
// q/r # dir=q elem=r
// q/s/ # dir=q elem=s
// q/v # dir=q elem=v
// q/s/t # dir=q/s elem=t
// q/s/u # dir=q/s elem=u
//
// This order brings directory contents together in contiguous sections
// of the list, allowing a directory read to use binary search to find
// the relevant sequence of entries.
files *[]file
}
// split splits the name into dir and elem as described in the
// comment in the FS struct above. isDir reports whether the
// final trailing slash was present, indicating that name is a directory.
func split(name string) (dir, elem string, isDir bool) {
if name[len(name)-1] == '/' {
isDir = true
name = name[:len(name)-1]
}
i := len(name) - 1
for i >= 0 && name[i] != '/' {
i--
}
if i < 0 {
return ".", name, isDir
}
return name[:i], name[i+1:], isDir
}
// trimSlash trims a trailing slash from name, if present,
// returning the possibly shortened name.
func trimSlash(name string) string {
if len(name) > 0 && name[len(name)-1] == '/' {
return name[:len(name)-1]
}
return name
}
var (
_ fs.ReadDirFS = FS{}
_ fs.ReadFileFS = FS{}
)
// A file is a single file in the FS.
// It implements fs.FileInfo and fs.DirEntry.
type file struct {
// The compiler knows the layout of this struct.
// See cmd/compile/internal/staticdata's WriteEmbed.
name string
data string
hash [16]byte // truncated SHA256 hash
}
var (
_ fs.FileInfo = (*file)(nil)
_ fs.DirEntry = (*file)(nil)
)
func (f *file) Name() string { _, elem, _ := split(f.name); return elem }
func (f *file) Size() int64 { return int64(len(f.data)) }
func (f *file) ModTime() time.Time { return time.Time{} }
func (f *file) IsDir() bool { _, _, isDir := split(f.name); return isDir }
func (f *file) Sys() any { return nil }
func (f *file) Type() fs.FileMode { return f.Mode().Type() }
func (f *file) Info() (fs.FileInfo, error) { return f, nil }
func (f *file) Mode() fs.FileMode {
if f.IsDir() {
return fs.ModeDir | 0555
}
return 0444
}
// dotFile is a file for the root directory,
// which is omitted from the files list in a FS.
var dotFile = &file{name: "./"}
// lookup returns the named file, or nil if it is not present.
func (f FS) lookup(name string) *file {
if !fs.ValidPath(name) {
// The compiler should never emit a file with an invalid name,
// so this check is not strictly necessary (if name is invalid,
// we shouldn't find a match below), but it's a good backstop anyway.
return nil
}
if name == "." {
return dotFile
}
if f.files == nil {
return nil
}
// Binary search to find where name would be in the list,
// and then check if name is at that position.
dir, elem, _ := split(name)
files := *f.files
i := sortSearch(len(files), func(i int) bool {
idir, ielem, _ := split(files[i].name)
return idir > dir || idir == dir && ielem >= elem
})
if i < len(files) && trimSlash(files[i].name) == name {
return &files[i]
}
return nil
}
// readDir returns the list of files corresponding to the directory dir.
func (f FS) readDir(dir string) []file {
if f.files == nil {
return nil
}
// Binary search to find where dir starts and ends in the list
// and then return that slice of the list.
files := *f.files
i := sortSearch(len(files), func(i int) bool {
idir, _, _ := split(files[i].name)
return idir >= dir
})
j := sortSearch(len(files), func(j int) bool {
jdir, _, _ := split(files[j].name)
return jdir > dir
})
return files[i:j]
}
// Open opens the named file for reading and returns it as an fs.File.
//
// The returned file implements io.Seeker when the file is not a directory.
func (f FS) Open(name string) (fs.File, error) {
file := f.lookup(name)
if file == nil {
return nil, &fs.PathError{Op: "open", Path: name, Err: fs.ErrNotExist}
}
if file.IsDir() {
return &openDir{file, f.readDir(name), 0}, nil
}
return &openFile{file, 0}, nil
}
// ReadDir reads and returns the entire named directory.
func (f FS) ReadDir(name string) ([]fs.DirEntry, error) {
file, err := f.Open(name)
if err != nil {
return nil, err
}
dir, ok := file.(*openDir)
if !ok {
return nil, &fs.PathError{Op: "read", Path: name, Err: errors.New("not a directory")}
}
list := make([]fs.DirEntry, len(dir.files))
for i := range list {
list[i] = &dir.files[i]
}
return list, nil
}
// ReadFile reads and returns the content of the named file.
func (f FS) ReadFile(name string) ([]byte, error) {
file, err := f.Open(name)
if err != nil {
return nil, err
}
ofile, ok := file.(*openFile)
if !ok {
return nil, &fs.PathError{Op: "read", Path: name, Err: errors.New("is a directory")}
}
return []byte(ofile.f.data), nil
}
// An openFile is a regular file open for reading.
type openFile struct {
f *file // the file itself
offset int64 // current read offset
}
var (
_ io.Seeker = (*openFile)(nil)
)
func (f *openFile) Close() error { return nil }
func (f *openFile) Stat() (fs.FileInfo, error) { return f.f, nil }
func (f *openFile) Read(b []byte) (int, error) {
if f.offset >= int64(len(f.f.data)) {
return 0, io.EOF
}
if f.offset < 0 {
return 0, &fs.PathError{Op: "read", Path: f.f.name, Err: fs.ErrInvalid}
}
n := copy(b, f.f.data[f.offset:])
f.offset += int64(n)
return n, nil
}
func (f *openFile) Seek(offset int64, whence int) (int64, error) {
switch whence {
case 0:
// offset += 0
case 1:
offset += f.offset
case 2:
offset += int64(len(f.f.data))
}
if offset < 0 || offset > int64(len(f.f.data)) {
return 0, &fs.PathError{Op: "seek", Path: f.f.name, Err: fs.ErrInvalid}
}
f.offset = offset
return offset, nil
}
// An openDir is a directory open for reading.
type openDir struct {
f *file // the directory file itself
files []file // the directory contents
offset int // the read offset, an index into the files slice
}
func (d *openDir) Close() error { return nil }
func (d *openDir) Stat() (fs.FileInfo, error) { return d.f, nil }
func (d *openDir) Read([]byte) (int, error) {
return 0, &fs.PathError{Op: "read", Path: d.f.name, Err: errors.New("is a directory")}
}
func (d *openDir) ReadDir(count int) ([]fs.DirEntry, error) {
n := len(d.files) - d.offset
if n == 0 {
if count <= 0 {
return nil, nil
}
return nil, io.EOF
}
if count > 0 && n > count {
n = count
}
list := make([]fs.DirEntry, n)
for i := range list {
list[i] = &d.files[d.offset+i]
}
d.offset += n
return list, nil
}
// sortSearch is like sort.Search, avoiding an import.
func sortSearch(n int, f func(int) bool) int {
// Define f(-1) == false and f(n) == true.
// Invariant: f(i-1) == false, f(j) == true.
i, j := 0, n
for i < j {
h := int(uint(i+j) >> 1) // avoid overflow when computing h
// i ≤ h < j
if !f(h) {
i = h + 1 // preserves f(i-1) == false
} else {
j = h // preserves f(j) == true
}
}
// i == j, f(i-1) == false, and f(j) (= f(i)) == true => answer is i.
return i
}

75
bootstrap/fs.go Normal file
View File

@@ -0,0 +1,75 @@
package bootstrap
import (
"archive/zip"
"crypto/sha256"
"github.com/cloudreve/Cloudreve/v3/pkg/util"
"github.com/pkg/errors"
"io"
"io/fs"
"sort"
"strings"
)
func NewFS(zipContent string) fs.FS {
zipReader, err := zip.NewReader(strings.NewReader(zipContent), int64(len(zipContent)))
if err != nil {
util.Log().Panic("Static resource is not a valid zip file: %s", err)
}
var files []file
err = fs.WalkDir(zipReader, ".", func(path string, d fs.DirEntry, err error) error {
if err != nil {
return errors.Errorf("无法获取[%s]的信息, %s, 跳过...", path, err)
}
if path == "." {
return nil
}
var f file
if d.IsDir() {
f.name = path + "/"
} else {
f.name = path
rc, err := zipReader.Open(path)
if err != nil {
return errors.Errorf("无法打开文件[%s], %s, 跳过...", path, err)
}
defer rc.Close()
data, err := io.ReadAll(rc)
if err != nil {
return errors.Errorf("无法读取文件[%s], %s, 跳过...", path, err)
}
f.data = string(data)
hash := sha256.Sum256(data)
for i := range f.hash {
f.hash[i] = ^hash[i]
}
}
files = append(files, f)
return nil
})
if err != nil {
util.Log().Panic("初始化静态资源失败: %s", err)
}
sort.Slice(files, func(i, j int) bool {
fi, fj := files[i], files[j]
di, ei, _ := split(fi.name)
dj, ej, _ := split(fj.name)
if di != dj {
return di < dj
}
return ei < ej
})
var embedFS FS
embedFS.files = &files
return embedFS
}

View File

@@ -98,16 +98,7 @@ func Init(path string, statics fs.FS) {
}
for _, dependency := range dependencies {
switch dependency.mode {
case "master":
if conf.SystemConfig.Mode == "master" {
dependency.factory()
}
case "slave":
if conf.SystemConfig.Mode == "slave" {
dependency.factory()
}
default:
if dependency.mode == conf.SystemConfig.Mode || dependency.mode == "both" {
dependency.factory()
}
}

View File

@@ -10,9 +10,9 @@ func RunScript(name string) {
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
if err := invoker.RunDBScript(name, ctx); err != nil {
util.Log().Error("数据库脚本执行失败: %s", err)
util.Log().Error("Failed to execute database script: %s", err)
return
}
util.Log().Info("数据库脚本 [%s] 执行完毕", name)
util.Log().Info("Finish executing database script %q.", name)
}

View File

@@ -46,13 +46,13 @@ func (b *GinFS) Exists(prefix string, filepath string) bool {
// InitStatic 初始化静态资源文件
func InitStatic(statics fs.FS) {
if util.Exists(util.RelativePath(StaticFolder)) {
util.Log().Info("检测到 statics 目录存在,将使用此目录下的静态资源文件")
util.Log().Info("Folder with name \"statics\" already exists, it will be used to serve static files.")
StaticFS = static.LocalFile(util.RelativePath("statics"), false)
} else {
// 初始化静态资源
embedFS, err := fs.Sub(statics, "assets/build")
if err != nil {
util.Log().Panic("无法初始化静态资源, %s", err)
util.Log().Panic("Failed to initialize static resources: %s", err)
}
StaticFS = &GinFS{
@@ -62,19 +62,19 @@ func InitStatic(statics fs.FS) {
// 检查静态资源的版本
f, err := StaticFS.Open("version.json")
if err != nil {
util.Log().Warning("静态资源版本标识文件不存在,请重新构建或删除 statics 目录")
util.Log().Warning("Missing version identifier file in static resources, please delete \"statics\" folder and rebuild it.")
return
}
b, err := io.ReadAll(f)
if err != nil {
util.Log().Warning("无法读取静态资源文件版本,请重新构建或删除 statics 目录")
util.Log().Warning("Failed to read version identifier file in static resources, please delete \"statics\" folder and rebuild it.")
return
}
var v staticVersion
if err := json.Unmarshal(b, &v); err != nil {
util.Log().Warning("无法解析静态资源文件版本, %s", err)
util.Log().Warning("Failed to parse version identifier file in static resources: %s", err)
return
}
@@ -84,12 +84,12 @@ func InitStatic(statics fs.FS) {
}
if v.Name != staticName {
util.Log().Warning("静态资源版本不匹配,请重新构建或删除 statics 目录")
util.Log().Warning("Static resource version mismatch, please delete \"statics\" folder and rebuild it.")
return
}
if v.Version != conf.RequiredStaticVersion {
util.Log().Warning("静态资源版本不匹配 [当前 %s, 需要: %s],请重新构建或删除 statics 目录", v.Version, conf.RequiredStaticVersion)
util.Log().Warning("Static resource version mismatch [Current %s, Desired: %s]please delete \"statics\" folder and rebuild it.", v.Version, conf.RequiredStaticVersion)
return
}
}
@@ -99,13 +99,13 @@ func Eject(statics fs.FS) {
// 初始化静态资源
embedFS, err := fs.Sub(statics, "assets/build")
if err != nil {
util.Log().Panic("无法初始化静态资源, %s", err)
util.Log().Panic("Failed to initialize static resources: %s", err)
}
var walk func(relPath string, d fs.DirEntry, err error) error
walk = func(relPath string, d fs.DirEntry, err error) error {
if err != nil {
return errors.Errorf("无法获取[%s]的信息, %s, 跳过...", relPath, err)
return errors.Errorf("Failed to read info of %q: %s, skipping...", relPath, err)
}
if !d.IsDir() {
@@ -114,13 +114,13 @@ func Eject(statics fs.FS) {
defer out.Close()
if err != nil {
return errors.Errorf("无法创建文件[%s], %s, 跳过...", relPath, err)
return errors.Errorf("Failed to create file %q: %s, skipping...", relPath, err)
}
util.Log().Info("导出 [%s]...", relPath)
util.Log().Info("Ejecting %q...", relPath)
obj, _ := embedFS.Open(relPath)
if _, err := io.Copy(out, bufio.NewReader(obj)); err != nil {
return errors.Errorf("无法写入文件[%s], %s, 跳过...", relPath, err)
return errors.Errorf("Cannot write file %q: %s, skipping...", relPath, err)
}
}
return nil
@@ -129,8 +129,8 @@ func Eject(statics fs.FS) {
// util.Log().Info("开始导出内置静态资源...")
err = fs.WalkDir(embedFS, ".", walk)
if err != nil {
util.Log().Error("导出内置静态资源遇到错误:%s", err)
util.Log().Error("Error occurs while ejecting static resources: %s", err)
return
}
util.Log().Info("内置静态资源导出完成")
util.Log().Info("Finish ejecting static resources.")
}

View File

@@ -24,19 +24,24 @@ buildAssets() {
rm -rf assets/build
export CI=false
export GENERATE_SOURCEMAP=false
cd $REPO/assets
yarn install --network-timeout 1000000
yarn install
yarn run build
cd build
find . -name "*.map" -type f -delete
cd $REPO
# please keep in mind that if this final output binary `assets.zip` name changed, please go and update the `Dockerfile` as well
zip -r - assets/build >assets.zip
}
buildBinary() {
cd $REPO
# same as assets, if this final output binary `cloudreve` name changed, please go and update the `Dockerfile`
go build -a -o cloudreve -ldflags " -X 'github.com/cloudreve/Cloudreve/v3/pkg/conf.BackendVersion=$VERSION' -X 'github.com/cloudreve/Cloudreve/v3/pkg/conf.LastCommit=$COMMIT_SHA'"
}
@@ -88,7 +93,7 @@ usage() {
exit 1
}
while getopts "bacr:d" o; do
while getopts "bacrd" o; do
case "${o}" in
b)
ASSETS="true"

137
go.mod
View File

@@ -1,24 +1,25 @@
module github.com/cloudreve/Cloudreve/v3
go 1.17
go 1.18
require (
github.com/DATA-DOG/go-sqlmock v1.3.3
github.com/HFO4/aliyun-oss-go-sdk v2.2.3+incompatible
github.com/aws/aws-sdk-go v1.31.5
github.com/duo-labs/webauthn v0.0.0-20191119193225-4bf9a0f776d4
github.com/fatih/color v1.7.0
github.com/duo-labs/webauthn v0.0.0-20220330035159-03696f3d4499
github.com/fatih/color v1.9.0
github.com/gin-contrib/cors v1.3.0
github.com/gin-contrib/gzip v0.0.2-0.20200226035851-25bef2ef21e8
github.com/gin-contrib/sessions v0.0.1
github.com/gin-contrib/sessions v0.0.5
github.com/gin-contrib/static v0.0.0-20191128031702-f81c604d8ac2
github.com/gin-gonic/gin v1.7.7
github.com/gin-gonic/gin v1.8.1
github.com/go-ini/ini v1.50.0
github.com/go-mail/mail v2.3.1+incompatible
github.com/go-playground/validator/v10 v10.11.0
github.com/gofrs/uuid v4.0.0+incompatible
github.com/gomodule/redigo v2.0.0+incompatible
github.com/google/go-querystring v1.0.0
github.com/gorilla/websocket v1.4.1
github.com/gorilla/websocket v1.4.2
github.com/hashicorp/go-version v1.3.0
github.com/jinzhu/gorm v1.9.11
github.com/juju/ratelimit v1.0.1
@@ -30,71 +31,137 @@ require (
github.com/rafaeljusto/redigomock v0.0.0-20191117212112-00b2509252a1
github.com/robfig/cron/v3 v3.0.1
github.com/speps/go-hashids v2.0.0+incompatible
github.com/stretchr/testify v1.6.1
github.com/tencentcloud/tencentcloud-sdk-go v3.0.125+incompatible
github.com/stretchr/testify v1.7.2
github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/captcha v1.0.393
github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/common v1.0.393
github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/scf v1.0.393
github.com/tencentyun/cos-go-sdk-v5 v0.0.0-20200120023323-87ff3bc489ac
github.com/upyun/go-sdk v2.1.0+incompatible
golang.org/x/image v0.0.0-20211028202545-6944b10bf410
golang.org/x/text v0.3.7
gopkg.in/go-playground/validator.v9 v9.29.1
golang.org/x/time v0.0.0-20210220033141-f8bda1e9f3ba
)
require (
cloud.google.com/go v0.37.4 // indirect
github.com/agl/ed25519 v0.0.0-20170116200512-5312a6153412 // indirect
cloud.google.com/go v0.81.0 // indirect
github.com/andybalholm/brotli v1.0.4 // indirect
github.com/baiyubin/aliyun-sts-go-sdk v0.0.0-20180326062324-cfa1a18b161f // indirect
github.com/beorn7/perks v1.0.1 // indirect
github.com/bgentry/speakeasy v0.1.0 // indirect
github.com/boj/redistore v0.0.0-20180917114910-cd5dcc76aeff // indirect
github.com/boombuler/barcode v1.0.1-0.20190219062509-6c824513bacc // indirect
github.com/cloudflare/cfssl v0.0.0-20190726000631-633726f6bcb7 // indirect
github.com/census-instrumentation/opencensus-proto v0.3.0 // indirect
github.com/cespare/xxhash/v2 v2.1.1 // indirect
github.com/cloudflare/cfssl v1.6.1 // indirect
github.com/cncf/udpa/go v0.0.0-20210322005330-6414d713912e // indirect
github.com/coreos/go-semver v0.3.0 // indirect
github.com/coreos/go-systemd/v22 v22.3.2 // indirect
github.com/cpuguy83/go-md2man/v2 v2.0.0 // indirect
github.com/davecgh/go-spew v1.1.1 // indirect
github.com/denisenkom/go-mssqldb v0.0.0-20190515213511-eb9f6a1743f3 // indirect
github.com/dgrijalva/jwt-go v3.2.0+incompatible // indirect
github.com/dsnet/compress v0.0.1 // indirect
github.com/dustin/go-humanize v1.0.0 // indirect
github.com/envoyproxy/go-control-plane v0.9.9-0.20210217033140-668b12f5399d // indirect
github.com/envoyproxy/protoc-gen-validate v0.6.1 // indirect
github.com/form3tech-oss/jwt-go v3.2.3+incompatible // indirect
github.com/fullstorydev/grpcurl v1.8.1 // indirect
github.com/fxamacker/cbor/v2 v2.4.0 // indirect
github.com/gin-contrib/sse v0.1.0 // indirect
github.com/go-playground/locales v0.14.0 // indirect
github.com/go-playground/universal-translator v0.18.0 // indirect
github.com/go-playground/validator/v10 v10.8.0 // indirect
github.com/go-sql-driver/mysql v1.5.0 // indirect
github.com/go-sql-driver/mysql v1.6.0 // indirect
github.com/goccy/go-json v0.9.8 // indirect
github.com/gogo/protobuf v1.3.2 // indirect
github.com/golang-jwt/jwt/v4 v4.1.0 // indirect
github.com/golang/freetype v0.0.0-20170609003504-e2365dfdc4a0 // indirect
github.com/golang/protobuf v1.3.3 // indirect
github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
github.com/golang/mock v1.5.0 // indirect
github.com/golang/protobuf v1.5.2 // indirect
github.com/golang/snappy v0.0.4 // indirect
github.com/google/certificate-transparency-go v1.0.21 // indirect
github.com/google/btree v1.0.1 // indirect
github.com/google/certificate-transparency-go v1.1.2-0.20210511102531-373a877eec92 // indirect
github.com/google/go-cmp v0.5.5 // indirect
github.com/google/uuid v1.3.0 // indirect
github.com/gorilla/context v1.1.1 // indirect
github.com/gorilla/securecookie v1.1.1 // indirect
github.com/gorilla/sessions v1.1.3 // indirect
github.com/gorilla/sessions v1.2.1 // indirect
github.com/grpc-ecosystem/go-grpc-middleware v1.3.0 // indirect
github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 // indirect
github.com/grpc-ecosystem/grpc-gateway v1.16.0 // indirect
github.com/inconshreveable/mousetrap v1.0.0 // indirect
github.com/jhump/protoreflect v1.8.2 // indirect
github.com/jinzhu/inflection v1.0.0 // indirect
github.com/jmespath/go-jmespath v0.3.0 // indirect
github.com/json-iterator/go v1.1.9 // indirect
github.com/katzenpost/core v0.0.7 // indirect
github.com/jonboulle/clockwork v0.2.2 // indirect
github.com/json-iterator/go v1.1.12 // indirect
github.com/klauspost/compress v1.15.1 // indirect
github.com/klauspost/pgzip v1.2.5 // indirect
github.com/leodido/go-urn v1.2.1 // indirect
github.com/lib/pq v1.1.1 // indirect
github.com/lib/pq v1.10.3 // indirect
github.com/mattn/go-colorable v0.1.4 // indirect
github.com/mattn/go-isatty v0.0.12 // indirect
github.com/mattn/go-sqlite3 v1.11.0 // indirect
github.com/mattn/go-isatty v0.0.14 // indirect
github.com/mattn/go-runewidth v0.0.12 // indirect
github.com/mattn/go-sqlite3 v2.0.3+incompatible // indirect
github.com/matttproud/golang_protobuf_extensions v1.0.1 // indirect
github.com/mitchellh/mapstructure v1.1.2 // indirect
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
github.com/modern-go/reflect2 v1.0.1 // indirect
github.com/modern-go/reflect2 v1.0.2 // indirect
github.com/mozillazg/go-httpheader v0.2.1 // indirect
github.com/nwaples/rardecode/v2 v2.0.0-beta.2 // indirect
github.com/olekukonko/tablewriter v0.0.5 // indirect
github.com/pelletier/go-toml/v2 v2.0.2 // indirect
github.com/pierrec/lz4/v4 v4.1.14 // indirect
github.com/pmezard/go-difflib v1.0.0 // indirect
github.com/quasoft/memstore v0.0.0-20180925164028-84a050167438 // indirect
github.com/prometheus/client_golang v1.10.0 // indirect
github.com/prometheus/client_model v0.2.0 // indirect
github.com/prometheus/common v0.24.0 // indirect
github.com/prometheus/procfs v0.6.0 // indirect
github.com/quasoft/memstore v0.0.0-20191010062613-2bce066d2b0b // indirect
github.com/rivo/uniseg v0.2.0 // indirect
github.com/russross/blackfriday/v2 v2.1.0 // indirect
github.com/satori/go.uuid v1.2.0 // indirect
github.com/smartystreets/goconvey v1.6.4 // indirect
github.com/sirupsen/logrus v1.8.1 // indirect
github.com/soheilhy/cmux v0.1.5 // indirect
github.com/spf13/cobra v1.1.3 // indirect
github.com/spf13/pflag v1.0.5 // indirect
github.com/stretchr/objx v0.2.0 // indirect
github.com/therootcompany/xz v1.0.1 // indirect
github.com/ugorji/go/codec v1.1.7 // indirect
github.com/tmc/grpc-websocket-proxy v0.0.0-20201229170055-e5319fda7802 // indirect
github.com/ugorji/go/codec v1.2.7 // indirect
github.com/ulikunitz/xz v0.5.10 // indirect
golang.org/x/crypto v0.0.0-20210921155107-089bfa567519 // indirect
golang.org/x/sync v0.0.0-20201207232520-09787c993a3a // indirect
golang.org/x/sys v0.0.0-20211020174200-9d6173849985 // indirect
golang.org/x/time v0.0.0-20181108054448-85acf8d2951c // indirect
github.com/urfave/cli v1.22.5 // indirect
github.com/x448/float16 v0.8.4 // indirect
github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2 // indirect
go.etcd.io/bbolt v1.3.5 // indirect
go.etcd.io/etcd/api/v3 v3.5.0-alpha.0 // indirect
go.etcd.io/etcd/client/v2 v2.305.0-alpha.0 // indirect
go.etcd.io/etcd/client/v3 v3.5.0-alpha.0 // indirect
go.etcd.io/etcd/etcdctl/v3 v3.5.0-alpha.0 // indirect
go.etcd.io/etcd/pkg/v3 v3.5.0-alpha.0 // indirect
go.etcd.io/etcd/raft/v3 v3.5.0-alpha.0 // indirect
go.etcd.io/etcd/server/v3 v3.5.0-alpha.0 // indirect
go.etcd.io/etcd/tests/v3 v3.5.0-alpha.0 // indirect
go.etcd.io/etcd/v3 v3.5.0-alpha.0 // indirect
go.uber.org/atomic v1.7.0 // indirect
go.uber.org/multierr v1.7.0 // indirect
go.uber.org/zap v1.16.0 // indirect
golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d // indirect
golang.org/x/mod v0.4.2 // indirect
golang.org/x/net v0.0.0-20220630215102-69896b714898 // indirect
golang.org/x/oauth2 v0.0.0-20210427180440-81ed05c6b58c // indirect
golang.org/x/sync v0.0.0-20210220032951-036812b2e83c // indirect
golang.org/x/sys v0.0.0-20220702020025-31831981b65f // indirect
golang.org/x/text v0.3.7 // indirect
golang.org/x/tools v0.1.0 // indirect
golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 // indirect
google.golang.org/appengine v1.6.7 // indirect
google.golang.org/genproto v0.0.0-20210510173355-fb37daa5cd7a // indirect
google.golang.org/grpc v1.37.0 // indirect
google.golang.org/protobuf v1.28.0 // indirect
gopkg.in/alexcesaro/quotedprintable.v3 v3.0.0-20150716171945-2caba252f4dc // indirect
gopkg.in/ini.v1 v1.51.0 // indirect
gopkg.in/cheggaaa/pb.v1 v1.0.28 // indirect
gopkg.in/mail.v2 v2.3.1 // indirect
gopkg.in/yaml.v2 v2.2.8 // indirect
gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b // indirect
gopkg.in/yaml.v2 v2.4.0 // indirect
gopkg.in/yaml.v3 v3.0.1 // indirect
sigs.k8s.io/yaml v1.2.0 // indirect
)

1161
go.sum

File diff suppressed because it is too large Load Diff

109
main.go
View File

@@ -1,18 +1,22 @@
package main
import (
"context"
_ "embed"
"flag"
"io"
"io/fs"
"strings"
"net"
"net/http"
"os"
"os/signal"
"syscall"
"time"
"github.com/cloudreve/Cloudreve/v3/bootstrap"
model "github.com/cloudreve/Cloudreve/v3/models"
"github.com/cloudreve/Cloudreve/v3/pkg/conf"
"github.com/cloudreve/Cloudreve/v3/pkg/util"
"github.com/cloudreve/Cloudreve/v3/routers"
"github.com/mholt/archiver/v4"
)
var (
@@ -27,19 +31,23 @@ var staticZip string
var staticFS fs.FS
func init() {
flag.StringVar(&confPath, "c", util.RelativePath("conf.ini"), "配置文件路径")
flag.BoolVar(&isEject, "eject", false, "导出内置静态资源")
flag.StringVar(&scriptName, "database-script", "", "运行内置数据库助手脚本")
flag.StringVar(&confPath, "c", util.RelativePath("conf.ini"), "Path to the config file.")
flag.BoolVar(&isEject, "eject", false, "Eject all embedded static files.")
flag.StringVar(&scriptName, "database-script", "", "Name of database util script.")
flag.Parse()
staticFS = archiver.ArchiveFS{
Stream: io.NewSectionReader(strings.NewReader(staticZip), 0, int64(len(staticZip))),
Format: archiver.Zip{},
}
staticFS = bootstrap.NewFS(staticZip)
bootstrap.Init(confPath, staticFS)
}
func main() {
// 关闭数据库连接
defer func() {
if model.DB != nil {
model.DB.Close()
}
}()
if isEject {
// 开始导出内置静态资源文件
bootstrap.Eject(staticFS)
@@ -53,29 +61,82 @@ func main() {
}
api := routers.InitRouter()
api.TrustedPlatform = conf.SystemConfig.ProxyHeader
server := &http.Server{Handler: api}
// 收到信号后关闭服务器
sigChan := make(chan os.Signal, 1)
signal.Notify(sigChan, os.Interrupt, syscall.SIGTERM, syscall.SIGHUP, syscall.SIGQUIT)
go func() {
sig := <-sigChan
util.Log().Info("Signal %s received, shutting down server...", sig)
ctx := context.Background()
if conf.SystemConfig.GracePeriod != 0 {
var cancel context.CancelFunc
ctx, cancel = context.WithTimeout(ctx, time.Duration(conf.SystemConfig.GracePeriod)*time.Second)
defer cancel()
}
err := server.Shutdown(ctx)
if err != nil {
util.Log().Error("Failed to shutdown server: %s", err)
}
}()
// 如果启用了SSL
if conf.SSLConfig.CertPath != "" {
go func() {
util.Log().Info("开始监听 %s", conf.SSLConfig.Listen)
if err := api.RunTLS(conf.SSLConfig.Listen,
conf.SSLConfig.CertPath, conf.SSLConfig.KeyPath); err != nil {
util.Log().Error("无法监听[%s]%s", conf.SSLConfig.Listen, err)
}
}()
util.Log().Info("Listening to %q", conf.SSLConfig.Listen)
server.Addr = conf.SSLConfig.Listen
if err := server.ListenAndServeTLS(conf.SSLConfig.CertPath, conf.SSLConfig.KeyPath); err != nil {
util.Log().Error("Failed to listen to %q: %s", conf.SSLConfig.Listen, err)
return
}
}
// 如果启用了Unix
if conf.UnixConfig.Listen != "" {
util.Log().Info("开始监听 %s", conf.UnixConfig.Listen)
if err := api.RunUnix(conf.UnixConfig.Listen); err != nil {
util.Log().Error("无法监听[%s]%s", conf.UnixConfig.Listen, err)
// delete socket file before listening
if _, err := os.Stat(conf.UnixConfig.Listen); err == nil {
if err = os.Remove(conf.UnixConfig.Listen); err != nil {
util.Log().Error("Failed to delete socket file: %s", err)
return
}
}
util.Log().Info("Listening to %q", conf.UnixConfig.Listen)
if err := RunUnix(server); err != nil {
util.Log().Error("Failed to listen to %q: %s", conf.UnixConfig.Listen, err)
}
return
}
util.Log().Info("开始监听 %s", conf.SystemConfig.Listen)
if err := api.Run(conf.SystemConfig.Listen); err != nil {
util.Log().Error("无法监听[%s]%s", conf.SystemConfig.Listen, err)
util.Log().Info("Listening to %q", conf.SystemConfig.Listen)
server.Addr = conf.SystemConfig.Listen
if err := server.ListenAndServe(); err != nil {
util.Log().Error("Failed to listen to %q: %s", conf.SystemConfig.Listen, err)
}
}
func RunUnix(server *http.Server) error {
listener, err := net.Listen("unix", conf.UnixConfig.Listen)
if err != nil {
return err
}
defer listener.Close()
defer os.Remove(conf.UnixConfig.Listen)
if conf.UnixConfig.Perm > 0 {
err = os.Chmod(conf.UnixConfig.Listen, os.FileMode(conf.UnixConfig.Perm))
if err != nil {
util.Log().Warning(
"Failed to set permission to %q for socket file %q: %s",
conf.UnixConfig.Perm,
conf.UnixConfig.Listen,
err,
)
}
}
return server.Serve(listener)
}

View File

@@ -142,18 +142,18 @@ func uploadCallbackCheck(c *gin.Context, policyType string) serializer.Response
// 验证 Callback Key
sessionID := c.Param("sessionID")
if sessionID == "" {
return serializer.ParamErr("Session ID 不能为空", nil)
return serializer.ParamErr("Session ID cannot be empty", nil)
}
callbackSessionRaw, exist := cache.Get(filesystem.UploadSessionCachePrefix + sessionID)
if !exist {
return serializer.ParamErr("上传会话不存在或已过期", nil)
return serializer.Err(serializer.CodeUploadSessionExpired, "上传会话不存在或已过期", nil)
}
callbackSession := callbackSessionRaw.(serializer.UploadSession)
c.Set(filesystem.UploadSessionCtx, &callbackSession)
if callbackSession.Policy.Type != policyType {
return serializer.Err(serializer.CodePolicyNotAllowed, "Policy not supported", nil)
return serializer.Err(serializer.CodePolicyNotAllowed, "", nil)
}
// 清理回调会话
@@ -162,7 +162,7 @@ func uploadCallbackCheck(c *gin.Context, policyType string) serializer.Response
// 查找用户
user, err := model.GetActiveUserByID(callbackSession.UID)
if err != nil {
return serializer.Err(serializer.CodeCheckLogin, "找不到用户", err)
return serializer.Err(serializer.CodeUserNotFound, "", err)
}
c.Set(filesystem.UserCtx, &user)
return serializer.Response{}
@@ -194,14 +194,14 @@ func QiniuCallbackAuth() gin.HandlerFunc {
mac := qbox.NewMac(session.Policy.AccessKey, session.Policy.SecretKey)
ok, err := mac.VerifyCallback(c.Request)
if err != nil {
util.Log().Debug("无法验证回调请求,%s", err)
c.JSON(401, serializer.GeneralUploadCallbackFailed{Error: "无法验证回调请求"})
util.Log().Debug("Failed to verify callback request: %s", err)
c.JSON(401, serializer.GeneralUploadCallbackFailed{Error: "Failed to verify callback request."})
c.Abort()
return
}
if !ok {
c.JSON(401, serializer.GeneralUploadCallbackFailed{Error: "回调签名无效"})
c.JSON(401, serializer.GeneralUploadCallbackFailed{Error: "Invalid signature."})
c.Abort()
return
}
@@ -215,8 +215,8 @@ func OSSCallbackAuth() gin.HandlerFunc {
return func(c *gin.Context) {
err := oss.VerifyCallbackSignature(c.Request)
if err != nil {
util.Log().Debug("回调签名验证失败,%s", err)
c.JSON(401, serializer.GeneralUploadCallbackFailed{Error: "回调签名验证失败"})
util.Log().Debug("Failed to verify callback request: %s", err)
c.JSON(401, serializer.GeneralUploadCallbackFailed{Error: "Failed to verify callback request."})
c.Abort()
return
}
@@ -250,7 +250,7 @@ func UpyunCallbackAuth() gin.HandlerFunc {
// 计算正文MD5
actualContentMD5 := fmt.Sprintf("%x", md5.Sum(body))
if actualContentMD5 != contentMD5 {
c.JSON(401, serializer.GeneralUploadCallbackFailed{Error: "MD5不一致"})
c.JSON(401, serializer.GeneralUploadCallbackFailed{Error: "MD5 mismatch."})
c.Abort()
return
}
@@ -265,7 +265,7 @@ func UpyunCallbackAuth() gin.HandlerFunc {
// 对比签名
if signature != actualSignature {
c.JSON(401, serializer.GeneralUploadCallbackFailed{Error: "鉴权失败"})
c.JSON(401, serializer.GeneralUploadCallbackFailed{Error: "Signature not match"})
c.Abort()
return
}
@@ -289,7 +289,7 @@ func IsAdmin() gin.HandlerFunc {
return func(c *gin.Context) {
user, _ := c.Get("user")
if user.(*model.User).Group.ID != 1 && user.(*model.User).ID != 1 {
c.JSON(200, serializer.Err(serializer.CodeAdminRequired, "您不是管理组成员", nil))
c.JSON(200, serializer.Err(serializer.CodeNoPermissionErr, "", nil))
c.Abort()
return
}

View File

@@ -24,6 +24,11 @@ type req struct {
Randstr string `json:"randstr"`
}
const (
captchaNotMatch = "CAPTCHA not match."
captchaRefresh = "Verification failed, please refresh the page and retry."
)
// CaptchaRequired 验证请求签名
func CaptchaRequired(configName string) gin.HandlerFunc {
return func(c *gin.Context) {
@@ -43,7 +48,7 @@ func CaptchaRequired(configName string) gin.HandlerFunc {
bodyCopy := new(bytes.Buffer)
_, err := io.Copy(bodyCopy, c.Request.Body)
if err != nil {
c.JSON(200, serializer.ParamErr("验证码错误", err))
c.JSON(200, serializer.Err(serializer.CodeCaptchaError, captchaNotMatch, err))
c.Abort()
return
}
@@ -51,7 +56,7 @@ func CaptchaRequired(configName string) gin.HandlerFunc {
bodyData := bodyCopy.Bytes()
err = json.Unmarshal(bodyData, &service)
if err != nil {
c.JSON(200, serializer.ParamErr("验证码错误", err))
c.JSON(200, serializer.Err(serializer.CodeCaptchaError, captchaNotMatch, err))
c.Abort()
return
}
@@ -62,7 +67,7 @@ func CaptchaRequired(configName string) gin.HandlerFunc {
captchaID := util.GetSession(c, "captchaID")
util.DeleteSession(c, "captchaID")
if captchaID == nil || !base64Captcha.VerifyCaptcha(captchaID.(string), service.CaptchaCode) {
c.JSON(200, serializer.ParamErr("验证码错误", nil))
c.JSON(200, serializer.Err(serializer.CodeCaptchaError, captchaNotMatch, err))
c.Abort()
return
}
@@ -71,15 +76,15 @@ func CaptchaRequired(configName string) gin.HandlerFunc {
case "recaptcha":
reCAPTCHA, err := recaptcha.NewReCAPTCHA(options["captcha_ReCaptchaSecret"], recaptcha.V2, 10*time.Second)
if err != nil {
util.Log().Warning("reCAPTCHA 验证错误, %s", err)
util.Log().Warning("reCAPTCHA verification failed, %s", err)
c.Abort()
break
}
err = reCAPTCHA.Verify(service.CaptchaCode)
if err != nil {
util.Log().Warning("reCAPTCHA 验证错误, %s", err)
c.JSON(200, serializer.ParamErr("验证失败,请刷新网页后再次验证", nil))
util.Log().Warning("reCAPTCHA verification failed, %s", err)
c.JSON(200, serializer.Err(serializer.CodeCaptchaRefreshNeeded, captchaRefresh, nil))
c.Abort()
return
}
@@ -103,13 +108,13 @@ func CaptchaRequired(configName string) gin.HandlerFunc {
request.UserIp = common.StringPtr(c.ClientIP())
response, err := client.DescribeCaptchaResult(request)
if err != nil {
util.Log().Warning("TCaptcha 验证错误, %s", err)
util.Log().Warning("TCaptcha verification failed, %s", err)
c.Abort()
break
}
if *response.Response.CaptchaCode != int64(1) {
c.JSON(200, serializer.ParamErr("验证失败,请刷新网页后再次验证", nil))
c.JSON(200, serializer.Err(serializer.CodeCaptchaRefreshNeeded, captchaRefresh, nil))
c.Abort()
return
}

View File

@@ -25,7 +25,7 @@ func UseSlaveAria2Instance(clusterController cluster.Controller) gin.HandlerFunc
// 获取对应主机节点的从机Aria2实例
caller, err := clusterController.GetAria2Instance(siteID.(string))
if err != nil {
c.JSON(200, serializer.Err(serializer.CodeNotSet, "无法获取 Aria2 实例", err))
c.JSON(200, serializer.Err(serializer.CodeNotSet, "Failed to get Aria2 instance", err))
c.Abort()
return
}
@@ -35,7 +35,7 @@ func UseSlaveAria2Instance(clusterController cluster.Controller) gin.HandlerFunc
return
}
c.JSON(200, serializer.ParamErr("未知的主机节点ID", nil))
c.JSON(200, serializer.ParamErr("Unknown master node ID", nil))
c.Abort()
}
}
@@ -44,14 +44,14 @@ func SlaveRPCSignRequired(nodePool cluster.Pool) gin.HandlerFunc {
return func(c *gin.Context) {
nodeID, err := strconv.ParseUint(c.GetHeader(auth.CrHeaderPrefix+"Node-Id"), 10, 64)
if err != nil {
c.JSON(200, serializer.ParamErr("未知的主机节点ID", err))
c.JSON(200, serializer.ParamErr("Unknown master node ID", err))
c.Abort()
return
}
slaveNode := nodePool.GetNodeByID(uint(nodeID))
if slaveNode == nil {
c.JSON(200, serializer.ParamErr("未知的主机节点ID", err))
c.JSON(200, serializer.ParamErr("Unknown master node ID", err))
c.Abort()
return
}

View File

@@ -1,10 +1,13 @@
package middleware
import (
"fmt"
model "github.com/cloudreve/Cloudreve/v3/models"
"github.com/cloudreve/Cloudreve/v3/pkg/auth"
"github.com/cloudreve/Cloudreve/v3/pkg/hashid"
"github.com/cloudreve/Cloudreve/v3/pkg/serializer"
"github.com/gin-gonic/gin"
"net/http"
)
// HashID 将给定对象的HashID转换为真实ID
@@ -17,7 +20,7 @@ func HashID(IDType int) gin.HandlerFunc {
c.Next()
return
}
c.JSON(200, serializer.ParamErr("无法解析对象ID", nil))
c.JSON(200, serializer.ParamErr("Failed to parse object ID", nil))
c.Abort()
return
@@ -30,7 +33,7 @@ func HashID(IDType int) gin.HandlerFunc {
func IsFunctionEnabled(key string) gin.HandlerFunc {
return func(c *gin.Context) {
if !model.IsTrueVal(model.GetSettingByName(key)) {
c.JSON(200, serializer.Err(serializer.CodeNoPermissionErr, "未开启此功能", nil))
c.JSON(200, serializer.Err(serializer.CodeFeatureNotEnabled, "This feature is not enabled", nil))
c.Abort()
return
}
@@ -45,3 +48,30 @@ func CacheControl() gin.HandlerFunc {
c.Header("Cache-Control", "private, no-cache")
}
}
func Sandbox() gin.HandlerFunc {
return func(c *gin.Context) {
c.Header("Content-Security-Policy", "sandbox")
}
}
// StaticResourceCache 使用静态资源缓存策略
func StaticResourceCache() gin.HandlerFunc {
return func(c *gin.Context) {
c.Header("Cache-Control", fmt.Sprintf("public, max-age=%d", model.GetIntSetting("public_resource_maxage", 86400)))
}
}
// MobileRequestOnly
func MobileRequestOnly() gin.HandlerFunc {
return func(c *gin.Context) {
if c.GetHeader(auth.CrHeaderPrefix+"ios") == "" {
c.Redirect(http.StatusMovedPermanently, model.GetSiteURL().String())
c.Abort()
return
}
c.Next()
}
}

View File

@@ -85,3 +85,21 @@ func TestCacheControl(t *testing.T) {
TestFunc(c)
a.Contains(c.Writer.Header().Get("Cache-Control"), "no-cache")
}
func TestSandbox(t *testing.T) {
a := assert.New(t)
TestFunc := Sandbox()
rec := httptest.NewRecorder()
c, _ := gin.CreateTestContext(rec)
TestFunc(c)
a.Contains(c.Writer.Header().Get("Content-Security-Policy"), "sandbox")
}
func TestStaticResourceCache(t *testing.T) {
a := assert.New(t)
TestFunc := StaticResourceCache()
rec := httptest.NewRecorder()
c, _ := gin.CreateTestContext(rec)
TestFunc(c)
a.Contains(c.Writer.Header().Get("Cache-Control"), "public, max-age")
}

30
middleware/file.go Normal file
View File

@@ -0,0 +1,30 @@
package middleware
import (
model "github.com/cloudreve/Cloudreve/v3/models"
"github.com/cloudreve/Cloudreve/v3/pkg/serializer"
"github.com/gin-gonic/gin"
)
// ValidateSourceLink validates if the perm source link is a valid redirect link
func ValidateSourceLink() gin.HandlerFunc {
return func(c *gin.Context) {
linkID, ok := c.Get("object_id")
if !ok {
c.JSON(200, serializer.Err(serializer.CodeFileNotFound, "", nil))
c.Abort()
return
}
sourceLink, err := model.GetSourceLinkByID(linkID)
if err != nil || sourceLink.File.ID == 0 || sourceLink.File.Name != c.Param("name") {
c.JSON(200, serializer.Err(serializer.CodeFileNotFound, "", nil))
c.Abort()
return
}
sourceLink.Downloaded()
c.Set("source_link", sourceLink)
c.Next()
}
}

57
middleware/file_test.go Normal file
View File

@@ -0,0 +1,57 @@
package middleware
import (
"github.com/DATA-DOG/go-sqlmock"
"github.com/gin-gonic/gin"
"github.com/stretchr/testify/assert"
"net/http/httptest"
"testing"
)
func TestValidateSourceLink(t *testing.T) {
a := assert.New(t)
rec := httptest.NewRecorder()
testFunc := ValidateSourceLink()
// ID 不存在
{
c, _ := gin.CreateTestContext(rec)
testFunc(c)
a.True(c.IsAborted())
}
// SourceLink 不存在
{
c, _ := gin.CreateTestContext(rec)
c.Set("object_id", 1)
mock.ExpectQuery("SELECT(.+)source_links(.+)").WithArgs(1).WillReturnRows(sqlmock.NewRows([]string{"id"}))
testFunc(c)
a.True(c.IsAborted())
a.NoError(mock.ExpectationsWereMet())
}
// 原文件不存在
{
c, _ := gin.CreateTestContext(rec)
c.Set("object_id", 1)
mock.ExpectQuery("SELECT(.+)source_links(.+)").WithArgs(1).WillReturnRows(sqlmock.NewRows([]string{"id"}).AddRow(1))
mock.ExpectQuery("SELECT(.+)files(.+)").WithArgs(0).WillReturnRows(sqlmock.NewRows([]string{"id"}))
testFunc(c)
a.True(c.IsAborted())
a.NoError(mock.ExpectationsWereMet())
}
// 成功
{
c, _ := gin.CreateTestContext(rec)
c.Set("object_id", 1)
mock.ExpectQuery("SELECT(.+)source_links(.+)").WithArgs(1).WillReturnRows(sqlmock.NewRows([]string{"id", "file_id"}).AddRow(1, 2))
mock.ExpectQuery("SELECT(.+)files(.+)").WithArgs(2).WillReturnRows(sqlmock.NewRows([]string{"id"}).AddRow(2))
mock.ExpectBegin()
mock.ExpectExec("UPDATE(.+)source_links").WillReturnResult(sqlmock.NewResult(1, 1))
testFunc(c)
a.False(c.IsAborted())
a.NoError(mock.ExpectationsWereMet())
}
}

View File

@@ -23,13 +23,13 @@ func FrontendFileHandler() gin.HandlerFunc {
// 读取index.html
file, err := bootstrap.StaticFS.Open("/index.html")
if err != nil {
util.Log().Warning("静态文件[index.html]不存在,可能会影响首页展示")
util.Log().Warning("Static file \"index.html\" does not exist, it might affect the display of the homepage.")
return ignoreFunc
}
fileContentBytes, err := ioutil.ReadAll(file)
if err != nil {
util.Log().Warning("静态文件[index.html]读取失败,可能会影响首页展示")
util.Log().Warning("Cannot read static file \"index.html\", it might affect the display of the homepage.")
return ignoreFunc
}
fileContent := string(fileContentBytes)
@@ -39,7 +39,11 @@ func FrontendFileHandler() gin.HandlerFunc {
path := c.Request.URL.Path
// API 跳过
if strings.HasPrefix(path, "/api") || strings.HasPrefix(path, "/custom") || strings.HasPrefix(path, "/dav") || path == "/manifest.json" {
if strings.HasPrefix(path, "/api") ||
strings.HasPrefix(path, "/custom") ||
strings.HasPrefix(path, "/dav") ||
strings.HasPrefix(path, "/f") ||
path == "/manifest.json" {
c.Next()
return
}

View File

@@ -1,6 +1,9 @@
package middleware
import (
"net/http"
"strings"
"github.com/cloudreve/Cloudreve/v3/pkg/conf"
"github.com/cloudreve/Cloudreve/v3/pkg/serializer"
"github.com/cloudreve/Cloudreve/v3/pkg/util"
@@ -20,17 +23,35 @@ func Session(secret string) gin.HandlerFunc {
var err error
Store, err = redis.NewStoreWithDB(10, conf.RedisConfig.Network, conf.RedisConfig.Server, conf.RedisConfig.Password, conf.RedisConfig.DB, []byte(secret))
if err != nil {
util.Log().Panic("无法连接到 Redis%s", err)
util.Log().Panic("Failed to connect to Redis%s", err)
}
util.Log().Info("已连接到 Redis 服务器:%s", conf.RedisConfig.Server)
util.Log().Info("Connect to Redis server %q.", conf.RedisConfig.Server)
} else {
Store = memstore.NewStore([]byte(secret))
}
sameSiteMode := http.SameSiteDefaultMode
switch strings.ToLower(conf.CORSConfig.SameSite) {
case "default":
sameSiteMode = http.SameSiteDefaultMode
case "none":
sameSiteMode = http.SameSiteNoneMode
case "strict":
sameSiteMode = http.SameSiteStrictMode
case "lax":
sameSiteMode = http.SameSiteLaxMode
}
// Also set Secure: true if using SSL, you should though
// TODO:same-site policy
Store.Options(sessions.Options{HttpOnly: true, MaxAge: 7 * 86400, Path: "/"})
Store.Options(sessions.Options{
HttpOnly: true,
MaxAge: 60 * 86400,
Path: "/",
SameSite: sameSiteMode,
Secure: conf.CORSConfig.Secure,
})
return sessions.Sessions("cloudreve-session", Store)
}
@@ -50,7 +71,7 @@ func CSRFCheck() gin.HandlerFunc {
return
}
c.JSON(200, serializer.Err(serializer.CodeNoPermissionErr, "来源非法", nil))
c.JSON(200, serializer.Err(serializer.CodeNoPermissionErr, "Invalid origin", nil))
c.Abort()
}
}

View File

@@ -16,14 +16,14 @@ func ShareOwner() gin.HandlerFunc {
if userCtx, ok := c.Get("user"); ok {
user = userCtx.(*model.User)
} else {
c.JSON(200, serializer.Err(serializer.CodeCheckLogin, "请先登录", nil))
c.JSON(200, serializer.Err(serializer.CodeCheckLogin, "", nil))
c.Abort()
return
}
if share, ok := c.Get("share"); ok {
if share.(*model.Share).Creator().ID != user.ID {
c.JSON(200, serializer.Err(serializer.CodeNotFound, "分享不存在", nil))
c.JSON(200, serializer.Err(serializer.CodeShareLinkNotFound, "", nil))
c.Abort()
return
}
@@ -46,7 +46,7 @@ func ShareAvailable() gin.HandlerFunc {
share := model.GetShareByHashID(c.Param("id"))
if share == nil || !share.IsAvailable() {
c.JSON(200, serializer.Err(serializer.CodeNotFound, "分享不存在或已失效", nil))
c.JSON(200, serializer.Err(serializer.CodeShareLinkNotFound, "", nil))
c.Abort()
return
}
@@ -65,7 +65,7 @@ func ShareCanPreview() gin.HandlerFunc {
c.Next()
return
}
c.JSON(200, serializer.Err(serializer.CodeNoPermissionErr, "此分享无法预览",
c.JSON(200, serializer.Err(serializer.CodeDisabledSharePreview, "",
nil))
c.Abort()
return
@@ -85,7 +85,7 @@ func CheckShareUnlocked() gin.HandlerFunc {
unlocked := util.GetSession(c, sessionKey) != nil
if !unlocked {
c.JSON(200, serializer.Err(serializer.CodeNoPermissionErr,
"无权访问此分享", nil))
"", nil))
c.Abort()
return
}
@@ -109,7 +109,7 @@ func BeforeShareDownload() gin.HandlerFunc {
// 检查用户是否可以下载此分享的文件
err := share.CanBeDownloadBy(user)
if err != nil {
c.JSON(200, serializer.Err(serializer.CodeNoPermissionErr, err.Error(),
c.JSON(200, serializer.Err(serializer.CodeGroupNotAllowed, err.Error(),
nil))
c.Abort()
return
@@ -118,7 +118,7 @@ func BeforeShareDownload() gin.HandlerFunc {
// 对积分、下载次数进行更新
err = share.DownloadBy(user, c)
if err != nil {
c.JSON(200, serializer.Err(serializer.CodeNoPermissionErr, err.Error(),
c.JSON(200, serializer.Err(serializer.CodeGroupNotAllowed, err.Error(),
nil))
c.Abort()
return

View File

@@ -9,12 +9,11 @@ import (
var defaultSettings = []Setting{
{Name: "siteURL", Value: `http://localhost`, Type: "basic"},
{Name: "siteName", Value: `Cloudreve`, Type: "basic"},
{Name: "siteICPId", Value: ``, Type: "basic"},
{Name: "register_enabled", Value: `1`, Type: "register"},
{Name: "default_group", Value: `2`, Type: "register"},
{Name: "siteKeywords", Value: `网盘,网盘`, Type: "basic"},
{Name: "siteKeywords", Value: `Cloudreve, cloud storage`, Type: "basic"},
{Name: "siteDes", Value: `Cloudreve`, Type: "basic"},
{Name: "siteTitle", Value: `平步云端`, Type: "basic"},
{Name: "siteTitle", Value: `Inclusive cloud storage for everyone`, Type: "basic"},
{Name: "siteScript", Value: ``, Type: "basic"},
{Name: "siteID", Value: uuid.Must(uuid.NewV4()).String(), Type: "basic"},
{Name: "fromName", Value: `Cloudreve`, Type: "mail"},
@@ -114,4 +113,6 @@ Neue',Helvetica,Arial,sans-serif; box-sizing: border-box; font-size: 14px; verti
{Name: "pwa_theme_color", Value: "#000000", Type: "pwa"},
{Name: "pwa_background_color", Value: "#ffffff", Type: "pwa"},
{Name: "office_preview_service", Value: "https://view.officeapps.live.com/op/view.aspx?src={$src}", Type: "preview"},
{Name: "show_app_promotion", Value: "1", Type: "mobile"},
{Name: "public_resource_maxage", Value: "86400", Type: "timeout"},
}

View File

@@ -32,6 +32,7 @@ type Download struct {
// 数据库忽略字段
StatusInfo rpc.StatusInfo `gorm:"-"`
Task *Task `gorm:"-"`
NodeName string `gorm:"-"`
}
// AfterFind 找到下载任务后的钩子处理Status结构
@@ -60,7 +61,7 @@ func (task *Download) BeforeSave() (err error) {
// Create 创建离线下载记录
func (task *Download) Create() (uint, error) {
if err := DB.Create(task).Error; err != nil {
util.Log().Warning("无法插入离线下载记录, %s", err)
util.Log().Warning("Failed to insert download record: %s", err)
return 0, err
}
return task.ID, nil
@@ -69,7 +70,7 @@ func (task *Download) Create() (uint, error) {
// Save 更新
func (task *Download) Save() error {
if err := DB.Save(task).Error; err != nil {
util.Log().Warning("无法更新离线下载记录, %s", err)
util.Log().Warning("Failed to update download record: %s", err)
return err
}
return nil

View File

@@ -4,6 +4,7 @@ import (
"encoding/gob"
"encoding/json"
"errors"
"fmt"
"path"
"time"
@@ -43,7 +44,7 @@ func (file *File) Create() error {
tx := DB.Begin()
if err := tx.Create(file).Error; err != nil {
util.Log().Warning("无法插入文件记录, %s", err)
util.Log().Warning("Failed to insert file record: %s", err)
tx.Rollback()
return err
}
@@ -186,15 +187,20 @@ func RemoveFilesWithSoftLinks(files []File) ([]File, error) {
// 结果值
filteredFiles := make([]File, 0)
// 查询软链接的文件
var filesWithSoftLinks []File
tx := DB
for _, value := range files {
tx = tx.Or("source_name = ? and policy_id = ? and id != ?", value.SourceName, value.PolicyID, value.ID)
if len(files) == 0 {
return filteredFiles, nil
}
result := tx.Find(&filesWithSoftLinks)
if result.Error != nil {
return nil, result.Error
// 查询软链接的文件
filesWithSoftLinks := make([]File, 0)
for _, file := range files {
var softLinkFile File
res := DB.
Where("source_name = ? and policy_id = ? and id != ?", file.SourceName, file.PolicyID, file.ID).
First(&softLinkFile)
if res.Error == nil {
filesWithSoftLinks = append(filesWithSoftLinks, softLinkFile)
}
}
// 过滤具有软连接的文件
@@ -334,6 +340,25 @@ func (file *File) CanCopy() bool {
return file.UploadSessionID == nil
}
// CreateOrGetSourceLink creates a SourceLink model. If the given model exists, the existing
// model will be returned.
func (file *File) CreateOrGetSourceLink() (*SourceLink, error) {
res := &SourceLink{}
err := DB.Set("gorm:auto_preload", true).Where("file_id = ?", file.ID).Find(&res).Error
if err == nil && res.ID > 0 {
return res, nil
}
res.FileID = file.ID
res.Name = file.Name
if err := DB.Save(res).Error; err != nil {
return nil, fmt.Errorf("failed to insert SourceLink: %w", err)
}
res.File = *file
return res, nil
}
/*
实现 webdav.FileInfo 接口
*/

View File

@@ -257,6 +257,19 @@ func TestFile_GetPolicy(t *testing.T) {
}
}
func TestRemoveFilesWithSoftLinks_EmptyArg(t *testing.T) {
asserts := assert.New(t)
// 传入空
{
mock.ExpectQuery("SELECT(.+)files(.+)")
file, err := RemoveFilesWithSoftLinks([]File{})
asserts.Error(mock.ExpectationsWereMet())
asserts.NoError(err)
asserts.Equal(len(file), 0)
DB.Find(&File{})
}
}
func TestRemoveFilesWithSoftLinks(t *testing.T) {
asserts := assert.New(t)
files := []File{
@@ -272,30 +285,34 @@ func TestRemoveFilesWithSoftLinks(t *testing.T) {
},
}
// 传入空文件列表
{
file, err := RemoveFilesWithSoftLinks([]File{})
asserts.NoError(err)
asserts.Empty(file)
}
// 全都没有
{
mock.ExpectQuery("SELECT(.+)files(.+)").
WithArgs("1.txt", 23, 1, "2.txt", 24, 2).
WithArgs("1.txt", 23, 1).
WillReturnRows(sqlmock.NewRows([]string{"id", "policy_id", "source_name"}))
mock.ExpectQuery("SELECT(.+)files(.+)").
WithArgs("2.txt", 24, 2).
WillReturnRows(sqlmock.NewRows([]string{"id", "policy_id", "source_name"}))
file, err := RemoveFilesWithSoftLinks(files)
asserts.NoError(mock.ExpectationsWereMet())
asserts.NoError(err)
asserts.Equal(files, file)
}
// 查询出错
{
mock.ExpectQuery("SELECT(.+)files(.+)").
WithArgs("1.txt", 23, 1, "2.txt", 24, 2).
WillReturnError(errors.New("error"))
file, err := RemoveFilesWithSoftLinks(files)
asserts.NoError(mock.ExpectationsWereMet())
asserts.Error(err)
asserts.Nil(file)
}
// 第二个是软链
{
mock.ExpectQuery("SELECT(.+)files(.+)").
WithArgs("1.txt", 23, 1, "2.txt", 24, 2).
WithArgs("1.txt", 23, 1).
WillReturnRows(sqlmock.NewRows([]string{"id", "policy_id", "source_name"}))
mock.ExpectQuery("SELECT(.+)files(.+)").
WithArgs("2.txt", 24, 2).
WillReturnRows(
sqlmock.NewRows([]string{"id", "policy_id", "source_name"}).
AddRow(3, 24, "2.txt"),
@@ -305,14 +322,18 @@ func TestRemoveFilesWithSoftLinks(t *testing.T) {
asserts.NoError(err)
asserts.Equal(files[:1], file)
}
// 第一个是软链
{
mock.ExpectQuery("SELECT(.+)files(.+)").
WithArgs("1.txt", 23, 1, "2.txt", 24, 2).
WithArgs("1.txt", 23, 1).
WillReturnRows(
sqlmock.NewRows([]string{"id", "policy_id", "source_name"}).
AddRow(3, 23, "1.txt"),
)
mock.ExpectQuery("SELECT(.+)files(.+)").
WithArgs("2.txt", 24, 2).
WillReturnRows(sqlmock.NewRows([]string{"id", "policy_id", "source_name"}))
file, err := RemoveFilesWithSoftLinks(files)
asserts.NoError(mock.ExpectationsWereMet())
asserts.NoError(err)
@@ -321,11 +342,16 @@ func TestRemoveFilesWithSoftLinks(t *testing.T) {
// 全部是软链
{
mock.ExpectQuery("SELECT(.+)files(.+)").
WithArgs("1.txt", 23, 1, "2.txt", 24, 2).
WithArgs("1.txt", 23, 1).
WillReturnRows(
sqlmock.NewRows([]string{"id", "policy_id", "source_name"}).
AddRow(3, 24, "2.txt").
AddRow(4, 23, "1.txt"),
AddRow(3, 23, "1.txt"),
)
mock.ExpectQuery("SELECT(.+)files(.+)").
WithArgs("2.txt", 24, 2).
WillReturnRows(
sqlmock.NewRows([]string{"id", "policy_id", "source_name"}).
AddRow(3, 24, "2.txt"),
)
file, err := RemoveFilesWithSoftLinks(files)
asserts.NoError(mock.ExpectationsWereMet())
@@ -585,3 +611,44 @@ func TestGetFilesByKeywords(t *testing.T) {
asserts.Len(res, 1)
}
}
func TestFile_CreateOrGetSourceLink(t *testing.T) {
a := assert.New(t)
file := &File{}
file.ID = 1
// 已存在,返回老的 SourceLink
{
mock.ExpectQuery("SELECT(.+)source_links(.+)").WithArgs(1).WillReturnRows(sqlmock.NewRows([]string{"id"}).AddRow(2))
res, err := file.CreateOrGetSourceLink()
a.NoError(err)
a.EqualValues(2, res.ID)
a.NoError(mock.ExpectationsWereMet())
}
// 不存在,插入失败
{
expectedErr := errors.New("error")
mock.ExpectQuery("SELECT(.+)source_links(.+)").WithArgs(1).WillReturnRows(sqlmock.NewRows([]string{"id"}))
mock.ExpectBegin()
mock.ExpectExec("INSERT(.+)source_links(.+)").WillReturnError(expectedErr)
mock.ExpectRollback()
res, err := file.CreateOrGetSourceLink()
a.Nil(res)
a.ErrorIs(err, expectedErr)
a.NoError(mock.ExpectationsWereMet())
}
// 成功
{
mock.ExpectQuery("SELECT(.+)source_links(.+)").WithArgs(1).WillReturnRows(sqlmock.NewRows([]string{"id"}))
mock.ExpectBegin()
mock.ExpectExec("INSERT(.+)source_links(.+)").WillReturnResult(sqlmock.NewResult(2, 1))
mock.ExpectCommit()
res, err := file.CreateOrGetSourceLink()
a.NoError(err)
a.EqualValues(2, res.ID)
a.EqualValues(file.ID, res.File.ID)
a.NoError(mock.ExpectationsWereMet())
}
}

View File

@@ -161,7 +161,7 @@ func (folder *Folder) MoveOrCopyFileTo(files []uint, dstFolder *Folder, isCopy b
// 复制文件记录
for _, oldFile := range originFiles {
if !oldFile.CanCopy() {
util.Log().Warning("无法复制正在上传中的文件 [%s] 跳过...", oldFile.Name)
util.Log().Warning("Cannot copy file %q because it's being uploaded now, skipping...", oldFile.Name)
continue
}
@@ -224,8 +224,8 @@ func (folder *Folder) CopyFolderTo(folderID uint, dstFolder *Folder) (size uint6
} else if IDCache, ok := newIDCache[*folder.ParentID]; ok {
newID = IDCache
} else {
util.Log().Warning("无法取得新的父目录:%d", folder.ParentID)
return size, errors.New("无法取得新的父目录")
util.Log().Warning("Failed to get parent folder %q", *folder.ParentID)
return size, errors.New("Failed to get parent folder")
}
// 插入新的目录记录
@@ -254,7 +254,7 @@ func (folder *Folder) CopyFolderTo(folderID uint, dstFolder *Folder) (size uint6
// 复制文件记录
for _, oldFile := range originFiles {
if !oldFile.CanCopy() {
util.Log().Warning("无法复制正在上传中的文件 [%s] 跳过...", oldFile.Name)
util.Log().Warning("Cannot copy file %q because it's being uploaded now, skipping...", oldFile.Name)
continue
}

View File

@@ -23,16 +23,17 @@ type Group struct {
// GroupOption 用户组其他配置
type GroupOption struct {
ArchiveDownload bool `json:"archive_download,omitempty"` // 打包下载
ArchiveTask bool `json:"archive_task,omitempty"` // 在线压缩
CompressSize uint64 `json:"compress_size,omitempty"` // 可压缩大小
DecompressSize uint64 `json:"decompress_size,omitempty"`
OneTimeDownload bool `json:"one_time_download,omitempty"`
ShareDownload bool `json:"share_download,omitempty"`
Aria2 bool `json:"aria2,omitempty"` // 离线下载
Aria2Options map[string]interface{} `json:"aria2_options,omitempty"` // 离线下载用户组配置
SourceBatchSize int `json:"source_batch,omitempty"`
Aria2BatchSize int `json:"aria2_batch,omitempty"`
ArchiveDownload bool `json:"archive_download,omitempty"` // 打包下载
ArchiveTask bool `json:"archive_task,omitempty"` // 在线压缩
CompressSize uint64 `json:"compress_size,omitempty"` // 可压缩大小
DecompressSize uint64 `json:"decompress_size,omitempty"`
OneTimeDownload bool `json:"one_time_download,omitempty"`
ShareDownload bool `json:"share_download,omitempty"`
Aria2 bool `json:"aria2,omitempty"` // 离线下载
Aria2Options map[string]interface{} `json:"aria2_options,omitempty"` // 离线下载用户组配置
SourceBatchSize int `json:"source_batch,omitempty"`
RedirectedSource bool `json:"redirected_source,omitempty"`
Aria2BatchSize int `json:"aria2_batch,omitempty"`
}
// GetGroupByID 用ID获取用户组
@@ -66,7 +67,7 @@ func (group *Group) BeforeSave() (err error) {
return err
}
//SerializePolicyList 将序列后的可选策略列表、配置写入数据库字段
// SerializePolicyList 将序列后的可选策略列表、配置写入数据库字段
// TODO 完善测试
func (group *Group) SerializePolicyList() (err error) {
policies, err := json.Marshal(&group.PolicyList)

View File

@@ -20,7 +20,7 @@ var DB *gorm.DB
// Init 初始化 MySQL 链接
func Init() {
util.Log().Info("初始化数据库连接")
util.Log().Info("Initializing database connection...")
var (
db *gorm.DB
@@ -51,13 +51,13 @@ func Init() {
conf.DatabaseConfig.Name,
conf.DatabaseConfig.Charset))
default:
util.Log().Panic("不支持数据库类型: %s", conf.DatabaseConfig.Type)
util.Log().Panic("Unsupported database type %q.", conf.DatabaseConfig.Type)
}
}
//db.SetLogger(util.Log())
if err != nil {
util.Log().Panic("连接数据库不成功, %s", err)
util.Log().Panic("Failed to connect to database: %s", err)
}
// 处理表前缀

View File

@@ -19,16 +19,16 @@ func needMigration() bool {
return DB.Where("name = ?", "db_version_"+conf.RequiredDBVersion).First(&setting).Error != nil
}
//执行数据迁移
// 执行数据迁移
func migration() {
// 确认是否需要执行迁移
if !needMigration() {
util.Log().Info("数据库版本匹配,跳过数据库迁移")
util.Log().Info("Database version fulfilled, skip schema migration.")
return
}
util.Log().Info("开始进行数据库初始化...")
util.Log().Info("Start initializing database schema...")
// 清除所有缓存
if instance, ok := cache.Store.(*cache.RedisStore); ok {
@@ -41,7 +41,7 @@ func migration() {
}
DB.AutoMigrate(&User{}, &Setting{}, &Group{}, &Policy{}, &Folder{}, &File{}, &Share{},
&Task{}, &Download{}, &Tag{}, &Webdav{}, &Node{})
&Task{}, &Download{}, &Tag{}, &Webdav{}, &Node{}, &SourceLink{})
// 创建初始存储策略
addDefaultPolicy()
@@ -61,7 +61,7 @@ func migration() {
// 执行数据库升级脚本
execUpgradeScripts()
util.Log().Info("数据库初始化结束")
util.Log().Info("Finish initializing database schema.")
}
@@ -70,7 +70,7 @@ func addDefaultPolicy() {
// 未找到初始存储策略时,则创建
if gorm.IsRecordNotFoundError(err) {
defaultPolicy := Policy{
Name: "默认存储策略",
Name: "Default storage policy",
Type: "local",
MaxSize: 0,
AutoRename: true,
@@ -82,7 +82,7 @@ func addDefaultPolicy() {
},
}
if err := DB.Create(&defaultPolicy).Error; err != nil {
util.Log().Panic("无法创建初始存储策略, %s", err)
util.Log().Panic("Failed to create default storage policy: %s", err)
}
}
}
@@ -98,22 +98,23 @@ func addDefaultGroups() {
// 未找到初始管理组时,则创建
if gorm.IsRecordNotFoundError(err) {
defaultAdminGroup := Group{
Name: "管理员",
Name: "Admin",
PolicyList: []uint{1},
MaxStorage: 1 * 1024 * 1024 * 1024,
ShareEnabled: true,
WebDAVEnabled: true,
OptionsSerialized: GroupOption{
ArchiveDownload: true,
ArchiveTask: true,
ShareDownload: true,
Aria2: true,
SourceBatchSize: 1000,
Aria2BatchSize: 50,
ArchiveDownload: true,
ArchiveTask: true,
ShareDownload: true,
Aria2: true,
SourceBatchSize: 1000,
Aria2BatchSize: 50,
RedirectedSource: true,
},
}
if err := DB.Create(&defaultAdminGroup).Error; err != nil {
util.Log().Panic("无法创建管理用户组, %s", err)
util.Log().Panic("Failed to create admin user group: %s", err)
}
}
@@ -122,19 +123,20 @@ func addDefaultGroups() {
// 未找到初始注册会员时,则创建
if gorm.IsRecordNotFoundError(err) {
defaultAdminGroup := Group{
Name: "注册会员",
Name: "User",
PolicyList: []uint{1},
MaxStorage: 1 * 1024 * 1024 * 1024,
ShareEnabled: true,
WebDAVEnabled: true,
OptionsSerialized: GroupOption{
ShareDownload: true,
SourceBatchSize: 10,
Aria2BatchSize: 1,
ShareDownload: true,
SourceBatchSize: 10,
Aria2BatchSize: 1,
RedirectedSource: true,
},
}
if err := DB.Create(&defaultAdminGroup).Error; err != nil {
util.Log().Panic("无法创建初始注册会员用户组, %s", err)
util.Log().Panic("Failed to create initial user group: %s", err)
}
}
@@ -143,7 +145,7 @@ func addDefaultGroups() {
// 未找到初始游客用户组时,则创建
if gorm.IsRecordNotFoundError(err) {
defaultAdminGroup := Group{
Name: "游客",
Name: "Anonymous",
PolicyList: []uint{},
Policies: "[]",
OptionsSerialized: GroupOption{
@@ -151,7 +153,7 @@ func addDefaultGroups() {
},
}
if err := DB.Create(&defaultAdminGroup).Error; err != nil {
util.Log().Panic("无法创建初始游客用户组, %s", err)
util.Log().Panic("Failed to create anonymous user group: %s", err)
}
}
}
@@ -169,15 +171,15 @@ func addDefaultUser() {
defaultUser.GroupID = 1
err := defaultUser.SetPassword(password)
if err != nil {
util.Log().Panic("无法创建密码, %s", err)
util.Log().Panic("Failed to create password: %s", err)
}
if err := DB.Create(&defaultUser).Error; err != nil {
util.Log().Panic("无法创建初始用户, %s", err)
util.Log().Panic("Failed to create initial root user: %s", err)
}
c := color.New(color.FgWhite).Add(color.BgBlack).Add(color.Bold)
util.Log().Info("初始管理员账号:" + c.Sprint("admin@cloudreve.org"))
util.Log().Info("初始管理员密码:" + c.Sprint(password))
util.Log().Info("Admin user name: " + c.Sprint("admin@cloudreve.org"))
util.Log().Info("Admin password: " + c.Sprint(password))
}
}
@@ -186,7 +188,7 @@ func addDefaultNode() {
if gorm.IsRecordNotFoundError(err) {
defaultAdminGroup := Node{
Name: "主机(本机)",
Name: "Master (Local machine)",
Status: NodeActive,
Type: MasterNodeType,
Aria2OptionsSerialized: Aria2Option{
@@ -195,7 +197,7 @@ func addDefaultNode() {
},
}
if err := DB.Create(&defaultAdminGroup).Error; err != nil {
util.Log().Panic("无法创建初始节点记录, %s", err)
util.Log().Panic("Failed to create initial node: %s", err)
}
}
}

View File

@@ -61,6 +61,13 @@ type PolicyOption struct {
ChunkSize uint64 `json:"chunk_size,omitempty"`
// 分片上传时是否需要预留空间
PlaceholderWithSize bool `json:"placeholder_with_size,omitempty"`
// 每秒对存储端的 API 请求上限
TPSLimit float64 `json:"tps_limit,omitempty"`
// 每秒 API 请求爆发上限
TPSLimitBurst int `json:"tps_limit_burst,omitempty"`
// Set this to `true` to force the request to use path-style addressing,
// i.e., `http://s3.amazonaws.com/BUCKET/KEY `
S3ForcePathStyle bool `json:"s3_path_style"`
}
// thumbSuffix 支持缩略图处理的文件扩展名
@@ -118,7 +125,7 @@ func (policy *Policy) BeforeSave() (err error) {
return err
}
//SerializeOptions 将序列后的Option写入到数据库字段
// SerializeOptions 将序列后的Option写入到数据库字段
func (policy *Policy) SerializeOptions() (err error) {
optionsValue, err := json.Marshal(&policy.OptionsSerialized)
policy.Options = string(optionsValue)

View File

@@ -15,12 +15,12 @@ var availableScripts = make(map[string]DBScript)
func RunDBScript(name string, ctx context.Context) error {
if script, ok := availableScripts[name]; ok {
util.Log().Info("开始执行数据库脚本 [%s]", name)
util.Log().Info("Start executing database script %q.", name)
script.Run(ctx)
return nil
}
return fmt.Errorf("数据库脚本 [%s] 不存在", name)
return fmt.Errorf("Database script %q not exist.", name)
}
func Register(name string, script DBScript) {

View File

@@ -14,7 +14,7 @@ func (script ResetAdminPassword) Run(ctx context.Context) {
// 查找用户
user, err := model.GetUserByID(1)
if err != nil {
util.Log().Panic("初始管理员用户不存在, %s", err)
util.Log().Panic("Initial admin user not exist: %s", err)
}
// 生成密码
@@ -23,9 +23,9 @@ func (script ResetAdminPassword) Run(ctx context.Context) {
// 更改为新密码
user.SetPassword(password)
if err := user.Update(map[string]interface{}{"password": user.Password}); err != nil {
util.Log().Panic("密码更改失败, %s", err)
util.Log().Panic("Failed to update password: %s", err)
}
c := color.New(color.FgWhite).Add(color.BgBlack).Add(color.Bold)
util.Log().Info("初始管理员密码已更改为:" + c.Sprint(password))
util.Log().Info("Initial admin user password changed to:" + c.Sprint(password))
}

View File

@@ -25,9 +25,9 @@ func (script UserStorageCalibration) Run(ctx context.Context) {
model.DB.Model(&model.File{}).Where("user_id = ?", user.ID).Select("sum(size) as total").Scan(&total)
// 更新用户的容量
if user.Storage != total.Total {
util.Log().Info("将用户 [%s] 的容量由 %d 校准为 %d", user.Email,
util.Log().Info("Calibrate used storage for user %q, from %d to %d.", user.Email,
user.Storage, total.Total)
model.DB.Model(&user).Update("storage", total.Total)
}
model.DB.Model(&user).Update("storage", total.Total)
}
}

View File

@@ -52,6 +52,9 @@ func TestUserStorageCalibration_Run(t *testing.T) {
mock.ExpectQuery("SELECT(.+)files(.+)").
WithArgs(1).
WillReturnRows(sqlmock.NewRows([]string{"total"}).AddRow(10))
mock.ExpectBegin()
mock.ExpectExec("UPDATE(.+)").WillReturnResult(sqlmock.NewResult(1, 1))
mock.ExpectCommit()
script.Run(context.Background())
asserts.NoError(mock.ExpectationsWereMet())
}

View File

@@ -36,7 +36,7 @@ type Share struct {
// Create 创建分享
func (share *Share) Create() (uint, error) {
if err := DB.Create(share).Error; err != nil {
util.Log().Warning("无法插入数据库记录, %s", err)
util.Log().Warning("Failed to insert share record: %s", err)
return 0, err
}
return share.ID, nil
@@ -131,9 +131,9 @@ func (share *Share) CanBeDownloadBy(user *User) error {
// 用户组权限
if !user.Group.OptionsSerialized.ShareDownload {
if user.IsAnonymous() {
return errors.New("未登录用户无法下载")
return errors.New("you must login to download")
}
return errors.New("您当前的用户组无权下载")
return errors.New("your group has no permission to download")
}
return nil
}

47
models/source_link.go Normal file
View File

@@ -0,0 +1,47 @@
package model
import (
"fmt"
"github.com/cloudreve/Cloudreve/v3/pkg/hashid"
"github.com/jinzhu/gorm"
"net/url"
)
// SourceLink represent a shared file source link
type SourceLink struct {
gorm.Model
FileID uint // corresponding file ID
Name string // name of the file while creating the source link, for annotation
Downloads int // 下载数
// 关联模型
File File `gorm:"save_associations:false:false"`
}
// Link gets the URL of a SourceLink
func (s *SourceLink) Link() (string, error) {
baseURL := GetSiteURL()
linkPath, err := url.Parse(fmt.Sprintf("/f/%s/%s", hashid.HashID(s.ID, hashid.SourceLinkID), s.File.Name))
if err != nil {
return "", err
}
return baseURL.ResolveReference(linkPath).String(), nil
}
// GetTasksByID queries source link based on ID
func GetSourceLinkByID(id interface{}) (*SourceLink, error) {
link := &SourceLink{}
result := DB.Where("id = ?", id).First(link)
files, _ := GetFilesByIDs([]uint{link.FileID}, 0)
if len(files) > 0 {
link.File = files[0]
}
return link, result.Error
}
// Viewed 增加访问次数
func (s *SourceLink) Downloaded() {
s.Downloads++
DB.Model(s).UpdateColumn("downloads", gorm.Expr("downloads + ?", 1))
}

View File

@@ -0,0 +1,52 @@
package model
import (
"github.com/DATA-DOG/go-sqlmock"
"github.com/stretchr/testify/assert"
"testing"
)
func TestSourceLink_Link(t *testing.T) {
a := assert.New(t)
s := &SourceLink{}
s.ID = 1
// 失败
{
s.File.Name = string([]byte{0x7f})
res, err := s.Link()
a.Error(err)
a.Empty(res)
}
// 成功
{
s.File.Name = "filename"
res, err := s.Link()
a.NoError(err)
a.Contains(res, s.Name)
}
}
func TestGetSourceLinkByID(t *testing.T) {
a := assert.New(t)
mock.ExpectQuery("SELECT(.+)source_links(.+)").WithArgs(1).WillReturnRows(sqlmock.NewRows([]string{"id", "file_id"}).AddRow(1, 2))
mock.ExpectQuery("SELECT(.+)files(.+)").WithArgs(2).WillReturnRows(sqlmock.NewRows([]string{"id"}).AddRow(2))
res, err := GetSourceLinkByID(1)
a.NoError(err)
a.NotNil(res)
a.EqualValues(2, res.File.ID)
a.NoError(mock.ExpectationsWereMet())
}
func TestSourceLink_Downloaded(t *testing.T) {
a := assert.New(t)
s := &SourceLink{}
s.ID = 1
mock.ExpectBegin()
mock.ExpectExec("UPDATE(.+)source_links(.+)").WillReturnResult(sqlmock.NewResult(1, 1))
mock.ExpectCommit()
s.Downloaded()
a.NoError(mock.ExpectationsWereMet())
}

View File

@@ -26,7 +26,7 @@ const (
// Create 创建标签记录
func (tag *Tag) Create() (uint, error) {
if err := DB.Create(tag).Error; err != nil {
util.Log().Warning("无法插入离线下载记录, %s", err)
util.Log().Warning("Failed to insert tag record: %s", err)
return 0, err
}
return tag.ID, nil

View File

@@ -19,7 +19,7 @@ type Task struct {
// Create 创建任务记录
func (task *Task) Create() (uint, error) {
if err := DB.Create(task).Error; err != nil {
util.Log().Warning("无法插入任务记录, %s", err)
util.Log().Warning("Failed to insert task record: %s", err)
return 0, err
}
return task.ID, nil

View File

@@ -3,8 +3,6 @@ package aria2
import (
"context"
"fmt"
"github.com/cloudreve/Cloudreve/v3/pkg/cluster"
"github.com/cloudreve/Cloudreve/v3/pkg/mq"
"net/url"
"sync"
"time"
@@ -14,6 +12,8 @@ import (
"github.com/cloudreve/Cloudreve/v3/pkg/aria2/monitor"
"github.com/cloudreve/Cloudreve/v3/pkg/aria2/rpc"
"github.com/cloudreve/Cloudreve/v3/pkg/balancer"
"github.com/cloudreve/Cloudreve/v3/pkg/cluster"
"github.com/cloudreve/Cloudreve/v3/pkg/mq"
)
// Instance 默认使用的Aria2处理实例
@@ -40,7 +40,7 @@ func Init(isReload bool, pool cluster.Pool, mqClient mq.MQ) {
if !isReload {
// 从数据库中读取未完成任务,创建监控
unfinished := model.GetDownloadsByStatus(common.Ready, common.Paused, common.Downloading)
unfinished := model.GetDownloadsByStatus(common.Ready, common.Paused, common.Downloading, common.Seeding)
for i := 0; i < len(unfinished); i++ {
// 创建任务监控

View File

@@ -46,13 +46,15 @@ const (
Canceled
// Unknown 未知状态
Unknown
// Seeding 做种中
Seeding
)
var (
// ErrNotEnabled 功能未开启错误
ErrNotEnabled = serializer.NewError(serializer.CodeNoPermissionErr, "离线下载功能未开启", nil)
ErrNotEnabled = serializer.NewError(serializer.CodeFeatureNotEnabled, "not enabled", nil)
// ErrUserNotFound 未找到下载任务创建者
ErrUserNotFound = serializer.NewError(serializer.CodeNotFound, "无法找到任务创建者", nil)
ErrUserNotFound = serializer.NewError(serializer.CodeUserNotFound, "", nil)
)
// DummyAria2 未开启Aria2功能时使用的默认处理器
@@ -94,11 +96,14 @@ func (instance *DummyAria2) DeleteTempFile(src *model.Download) error {
}
// GetStatus 将给定的状态字符串转换为状态标识数字
func GetStatus(status string) int {
switch status {
func GetStatus(status rpc.StatusInfo) int {
switch status.Status {
case "complete":
return Complete
case "active":
if status.BitTorrent.Mode != "" && status.CompletedLength == status.TotalLength {
return Seeding
}
return Downloading
case "waiting":
return Ready

View File

@@ -1,9 +1,11 @@
package common
import (
model "github.com/cloudreve/Cloudreve/v3/models"
"github.com/stretchr/testify/assert"
"testing"
model "github.com/cloudreve/Cloudreve/v3/models"
"github.com/cloudreve/Cloudreve/v3/pkg/aria2/rpc"
"github.com/stretchr/testify/assert"
)
func TestDummyAria2(t *testing.T) {
@@ -35,11 +37,18 @@ func TestDummyAria2(t *testing.T) {
func TestGetStatus(t *testing.T) {
a := assert.New(t)
a.Equal(GetStatus("complete"), Complete)
a.Equal(GetStatus("active"), Downloading)
a.Equal(GetStatus("waiting"), Ready)
a.Equal(GetStatus("paused"), Paused)
a.Equal(GetStatus("error"), Error)
a.Equal(GetStatus("removed"), Canceled)
a.Equal(GetStatus("unknown"), Unknown)
a.Equal(GetStatus(rpc.StatusInfo{Status: "complete"}), Complete)
a.Equal(GetStatus(rpc.StatusInfo{Status: "active",
BitTorrent: rpc.BitTorrentInfo{Mode: ""}}), Downloading)
a.Equal(GetStatus(rpc.StatusInfo{Status: "active",
BitTorrent: rpc.BitTorrentInfo{Mode: "single"},
TotalLength: "100", CompletedLength: "50"}), Downloading)
a.Equal(GetStatus(rpc.StatusInfo{Status: "active",
BitTorrent: rpc.BitTorrentInfo{Mode: "multi"},
TotalLength: "100", CompletedLength: "100"}), Seeding)
a.Equal(GetStatus(rpc.StatusInfo{Status: "waiting"}), Ready)
a.Equal(GetStatus(rpc.StatusInfo{Status: "paused"}), Paused)
a.Equal(GetStatus(rpc.StatusInfo{Status: "error"}), Error)
a.Equal(GetStatus(rpc.StatusInfo{Status: "removed"}), Canceled)
a.Equal(GetStatus(rpc.StatusInfo{Status: "unknown"}), Unknown)
}

View File

@@ -45,7 +45,7 @@ func NewMonitor(task *model.Download, pool cluster.Pool, mqClient mq.MQ) {
monitor.notifier = mqClient.Subscribe(monitor.Task.GID, 0)
} else {
monitor.setErrorStatus(errors.New("节点不可用"))
monitor.setErrorStatus(errors.New("node not avaliable"))
}
}
@@ -77,11 +77,12 @@ func (monitor *Monitor) Update() bool {
if err != nil {
monitor.retried++
util.Log().Warning("无法获取下载任务[%s]的状态,%s", monitor.Task.GID, err)
util.Log().Warning("Cannot get status of download task %q: %s", monitor.Task.GID, err)
// 十次重试后认定为任务失败
if monitor.retried > MAX_RETRY {
util.Log().Warning("无法获取下载任务[%s]的状态,超过最大重试次数限制,%s", monitor.Task.GID, err)
util.Log().Warning("Cannot get status of download task %qexceed maximum retry threshold: %s",
monitor.Task.GID, err)
monitor.setErrorStatus(err)
monitor.RemoveTempFolder()
return true
@@ -93,7 +94,7 @@ func (monitor *Monitor) Update() bool {
// 磁力链下载需要跟随
if len(status.FollowedBy) > 0 {
util.Log().Debug("离线下载[%s]重定向至[%s]", monitor.Task.GID, status.FollowedBy[0])
util.Log().Debug("Redirected download task from %q to %q.", monitor.Task.GID, status.FollowedBy[0])
monitor.Task.GID = status.FollowedBy[0]
monitor.Task.Save()
return false
@@ -101,28 +102,28 @@ func (monitor *Monitor) Update() bool {
// 更新任务信息
if err := monitor.UpdateTaskInfo(status); err != nil {
util.Log().Warning("无法更新下载任务[%s]的任务信息[%s]", monitor.Task.GID, err)
util.Log().Warning("Failed to update status of download task %q: %s", monitor.Task.GID, err)
monitor.setErrorStatus(err)
monitor.RemoveTempFolder()
return true
}
util.Log().Debug("离线下载[%s]更新状态[%s]", status.Gid, status.Status)
util.Log().Debug("Remote download %q status updated to %q.", status.Gid, status.Status)
switch status.Status {
case "complete":
switch common.GetStatus(status) {
case common.Complete, common.Seeding:
return monitor.Complete(task.TaskPoll)
case "error":
case common.Error:
return monitor.Error(status)
case "active", "waiting", "paused":
case common.Downloading, common.Ready, common.Paused:
return false
case "removed":
case common.Canceled:
monitor.Task.Status = common.Canceled
monitor.Task.Save()
monitor.RemoveTempFolder()
return true
default:
util.Log().Warning("下载任务[%s]返回未知状态信息[%s]", monitor.Task.GID, status.Status)
util.Log().Warning("Download task %q returns unknown status %q.", monitor.Task.GID, status.Status)
return true
}
}
@@ -132,7 +133,7 @@ func (monitor *Monitor) UpdateTaskInfo(status rpc.StatusInfo) error {
originSize := monitor.Task.TotalSize
monitor.Task.GID = status.Gid
monitor.Task.Status = common.GetStatus(status.Status)
monitor.Task.Status = common.GetStatus(status)
// 文件大小、已下载大小
total, err := strconv.ParseUint(status.TotalLength, 10, 64)
@@ -235,6 +236,40 @@ func (monitor *Monitor) RemoveTempFolder() {
// Complete 完成下载,返回是否中断监控
func (monitor *Monitor) Complete(pool task.Pool) bool {
// 未开始转存,提交转存任务
if monitor.Task.TaskID == 0 {
return monitor.transfer(pool)
}
// 做种完成
if common.GetStatus(monitor.Task.StatusInfo) == common.Complete {
transferTask, err := model.GetTasksByID(monitor.Task.TaskID)
if err != nil {
monitor.setErrorStatus(err)
monitor.RemoveTempFolder()
return true
}
// 转存完成,回收下载目录
if transferTask.Type == task.TransferTaskType && transferTask.Status >= task.Error {
job, err := task.NewRecycleTask(monitor.Task)
if err != nil {
monitor.setErrorStatus(err)
monitor.RemoveTempFolder()
return true
}
// 提交回收任务
pool.Submit(job)
return true
}
}
return false
}
func (monitor *Monitor) transfer(pool task.Pool) bool {
// 创建中转任务
file := make([]string, 0, len(monitor.Task.StatusInfo.Files))
sizes := make(map[string]uint64, len(monitor.Task.StatusInfo.Files))
@@ -269,7 +304,7 @@ func (monitor *Monitor) Complete(pool task.Pool) bool {
monitor.Task.TaskID = job.Model().ID
monitor.Task.Save()
return true
return false
}
func (monitor *Monitor) setErrorStatus(err error) {

View File

@@ -3,6 +3,8 @@ package monitor
import (
"database/sql"
"errors"
"testing"
"github.com/DATA-DOG/go-sqlmock"
model "github.com/cloudreve/Cloudreve/v3/models"
"github.com/cloudreve/Cloudreve/v3/pkg/aria2/common"
@@ -13,7 +15,6 @@ import (
"github.com/jinzhu/gorm"
"github.com/stretchr/testify/assert"
testMock "github.com/stretchr/testify/mock"
"testing"
)
var mock sqlmock.Sqlmock
@@ -431,6 +432,14 @@ func TestMonitor_Complete(t *testing.T) {
mock.ExpectExec("UPDATE(.+)downloads").WillReturnResult(sqlmock.NewResult(1, 1))
mock.ExpectCommit()
mock.ExpectQuery("SELECT(.+)tasks").WillReturnRows(sqlmock.NewRows([]string{"id", "type", "status"}).AddRow(1, 2, 4))
mock.ExpectQuery("SELECT(.+)users").WillReturnRows(sqlmock.NewRows([]string{"id"}).AddRow(9414))
mock.ExpectBegin()
mock.ExpectExec("INSERT(.+)tasks").WillReturnResult(sqlmock.NewResult(2, 1))
mock.ExpectCommit()
a.False(m.Complete(mockPool))
m.Task.StatusInfo.Status = "complete"
a.True(m.Complete(mockPool))
a.NoError(mock.ExpectationsWereMet())
mockNode.AssertExpectations(t)

View File

@@ -4,35 +4,27 @@ package rpc
// StatusInfo represents response of aria2.tellStatus
type StatusInfo struct {
Gid string `json:"gid"` // GID of the download.
Status string `json:"status"` // active for currently downloading/seeding downloads. waiting for downloads in the queue; download is not started. paused for paused downloads. error for downloads that were stopped because of error. complete for stopped and completed downloads. removed for the downloads removed by user.
TotalLength string `json:"totalLength"` // Total length of the download in bytes.
CompletedLength string `json:"completedLength"` // Completed length of the download in bytes.
UploadLength string `json:"uploadLength"` // Uploaded length of the download in bytes.
BitField string `json:"bitfield"` // Hexadecimal representation of the download progress. The highest bit corresponds to the piece at index 0. Any set bits indicate loaded pieces, while unset bits indicate not yet loaded and/or missing pieces. Any overflow bits at the end are set to zero. When the download was not started yet, this key will not be included in the response.
DownloadSpeed string `json:"downloadSpeed"` // Download speed of this download measured in bytes/sec.
UploadSpeed string `json:"uploadSpeed"` // LocalUpload speed of this download measured in bytes/sec.
InfoHash string `json:"infoHash"` // InfoHash. BitTorrent only.
NumSeeders string `json:"numSeeders"` // The number of seeders aria2 has connected to. BitTorrent only.
Seeder string `json:"seeder"` // true if the local endpoint is a seeder. Otherwise false. BitTorrent only.
PieceLength string `json:"pieceLength"` // Piece length in bytes.
NumPieces string `json:"numPieces"` // The number of pieces.
Connections string `json:"connections"` // The number of peers/servers aria2 has connected to.
ErrorCode string `json:"errorCode"` // The code of the last error for this item, if any. The value is a string. The error codes are defined in the EXIT STATUS section. This value is only available for stopped/completed downloads.
ErrorMessage string `json:"errorMessage"` // The (hopefully) human readable error message associated to errorCode.
FollowedBy []string `json:"followedBy"` // List of GIDs which are generated as the result of this download. For example, when aria2 downloads a Metalink file, it generates downloads described in the Metalink (see the --follow-metalink option). This value is useful to track auto-generated downloads. If there are no such downloads, this key will not be included in the response.
BelongsTo string `json:"belongsTo"` // GID of a parent download. Some downloads are a part of another download. For example, if a file in a Metalink has BitTorrent resources, the downloads of ".torrent" files are parts of that parent. If this download has no parent, this key will not be included in the response.
Dir string `json:"dir"` // Directory to save files.
Files []FileInfo `json:"files"` // Returns the list of files. The elements of this list are the same structs used in aria2.getFiles() method.
BitTorrent struct {
AnnounceList [][]string `json:"announceList"` // List of lists of announce URIs. If the torrent contains announce and no announce-list, announce is converted to the announce-list format.
Comment string `json:"comment"` // The comment of the torrent. comment.utf-8 is used if available.
CreationDate int64 `json:"creationDate"` // The creation time of the torrent. The value is an integer since the epoch, measured in seconds.
Mode string `json:"mode"` // File mode of the torrent. The value is either single or multi.
Info struct {
Name string `json:"name"` // name in info dictionary. name.utf-8 is used if available.
} `json:"info"` // Struct which contains data from Info dictionary. It contains following keys.
} `json:"bittorrent"` // Struct which contains information retrieved from the .torrent (file). BitTorrent only. It contains following keys.
Gid string `json:"gid"` // GID of the download.
Status string `json:"status"` // active for currently downloading/seeding downloads. waiting for downloads in the queue; download is not started. paused for paused downloads. error for downloads that were stopped because of error. complete for stopped and completed downloads. removed for the downloads removed by user.
TotalLength string `json:"totalLength"` // Total length of the download in bytes.
CompletedLength string `json:"completedLength"` // Completed length of the download in bytes.
UploadLength string `json:"uploadLength"` // Uploaded length of the download in bytes.
BitField string `json:"bitfield"` // Hexadecimal representation of the download progress. The highest bit corresponds to the piece at index 0. Any set bits indicate loaded pieces, while unset bits indicate not yet loaded and/or missing pieces. Any overflow bits at the end are set to zero. When the download was not started yet, this key will not be included in the response.
DownloadSpeed string `json:"downloadSpeed"` // Download speed of this download measured in bytes/sec.
UploadSpeed string `json:"uploadSpeed"` // LocalUpload speed of this download measured in bytes/sec.
InfoHash string `json:"infoHash"` // InfoHash. BitTorrent only.
NumSeeders string `json:"numSeeders"` // The number of seeders aria2 has connected to. BitTorrent only.
Seeder string `json:"seeder"` // true if the local endpoint is a seeder. Otherwise false. BitTorrent only.
PieceLength string `json:"pieceLength"` // Piece length in bytes.
NumPieces string `json:"numPieces"` // The number of pieces.
Connections string `json:"connections"` // The number of peers/servers aria2 has connected to.
ErrorCode string `json:"errorCode"` // The code of the last error for this item, if any. The value is a string. The error codes are defined in the EXIT STATUS section. This value is only available for stopped/completed downloads.
ErrorMessage string `json:"errorMessage"` // The (hopefully) human readable error message associated to errorCode.
FollowedBy []string `json:"followedBy"` // List of GIDs which are generated as the result of this download. For example, when aria2 downloads a Metalink file, it generates downloads described in the Metalink (see the --follow-metalink option). This value is useful to track auto-generated downloads. If there are no such downloads, this key will not be included in the response.
BelongsTo string `json:"belongsTo"` // GID of a parent download. Some downloads are a part of another download. For example, if a file in a Metalink has BitTorrent resources, the downloads of ".torrent" files are parts of that parent. If this download has no parent, this key will not be included in the response.
Dir string `json:"dir"` // Directory to save files.
Files []FileInfo `json:"files"` // Returns the list of files. The elements of this list are the same structs used in aria2.getFiles() method.
BitTorrent BitTorrentInfo `json:"bittorrent"` // Struct which contains information retrieved from the .torrent (file). BitTorrent only. It contains following keys.
}
// URIInfo represents an element of response of aria2.getUris
@@ -100,3 +92,13 @@ type Method struct {
Name string `json:"methodName"` // Method name to call
Params []interface{} `json:"params"` // Array containing parameters to the method call
}
type BitTorrentInfo struct {
AnnounceList [][]string `json:"announceList"` // List of lists of announce URIs. If the torrent contains announce and no announce-list, announce is converted to the announce-list format.
Comment string `json:"comment"` // The comment of the torrent. comment.utf-8 is used if available.
CreationDate int64 `json:"creationDate"` // The creation time of the torrent. The value is an integer since the epoch, measured in seconds.
Mode string `json:"mode"` // File mode of the torrent. The value is either single or multi.
Info struct {
Name string `json:"name"` // name in info dictionary. name.utf-8 is used if available.
} `json:"info"` // Struct which contains data from Info dictionary. It contains following keys.
}

View File

@@ -17,10 +17,10 @@ import (
)
var (
ErrAuthFailed = serializer.NewError(serializer.CodeNoPermissionErr, "鉴权失败", nil)
ErrAuthFailed = serializer.NewError(serializer.CodeInvalidSign, "invalid sign", nil)
ErrAuthHeaderMissing = serializer.NewError(serializer.CodeNoPermissionErr, "authorization header is missing", nil)
ErrExpiresMissing = serializer.NewError(serializer.CodeNoPermissionErr, "expire timestamp is missing", nil)
ErrExpired = serializer.NewError(serializer.CodeSignExpired, "签名已过期", nil)
ErrExpired = serializer.NewError(serializer.CodeSignExpired, "signature expired", nil)
)
const CrHeaderPrefix = "X-Cr-"
@@ -136,7 +136,7 @@ func Init() {
} else {
secretKey = conf.SlaveConfig.Secret
if secretKey == "" {
util.Log().Panic("未指定 SlaveSecret,请前往配置文件中指定")
util.Log().Panic("SlaveSecret is not set, please specify it in config file.")
}
}
General = HMACAuth{

2
pkg/cache/driver.go vendored
View File

@@ -24,7 +24,7 @@ func Init(isSlave bool) {
if isSlave {
err := Store.Sets(conf.OptionOverwrite, "setting_")
if err != nil {
util.Log().Warning("无法覆盖数据库设置: %s", err)
util.Log().Warning("Failed to overwrite database setting: %s", err)
}
}
}

2
pkg/cache/memo.go vendored
View File

@@ -53,7 +53,7 @@ func (store *MemoStore) GarbageCollect() {
store.Store.Range(func(key, value interface{}) bool {
if item, ok := value.(itemWithTTL); ok {
if item.expires > 0 && item.expires < time.Now().Unix() {
util.Log().Debug("回收垃圾[%s]", key.(string))
util.Log().Debug("Cache %q is garbage collected.", key.(string))
store.Store.Delete(key)
}
}

2
pkg/cache/redis.go vendored
View File

@@ -66,7 +66,7 @@ func NewRedisStore(size int, network, address, password, database string) *Redis
redis.DialPassword(password),
)
if err != nil {
util.Log().Warning("无法创建Redis连接%s", err)
util.Log().Warning("Failed to create Redis connection: %s", err)
return nil, err
}
return c, nil

View File

@@ -8,5 +8,5 @@ import (
var (
ErrFeatureNotExist = errors.New("No nodes in nodepool match the feature specificed")
ErrIlegalPath = errors.New("path out of boundary of setting temp folder")
ErrMasterNotFound = serializer.NewError(serializer.CodeMasterNotFound, "未知的主机节点", nil)
ErrMasterNotFound = serializer.NewError(serializer.CodeMasterNotFound, "Unknown master node id", nil)
)

View File

@@ -161,7 +161,7 @@ func (r *rpcService) Init() error {
// 解析RPC服务地址
server, err := url.Parse(r.parent.Model.Aria2OptionsSerialized.Server)
if err != nil {
util.Log().Warning("无法解析主机 Aria2 RPC 服务地址,%s", err)
util.Log().Warning("Failed to parse Aria2 RPC server URL: %s", err)
return err
}
server.Path = "/jsonrpc"
@@ -171,7 +171,7 @@ func (r *rpcService) Init() error {
if r.parent.Model.Aria2OptionsSerialized.Options != "" {
err = json.Unmarshal([]byte(r.parent.Model.Aria2OptionsSerialized.Options), &globalOptions)
if err != nil {
util.Log().Warning("无法解析主机 Aria2 配置,%s", err)
util.Log().Warning("Failed to parse aria2 options: %s", err)
return err
}
}
@@ -221,7 +221,7 @@ func (r *rpcService) Status(task *model.Download) (rpc.StatusInfo, error) {
res, err := r.Caller.TellStatus(task.GID)
if err != nil {
// 失败后重试
util.Log().Debug("无法获取离线下载状态,%s稍后重试", err)
util.Log().Debug("Failed to get download task status, please retry later: %s", err)
time.Sleep(r.retryDuration)
res, err = r.Caller.TellStatus(task.GID)
}
@@ -233,7 +233,7 @@ func (r *rpcService) Cancel(task *model.Download) error {
// 取消下载任务
_, err := r.Caller.Remove(task.GID)
if err != nil {
util.Log().Warning("无法取消离线下载任务[%s], %s", task.GID, err)
util.Log().Warning("Failed to cancel task %q: %s", task.GID, err)
}
return err
@@ -264,7 +264,7 @@ func (s *rpcService) DeleteTempFile(task *model.Download) error {
time.Sleep(d)
err := os.RemoveAll(src)
if err != nil {
util.Log().Warning("无法删除离线下载临时目录[%s], %s", src, err)
util.Log().Warning("Failed to delete temp download folder: %q: %s", src, err)
}
}(s.deletePaddingDuration, task.Parent)

View File

@@ -42,7 +42,7 @@ func Init() {
Default = &NodePool{}
Default.Init()
if err := Default.initFromDB(); err != nil {
util.Log().Warning("节点池初始化失败, %s", err)
util.Log().Warning("Failed to initialize node pool: %s", err)
}
}
@@ -83,7 +83,7 @@ func (pool *NodePool) GetNodeByID(id uint) Node {
}
func (pool *NodePool) nodeStatusChange(isActive bool, id uint) {
util.Log().Debug("从机节点 [ID=%d] 状态变更 [Active=%t]", id, isActive)
util.Log().Debug("Slave node [ID=%d] status changed to [Active=%t].", id, isActive)
var node Node
pool.lock.Lock()
if n, ok := pool.inactive[id]; ok {

View File

@@ -172,7 +172,7 @@ func (node *SlaveNode) StartPingLoop() {
recoverDuration := time.Duration(model.GetIntSetting("slave_recover_interval", 600)) * time.Second
pingTicker := time.Duration(0)
util.Log().Debug("从机节点 [%s] 启动心跳循环", node.Model.Name)
util.Log().Debug("Slave node %q heartbeat loop started.", node.Model.Name)
retry := 0
recoverMode := false
isFirstLoop := true
@@ -185,39 +185,39 @@ loop:
pingTicker = tickDuration
}
util.Log().Debug("从机节点 [%s] 发送Ping", node.Model.Name)
util.Log().Debug("Slave node %q send ping.", node.Model.Name)
res, err := node.Ping(node.getHeartbeatContent(isFirstLoop))
isFirstLoop = false
if err != nil {
util.Log().Debug("Ping从机节点 [%s] 时发生错误: %s", node.Model.Name, err)
util.Log().Debug("Error while ping slave node %q: %s", node.Model.Name, err)
retry++
if retry >= model.GetIntSetting("slave_node_retry", 3) {
util.Log().Debug("从机节点 [%s] Ping 重试已达到最大限制,将从机节点标记为不可用", node.Model.Name)
util.Log().Debug("Retry threshold for pinging slave node %q exceeded, mark it as offline.", node.Model.Name)
node.changeStatus(false)
if !recoverMode {
// 启动恢复监控循环
util.Log().Debug("从机节点 [%s] 进入恢复模式", node.Model.Name)
util.Log().Debug("Slave node %q entered recovery mode.", node.Model.Name)
pingTicker = recoverDuration
recoverMode = true
}
}
} else {
if recoverMode {
util.Log().Debug("从机节点 [%s] 复活", node.Model.Name)
util.Log().Debug("Slave node %q recovered.", node.Model.Name)
pingTicker = tickDuration
recoverMode = false
isFirstLoop = true
}
util.Log().Debug("从机节点 [%s] 状态: %s", node.Model.Name, res)
util.Log().Debug("Status of slave node %q: %s", node.Model.Name, res)
node.changeStatus(true)
retry = 0
}
case <-node.close:
util.Log().Debug("从机节点 [%s] 收到关闭信号", node.Model.Name)
util.Log().Debug("Slave node %q received shutdown signal.", node.Model.Name)
break loop
}
}
@@ -421,7 +421,7 @@ func RemoteCallback(url string, body serializer.UploadCallback) error {
Data: body,
})
if err != nil {
return serializer.NewError(serializer.CodeCallbackError, "无法编码回调正文", err)
return serializer.NewError(serializer.CodeCallbackError, "Failed to encode callback content", err)
}
resp := request.GeneralClient.Request(
@@ -433,13 +433,13 @@ func RemoteCallback(url string, body serializer.UploadCallback) error {
)
if resp.Err != nil {
return serializer.NewError(serializer.CodeCallbackError, "从机无法发起回调请求", resp.Err)
return serializer.NewError(serializer.CodeCallbackError, "Slave cannot send callback request", resp.Err)
}
// 解析回调服务端响应
response, err := resp.DecodeResponse()
if err != nil {
msg := fmt.Sprintf("从机无法解析主机返回的响应 (StatusCode=%d)", resp.Response.StatusCode)
msg := fmt.Sprintf("Slave cannot parse callback response from master (StatusCode=%d).", resp.Response.StatusCode)
return serializer.NewError(serializer.CodeCallbackError, msg, err)
}

View File

@@ -3,7 +3,7 @@ package conf
import (
"github.com/cloudreve/Cloudreve/v3/pkg/util"
"github.com/go-ini/ini"
"gopkg.in/go-playground/validator.v9"
"github.com/go-playground/validator/v10"
)
// database 数据库
@@ -26,6 +26,8 @@ type system struct {
Debug bool
SessionSecret string
HashIDSalt string
GracePeriod int `validate:"gte=0"`
ProxyHeader string `validate:"required_with=Listen"`
}
type ssl struct {
@@ -36,6 +38,7 @@ type ssl struct {
type unix struct {
Listen string
Perm uint32
}
// slave 作为slave存储端配置
@@ -60,6 +63,8 @@ type cors struct {
AllowHeaders []string
AllowCredentials bool
ExposeHeaders []string
SameSite string
Secure bool
}
var cfg *ini.File
@@ -84,13 +89,13 @@ func Init(path string) {
}, defaultConf)
f, err := util.CreatNestedFile(path)
if err != nil {
util.Log().Panic("无法创建配置文件, %s", err)
util.Log().Panic("Failed to create config file: %s", err)
}
// 写入配置文件
_, err = f.WriteString(confContent)
if err != nil {
util.Log().Panic("无法写入配置文件, %s", err)
util.Log().Panic("Failed to write config file: %s", err)
}
f.Close()
@@ -98,7 +103,7 @@ func Init(path string) {
cfg, err = ini.Load(path)
if err != nil {
util.Log().Panic("无法解析配置文件 '%s': %s", path, err)
util.Log().Panic("Failed to parse config file %q: %s", path, err)
}
sections := map[string]interface{}{
@@ -113,7 +118,7 @@ func Init(path string) {
for sectionName, sectionStruct := range sections {
err = mapSection(sectionName, sectionStruct)
if err != nil {
util.Log().Panic("配置文件 %s 分区解析失败: %s", sectionName, err)
util.Log().Panic("Failed to parse config section %q: %s", sectionName, err)
}
}

View File

@@ -18,9 +18,10 @@ var DatabaseConfig = &database{
// SystemConfig 系统公用配置
var SystemConfig = &system{
Debug: false,
Mode: "master",
Listen: ":5212",
Debug: false,
Mode: "master",
Listen: ":5212",
ProxyHeader: "X-Forwarded-For",
}
// CORSConfig 跨域配置
@@ -30,6 +31,8 @@ var CORSConfig = &cors{
AllowHeaders: []string{"Cookie", "X-Cr-Policy", "Authorization", "Content-Length", "Content-Type", "X-Cr-Path", "X-Cr-FileName"},
AllowCredentials: false,
ExposeHeaders: nil,
SameSite: "Default",
Secure: false,
}
// SlaveConfig 从机配置

View File

@@ -1,13 +1,13 @@
package conf
// BackendVersion 当前后端版本号
var BackendVersion = "3.5.3"
var BackendVersion = "3.6.2"
// RequiredDBVersion 与当前版本匹配的数据库版本
var RequiredDBVersion = "3.5.2"
var RequiredDBVersion = "3.6.0"
// RequiredStaticVersion 与当前版本匹配的静态资源版本
var RequiredStaticVersion = "3.5.3"
var RequiredStaticVersion = "3.6.2"
// IsPro 是否为Pro版本
var IsPro = "false"

View File

@@ -22,7 +22,7 @@ func garbageCollect() {
collectCache(store)
}
util.Log().Info("定时任务 [cron_garbage_collect] 执行完毕")
util.Log().Info("Crontab job \"cron_garbage_collect\" complete.")
}
func collectArchiveFile() {
@@ -36,23 +36,23 @@ func collectArchiveFile() {
if err == nil && !info.IsDir() &&
strings.HasPrefix(filepath.Base(path), "archive_") &&
time.Now().Sub(info.ModTime()).Seconds() > float64(expires) {
util.Log().Debug("删除过期打包下载临时文件 [%s]", path)
util.Log().Debug("Delete expired batch download temp file %q.", path)
// 删除符合条件的文件
if err := os.Remove(path); err != nil {
util.Log().Debug("临时文件 [%s] 删除失败 , %s", path, err)
util.Log().Debug("Failed to delete temp file %q: %s", path, err)
}
}
return nil
})
if err != nil {
util.Log().Debug("[定时任务] 无法列取临时打包目录")
util.Log().Debug("Crontab job cannot list temp batch download folder: %s", err)
}
}
func collectCache(store *cache.MemoStore) {
util.Log().Debug("清理内存缓存")
util.Log().Debug("Cleanup memory cache.")
store.GarbageCollect()
}
@@ -78,22 +78,22 @@ func uploadSessionCollect() {
for uid, filesIDs := range userToFiles {
user, err := model.GetUserByID(uid)
if err != nil {
util.Log().Warning("上传会话所属用户不存在, %s", err)
util.Log().Warning("Owner of the upload session cannot be found: %s", err)
continue
}
fs, err := filesystem.NewFileSystem(&user)
if err != nil {
util.Log().Warning("无法初始化文件系统, %s", err)
util.Log().Warning("Failed to initialize filesystem: %s", err)
continue
}
if err = fs.Delete(context.Background(), []uint{}, filesIDs, false); err != nil {
util.Log().Warning("无法删除上传会话, %s", err)
util.Log().Warning("Failed to delete upload session: %s", err)
}
fs.Recycle()
}
util.Log().Info("定时任务 [cron_recycle_upload_session] 执行完毕")
util.Log().Info("Crontab job \"cron_recycle_upload_session\" complete.")
}

View File

@@ -19,7 +19,7 @@ func Reload() {
// Init 初始化定时任务
func Init() {
util.Log().Info("初始化定时任务...")
util.Log().Info("Initialize crontab jobs...")
// 读取cron日程设置
options := model.GetSettingByNames(
"cron_garbage_collect",
@@ -34,12 +34,12 @@ func Init() {
case "cron_recycle_upload_session":
handler = uploadSessionCollect
default:
util.Log().Warning("未知定时任务类型 [%s],跳过", k)
util.Log().Warning("Unknown crontab job type %q, skipping...", k)
continue
}
if _, err := Cron.AddFunc(v, handler); err != nil {
util.Log().Warning("无法启动定时任务 [%s] , %s", k, err)
util.Log().Warning("Failed to start crontab job %q: %s", k, err)
}
}

View File

@@ -15,7 +15,7 @@ var Lock sync.RWMutex
// Init 初始化
func Init() {
util.Log().Debug("邮件队列初始化")
util.Log().Debug("Initializing email sending queue...")
Lock.Lock()
defer Lock.Unlock()

View File

@@ -15,9 +15,9 @@ type Driver interface {
var (
// ErrChanNotOpen 邮件队列未开启
ErrChanNotOpen = errors.New("邮件队列未开启")
ErrChanNotOpen = errors.New("email queue is not started")
// ErrNoActiveDriver 无可用邮件发送服务
ErrNoActiveDriver = errors.New("无可用邮件发送服务")
ErrNoActiveDriver = errors.New("no avaliable email provider")
)
// Send 发送邮件

View File

@@ -68,7 +68,7 @@ func (client *SMTP) Init() {
defer func() {
if err := recover(); err != nil {
client.chOpen = false
util.Log().Error("邮件发送队列出现异常, %s ,10 秒后重置", err)
util.Log().Error("Exception while sending email: %s, queue will be reset in 10 seconds.", err)
time.Sleep(time.Duration(10) * time.Second)
client.Init()
}
@@ -91,7 +91,7 @@ func (client *SMTP) Init() {
select {
case m, ok := <-client.ch:
if !ok {
util.Log().Debug("邮件队列关闭")
util.Log().Debug("Email queue closing...")
client.chOpen = false
return
}
@@ -102,15 +102,15 @@ func (client *SMTP) Init() {
open = true
}
if err := mail.Send(s, m); err != nil {
util.Log().Warning("邮件发送失败, %s", err)
util.Log().Warning("Failed to send email: %s", err)
} else {
util.Log().Debug("邮件已发送")
util.Log().Debug("Email sent.")
}
// 长时间没有新邮件则关闭SMTP连接
case <-time.After(time.Duration(client.Config.Keepalive) * time.Second):
if open {
if err := s.Close(); err != nil {
util.Log().Warning("无法关闭 SMTP 连接 %s", err)
util.Log().Warning("Failed to close SMTP connection: %s", err)
}
open = false
}

View File

@@ -107,7 +107,7 @@ func (fs *FileSystem) doCompress(ctx context.Context, file *model.File, folder *
fs.Policy = file.GetPolicy()
err := fs.DispatchHandler()
if err != nil {
util.Log().Warning("无法压缩文件%s%s", file.Name, err)
util.Log().Warning("Failed to compress file %q: %s", file.Name, err)
return
}
@@ -117,7 +117,7 @@ func (fs *FileSystem) doCompress(ctx context.Context, file *model.File, folder *
file.SourceName,
)
if err != nil {
util.Log().Debug("Open%s%s", file.Name, err)
util.Log().Debug("Failed to open %q: %s", file.Name, err)
return
}
if closer, ok := fileToZip.(io.Closer); ok {
@@ -176,7 +176,7 @@ func (fs *FileSystem) Decompress(ctx context.Context, src, dst, encoding string)
// 结束时删除临时压缩文件
if tempZipFilePath != "" {
if err := os.Remove(tempZipFilePath); err != nil {
util.Log().Warning("无法删除临时压缩文件 %s , %s", tempZipFilePath, err)
util.Log().Warning("Failed to delete temp archive file %q: %s", tempZipFilePath, err)
}
}
}()
@@ -197,7 +197,7 @@ func (fs *FileSystem) Decompress(ctx context.Context, src, dst, encoding string)
zipFile, err := util.CreatNestedFile(tempZipFilePath)
if err != nil {
util.Log().Warning("无法创建临时压缩文件 %s , %s", tempZipFilePath, err)
util.Log().Warning("Failed to create temp archive file %q: %s", tempZipFilePath, err)
tempZipFilePath = ""
return err
}
@@ -206,7 +206,7 @@ func (fs *FileSystem) Decompress(ctx context.Context, src, dst, encoding string)
// 下载前先判断是否是可解压的格式
format, readStream, err := archiver.Identify(fs.FileTarget[0].SourceName, fileStream)
if err != nil {
util.Log().Warning("无法识别文件格式 %s , %s", fs.FileTarget[0].SourceName, err)
util.Log().Warning("Failed to detect compressed format of file %q: %s", fs.FileTarget[0].SourceName, err)
return err
}
@@ -228,7 +228,7 @@ func (fs *FileSystem) Decompress(ctx context.Context, src, dst, encoding string)
if isZip {
_, err = io.Copy(zipFile, readStream)
if err != nil {
util.Log().Warning("无法写入临时压缩文件 %s , %s", tempZipFilePath, err)
util.Log().Warning("Failed to write temp archive file %q: %s", tempZipFilePath, err)
return err
}
@@ -261,7 +261,7 @@ func (fs *FileSystem) Decompress(ctx context.Context, src, dst, encoding string)
wg.Done()
}
if err := recover(); err != nil {
util.Log().Warning("上传压缩包内文件时出错")
util.Log().Warning("Error while uploading files inside of archive file.")
fmt.Println(err)
}
}()
@@ -274,7 +274,7 @@ func (fs *FileSystem) Decompress(ctx context.Context, src, dst, encoding string)
}, true)
fileStream.Close()
if err != nil {
util.Log().Debug("无法上传压缩包内的文件%s , %s , 跳过", rawPath, err)
util.Log().Debug("Failed to upload file %q in archive file: %s, skipping...", rawPath, err)
}
}
@@ -297,7 +297,7 @@ func (fs *FileSystem) Decompress(ctx context.Context, src, dst, encoding string)
// 上传文件
fileStream, err := f.Open()
if err != nil {
util.Log().Warning("无法打开压缩包内文件%s , %s , 跳过", rawPath, err)
util.Log().Warning("Failed to open file %q in archive file: %s, skipping...", rawPath, err)
return nil
}

View File

@@ -1,14 +1,22 @@
package backoff
import "time"
import (
"errors"
"fmt"
"github.com/cloudreve/Cloudreve/v3/pkg/util"
"net/http"
"strconv"
"time"
)
// Backoff used for retry sleep backoff
type Backoff interface {
Next() bool
Next(err error) bool
Reset()
}
// ConstantBackoff implements Backoff interface with constant sleep time
// ConstantBackoff implements Backoff interface with constant sleep time. If the error
// is retryable and with `RetryAfter` defined, the `RetryAfter` will be used as sleep duration.
type ConstantBackoff struct {
Sleep time.Duration
Max int
@@ -16,16 +24,51 @@ type ConstantBackoff struct {
tried int
}
func (c *ConstantBackoff) Next() bool {
func (c *ConstantBackoff) Next(err error) bool {
c.tried++
if c.tried > c.Max {
return false
}
time.Sleep(c.Sleep)
var e *RetryableError
if errors.As(err, &e) && e.RetryAfter > 0 {
util.Log().Warning("Retryable error %q occurs in backoff, will sleep after %s.", e, e.RetryAfter)
time.Sleep(e.RetryAfter)
} else {
time.Sleep(c.Sleep)
}
return true
}
func (c *ConstantBackoff) Reset() {
c.tried = 0
}
type RetryableError struct {
Err error
RetryAfter time.Duration
}
// NewRetryableErrorFromHeader constructs a new RetryableError from http response header
// and existing error.
func NewRetryableErrorFromHeader(err error, header http.Header) *RetryableError {
retryAfter := header.Get("retry-after")
if retryAfter == "" {
retryAfter = "0"
}
res := &RetryableError{
Err: err,
}
if retryAfterSecond, err := strconv.ParseInt(retryAfter, 10, 64); err == nil {
res.RetryAfter = time.Duration(retryAfterSecond) * time.Second
}
return res
}
func (e *RetryableError) Error() string {
return fmt.Sprintf("retryable error with retry-after=%s: %s", e.RetryAfter, e.Err)
}

View File

@@ -1,7 +1,9 @@
package backoff
import (
"errors"
"github.com/stretchr/testify/assert"
"net/http"
"testing"
"time"
)
@@ -9,14 +11,51 @@ import (
func TestConstantBackoff_Next(t *testing.T) {
a := assert.New(t)
b := &ConstantBackoff{Sleep: time.Duration(0), Max: 3}
a.True(b.Next())
a.True(b.Next())
a.True(b.Next())
a.False(b.Next())
b.Reset()
a.True(b.Next())
a.True(b.Next())
a.True(b.Next())
a.False(b.Next())
// General error
{
err := errors.New("error")
b := &ConstantBackoff{Sleep: time.Duration(0), Max: 3}
a.True(b.Next(err))
a.True(b.Next(err))
a.True(b.Next(err))
a.False(b.Next(err))
b.Reset()
a.True(b.Next(err))
a.True(b.Next(err))
a.True(b.Next(err))
a.False(b.Next(err))
}
// Retryable error
{
err := &RetryableError{RetryAfter: time.Duration(1)}
b := &ConstantBackoff{Sleep: time.Duration(0), Max: 3}
a.True(b.Next(err))
a.True(b.Next(err))
a.True(b.Next(err))
a.False(b.Next(err))
b.Reset()
a.True(b.Next(err))
a.True(b.Next(err))
a.True(b.Next(err))
a.False(b.Next(err))
}
}
func TestNewRetryableErrorFromHeader(t *testing.T) {
a := assert.New(t)
// no retry-after header
{
err := NewRetryableErrorFromHeader(nil, http.Header{})
a.Empty(err.RetryAfter)
}
// with retry-after header
{
header := http.Header{}
header.Add("retry-after", "120")
err := NewRetryableErrorFromHeader(nil, header)
a.EqualValues(time.Duration(120)*time.Second, err.RetryAfter)
}
}

View File

@@ -5,6 +5,7 @@ import (
"fmt"
"github.com/cloudreve/Cloudreve/v3/pkg/filesystem/chunk/backoff"
"github.com/cloudreve/Cloudreve/v3/pkg/filesystem/fsctx"
"github.com/cloudreve/Cloudreve/v3/pkg/request"
"github.com/cloudreve/Cloudreve/v3/pkg/util"
"io"
"os"
@@ -66,7 +67,7 @@ func (c *ChunkGroup) TempAvailable() bool {
// Process a chunk with retry logic
func (c *ChunkGroup) Process(processor ChunkProcessFunc) error {
reader := io.LimitReader(c.file, int64(c.chunkSize))
reader := io.LimitReader(c.file, c.Length())
// If useBuffer is enabled, tee the reader to a temp file
if c.enableRetryBuffer && c.bufferTemp == nil && !c.file.Seekable() {
@@ -90,13 +91,17 @@ func (c *ChunkGroup) Process(processor ChunkProcessFunc) error {
}
util.Log().Debug("Chunk %d will be read from temp file %q.", c.Index(), c.bufferTemp.Name())
reader = c.bufferTemp
reader = io.NopCloser(c.bufferTemp)
}
}
err := processor(c, reader)
if err != nil {
if err != context.Canceled && (c.file.Seekable() || c.TempAvailable()) && c.backoff.Next() {
if c.enableRetryBuffer {
request.BlackHole(reader)
}
if err != context.Canceled && (c.file.Seekable() || c.TempAvailable()) && c.backoff.Next(err) {
if c.file.Seekable() {
if _, seekErr := c.file.Seek(c.Start(), io.SeekStart); seekErr != nil {
return fmt.Errorf("failed to seek back to chunk start: %w, last error: %s", seekErr, err)

View File

@@ -218,7 +218,7 @@ func (handler Driver) Delete(ctx context.Context, files []string) ([]string, err
return failed, nil
}
return failed, errors.New("删除失败")
return failed, errors.New("delete failed")
}
// Thumb 获取文件缩略图

View File

@@ -43,7 +43,7 @@ func (handler Driver) List(ctx context.Context, path string, recursive bool) ([]
}
if err != nil {
util.Log().Warning("无法遍历目录 %s, %s", path, err)
util.Log().Warning("Failed to walk folder %q: %s", path, err)
return filepath.SkipDir
}
@@ -78,7 +78,7 @@ func (handler Driver) Get(ctx context.Context, path string) (response.RSCloser,
// 打开文件
file, err := os.Open(util.RelativePath(path))
if err != nil {
util.Log().Debug("无法打开文件:%s", err)
util.Log().Debug("Failed to open file: %s", err)
return nil, err
}
@@ -94,8 +94,8 @@ func (handler Driver) Put(ctx context.Context, file fsctx.FileHeader) error {
// 如果非 Overwrite则检查是否有重名冲突
if fileInfo.Mode&fsctx.Overwrite != fsctx.Overwrite {
if util.Exists(dst) {
util.Log().Warning("物理同名文件已存在或不可用: %s", dst)
return errors.New("物理同名文件已存在或不可用")
util.Log().Warning("File with the same name existed or unavailable: %s", dst)
return errors.New("file with the same name existed or unavailable")
}
}
@@ -104,7 +104,7 @@ func (handler Driver) Put(ctx context.Context, file fsctx.FileHeader) error {
if !util.Exists(basePath) {
err := os.MkdirAll(basePath, Perm)
if err != nil {
util.Log().Warning("无法创建目录,%s", err)
util.Log().Warning("Failed to create directory: %s", err)
return err
}
}
@@ -123,7 +123,7 @@ func (handler Driver) Put(ctx context.Context, file fsctx.FileHeader) error {
out, err = os.OpenFile(dst, openMode, Perm)
if err != nil {
util.Log().Warning("无法打开或创建文件,%s", err)
util.Log().Warning("Failed to open or create file: %s", err)
return err
}
defer out.Close()
@@ -131,22 +131,22 @@ func (handler Driver) Put(ctx context.Context, file fsctx.FileHeader) error {
if fileInfo.Mode&fsctx.Append == fsctx.Append {
stat, err := out.Stat()
if err != nil {
util.Log().Warning("无法读取文件信息,%s", err)
util.Log().Warning("Failed to read file info: %s", err)
return err
}
if uint64(stat.Size()) < fileInfo.AppendStart {
return errors.New("未上传完成的文件分片与预期大小不一致")
return errors.New("size of unfinished uploaded chunks is not as expected")
} else if uint64(stat.Size()) > fileInfo.AppendStart {
out.Close()
if err := handler.Truncate(ctx, dst, fileInfo.AppendStart); err != nil {
return fmt.Errorf("覆盖分片时发生错误: %w", err)
return fmt.Errorf("failed to overwrite chunk: %w", err)
}
out, err = os.OpenFile(dst, openMode, Perm)
defer out.Close()
if err != nil {
util.Log().Warning("无法打开或创建文件,%s", err)
util.Log().Warning("Failed to create or open file: %s", err)
return err
}
}
@@ -158,10 +158,10 @@ func (handler Driver) Put(ctx context.Context, file fsctx.FileHeader) error {
}
func (handler Driver) Truncate(ctx context.Context, src string, size uint64) error {
util.Log().Warning("截断文件 [%s] 至 [%d]", src, size)
util.Log().Warning("Truncate file %q to [%d].", src, size)
out, err := os.OpenFile(src, os.O_WRONLY, Perm)
if err != nil {
util.Log().Warning("无法打开文件,%s", err)
util.Log().Warning("Failed to open file: %s", err)
return err
}
@@ -180,7 +180,7 @@ func (handler Driver) Delete(ctx context.Context, files []string) ([]string, err
if util.Exists(filePath) {
err := os.Remove(filePath)
if err != nil {
util.Log().Warning("无法删除文件,%s", err)
util.Log().Warning("Failed to delete file: %s", err)
retErr = err
deleteFailed = append(deleteFailed, value)
}
@@ -217,7 +217,7 @@ func (handler Driver) Source(
) (string, error) {
file, ok := ctx.Value(fsctx.FileModelCtx).(model.File)
if !ok {
return "", errors.New("无法获取文件记录上下文")
return "", errors.New("failed to read file model context")
}
// 是否启用了CDN
@@ -238,7 +238,7 @@ func (handler Driver) Source(
downloadSessionID := util.RandStringRunes(16)
err = cache.Set("download_"+downloadSessionID, file, int(ttl))
if err != nil {
return "", serializer.NewError(serializer.CodeCacheOperation, "无法创建下载会话", err)
return "", serializer.NewError(serializer.CodeCacheOperation, "Failed to create download session", err)
}
// 签名生成文件记录
@@ -257,7 +257,7 @@ func (handler Driver) Source(
}
if err != nil {
return "", serializer.NewError(serializer.CodeEncryptError, "无法对URL进行签名", err)
return "", serializer.NewError(serializer.CodeEncryptError, "Failed to sign url", err)
}
finalURL := baseURL.ResolveReference(signedURI).String()

View File

@@ -36,7 +36,7 @@ func TestHandler_Put(t *testing.T) {
{&fsctx.FileStream{
SavePath: "TestHandler_Put.txt",
File: io.NopCloser(strings.NewReader("")),
}, "物理同名文件已存在或不可用"},
}, "file with the same name existed or unavailable"},
{&fsctx.FileStream{
SavePath: "inner/TestHandler_Put.txt",
File: io.NopCloser(strings.NewReader("")),
@@ -51,7 +51,7 @@ func TestHandler_Put(t *testing.T) {
Mode: fsctx.Append | fsctx.Overwrite,
SavePath: "inner/TestHandler_Put.txt",
File: io.NopCloser(strings.NewReader("123")),
}, "未上传完成的文件分片与预期大小不一致"},
}, "size of unfinished uploaded chunks is not as expected"},
{&fsctx.FileStream{
Mode: fsctx.Append | fsctx.Overwrite,
SavePath: "inner/TestHandler_Put.txt",

View File

@@ -7,7 +7,6 @@ import (
"fmt"
"github.com/cloudreve/Cloudreve/v3/pkg/conf"
"io"
"io/ioutil"
"net/http"
"net/url"
"path"
@@ -37,24 +36,18 @@ const (
// GetSourcePath 获取文件的绝对路径
func (info *FileInfo) GetSourcePath() string {
res, err := url.PathUnescape(
strings.TrimPrefix(
path.Join(
strings.TrimPrefix(info.ParentReference.Path, "/drive/root:"),
info.Name,
),
"/",
),
)
res, err := url.PathUnescape(info.ParentReference.Path)
if err != nil {
return ""
}
return res
}
// Error 实现error接口
func (err RespError) Error() string {
return err.APIError.Message
return strings.TrimPrefix(
path.Join(
strings.TrimPrefix(res, "/drive/root:"),
info.Name,
),
"/",
)
}
func (client *Client) getRequestURL(api string, opts ...Option) string {
@@ -95,7 +88,7 @@ func (client *Client) ListChildren(ctx context.Context, path string) ([]FileInfo
}
if retried < ListRetry {
retried++
util.Log().Debug("路径[%s]列取请求失败[%s]5秒钟后重试", path, err)
util.Log().Debug("Failed to list path %q: %s, will retry in 5 seconds.", path, err)
time.Sleep(time.Duration(5) * time.Second)
return client.ListChildren(context.WithValue(ctx, fsctx.RetryCtx, retried), path)
}
@@ -445,7 +438,7 @@ func (client *Client) GetThumbURL(ctx context.Context, dst string, w, h uint) (s
}
}
return "", errors.New("无法生成缩略图")
return "", errors.New("failed to generate thumb")
}
// MonitorUpload 监控客户端分片上传进度
@@ -460,39 +453,39 @@ func (client *Client) MonitorUpload(uploadURL, callbackKey, path string, size ui
for {
select {
case <-callbackChan:
util.Log().Debug("客户端完成回调")
util.Log().Debug("Client finished OneDrive callback.")
return
case <-time.After(time.Duration(ttl) * time.Second):
// 上传会话到期,仍未完成上传,创建占位符
client.DeleteUploadSession(context.Background(), uploadURL)
_, err := client.SimpleUpload(context.Background(), path, strings.NewReader(""), 0, WithConflictBehavior("replace"))
if err != nil {
util.Log().Debug("无法创建占位文件,%s", err)
util.Log().Debug("Failed to create placeholder file: %s", err)
}
return
case <-time.After(time.Duration(timeout) * time.Second):
util.Log().Debug("检查上传情况")
util.Log().Debug("Checking OneDrive upload status.")
status, err := client.GetUploadSessionStatus(context.Background(), uploadURL)
if err != nil {
if resErr, ok := err.(*RespError); ok {
if resErr.APIError.Code == "itemNotFound" {
util.Log().Debug("上传会话已完成,稍后检查回调")
util.Log().Debug("Upload completed, will check upload callback later.")
select {
case <-time.After(time.Duration(interval) * time.Second):
util.Log().Warning("未发送回调,删除文件")
util.Log().Warning("No callback is made, file will be deleted.")
cache.Deletes([]string{callbackKey}, "callback_")
_, err = client.Delete(context.Background(), []string{path})
if err != nil {
util.Log().Warning("无法删除未回调的文件,%s", err)
util.Log().Warning("Failed to delete file without callback: %s", err)
}
case <-callbackChan:
util.Log().Debug("客户端完成回调")
util.Log().Debug("Client finished callback.")
}
return
}
}
util.Log().Debug("无法获取上传会话状态,继续下一轮,%s", err.Error())
util.Log().Debug("Failed to get upload session status: %s, continue next iteration.", err.Error())
continue
}
@@ -509,7 +502,7 @@ func (client *Client) MonitorUpload(uploadURL, callbackKey, path string, size ui
}
uploadFullSize, _ := strconv.ParseUint(sizeRange[1], 10, 64)
if (sizeRange[0] == "0" && sizeRange[1] == "") || uploadFullSize+1 != size {
util.Log().Debug("未开始上传或文件大小不一致,取消上传会话")
util.Log().Debug("Upload has not started, or uploaded file size not match, canceling upload session...")
// 取消上传会话实测OneDrive取消上传会话后客户端还是可以上传
// 所以上传一个空文件占位,阻止客户端上传
client.DeleteUploadSession(context.Background(), uploadURL)
@@ -531,7 +524,7 @@ func sysError(err error) *RespError {
}}
}
func (client *Client) request(ctx context.Context, method string, url string, body io.Reader, option ...request.Option) (string, *RespError) {
func (client *Client) request(ctx context.Context, method string, url string, body io.Reader, option ...request.Option) (string, error) {
// 获取凭证
err := client.UpdateCredential(ctx, conf.SystemConfig.Mode == "slave")
if err != nil {
@@ -544,6 +537,11 @@ func (client *Client) request(ctx context.Context, method string, url string, bo
"Content-Type": {"application/json"},
}),
request.WithContext(ctx),
request.WithTPSLimit(
fmt.Sprintf("policy_%d", client.Policy.ID),
client.Policy.OptionsSerialized.TPSLimit,
client.Policy.OptionsSerialized.TPSLimitBurst,
),
)
// 发送请求
@@ -572,18 +570,24 @@ func (client *Client) request(ctx context.Context, method string, url string, bo
if res.Response.StatusCode < 200 || res.Response.StatusCode >= 300 {
decodeErr = json.Unmarshal([]byte(respBody), &errResp)
if decodeErr != nil {
util.Log().Debug("Onedrive返回未知响应[%s]", respBody)
util.Log().Debug("Onedrive returns unknown response: %s", respBody)
return "", sysError(decodeErr)
}
if res.Response.StatusCode == 429 {
util.Log().Warning("OneDrive request is throttled.")
return "", backoff.NewRetryableErrorFromHeader(&errResp, res.Response.Header)
}
return "", &errResp
}
return respBody, nil
}
func (client *Client) requestWithStr(ctx context.Context, method string, url string, body string, expectedCode int) (string, *RespError) {
func (client *Client) requestWithStr(ctx context.Context, method string, url string, body string, expectedCode int) (string, error) {
// 发送请求
bodyReader := ioutil.NopCloser(strings.NewReader(body))
bodyReader := io.NopCloser(strings.NewReader(body))
return client.request(ctx, method, url, bodyReader,
request.WithContentLength(int64(len(body))),
)

View File

@@ -112,6 +112,35 @@ func TestRequest(t *testing.T) {
asserts.Equal("error msg", err.Error())
}
// OneDrive返回429错误
{
header := http.Header{}
header.Add("retry-after", "120")
clientMock := ClientMock{}
clientMock.On(
"Request",
"POST",
"http://dev.com",
testMock.Anything,
testMock.Anything,
).Return(&request.Response{
Err: nil,
Response: &http.Response{
StatusCode: 429,
Header: header,
Body: ioutil.NopCloser(strings.NewReader(`{"error":{"message":"error msg"}}`)),
},
})
client.Request = clientMock
res, err := client.request(context.Background(), "POST", "http://dev.com", strings.NewReader(""))
clientMock.AssertExpectations(t)
asserts.Error(err)
asserts.Empty(res)
var retryErr *backoff.RetryableError
asserts.ErrorAs(err, &retryErr)
asserts.EqualValues(time.Duration(120)*time.Second, retryErr.RetryAfter)
}
// OneDrive返回未知响应
{
clientMock := ClientMock{}
@@ -144,18 +173,18 @@ func TestFileInfo_GetSourcePath(t *testing.T) {
fileInfo := FileInfo{
Name: "%e6%96%87%e4%bb%b6%e5%90%8d.jpg",
ParentReference: parentReference{
Path: "/drive/root:/123/321",
Path: "/drive/root:/123/32%201",
},
}
asserts.Equal("123/321/文件名.jpg", fileInfo.GetSourcePath())
asserts.Equal("123/32 1/%e6%96%87%e4%bb%b6%e5%90%8d.jpg", fileInfo.GetSourcePath())
}
// 失败
{
fileInfo := FileInfo{
Name: "%e6%96%87%e4%bb%b6%e5%90%8g.jpg",
Name: "123.jpg",
ParentReference: parentReference{
Path: "/drive/root:/123/321",
Path: "/drive/root:/123/%e6%96%87%e4%bb%b6%e5%90%8g",
},
}
asserts.Equal("", fileInfo.GetSourcePath())

View File

@@ -10,13 +10,13 @@ import (
var (
// ErrAuthEndpoint 无法解析授权端点地址
ErrAuthEndpoint = errors.New("无法解析授权端点地址")
ErrAuthEndpoint = errors.New("failed to parse endpoint url")
// ErrInvalidRefreshToken 上传策略无有效的RefreshToken
ErrInvalidRefreshToken = errors.New("上传策略无有效的RefreshToken")
ErrInvalidRefreshToken = errors.New("no valid refresh token in this policy")
// ErrDeleteFile 无法删除文件
ErrDeleteFile = errors.New("无法删除文件")
ErrDeleteFile = errors.New("cannot delete file")
// ErrClientCanceled 客户端取消操作
ErrClientCanceled = errors.New("客户端取消操作")
ErrClientCanceled = errors.New("client canceled")
)
// Client OneDrive客户端

View File

@@ -11,7 +11,6 @@ import (
"time"
model "github.com/cloudreve/Cloudreve/v3/models"
"github.com/cloudreve/Cloudreve/v3/pkg/auth"
"github.com/cloudreve/Cloudreve/v3/pkg/cache"
"github.com/cloudreve/Cloudreve/v3/pkg/filesystem/driver"
"github.com/cloudreve/Cloudreve/v3/pkg/filesystem/fsctx"
@@ -143,7 +142,7 @@ func (handler Driver) Thumb(ctx context.Context, path string) (*response.Content
ok = false
)
if thumbSize, ok = ctx.Value(fsctx.ThumbSizeCtx).([2]uint); !ok {
return nil, errors.New("无法获取缩略图尺寸设置")
return nil, errors.New("failed to get thumbnail size")
}
res, err := handler.Client.GetThumbURL(ctx, path, thumbSize[0], thumbSize[1])
@@ -171,19 +170,6 @@ func (handler Driver) Source(
cacheKey := fmt.Sprintf("onedrive_source_%d_%s", handler.Policy.ID, path)
if file, ok := ctx.Value(fsctx.FileModelCtx).(model.File); ok {
cacheKey = fmt.Sprintf("onedrive_source_file_%d_%d", file.UpdatedAt.Unix(), file.ID)
// 如果是永久链接,则返回签名后的中转外链
if ttl == 0 {
signedURI, err := auth.SignURI(
auth.General,
fmt.Sprintf("/api/v3/file/source/%d/%s", file.ID, file.Name),
ttl,
)
if err != nil {
return "", err
}
return baseURL.ResolveReference(signedURI).String(), nil
}
}
// 尝试从缓存中查找

View File

@@ -3,7 +3,6 @@ package onedrive
import (
"context"
"fmt"
"github.com/cloudreve/Cloudreve/v3/pkg/auth"
"github.com/cloudreve/Cloudreve/v3/pkg/mq"
"github.com/cloudreve/Cloudreve/v3/pkg/serializer"
"github.com/jinzhu/gorm"
@@ -161,21 +160,6 @@ func TestDriver_Source(t *testing.T) {
asserts.NoError(err)
asserts.Equal("123321", res)
}
// 成功 永久直链
{
file := model.File{}
file.ID = 1
file.Name = "123.jpg"
file.UpdatedAt = time.Now()
ctx := context.WithValue(context.Background(), fsctx.FileModelCtx, file)
handler.Client.Credential.ExpiresIn = time.Now().Add(time.Duration(100) * time.Hour).Unix()
auth.General = auth.HMACAuth{}
handler.Client.Credential.AccessToken = "1"
res, err := handler.Source(ctx, "123.jpg", url.URL{}, 0, true, 0)
asserts.NoError(err)
asserts.Contains(res, "/api/v3/file/source/1/123.jpg?sign")
}
}
func TestDriver_List(t *testing.T) {

View File

@@ -152,7 +152,7 @@ func (client *Client) UpdateCredential(ctx context.Context, isSlave bool) error
// 获取新的凭证
if client.Credential == nil || client.Credential.RefreshToken == "" {
// 无有效的RefreshToken
util.Log().Error("上传策略[%s]凭证刷新失败请重新授权OneDrive账号", client.Policy.Name)
util.Log().Error("Failed to refresh credential for policy %q, please login your Microsoft account again.", client.Policy.Name)
return ErrInvalidRefreshToken
}

View File

@@ -133,3 +133,8 @@ type Site struct {
func init() {
gob.Register(Credential{})
}
// Error 实现error接口
func (err RespError) Error() string {
return err.APIError.Message
}

View File

@@ -38,7 +38,7 @@ func GetPublicKey(r *http.Request) ([]byte, error) {
// 确保这个 public key 是由 OSS 颁发的
if !strings.HasPrefix(string(pubURL), "http://gosspublic.alicdn.com/") &&
!strings.HasPrefix(string(pubURL), "https://gosspublic.alicdn.com/") {
return pubKey, errors.New("公钥URL无效")
return pubKey, errors.New("public key url invalid")
}
// 获取公钥

View File

@@ -91,7 +91,7 @@ func (handler *Driver) CORS() error {
// InitOSSClient 初始化OSS鉴权客户端
func (handler *Driver) InitOSSClient(forceUsePublicEndpoint bool) error {
if handler.Policy == nil {
return errors.New("存储策略为空")
return errors.New("empty policy")
}
// 决定是否使用内网 Endpoint
@@ -286,7 +286,7 @@ func (handler *Driver) Delete(ctx context.Context, files []string) ([]string, er
// 统计未删除的文件
failed := util.SliceDifference(files, delRes.DeletedObjects)
if len(failed) > 0 {
return failed, errors.New("删除失败")
return failed, errors.New("failed to delete")
}
return []string{}, nil
@@ -304,7 +304,7 @@ func (handler *Driver) Thumb(ctx context.Context, path string) (*response.Conten
ok = false
)
if thumbSize, ok = ctx.Value(fsctx.ThumbSizeCtx).([2]uint); !ok {
return nil, errors.New("无法获取缩略图尺寸设置")
return nil, errors.New("failed to get thumbnail size")
}
thumbParam := fmt.Sprintf("image/resize,m_lfit,h_%d,w_%d", thumbSize[1], thumbSize[0])

View File

@@ -197,7 +197,7 @@ func (handler *Driver) Delete(ctx context.Context, files []string) ([]string, er
return failedResp.Files, errors.New(reqResp.Error)
}
}
return files, errors.New("未知的返回结果格式")
return files, errors.New("unknown format of returned response")
}
return []string{}, nil
@@ -265,7 +265,7 @@ func (handler *Driver) Source(
)
if err != nil {
return "", serializer.NewError(serializer.CodeEncryptError, "无法对URL进行签名", err)
return "", serializer.NewError(serializer.CodeEncryptError, "Failed to sign URL", err)
}
finalURL := serverURL.ResolveReference(signedURI).String()

View File

@@ -4,9 +4,6 @@ import (
"context"
"errors"
"fmt"
"github.com/cloudreve/Cloudreve/v3/pkg/filesystem/chunk"
"github.com/cloudreve/Cloudreve/v3/pkg/filesystem/chunk/backoff"
"github.com/cloudreve/Cloudreve/v3/pkg/util"
"io"
"net/http"
"net/url"
@@ -15,6 +12,10 @@ import (
"strings"
"time"
"github.com/cloudreve/Cloudreve/v3/pkg/filesystem/chunk"
"github.com/cloudreve/Cloudreve/v3/pkg/filesystem/chunk/backoff"
"github.com/cloudreve/Cloudreve/v3/pkg/util"
"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/aws/credentials"
"github.com/aws/aws-sdk-go/aws/session"
@@ -40,7 +41,7 @@ type UploadPolicy struct {
Conditions []interface{} `json:"conditions"`
}
//MetaData 文件信息
// MetaData 文件信息
type MetaData struct {
Size uint64
Etag string
@@ -61,7 +62,7 @@ func NewDriver(policy *model.Policy) (*Driver, error) {
// InitS3Client 初始化S3会话
func (handler *Driver) InitS3Client() error {
if handler.Policy == nil {
return errors.New("存储策略为空")
return errors.New("empty policy")
}
if handler.svc == nil {
@@ -70,7 +71,7 @@ func (handler *Driver) InitS3Client() error {
Credentials: credentials.NewStaticCredentials(handler.Policy.AccessKey, handler.Policy.SecretKey, ""),
Endpoint: &handler.Policy.Server,
Region: &handler.Policy.OptionsSerialized.Region,
S3ForcePathStyle: aws.Bool(true),
S3ForcePathStyle: &handler.Policy.OptionsSerialized.S3ForcePathStyle,
})
if err != nil {
@@ -288,19 +289,22 @@ func (handler *Driver) Source(
return "", err
}
contentDescription := aws.String("attachment; filename=\"" + url.PathEscape(fileName) + "\"")
if !isDownload {
contentDescription = nil
}
req, _ := handler.svc.GetObjectRequest(
&s3.GetObjectInput{
Bucket: &handler.Policy.BucketName,
Key: &path,
ResponseContentDisposition: aws.String("attachment; filename=\"" + url.PathEscape(fileName) + "\""),
ResponseContentDisposition: contentDescription,
})
if ttl == 0 {
ttl = 3600
signedURL, err := req.Presign(time.Duration(ttl) * time.Second)
if err != nil {
return "", err
}
signedURL, _ := req.Presign(time.Duration(ttl) * time.Second)
// 将最终生成的签名URL域名换成用户自定义的加速域名如果有
finalURL, err := url.Parse(signedURL)
if err != nil {
@@ -394,8 +398,8 @@ func (handler *Driver) Token(ctx context.Context, ttl int64, uploadSession *seri
// Meta 获取文件信息
func (handler *Driver) Meta(ctx context.Context, path string) (*MetaData, error) {
res, err := handler.svc.GetObject(
&s3.GetObjectInput{
res, err := handler.svc.HeadObject(
&s3.HeadObjectInput{
Bucket: &handler.Policy.BucketName,
Key: &path,
})

View File

@@ -5,6 +5,9 @@ import (
"context"
"encoding/json"
"errors"
"net/url"
"time"
model "github.com/cloudreve/Cloudreve/v3/models"
"github.com/cloudreve/Cloudreve/v3/pkg/cluster"
"github.com/cloudreve/Cloudreve/v3/pkg/filesystem/driver"
@@ -13,8 +16,6 @@ import (
"github.com/cloudreve/Cloudreve/v3/pkg/mq"
"github.com/cloudreve/Cloudreve/v3/pkg/request"
"github.com/cloudreve/Cloudreve/v3/pkg/serializer"
"net/url"
"time"
)
// Driver 影子存储策略,将上传任务指派给从机节点处理,并等待从机通知上传结果
@@ -118,6 +119,6 @@ func (d *Driver) List(ctx context.Context, path string, recursive bool) ([]respo
}
// 取消上传凭证
func (handler Driver) CancelToken(ctx context.Context, uploadSession *serializer.UploadSession) error {
func (d *Driver) CancelToken(ctx context.Context, uploadSession *serializer.UploadSession) error {
return nil
}

View File

@@ -226,7 +226,7 @@ func (handler Driver) Thumb(ctx context.Context, path string) (*response.Content
ok = false
)
if thumbSize, ok = ctx.Value(fsctx.ThumbSizeCtx).([2]uint); !ok {
return nil, errors.New("无法获取缩略图尺寸设置")
return nil, errors.New("failed to get thumbnail size")
}
thumbParam := fmt.Sprintf("!/fwfh/%dx%d", thumbSize[0], thumbSize[1])

View File

@@ -7,20 +7,20 @@ import (
)
var (
ErrUnknownPolicyType = errors.New("未知存储策略类型")
ErrFileSizeTooBig = errors.New("单个文件尺寸太大")
ErrFileExtensionNotAllowed = errors.New("不允许上传此类型的文件")
ErrInsufficientCapacity = errors.New("容量空间不足")
ErrIllegalObjectName = errors.New("目标名称非法")
ErrClientCanceled = errors.New("客户端取消操作")
ErrRootProtected = errors.New("无法对根目录进行操作")
ErrInsertFileRecord = serializer.NewError(serializer.CodeDBError, "无法插入文件记录", nil)
ErrFileExisted = serializer.NewError(serializer.CodeObjectExist, "同名文件或目录已存在", nil)
ErrFileUploadSessionExisted = serializer.NewError(serializer.CodeObjectExist, "当前目录下已经有同名文件正在上传中,请尝试清空上传会话", nil)
ErrFolderExisted = serializer.NewError(serializer.CodeObjectExist, "同名目录已存在", nil)
ErrPathNotExist = serializer.NewError(404, "路径不存在", nil)
ErrObjectNotExist = serializer.NewError(404, "文件不存在", nil)
ErrIO = serializer.NewError(serializer.CodeIOFailed, "无法读取文件数据", nil)
ErrDBListObjects = serializer.NewError(serializer.CodeDBError, "无法列取对象记录", nil)
ErrDBDeleteObjects = serializer.NewError(serializer.CodeDBError, "无法删除对象记录", nil)
ErrUnknownPolicyType = serializer.NewError(serializer.CodeInternalSetting, "Unknown policy type", nil)
ErrFileSizeTooBig = serializer.NewError(serializer.CodeFileTooLarge, "File is too large", nil)
ErrFileExtensionNotAllowed = serializer.NewError(serializer.CodeFileTypeNotAllowed, "File type not allowed", nil)
ErrInsufficientCapacity = serializer.NewError(serializer.CodeInsufficientCapacity, "Insufficient capacity", nil)
ErrIllegalObjectName = serializer.NewError(serializer.CodeIllegalObjectName, "Invalid object name", nil)
ErrClientCanceled = errors.New("Client canceled operation")
ErrRootProtected = serializer.NewError(serializer.CodeRootProtected, "Root protected", nil)
ErrInsertFileRecord = serializer.NewError(serializer.CodeDBError, "Failed to create file record", nil)
ErrFileExisted = serializer.NewError(serializer.CodeObjectExist, "Object existed", nil)
ErrFileUploadSessionExisted = serializer.NewError(serializer.CodeConflictUploadOngoing, "Upload session existed", nil)
ErrPathNotExist = serializer.NewError(serializer.CodeParentNotExist, "Path not exist", nil)
ErrObjectNotExist = serializer.NewError(serializer.CodeParentNotExist, "Object not exist", nil)
ErrIO = serializer.NewError(serializer.CodeIOFailed, "Failed to read file data", nil)
ErrDBListObjects = serializer.NewError(serializer.CodeDBError, "Failed to list object records", nil)
ErrDBDeleteObjects = serializer.NewError(serializer.CodeDBError, "Failed to delete object records", nil)
ErrOneObjectOnly = serializer.ParamErr("You can only copy one object at the same time", nil)
)

View File

@@ -72,7 +72,7 @@ func (fs *FileSystem) AddFile(ctx context.Context, parent *model.Folder, file fs
if err != nil {
if err := fs.Trigger(ctx, "AfterValidateFailed", file); err != nil {
util.Log().Debug("AfterValidateFailed 钩子执行失败,%s", err)
util.Log().Debug("AfterValidateFailed hook execution failed: %s", err)
}
return nil, ErrFileExisted.WithError(err)
}
@@ -203,7 +203,7 @@ func (fs *FileSystem) deleteGroupedFile(ctx context.Context, files map[uint][]*m
// 取消上传会话
for _, upSession := range uploadSessions {
if err := fs.Handler.CancelToken(ctx, upSession); err != nil {
util.Log().Warning("无法取消 [%s] 的上传会话: %s", upSession.Name, err)
util.Log().Warning("Failed to cancel upload session for %q: %s", upSession.Name, err)
}
cache.Deletes([]string{upSession.Key}, UploadSessionCachePrefix)
@@ -270,14 +270,14 @@ func (fs *FileSystem) GetSource(ctx context.Context, fileID uint) (string, error
if !fs.Policy.IsOriginLinkEnable {
return "", serializer.NewError(
serializer.CodePolicyNotAllowed,
"当前存储策略无法获得外链",
"This policy is not enabled for getting source link",
nil,
)
}
source, err := fs.SignURL(ctx, &fs.FileTarget[0], 0, false)
if err != nil {
return "", serializer.NewError(serializer.CodeNotSet, "无法获取外链", err)
return "", serializer.NewError(serializer.CodeNotSet, "Failed to get source link", err)
}
return source, nil
@@ -298,7 +298,7 @@ func (fs *FileSystem) SignURL(ctx context.Context, file *model.File, ttl int64,
siteURL := model.GetSiteURL()
source, err := fs.Handler.Source(ctx, fs.FileTarget[0].SourceName, *siteURL, ttl, isDownload, fs.User.Group.SpeedLimit)
if err != nil {
return "", serializer.NewError(serializer.CodeNotSet, "无法获取外链", err)
return "", serializer.NewError(serializer.CodeNotSet, "Failed to get source link", err)
}
return source, nil

View File

@@ -203,7 +203,7 @@ func NewFileSystemFromCallback(c *gin.Context) (*FileSystem, error) {
// 获取回调会话
callbackSessionRaw, ok := c.Get(UploadSessionCtx)
if !ok {
return nil, errors.New("找不到回调会话")
return nil, errors.New("upload session not exist")
}
callbackSession := callbackSessionRaw.(*serializer.UploadSession)

View File

@@ -44,7 +44,7 @@ func (fs *FileSystem) Trigger(ctx context.Context, name string, file fsctx.FileH
for _, hook := range hooks {
err := hook(ctx, fs, file)
if err != nil {
util.Log().Warning("钩子执行失败%s", err)
util.Log().Warning("Failed to execute hook%s", err)
return err
}
}
@@ -112,7 +112,7 @@ func HookDeleteTempFile(ctx context.Context, fs *FileSystem, file fsctx.FileHead
// 删除临时文件
_, err := fs.Handler.Delete(ctx, []string{file.Info().SavePath})
if err != nil {
util.Log().Warning("无法清理上传临时文件,%s", err)
util.Log().Warning("Failed to clean-up temp files: %s", err)
}
return nil

View File

@@ -71,16 +71,16 @@ func getThumbWorker() *Pool {
thumbPool = &Pool{
worker: make(chan int, maxWorker),
}
util.Log().Debug("初始化Thumb任务队列WorkerNum = %d", maxWorker)
util.Log().Debug("Initialize thumbnails task queue with: WorkerNum = %d", maxWorker)
})
return thumbPool
}
func (pool *Pool) addWorker() {
pool.worker <- 1
util.Log().Debug("Thumb任务队列addWorker")
util.Log().Debug("Worker added to thumbnails task queue.")
}
func (pool *Pool) releaseWorker() {
util.Log().Debug("Thumb任务队列releaseWorker")
util.Log().Debug("Worker released from thumbnails task queue.")
<-pool.worker
}
@@ -107,7 +107,7 @@ func (fs *FileSystem) GenerateThumbnail(ctx context.Context, file *model.File) {
image, err := thumb.NewThumbFromFile(source, file.Name)
if err != nil {
util.Log().Warning("生成缩略图时无法解析 [%s] 图像数据:%s", file.SourceName, err)
util.Log().Warning("Cannot generate thumb because of failed to parse image %q: %s", file.SourceName, err)
return
}
@@ -125,7 +125,7 @@ func (fs *FileSystem) GenerateThumbnail(ctx context.Context, file *model.File) {
}
if err != nil {
util.Log().Warning("无法保存缩略图:%s", err)
util.Log().Warning("Failed to save thumb: %s", err)
return
}

View File

@@ -73,7 +73,7 @@ func (fs *FileSystem) Copy(ctx context.Context, dirs, files []uint, src, dst str
if len(dirs) > 0 {
subFileSizes, err := srcFolder.CopyFolderTo(dirs[0], dstFolder)
if err != nil {
return serializer.NewError(serializer.CodeDBError, "操作失败,可能有重名冲突", err)
return ErrObjectNotExist.WithError(err)
}
newUsedStorage += subFileSizes
}
@@ -82,7 +82,7 @@ func (fs *FileSystem) Copy(ctx context.Context, dirs, files []uint, src, dst str
if len(files) > 0 {
subFileSizes, err := srcFolder.MoveOrCopyFileTo(files, dstFolder, true)
if err != nil {
return serializer.NewError(serializer.CodeDBError, "操作失败,可能有重名冲突", err)
return ErrObjectNotExist.WithError(err)
}
newUsedStorage += subFileSizes
}
@@ -106,13 +106,13 @@ func (fs *FileSystem) Move(ctx context.Context, dirs, files []uint, src, dst str
// 处理目录及子文件移动
err := srcFolder.MoveFolderTo(dirs, dstFolder)
if err != nil {
return serializer.NewError(serializer.CodeDBError, "操作失败,可能有重名冲突", err)
return ErrFileExisted.WithError(err)
}
// 处理文件移动
_, err = srcFolder.MoveOrCopyFileTo(files, dstFolder, false)
if err != nil {
return serializer.NewError(serializer.CodeDBError, "操作失败,可能有重名冲突", err)
return ErrFileExisted.WithError(err)
}
// 移动文件
@@ -206,7 +206,7 @@ func (fs *FileSystem) Delete(ctx context.Context, dirs, files []uint, force bool
if notDeleted := len(fs.FileTarget) - len(deletedFiles); notDeleted > 0 {
return serializer.NewError(
serializer.CodeNotFullySuccess,
fmt.Sprintf("有 %d 个文件未能成功删除", notDeleted),
fmt.Sprintf("Failed to delete %d file(s).", notDeleted),
nil,
)
}

View File

@@ -472,6 +472,9 @@ func TestFileSystem_Delete(t *testing.T) {
AddRow(4, "1.txt", "1.txt", 365, 1),
)
mock.ExpectQuery("SELECT(.+)").WillReturnRows(sqlmock.NewRows([]string{"id", "name", "source_name", "policy_id", "size"}).AddRow(1, "2.txt", "2.txt", 365, 2))
// 两次查询软连接
mock.ExpectQuery("SELECT(.+)files(.+)").
WillReturnRows(sqlmock.NewRows([]string{"id", "policy_id", "source_name"}))
mock.ExpectQuery("SELECT(.+)files(.+)").
WillReturnRows(sqlmock.NewRows([]string{"id", "policy_id", "source_name"}))
// 查询上传策略
@@ -527,6 +530,9 @@ func TestFileSystem_Delete(t *testing.T) {
AddRow(4, "1.txt", "1.txt", 602, 1),
)
mock.ExpectQuery("SELECT(.+)").WillReturnRows(sqlmock.NewRows([]string{"id", "name", "source_name", "policy_id", "size"}).AddRow(1, "2.txt", "2.txt", 602, 2))
// 两次查询软连接
mock.ExpectQuery("SELECT(.+)files(.+)").
WillReturnRows(sqlmock.NewRows([]string{"id", "policy_id", "source_name"}))
mock.ExpectQuery("SELECT(.+)files(.+)").
WillReturnRows(sqlmock.NewRows([]string{"id", "policy_id", "source_name"}))
// 查询上传策略

View File

@@ -69,7 +69,7 @@ func (fs *FileSystem) Upload(ctx context.Context, file *fsctx.FileStream) (err e
followUpErr := fs.Trigger(ctx, "AfterValidateFailed", file)
// 失败后再失败...
if followUpErr != nil {
util.Log().Debug("AfterValidateFailed 钩子执行失败,%s", followUpErr)
util.Log().Debug("AfterValidateFailed hook execution failed: %s", followUpErr)
}
return err
@@ -113,13 +113,13 @@ func (fs *FileSystem) CancelUpload(ctx context.Context, path string, file fsctx.
// 客户端正常关闭,不执行操作
default:
// 客户端取消上传,删除临时文件
util.Log().Debug("客户端取消上传")
util.Log().Debug("Client canceled upload.")
if fs.Hooks["AfterUploadCanceled"] == nil {
return
}
err := fs.Trigger(ctx, "AfterUploadCanceled", file)
if err != nil {
util.Log().Debug("执行 AfterUploadCanceled 钩子出错,%s", err)
util.Log().Debug("AfterUploadCanceled hook execution failed: %s", err)
}
}

View File

@@ -15,11 +15,12 @@ const (
FolderID // 目录ID
TagID // 标签ID
PolicyID // 存储策略ID
SourceLinkID
)
var (
// ErrTypeNotMatch ID类型不匹配
ErrTypeNotMatch = errors.New("ID类型不匹配")
ErrTypeNotMatch = errors.New("mismatched ID type.")
)
// HashEncode 对给定数据计算HashID

View File

@@ -15,15 +15,18 @@ type Option interface {
}
type options struct {
timeout time.Duration
header http.Header
sign auth.Auth
signTTL int64
ctx context.Context
contentLength int64
masterMeta bool
endpoint *url.URL
slaveNodeID string
timeout time.Duration
header http.Header
sign auth.Auth
signTTL int64
ctx context.Context
contentLength int64
masterMeta bool
endpoint *url.URL
slaveNodeID string
tpsLimiterToken string
tps float64
tpsBurst int
}
type optionFunc func(*options)
@@ -37,9 +40,16 @@ func newDefaultOption() *options {
header: http.Header{},
timeout: time.Duration(30) * time.Second,
contentLength: -1,
ctx: context.Background(),
}
}
func (o *options) clone() options {
newOptions := *o
newOptions.header = o.header.Clone()
return newOptions
}
// WithTimeout 设置请求超时
func WithTimeout(t time.Duration) Option {
return optionFunc(func(o *options) {
@@ -113,3 +123,15 @@ func WithEndpoint(endpoint string) Option {
o.endpoint = endpointURL
})
}
// WithTPSLimit 请求时使用全局流量限制
func WithTPSLimit(token string, tps float64, burst int) Option {
return optionFunc(func(o *options) {
o.tpsLimiterToken = token
o.tps = tps
if burst < 1 {
burst = 1
}
o.tpsBurst = burst
})
}

View File

@@ -34,13 +34,15 @@ type Client interface {
// HTTPClient 实现 Client 接口
type HTTPClient struct {
mu sync.Mutex
options *options
mu sync.Mutex
options *options
tpsLimiter TPSLimiter
}
func NewClient(opts ...Option) Client {
client := &HTTPClient{
options: newDefaultOption(),
options: newDefaultOption(),
tpsLimiter: globalTPSLimiter,
}
for _, o := range opts {
@@ -54,7 +56,7 @@ func NewClient(opts ...Option) Client {
func (c *HTTPClient) Request(method, target string, body io.Reader, opts ...Option) *Response {
// 应用额外设置
c.mu.Lock()
options := *c.options
options := c.options.clone()
c.mu.Unlock()
for _, o := range opts {
o.apply(&options)
@@ -126,6 +128,10 @@ func (c *HTTPClient) Request(method, target string, body io.Reader, opts ...Opti
}
}
if options.tps > 0 {
c.tpsLimiter.Limit(options.ctx, options.tpsLimiterToken, options.tps, options.tpsBurst)
}
// 发送请求
resp, err := client.Do(req)
if err != nil {
@@ -173,7 +179,7 @@ func (resp *Response) DecodeResponse() (*serializer.Response, error) {
var res serializer.Response
err = json.Unmarshal([]byte(respString), &res)
if err != nil {
util.Log().Debug("无法解析回调服务端响应:%s", string(respString))
util.Log().Debug("Failed to parse response: %s", string(respString))
return nil, err
}
return &res, nil
@@ -245,7 +251,7 @@ func (instance NopRSCloser) Seek(offset int64, whence int) (int64, error) {
return instance.status.Size, nil
}
}
return 0, errors.New("未实现")
return 0, errors.New("not implemented")
}

View File

@@ -238,3 +238,41 @@ func TestBlackHole(t *testing.T) {
BlackHole(strings.NewReader("TestBlackHole"))
})
}
func TestHTTPClient_TPSLimit(t *testing.T) {
a := assert.New(t)
client := NewClient()
finished := make(chan struct{})
go func() {
client.Request(
"POST",
"/test",
strings.NewReader(""),
WithTPSLimit("TestHTTPClient_TPSLimit", 1, 1),
)
close(finished)
}()
select {
case <-finished:
case <-time.After(10 * time.Second):
a.Fail("Request should be finished instantly.")
}
finished = make(chan struct{})
go func() {
client.Request(
"POST",
"/test",
strings.NewReader(""),
WithTPSLimit("TestHTTPClient_TPSLimit", 1, 1),
)
close(finished)
}()
select {
case <-finished:
case <-time.After(2 * time.Second):
a.Fail("Request should be finished in 1 second.")
}
}

Some files were not shown because too many files have changed in this diff Show More