Bump version to 0.4.1

Bye Travis!
maybe we don't have to install docker.io
2025-12-06 14:36:47 +00:00 · 2020-03-13 17:53:28 +08:00 · 2020-03-13 17:24:57 +08:00 · 2020-03-13 17:17:02 +08:00 · 2020-03-13 17:14:49 +08:00 · 2020-03-13 17:08:39 +08:00
--- a/.github/workflows/release.yml
+++ b/.github/workflows/release.yml
@@ -0,0 +1,54 @@
+name: release
+on:
+  push:
+    # Sequence of patterns matched against refs/tags
+    tags:
+    - 'v*' # Push events to matching v*, i.e. v1.0, v20.15.10
+
+jobs:
+
+  build:
+    name: Build
+    runs-on: ubuntu-latest
+    steps:
+
+    - name: Set up Go 1.13
+      uses: actions/setup-go@v1
+      with:
+        go-version: 1.13
+      id: go
+
+    - name: Check out code into the Go module directory
+      uses: actions/checkout@v2
+
+    - name: Get dependencies
+      run: |
+        go get -v -t -d ./cmd/tunasync
+        go get -v -t -d ./cmd/tunasynctl
+
+    - name: Build
+      run: |
+        make tunasync
+        make tunasynctl
+        tar -jcf build/tunasync-linux-bin.tar.bz2 -C build tunasync tunasynctl
+
+    - name: Create Release
+      id: create_release
+      uses: actions/create-release@v1
+      env:
+        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+      with:
+        tag_name: ${{ github.ref }}
+        release_name: Release ${{ github.ref }}
+        draft: false
+        prerelease: false
+    - name: Upload Release Asset
+      id: upload-release-asset 
+      uses: actions/upload-release-asset@v1
+      env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+      with:
+          upload_url: ${{ steps.create_release.outputs.upload_url }} # This pulls from the CREATE RELEASE step above, referencing it's ID to get its outputs object, which include a `upload_url`. See this blog post for more info: https://jasonet.co/posts/new-features-of-github-actions/#passing-data-to-future-steps 
+          asset_path: ./build/tunasync-linux-bin.tar.bz2
+          asset_name: tunasync-linux-bin.tar.bz2
+          asset_content_type: application/x-bzip2
--- a/.github/workflows/tunasync.yml
+++ b/.github/workflows/tunasync.yml
@@ -0,0 +1,69 @@
+name: tunasync
+
+on: [push]
+
+jobs:
+
+  build:
+    name: Build
+    runs-on: ubuntu-latest
+    steps:
+
+    - name: Set up Go 1.13
+      uses: actions/setup-go@v1
+      with:
+        go-version: 1.13
+      id: go
+
+    - name: Check out code into the Go module directory
+      uses: actions/checkout@v2
+
+    - name: Get dependencies
+      run: |
+        go get -v -t -d ./cmd/tunasync
+        go get -v -t -d ./cmd/tunasynctl
+
+    - name: Build
+      run: |
+        make tunasync
+        make tunasynctl
+
+
+  test:
+    name: Test
+    runs-on: ubuntu-latest
+    steps:
+
+    - name: Setup test dependencies
+      run: |
+        sudo apt update
+        sudo apt install -y cgroup-bin
+        /usr/bin/docker pull alpine
+        /usr/bin/docker images
+        lssubsys -am
+        sudo cgcreate -a $USER -t $USER -g cpu:tunasync
+        sudo cgcreate -a $USER -t $USER -g memory:tunasync
+
+    - name: Set up Go 1.13
+      uses: actions/setup-go@v1
+      with:
+        go-version: 1.13
+      id: go
+
+    - name: Check out code into the Go module directory
+      uses: actions/checkout@v2
+
+    - name: Run Unit tests.
+      run: make test
+
+    - name: Convert coverage to lcov
+      uses: jandelgado/gcov2lcov-action@v1.0.0
+      with:
+          infile: profile.cov
+          outfile: coverage.lcov
+
+    - name: Coveralls
+      uses: coverallsapp/github-action@v1.0.1
+      with:
+          github-token: ${{ secrets.github_token }}
+          path-to-lcov: coverage.lcov
--- a/.testandcover.bash
+++ b/.testandcover.bash
@@ -1,30 +0,0 @@
-#!/bin/bash
-function die() {
-  echo $*
-  exit 1
-}
-
-export GOPATH=`pwd`:$GOPATH
-
-make travis
-
-# Initialize profile.cov
-echo "mode: count" > profile.cov
-
-# Initialize error tracking
-ERROR=""
-
-# Test each package and append coverage profile info to profile.cov
-for pkg in `cat .testpackages.txt`
-do
-	go test -v -covermode=count -coverprofile=profile_tmp.cov $pkg || ERROR="Error testing $pkg"
-	
-	[ -f profile_tmp.cov ] && {
-		tail -n +2 profile_tmp.cov >> profile.cov || die "Unable to append coverage for $pkg"
-	}
-done
-
-if [ ! -z "$ERROR" ]
-then
-    die "Encountered error, last error was: $ERROR"
-fi
--- a/.travis.yml
+++ b/.travis.yml
@@ -1,44 +0,0 @@
-sudo: required
-
-language: go
-go:
-    - 1.11
-
-before_install:
-    - sudo apt-get install cgroup-bin
-    - go get github.com/smartystreets/goconvey
-    - go get golang.org/x/tools/cmd/cover
-    - go get -v github.com/mattn/goveralls
-
-os:
-    - linux
-
-services:
-    - docker
-
-before_script:
-    - lssubsys -am
-    - sudo cgcreate -a $USER -t $USER -g cpu:tunasync
-    - sudo cgcreate -a $USER -t $USER -g memory:tunasync
-    - docker pull alpine
-
-script:
-    - ./.testandcover.bash
-
-after_success:
-    - goveralls -coverprofile=profile.cov -service=travis-ci
-
-before_deploy: "echo 'ready to deploy?'"
-
-deploy:
-    provider: releases
-    file:
-        - "build/tunasync-linux-bin.tar.gz"
-    api_key:
-        secure: ZOYL/CALrVJsZzbZqUMSI89Gw4zsBJH1StD/2yTyG45GfKgvtK4hG0S5cQM/L0wcikjEkgxSMsmr4ycq+OwbN++gc0umfoAQ/VSjzetiobAlT1E854aRKRjT82WxYdnPW2fsFjuEJTcyZmcbgJGTMi86MDt7w8tEjLomhd1+rUo=
-    skip_cleanup: true
-    overwrite: true
-    on:
-        tags: true
-        all_branches: true
-        repo: tuna/tunasync
--- a/6
+++ b/6
@@ -2,8 +2,6 @@ LDFLAGS="-X main.buildstamp=`date -u '+%s'` -X main.githash=`git rev-parse HEAD`

 all: get tunasync tunasynctl

-travis: get tunasync tunasynctl travis-package
-
 get: 
 	go get ./cmd/tunasync
 	go get ./cmd/tunasynctl
@@ -17,5 +15,5 @@ tunasync: build
 tunasynctl: build
 	go build -o build/tunasynctl -ldflags ${LDFLAGS} github.com/tuna/tunasync/cmd/tunasynctl

-travis-package: tunasync tunasynctl
-	tar zcf build/tunasync-linux-bin.tar.gz -C build tunasync tunasynctl
+test:
+	go test -v -covermode=count -coverprofile=profile.cov ./...
--- a/README.md
+++ b/README.md
@@ -1,8 +1,8 @@
 tunasync
 ========

-[![Build Status](https://travis-ci.org/tuna/tunasync.svg?branch=dev)](https://travis-ci.org/tuna/tunasync)
-[![Coverage Status](https://coveralls.io/repos/github/tuna/tunasync/badge.svg?branch=dev)](https://coveralls.io/github/tuna/tunasync?branch=dev)
+![Build Status](https://github.com/tuna/tunasync/workflows/tunasync/badge.svg)
+[![Coverage Status](https://coveralls.io/repos/github/tuna/tunasync/badge.svg?branch=master)](https://coveralls.io/github/tuna/tunasync?branch=master)
 [![Commitizen friendly](https://img.shields.io/badge/commitizen-friendly-brightgreen.svg)](http://commitizen.github.io/cz-cli/)
 ![GPLv3](https://img.shields.io/badge/license-GPLv3-blue.svg)

@@ -53,14 +53,10 @@ PreSyncing                           Syncing                               Succe

 ## Building

-Setup GOPATH like [this](https://golang.org/cmd/go/#hdr-GOPATH_environment_variable).
-
-Then:
+Go version: 1.13

 ```
-go get -d github.com/tuna/tunasync/cmd/tunasync
-cd $GOPATH/src/github.com/tuna/tunasync
-make
+make all
 ```

-If you have multiple `GOPATH`s, replace the `$GOPATH` with your first one.
+Binaries in the `build/`.
--- a/cmd/tunasync/tunasync.go
+++ b/cmd/tunasync/tunasync.go
@@ -11,7 +11,7 @@ import (
 	"github.com/gin-gonic/gin"
 	"github.com/pkg/profile"
 	"gopkg.in/op/go-logging.v1"
-	"gopkg.in/urfave/cli.v1"
+	"github.com/urfave/cli"

 	tunasync "github.com/tuna/tunasync/internal"
 	"github.com/tuna/tunasync/manager"
--- a/cmd/tunasynctl/tunasynctl.go
+++ b/cmd/tunasynctl/tunasynctl.go
@@ -11,8 +11,8 @@ import (
 	"time"

 	"github.com/BurntSushi/toml"
+	"github.com/urfave/cli"
 	"gopkg.in/op/go-logging.v1"
-	"gopkg.in/urfave/cli.v1"

 	tunasync "github.com/tuna/tunasync/internal"
 )
--- a/docs/zh_CN/tips.md
+++ b/docs/zh_CN/tips.md
@@ -2,7 +2,7 @@

 先确定已经给tunasynctl写好config文件：`~/.config/tunasync/ctl.conf`

-```
+```toml
 manager_addr = "127.0.0.1"
 manager_port = 12345
 ca_cert = ""
@@ -10,7 +10,7 @@ ca_cert = ""

 接着

-```
+```shell
 $ tunasynctl disable -w <worker_id> <mirror_name>
 $ tunasynctl flush
 ```
@@ -18,8 +18,9 @@ $ tunasynctl flush

 ## 热重载 `worker.conf`

-`$ tunasynctl reload -w <worker_id>`
-
+```shell
+$ tunasynctl reload -w <worker_id>
+```

 e.g. 删除 `test_worker` 的 `elvish` 镜像：

@@ -29,7 +30,7 @@ e.g. 删除 `test_worker` 的 `elvish` 镜像：

 3. 接着操作：

-```
+```shell
 $ tunasynctl reload -w test_worker
 $ tunasynctl disable -w test_worker elvish
 $ tunasynctl flush
@@ -40,15 +41,53 @@ $ tunasynctl flush

 ## 删除worker

-`$ tunasynctl rm-worker -w <worker_id>`
+```shell
+$ tunasynctl rm-worker -w <worker_id>
+```

-e.g. `$ tunasynctl rm-worker -w test_worker`
+e.g.
+
+```shell
+$ tunasynctl rm-worker -w test_worker
+```


 ## 更新镜像的大小

-`$ tunasynctl set-size -w <worker_id> <mirror_name> <size>`
+```shell
+$ tunasynctl set-size -w <worker_id> <mirror_name> <size>
+```

 其中，末尾的 <size> 参数，由操作者设定，或由某定时脚本生成

 由于 `du -s` 比较耗时，故镜像大小可直接由rsync的日志文件读出
+
+
+## Btrfs 文件系统快照
+
+如果镜像文件存放在以 Btrfs 为文件系统的分区中，可启用由 Btrfs 提供的快照 (Snapshot) 功能。对于每一个镜像，tunasync 在每次成功同步后更新其快照。
+
+在 `worker.conf` 中添加如下配置，即可启用 Btrfs 快照功能：
+
+```toml
+[btrfs_snapshot]
+enable = true
+snapshot_path = "/path/to/snapshot/directory"
+```
+
+其中 `snapshot_path` 为快照所在目录。如将其作为发布版本，则镜像同步过程对于镜像站用户而言具有原子性。如此可避免用户接收到仍处于“中间态”的（未完成同步的）文件。
+
+也可以在 `[[mirrors]]` 中为特定镜像单独指定快照路径，如：
+
+```toml
+[[mirrors]]
+name = "elvish"
+provider = "rsync"
+upstream = "rsync://rsync.elvish.io/elvish/"
+interval = 1440
+snapshot_path = "/data/publish/elvish"
+```
+
+**提示：** 
+
+若运行 tunasync 的用户无 root 权限，请确保该用户对镜像同步目录和快照目录均具有写和执行权限，并使用 [`user_subvol_rm_allowed` 选项](https://btrfs.wiki.kernel.org/index.php/Manpage/btrfs(5)#MOUNT_OPTIONS)挂载相应的 Btrfs 分区。
--- a/docs/zh_CN/workers.conf
+++ b/docs/zh_CN/workers.conf
@@ -45,6 +45,7 @@ docker_image = "tunathu/tunasync-scripts:latest"
 name = "gnu"
 provider = "rsync"
 upstream = "rsync://mirrors.ocf.berkeley.edu/gnu/"
+rsync_options = [ "--delete-excluded" ]
 memory_limit = "256M"

 [[mirrors]]
@@ -72,6 +73,7 @@ name = "ubuntu"
 provider = "two-stage-rsync"
 stage1_profile = "debian"
 upstream = "rsync://archive.ubuntu.com/ubuntu/"
+rsync_options = [ "--delete-excluded" ]
 memory_limit = "256M"

 # vim: ft=toml
--- a/go.mod
+++ b/go.mod
@@ -0,0 +1,20 @@
+module github.com/tuna/tunasync
+
+go 1.13
+
+require (
+	github.com/BurntSushi/toml v0.3.1
+	github.com/anmitsu/go-shlex v0.0.0-20161002113705-648efa622239
+	github.com/boltdb/bolt v1.3.1
+	github.com/codeskyblue/go-sh v0.0.0-20190412065543-76bd3d59ff27
+	github.com/dennwc/btrfs v0.0.0-20190517175702-d917b30ff035
+	github.com/gin-gonic/gin v1.5.0
+	github.com/mattn/goveralls v0.0.5 // indirect
+	github.com/pkg/profile v1.4.0
+	github.com/ryszard/goskiplist v0.0.0-20150312221310-2dfbae5fcf46
+	github.com/smartystreets/goconvey v1.6.4
+	github.com/urfave/cli v1.22.3
+	golang.org/x/sys v0.0.0-20200302150141-5c8b2ff67527
+	golang.org/x/tools v0.0.0-20200312194400-c312e98713c2 // indirect
+	gopkg.in/op/go-logging.v1 v1.0.0-20160211212156-b2cb9fa56473
+)
--- a/go.sum
+++ b/go.sum
@@ -0,0 +1,99 @@
+github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ=
+github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
+github.com/anmitsu/go-shlex v0.0.0-20161002113705-648efa622239 h1:kFOfPq6dUM1hTo4JG6LR5AXSUEsOjtdm0kw0FtQtMJA=
+github.com/anmitsu/go-shlex v0.0.0-20161002113705-648efa622239/go.mod h1:2FmKhYUyUczH0OGQWaF5ceTx0UBShxjsH6f8oGKYe2c=
+github.com/boltdb/bolt v1.3.1 h1:JQmyP4ZBrce+ZQu0dY660FMfatumYDLun9hBCUVIkF4=
+github.com/boltdb/bolt v1.3.1/go.mod h1:clJnj/oiGkjum5o1McbSZDSLxVThjynRyGBgiAx27Ps=
+github.com/codegangsta/inject v0.0.0-20150114235600-33e0aa1cb7c0 h1:sDMmm+q/3+BukdIpxwO365v/Rbspp2Nt5XntgQRXq8Q=
+github.com/codegangsta/inject v0.0.0-20150114235600-33e0aa1cb7c0/go.mod h1:4Zcjuz89kmFXt9morQgcfYZAYZ5n8WHjt81YYWIwtTM=
+github.com/codeskyblue/go-sh v0.0.0-20190412065543-76bd3d59ff27 h1:HHUr4P/aKh4quafGxDT9LDasjGdlGkzLbfmmrlng3kA=
+github.com/codeskyblue/go-sh v0.0.0-20190412065543-76bd3d59ff27/go.mod h1:VQx0hjo2oUeQkQUET7wRwradO6f+fN5jzXgB/zROxxE=
+github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d h1:U+s90UTSYgptZMwQh2aRr3LuazLJIa+Pg3Kc1ylSYVY=
+github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU=
+github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
+github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/dennwc/btrfs v0.0.0-20190517175702-d917b30ff035 h1:4e+UEZaKPx0ZEiCMPUHMV51RGwbb1VJGCYqRFn/qmWM=
+github.com/dennwc/btrfs v0.0.0-20190517175702-d917b30ff035/go.mod h1:MYsOV9Dgsec3FFSOjywi0QK5r6TeBbdWxdrMGtiYXHA=
+github.com/dennwc/ioctl v1.0.0 h1:DsWAAjIxRqNcLn9x6mwfuf2pet3iB7aK90K4tF16rLg=
+github.com/dennwc/ioctl v1.0.0/go.mod h1:ellh2YB5ldny99SBU/VX7Nq0xiZbHphf1DrtHxxjMk0=
+github.com/gin-contrib/sse v0.1.0 h1:Y/yl/+YNO8GZSjAhjMsSuLt29uWRFHdHYUb5lYOV9qE=
+github.com/gin-contrib/sse v0.1.0/go.mod h1:RHrZQHXnP2xjPF+u1gW/2HnVO7nvIa9PG3Gm+fLHvGI=
+github.com/gin-gonic/gin v1.5.0 h1:fi+bqFAx/oLK54somfCtEZs9HeH1LHVoEPUgARpTqyc=
+github.com/gin-gonic/gin v1.5.0/go.mod h1:Nd6IXA8m5kNZdNEHMBd93KT+mdY3+bewLgRvmCsR2Do=
+github.com/go-playground/locales v0.12.1 h1:2FITxuFt/xuCNP1Acdhv62OzaCiviiE4kotfhkmOqEc=
+github.com/go-playground/locales v0.12.1/go.mod h1:IUMDtCfWo/w/mtMfIE/IG2K+Ey3ygWanZIBtBW0W2TM=
+github.com/go-playground/universal-translator v0.16.0 h1:X++omBR/4cE2MNg91AoC3rmGrCjJ8eAeUP/K/EKx4DM=
+github.com/go-playground/universal-translator v0.16.0/go.mod h1:1AnU7NaIRDWWzGEKwgtJRd2xk99HeFyHw3yid4rvQIY=
+github.com/golang/protobuf v1.3.2 h1:6nsPYzhq5kReh6QImI3k5qWzO4PEbvbIW2cwSfR/6xs=
+github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
+github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
+github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY=
+github.com/json-iterator/go v1.1.7/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
+github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo=
+github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU=
+github.com/leodido/go-urn v1.1.0 h1:Sm1gr51B1kKyfD2BlRcLSiEkffoG96g6TPv6eRoEiB8=
+github.com/leodido/go-urn v1.1.0/go.mod h1:+cyI34gQWZcE1eQU7NVgKkkzdXDQHr1dBMtdAPozLkw=
+github.com/mattn/go-isatty v0.0.9 h1:d5US/mDsogSGW37IV293h//ZFaeajb69h+EHFsv2xGg=
+github.com/mattn/go-isatty v0.0.9/go.mod h1:YNRxwqDuOph6SZLI9vUUz6OYw3QyUt7WiY2yME+cCiQ=
+github.com/mattn/goveralls v0.0.5 h1:spfq8AyZ0cCk57Za6/juJ5btQxeE1FaEGMdfcI+XO48=
+github.com/mattn/goveralls v0.0.5/go.mod h1:Xg2LHi51faXLyKXwsndxiW6uxEEQT9+3sjGzzwU4xy0=
+github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
+github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
+github.com/pkg/profile v1.4.0 h1:uCmaf4vVbWAOZz36k1hrQD7ijGRzLwaME8Am/7a4jZI=
+github.com/pkg/profile v1.4.0/go.mod h1:NWz/XGvpEW1FyYQ7fCx4dqYBLlfTcE+A9FLAkNKqjFE=
+github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
+github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/russross/blackfriday/v2 v2.0.1 h1:lPqVAte+HuHNfhJ/0LC98ESWRz8afy9tM/0RK8m9o+Q=
+github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
+github.com/ryszard/goskiplist v0.0.0-20150312221310-2dfbae5fcf46 h1:GHRpF1pTW19a8tTFrMLUcfWwyC0pnifVo2ClaLq+hP8=
+github.com/ryszard/goskiplist v0.0.0-20150312221310-2dfbae5fcf46/go.mod h1:uAQ5PCi+MFsC7HjREoAz1BU+Mq60+05gifQSsHSDG/8=
+github.com/shurcooL/sanitized_anchor_name v1.0.0 h1:PdmoCO6wvbs+7yrJyMORt4/BmY5IYyJwS/kOiWx8mHo=
+github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc=
+github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d h1:zE9ykElWQ6/NYmHa3jpm/yHnI4xSofP+UP6SpjHcSeM=
+github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc=
+github.com/smartystreets/goconvey v1.6.4 h1:fv0U8FUIMPNf1L9lnHLvLhgicrIVChEkdzIKYqbNC9s=
+github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA=
+github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
+github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk=
+github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
+github.com/ugorji/go v1.1.7 h1:/68gy2h+1mWMrwZFeD1kQialdSzAb432dtpeJ42ovdo=
+github.com/ugorji/go v1.1.7/go.mod h1:kZn38zHttfInRq0xu/PH0az30d+z6vm202qpg1oXVMw=
+github.com/ugorji/go/codec v1.1.7 h1:2SvQaVZ1ouYrrKKwoSk2pzd4A9evlKJb9oTL+OaLUSs=
+github.com/ugorji/go/codec v1.1.7/go.mod h1:Ax+UKWsSmolVDwsd+7N3ZtXu+yMGCf907BLYF3GoBXY=
+github.com/urfave/cli v1.22.3 h1:FpNT6zq26xNpHZy08emi755QwzLPs6Pukqjlc7RfOMU=
+github.com/urfave/cli v1.22.3/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0=
+golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
+golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
+golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg=
+golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
+golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
+golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
+golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190813064441-fde4db37ae7a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200302150141-5c8b2ff67527 h1:uYVVQ9WP/Ds2ROhcaGPeIdVq0RIXVLwsHlnvJ+cT1So=
+golang.org/x/sys v0.0.0-20200302150141-5c8b2ff67527/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
+golang.org/x/tools v0.0.0-20190328211700-ab21143f2384 h1:TFlARGu6Czu1z7q93HTxcP1P+/ZFC/IKythI5RzrnRg=
+golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
+golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.0.0-20200113040837-eac381796e91/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
+golang.org/x/tools v0.0.0-20200312194400-c312e98713c2 h1:6TB4+MaZlkcSsJDu+BS5yxSEuZIYhjWz+jhbSLEZylI=
+golang.org/x/tools v0.0.0-20200312194400-c312e98713c2/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw=
+golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/go-playground/assert.v1 v1.2.1/go.mod h1:9RXL0bg/zibRAgZUYszZSwO/z8Y/a8bDuhia5mkpMnE=
+gopkg.in/go-playground/validator.v9 v9.29.1 h1:SvGtYmN60a5CVKTOzMSyfzWDeZRxRuGvRQyEAKbw1xc=
+gopkg.in/go-playground/validator.v9 v9.29.1/go.mod h1:+c9/zcJMFNgbLvly1L1V+PpxWdVbfP1avr/N00E2vyQ=
+gopkg.in/op/go-logging.v1 v1.0.0-20160211212156-b2cb9fa56473 h1:6D+BvnJ/j6e222UW8s2qTSe3wGBtvo0MbVQG/c5k8RE=
+gopkg.in/op/go-logging.v1 v1.0.0-20160211212156-b2cb9fa56473/go.mod h1:N1eN2tsCx0Ydtgjl4cqmbRCsY4/+z4cYDeqwZTk6zog=
+gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw=
+gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
--- a/internal/version.go
+++ b/internal/version.go
@@ -1,3 +1,3 @@
 package internal

-const Version string = "0.3.3"
+const Version string = "0.4.1"
--- a/manager/config.go
+++ b/manager/config.go
@@ -2,7 +2,7 @@ package manager

 import (
 	"github.com/BurntSushi/toml"
-	"gopkg.in/urfave/cli.v1"
+	"github.com/urfave/cli"
 )

 // A Config is the top-level toml-serializaible config struct
--- a/manager/config_test.go
+++ b/manager/config_test.go
@@ -9,7 +9,7 @@ import (

 	"github.com/BurntSushi/toml"
 	. "github.com/smartystreets/goconvey/convey"
-	"gopkg.in/urfave/cli.v1"
+	"github.com/urfave/cli"
 )

 func TestConfig(t *testing.T) {
--- a/worker/btrfs_snapshot_hook.go
+++ b/worker/btrfs_snapshot_hook.go
@@ -0,0 +1,90 @@
+package worker
+
+import (
+	"fmt"
+	"os"
+	"path/filepath"
+
+	"github.com/dennwc/btrfs"
+)
+
+type btrfsSnapshotHook struct {
+	provider           mirrorProvider
+	mirrorSnapshotPath string
+}
+
+// the user who runs the jobs (typically `tunasync`) should be granted the permission to run btrfs commands
+// TODO: check if the filesystem is Btrfs
+func newBtrfsSnapshotHook(provider mirrorProvider, snapshotPath string, mirror mirrorConfig) *btrfsSnapshotHook {
+	mirrorSnapshotPath := mirror.SnapshotPath
+	if mirrorSnapshotPath == "" {
+		mirrorSnapshotPath = filepath.Join(snapshotPath, provider.Name())
+	}
+	return &btrfsSnapshotHook{
+		provider:           provider,
+		mirrorSnapshotPath: mirrorSnapshotPath,
+	}
+}
+
+// check if path `snapshotPath/providerName` exists
+// Case 1: Not exists => create a new subvolume
+// Case 2: Exists as a subvolume => nothing to do
+// Case 3: Exists as a directory => error detected
+func (h *btrfsSnapshotHook) preJob() error {
+	path := h.provider.WorkingDir()
+	if _, err := os.Stat(path); os.IsNotExist(err) {
+		// create subvolume
+		err := btrfs.CreateSubVolume(path)
+		if err != nil {
+			logger.Errorf("failed to create Btrfs subvolume %s: %s", path, err.Error())
+			return err
+		}
+		logger.Noticef("created new Btrfs subvolume %s", path)
+	} else {
+		if is, err := btrfs.IsSubVolume(path); err != nil {
+			return err
+		} else if !is {
+			return fmt.Errorf("path %s exists but isn't a Btrfs subvolume", path)
+		}
+	}
+	return nil
+}
+
+func (h *btrfsSnapshotHook) preExec() error {
+	return nil
+}
+
+func (h *btrfsSnapshotHook) postExec() error {
+	return nil
+}
+
+// delete old snapshot if exists, then create a new snapshot
+func (h *btrfsSnapshotHook) postSuccess() error {
+	if _, err := os.Stat(h.mirrorSnapshotPath); !os.IsNotExist(err) {
+		isSubVol, err := btrfs.IsSubVolume(h.mirrorSnapshotPath)
+		if err != nil {
+			return err
+		} else if !isSubVol {
+			return fmt.Errorf("path %s exists and isn't a Btrfs snapshot", h.mirrorSnapshotPath)
+		}
+		// is old snapshot => delete it
+		if err := btrfs.DeleteSubVolume(h.mirrorSnapshotPath); err != nil {
+			logger.Errorf("failed to delete old Btrfs snapshot %s", h.mirrorSnapshotPath)
+			return err
+		}
+		logger.Noticef("deleted old snapshot %s", h.mirrorSnapshotPath)
+	}
+	// create a new writable snapshot
+	// (the snapshot is writable so that it can be deleted easily)
+	if err := btrfs.SnapshotSubVolume(h.provider.WorkingDir(), h.mirrorSnapshotPath, false); err != nil {
+		logger.Errorf("failed to create new Btrfs snapshot %s", h.mirrorSnapshotPath)
+		return err
+	}
+	logger.Noticef("created new Btrfs snapshot %s", h.mirrorSnapshotPath)
+	return nil
+}
+
+// keep the old snapshot => nothing to do
+func (h *btrfsSnapshotHook) postFail() error {
+	return nil
+}
--- a/worker/cmd_provider.go
+++ b/worker/cmd_provider.go
@@ -2,6 +2,9 @@ package worker

 import (
 	"errors"
+	"fmt"
+	"io/ioutil"
+	"regexp"
 	"time"

 	"github.com/anmitsu/go-shlex"
@@ -14,12 +17,14 @@ type cmdConfig struct {
 	interval                    time.Duration
 	retry                       int
 	env                         map[string]string
+	failOnMatch                 string
 }

 type cmdProvider struct {
 	baseProvider
 	cmdConfig
-	command []string
+	command     []string
+	failOnMatch *regexp.Regexp
 }

 func newCmdProvider(c cmdConfig) (*cmdProvider, error) {
@@ -46,6 +51,14 @@ func newCmdProvider(c cmdConfig) (*cmdProvider, error) {
 		return nil, err
 	}
 	provider.command = cmd
+	if len(c.failOnMatch) > 0 {
+		var err error
+		failOnMatch, err := regexp.Compile(c.failOnMatch)
+		if err != nil {
+			return nil, errors.New("fail-on-match regexp error: " + err.Error())
+		}
+		provider.failOnMatch = failOnMatch
+	}

 	return provider, nil
 }
@@ -62,7 +75,22 @@ func (p *cmdProvider) Run() error {
 	if err := p.Start(); err != nil {
 		return err
 	}
-	return p.Wait()
+	if err := p.Wait(); err != nil {
+		return err
+	}
+	if p.failOnMatch != nil {
+		if logContent, err := ioutil.ReadFile(p.LogFile()); err == nil {
+			matches := p.failOnMatch.FindAllSubmatch(logContent, -1)
+			if len(matches) != 0 {
+				logger.Debug("Fail-on-match: %r", matches)
+				return errors.New(
+					fmt.Sprintf("Fail-on-match regexp found %d matches", len(matches)))
+			}
+		} else {
+			return err
+		}
+	}
+	return nil
 }

 func (p *cmdProvider) Start() error {
--- a/worker/config.go
+++ b/worker/config.go
@@ -33,14 +33,15 @@ func (p *providerEnum) UnmarshalText(text []byte) error {

 // Config represents worker config options
 type Config struct {
-	Global  globalConfig   `toml:"global"`
-	Manager managerConfig  `toml:"manager"`
-	Server  serverConfig   `toml:"server"`
-	Cgroup  cgroupConfig   `toml:"cgroup"`
-	ZFS     zfsConfig      `toml:"zfs"`
-	Docker  dockerConfig   `toml:"docker"`
-	Include includeConfig  `toml:"include"`
-	Mirrors []mirrorConfig `toml:"mirrors"`
+	Global        globalConfig        `toml:"global"`
+	Manager       managerConfig       `toml:"manager"`
+	Server        serverConfig        `toml:"server"`
+	Cgroup        cgroupConfig        `toml:"cgroup"`
+	ZFS           zfsConfig           `toml:"zfs"`
+	BtrfsSnapshot btrfsSnapshotConfig `toml:"btrfs_snapshot"`
+	Docker        dockerConfig        `toml:"docker"`
+	Include       includeConfig       `toml:"include"`
+	Mirrors       []mirrorConfig      `toml:"mirrors"`
 }

 type globalConfig struct {
@@ -96,6 +97,11 @@ type zfsConfig struct {
 	Zpool  string `toml:"zpool"`
 }

+type btrfsSnapshotConfig struct {
+	Enable       bool   `toml:"enable"`
+	SnapshotPath string `toml:"snapshot_path"`
+}
+
 type includeConfig struct {
 	IncludeMirrors string `toml:"include_mirrors"`
 }
@@ -123,19 +129,24 @@ type mirrorConfig struct {
 	ExecOnSuccessExtra []string `toml:"exec_on_success_extra"`
 	ExecOnFailureExtra []string `toml:"exec_on_failure_extra"`

-	Command       string `toml:"command"`
-	UseIPv6       bool   `toml:"use_ipv6"`
-	UseIPv4       bool   `toml:"use_ipv4"`
-	ExcludeFile   string `toml:"exclude_file"`
-	Username      string `toml:"username"`
-	Password      string `toml:"password"`
-	Stage1Profile string `toml:"stage1_profile"`
+	Command       string   `toml:"command"`
+	FailOnMatch   string   `toml:"fail_on_match"`
+	UseIPv6       bool     `toml:"use_ipv6"`
+	UseIPv4       bool     `toml:"use_ipv4"`
+	ExcludeFile   string   `toml:"exclude_file"`
+	Username      string   `toml:"username"`
+	Password      string   `toml:"password"`
+	RsyncOptions  []string `toml:"rsync_options"`
+	RsyncOverride []string `toml:"rsync_override"`
+	Stage1Profile string   `toml:"stage1_profile"`

 	MemoryLimit string `toml:"memory_limit"`

 	DockerImage   string   `toml:"docker_image"`
 	DockerVolumes []string `toml:"docker_volumes"`
 	DockerOptions []string `toml:"docker_options"`
+
+	SnapshotPath string `toml:"snapshot_path"`
 }

 // LoadConfig loads configuration
--- a/worker/docker_test.go
+++ b/worker/docker_test.go
@@ -1,99 +0,0 @@
-package worker
-
-import (
-	"fmt"
-	"io/ioutil"
-	"os"
-	"path/filepath"
-	"testing"
-	"time"
-
-	"github.com/codeskyblue/go-sh"
-	. "github.com/smartystreets/goconvey/convey"
-)
-
-func getDockerByName(name string) (string, error) {
-	// docker ps -f 'name=$name' --format '{{.Names}}'
-	out, err := sh.Command(
-		"docker", "ps",
-		"--filter", "name="+name,
-		"--format", "{{.Names}}",
-	).Output()
-	return string(out), err
-}
-
-func TestDocker(t *testing.T) {
-	Convey("Docker Should Work", t, func(ctx C) {
-		tmpDir, err := ioutil.TempDir("", "tunasync")
-		defer os.RemoveAll(tmpDir)
-		So(err, ShouldBeNil)
-		cmdScript := filepath.Join(tmpDir, "cmd.sh")
-		tmpFile := filepath.Join(tmpDir, "log_file")
-		expectedOutput := "HELLO_WORLD"
-
-		c := cmdConfig{
-			name:        "tuna-docker",
-			upstreamURL: "http://mirrors.tuna.moe/",
-			command:     "/bin/cmd.sh",
-			workingDir:  tmpDir,
-			logDir:      tmpDir,
-			logFile:     tmpFile,
-			interval:    600 * time.Second,
-			env: map[string]string{
-				"TEST_CONTENT": expectedOutput,
-			},
-		}
-
-		cmdScriptContent := `#!/bin/sh
-echo ${TEST_CONTENT}
-sleep 10
-`
-		err = ioutil.WriteFile(cmdScript, []byte(cmdScriptContent), 0755)
-		So(err, ShouldBeNil)
-
-		provider, err := newCmdProvider(c)
-		So(err, ShouldBeNil)
-
-		d := &dockerHook{
-			emptyHook: emptyHook{
-				provider: provider,
-			},
-			image: "alpine",
-			volumes: []string{
-				fmt.Sprintf("%s:%s", cmdScript, "/bin/cmd.sh"),
-			},
-		}
-		provider.AddHook(d)
-		So(provider.Docker(), ShouldNotBeNil)
-
-		err = d.preExec()
-		So(err, ShouldBeNil)
-
-		go func() {
-			err = provider.Run()
-			ctx.So(err, ShouldNotBeNil)
-		}()
-
-		time.Sleep(1 * time.Second)
-
-		// assert container running
-		names, err := getDockerByName(d.Name())
-		So(err, ShouldBeNil)
-		So(names, ShouldEqual, d.Name()+"\n")
-
-		err = provider.Terminate()
-		So(err, ShouldBeNil)
-
-		// container should be terminated and removed
-		names, err = getDockerByName(d.Name())
-		So(err, ShouldBeNil)
-		So(names, ShouldEqual, "")
-
-		// check log content
-		loggedContent, err := ioutil.ReadFile(provider.LogFile())
-		So(err, ShouldBeNil)
-		So(string(loggedContent), ShouldEqual, expectedOutput+"\n")
-
-		d.postExec()
-	})
-}
--- a/worker/job.go
+++ b/worker/job.go
@@ -113,7 +113,7 @@ func (m *mirrorJob) Run(managerChan chan<- jobMessage, semaphore chan empty) err
 				managerChan <- jobMessage{
 					tunasync.Failed, m.Name(),
 					fmt.Sprintf("error exec hook %s: %s", hookname, err.Error()),
-					false,
+					true,
 				}
 				return err
 			}
--- a/worker/job_test.go
+++ b/worker/job_test.go
@@ -112,6 +112,74 @@ func TestMirrorJob(t *testing.T) {

 		})

+		Convey("When running long jobs with post-fail hook", func(ctx C) {
+			scriptContent := `#!/bin/bash
+echo '++++++'
+echo $TUNASYNC_WORKING_DIR
+echo $0 sleeping
+sleep 3
+echo $TUNASYNC_WORKING_DIR
+echo '------'
+			`
+			err = ioutil.WriteFile(scriptFile, []byte(scriptContent), 0755)
+			So(err, ShouldBeNil)
+
+			hookScriptFile := filepath.Join(tmpDir, "hook.sh")
+			err = ioutil.WriteFile(hookScriptFile, []byte(scriptContent), 0755)
+			So(err, ShouldBeNil)
+
+			h, err := newExecPostHook(provider, execOnFailure, hookScriptFile)
+			So(err, ShouldBeNil)
+			provider.AddHook(h)
+
+			managerChan := make(chan jobMessage, 10)
+			semaphore := make(chan empty, 1)
+			job := newMirrorJob(provider)
+
+			Convey("If we kill it", func(ctx C) {
+				go job.Run(managerChan, semaphore)
+				job.ctrlChan <- jobStart
+
+				time.Sleep(1 * time.Second)
+				msg := <-managerChan
+				So(msg.status, ShouldEqual, PreSyncing)
+				msg = <-managerChan
+				So(msg.status, ShouldEqual, Syncing)
+
+				job.ctrlChan <- jobStop
+
+				msg = <-managerChan
+				So(msg.status, ShouldEqual, Failed)
+
+				job.ctrlChan <- jobDisable
+				<-job.disabled
+			})
+
+			Convey("If we kill it then start it", func(ctx C) {
+				go job.Run(managerChan, semaphore)
+				job.ctrlChan <- jobStart
+
+				time.Sleep(1 * time.Second)
+				msg := <-managerChan
+				So(msg.status, ShouldEqual, PreSyncing)
+				msg = <-managerChan
+				So(msg.status, ShouldEqual, Syncing)
+
+				job.ctrlChan <- jobStop
+
+				time.Sleep(2 * time.Second)
+				logger.Debugf("Now starting...\n")
+				job.ctrlChan <- jobStart
+
+				msg = <-managerChan
+				So(msg.status, ShouldEqual, Failed)
+
+				job.ctrlChan <- jobDisable
+				<-job.disabled
+			})
+
+		})
+
 		Convey("When running long jobs", func(ctx C) {
 			scriptContent := `#!/bin/bash
 echo $TUNASYNC_WORKING_DIR
--- a/worker/provider.go
+++ b/worker/provider.go
@@ -112,6 +112,7 @@ func newMirrorProvider(mirror mirrorConfig, cfg *Config) mirrorProvider {
 			upstreamURL: mirror.Upstream,
 			command:     mirror.Command,
 			workingDir:  mirrorDir,
+			failOnMatch: mirror.FailOnMatch,
 			logDir:      logDir,
 			logFile:     filepath.Join(logDir, "latest.log"),
 			interval:    time.Duration(mirror.Interval) * time.Minute,
@@ -126,19 +127,21 @@ func newMirrorProvider(mirror mirrorConfig, cfg *Config) mirrorProvider {
 		provider = p
 	case provRsync:
 		rc := rsyncConfig{
-			name:        mirror.Name,
-			upstreamURL: mirror.Upstream,
-			rsyncCmd:    mirror.Command,
-			username:    mirror.Username,
-			password:    mirror.Password,
-			excludeFile: mirror.ExcludeFile,
-			workingDir:  mirrorDir,
-			logDir:      logDir,
-			logFile:     filepath.Join(logDir, "latest.log"),
-			useIPv6:     mirror.UseIPv6,
-			useIPv4:     mirror.UseIPv4,
-			interval:    time.Duration(mirror.Interval) * time.Minute,
-			retry:       mirror.Retry,
+			name:              mirror.Name,
+			upstreamURL:       mirror.Upstream,
+			rsyncCmd:          mirror.Command,
+			username:          mirror.Username,
+			password:          mirror.Password,
+			excludeFile:       mirror.ExcludeFile,
+			extraOptions:      mirror.RsyncOptions,
+			overriddenOptions: mirror.RsyncOverride,
+			workingDir:        mirrorDir,
+			logDir:            logDir,
+			logFile:           filepath.Join(logDir, "latest.log"),
+			useIPv6:           mirror.UseIPv6,
+			useIPv4:           mirror.UseIPv4,
+			interval:          time.Duration(mirror.Interval) * time.Minute,
+			retry:             mirror.Retry,
 		}
 		p, err := newRsyncProvider(rc)
 		if err != nil {
@@ -155,6 +158,7 @@ func newMirrorProvider(mirror mirrorConfig, cfg *Config) mirrorProvider {
 			username:      mirror.Username,
 			password:      mirror.Password,
 			excludeFile:   mirror.ExcludeFile,
+			extraOptions:  mirror.RsyncOptions,
 			workingDir:    mirrorDir,
 			logDir:        logDir,
 			logFile:       filepath.Join(logDir, "latest.log"),
@@ -180,6 +184,11 @@ func newMirrorProvider(mirror mirrorConfig, cfg *Config) mirrorProvider {
 		provider.AddHook(newZfsHook(provider, cfg.ZFS.Zpool))
 	}

+	// Add Btrfs Snapshot Hook
+	if cfg.BtrfsSnapshot.Enable {
+		provider.AddHook(newBtrfsSnapshotHook(provider, cfg.BtrfsSnapshot.SnapshotPath, mirror))
+	}
+
 	// Add Docker Hook
 	if cfg.Docker.Enable && len(mirror.DockerImage) > 0 {
 		provider.AddHook(newDockerHook(provider, cfg.Docker, mirror))
--- a/worker/provider_test.go
+++ b/worker/provider_test.go
@@ -116,16 +116,17 @@ func TestRsyncProviderWithAuthentication(t *testing.T) {
 		tmpFile := filepath.Join(tmpDir, "log_file")

 		c := rsyncConfig{
-			name:        "tuna",
-			upstreamURL: "rsync://rsync.tuna.moe/tuna/",
-			rsyncCmd:    scriptFile,
-			username:    "tunasync",
-			password:    "tunasyncpassword",
-			workingDir:  tmpDir,
-			logDir:      tmpDir,
-			logFile:     tmpFile,
-			useIPv6:     true,
-			interval:    600 * time.Second,
+			name:         "tuna",
+			upstreamURL:  "rsync://rsync.tuna.moe/tuna/",
+			rsyncCmd:     scriptFile,
+			username:     "tunasync",
+			password:     "tunasyncpassword",
+			workingDir:   tmpDir,
+			extraOptions: []string{"--delete-excluded"},
+			logDir:       tmpDir,
+			logFile:      tmpFile,
+			useIPv4:      true,
+			interval:     600 * time.Second,
 		}

 		provider, err := newRsyncProvider(c)
@@ -157,7 +158,7 @@ exit 0
 				fmt.Sprintf(
 					"%s %s -aHvh --no-o --no-g --stats --exclude .~tmp~/ "+
 						"--delete --delete-after --delay-updates --safe-links "+
-						"--timeout=120 --contimeout=120 -6 %s %s",
+						"--timeout=120 --contimeout=120 -4 --delete-excluded %s %s",
 					provider.username, provider.password, provider.upstreamURL, provider.WorkingDir(),
 				),
 			)
@@ -173,6 +174,68 @@ exit 0
 	})
 }

+func TestRsyncProviderWithOverriddenOptions(t *testing.T) {
+	Convey("Rsync Provider with overridden options should work", t, func() {
+		tmpDir, err := ioutil.TempDir("", "tunasync")
+		defer os.RemoveAll(tmpDir)
+		So(err, ShouldBeNil)
+		scriptFile := filepath.Join(tmpDir, "myrsync")
+		tmpFile := filepath.Join(tmpDir, "log_file")
+
+		c := rsyncConfig{
+			name:              "tuna",
+			upstreamURL:       "rsync://rsync.tuna.moe/tuna/",
+			rsyncCmd:          scriptFile,
+			workingDir:        tmpDir,
+			overriddenOptions: []string{"-aHvh", "--no-o", "--no-g", "--stats"},
+			extraOptions:      []string{"--delete-excluded"},
+			logDir:            tmpDir,
+			logFile:           tmpFile,
+			useIPv6:           true,
+			interval:          600 * time.Second,
+		}
+
+		provider, err := newRsyncProvider(c)
+		So(err, ShouldBeNil)
+
+		So(provider.Name(), ShouldEqual, c.name)
+		So(provider.WorkingDir(), ShouldEqual, c.workingDir)
+		So(provider.LogDir(), ShouldEqual, c.logDir)
+		So(provider.LogFile(), ShouldEqual, c.logFile)
+		So(provider.Interval(), ShouldEqual, c.interval)
+
+		Convey("Let's try a run", func() {
+			scriptContent := `#!/bin/bash
+echo "syncing to $(pwd)"
+echo $@
+sleep 1
+echo "Done"
+exit 0
+			`
+			err = ioutil.WriteFile(scriptFile, []byte(scriptContent), 0755)
+			So(err, ShouldBeNil)
+
+			targetDir, _ := filepath.EvalSymlinks(provider.WorkingDir())
+			expectedOutput := fmt.Sprintf(
+				"syncing to %s\n"+
+					"-aHvh --no-o --no-g --stats -6 --delete-excluded %s %s\n"+
+					"Done\n",
+				targetDir,
+				provider.upstreamURL,
+				provider.WorkingDir(),
+			)
+
+			err = provider.Run()
+			So(err, ShouldBeNil)
+			loggedContent, err := ioutil.ReadFile(provider.LogFile())
+			So(err, ShouldBeNil)
+			So(string(loggedContent), ShouldEqual, expectedOutput)
+			// fmt.Println(string(loggedContent))
+		})
+
+	})
+}
+
 func TestCmdProvider(t *testing.T) {
 	Convey("Command Provider should work", t, func(ctx C) {
 		tmpDir, err := ioutil.TempDir("", "tunasync")
@@ -299,6 +362,41 @@ sleep 5

 		})
 	})
+	Convey("Command Provider with fail-on-match regexp should work", t, func(ctx C) {
+		tmpDir, err := ioutil.TempDir("", "tunasync")
+		defer os.RemoveAll(tmpDir)
+		So(err, ShouldBeNil)
+		tmpFile := filepath.Join(tmpDir, "log_file")
+
+		c := cmdConfig{
+			name:        "run-uptime",
+			upstreamURL: "http://mirrors.tuna.moe/",
+			command:     "uptime",
+			failOnMatch: "",
+			workingDir:  tmpDir,
+			logDir:      tmpDir,
+			logFile:     tmpFile,
+			interval:    600 * time.Second,
+		}
+
+		Convey("when regexp matches", func() {
+			c.failOnMatch = `[a-z]+`
+			provider, err := newCmdProvider(c)
+			So(err, ShouldBeNil)
+
+			err = provider.Run()
+			So(err, ShouldNotBeNil)
+		})
+
+		Convey("when regexp does not match", func() {
+			c.failOnMatch = `load average_`
+			provider, err := newCmdProvider(c)
+			So(err, ShouldBeNil)
+
+			err = provider.Run()
+			So(err, ShouldBeNil)
+		})
+	})
 }

 func TestTwoStageRsyncProvider(t *testing.T) {
@@ -319,6 +417,7 @@ func TestTwoStageRsyncProvider(t *testing.T) {
 			logFile:       tmpFile,
 			useIPv6:       true,
 			excludeFile:   tmpFile,
+			extraOptions:  []string{"--delete-excluded", "--cache"},
 			username:      "hello",
 			password:      "world",
 		}
@@ -359,14 +458,14 @@ exit 0
 				fmt.Sprintf(
 					"-aHvh --no-o --no-g --stats --exclude .~tmp~/ --safe-links "+
 						"--timeout=120 --contimeout=120 --exclude dists/ -6 "+
-						"--exclude-from %s %s %s",
+						"--exclude-from %s --delete-excluded --cache %s %s",
 					provider.excludeFile, provider.upstreamURL, provider.WorkingDir(),
 				),
 				targetDir,
 				fmt.Sprintf(
 					"-aHvh --no-o --no-g --stats --exclude .~tmp~/ "+
 						"--delete --delete-after --delay-updates --safe-links "+
-						"--timeout=120 --contimeout=120 -6 --exclude-from %s %s %s",
+						"--timeout=120 --contimeout=120 -6 --exclude-from %s --delete-excluded --cache %s %s",
 					provider.excludeFile, provider.upstreamURL, provider.WorkingDir(),
 				),
 			)
@@ -398,7 +497,7 @@ exit 0
 			expectedOutput := fmt.Sprintf(
 				"-aHvh --no-o --no-g --stats --exclude .~tmp~/ --safe-links "+
 					"--timeout=120 --contimeout=120 --exclude dists/ -6 "+
-					"--exclude-from %s %s %s\n",
+					"--exclude-from %s --delete-excluded --cache %s %s\n",
 				provider.excludeFile, provider.upstreamURL, provider.WorkingDir(),
 			)

--- a/worker/rsync_provider.go
+++ b/worker/rsync_provider.go
@@ -13,6 +13,8 @@ type rsyncConfig struct {
 	name                                         string
 	rsyncCmd                                     string
 	upstreamURL, username, password, excludeFile string
+	extraOptions                                 []string
+	overriddenOptions                            []string
 	workingDir, logDir, logFile                  string
 	useIPv6, useIPv4                             bool
 	interval                                     time.Duration
@@ -55,6 +57,9 @@ func newRsyncProvider(c rsyncConfig) (*rsyncProvider, error) {
 		"--delete", "--delete-after", "--delay-updates",
 		"--safe-links", "--timeout=120", "--contimeout=120",
 	}
+	if c.overriddenOptions != nil {
+		options = c.overriddenOptions
+	}

 	if c.useIPv6 {
 		options = append(options, "-6")
@@ -65,6 +70,9 @@ func newRsyncProvider(c rsyncConfig) (*rsyncProvider, error) {
 	if c.excludeFile != "" {
 		options = append(options, "--exclude-from", c.excludeFile)
 	}
+	if c.extraOptions != nil {
+		options = append(options, c.extraOptions...)
+	}
 	provider.options = options

 	provider.ctx.Set(_WorkingDirKey, c.workingDir)
--- a/worker/two_stage_rsync_provider.go
+++ b/worker/two_stage_rsync_provider.go
@@ -15,6 +15,7 @@ type twoStageRsyncConfig struct {
 	rsyncCmd                                     string
 	stage1Profile                                string
 	upstreamURL, username, password, excludeFile string
+	extraOptions                                 []string
 	workingDir, logDir, logFile                  string
 	useIPv6                                      bool
 	interval                                     time.Duration
@@ -116,6 +117,9 @@ func (p *twoStageRsyncProvider) Options(stage int) ([]string, error) {
 	if p.excludeFile != "" {
 		options = append(options, "--exclude-from", p.excludeFile)
 	}
+	if p.extraOptions != nil {
+		options = append(options, p.extraOptions...)
+	}

 	return options, nil
 }
--- a/worker/worker_test.go
+++ b/worker/worker_test.go
@@ -93,6 +93,8 @@ func TestWorker(t *testing.T) {
 		err := httpServer.ListenAndServe()
 		So(err, ShouldBeNil)
 	}()
+	// Wait for http server starting
+	time.Sleep(500 * time.Millisecond)

 	Convey("Worker should work", t, func(ctx C) {

@@ -133,7 +135,7 @@ func TestWorker(t *testing.T) {
 						} else if sch, ok := data.(MirrorSchedules); ok {
 							So(len(sch.Schedules), ShouldEqual, 0)
 						}
-					case <-time.After(1 * time.Second):
+					case <-time.After(2 * time.Second):
 						So(registered, ShouldBeTrue)
 						return
 					}
@@ -178,7 +180,7 @@ func TestWorker(t *testing.T) {
 							So(status.Status, ShouldNotEqual, Failed)
 							lastStatus = status.Status
 						}
-					case <-time.After(1 * time.Second):
+					case <-time.After(2 * time.Second):
 						So(url, ShouldNotEqual, "")
 						So(jobRunning, ShouldBeFalse)
 						So(lastStatus, ShouldEqual, Success)
@@ -239,7 +241,7 @@ func TestWorker(t *testing.T) {
 							}
 							lastStatus[status.Name] = status.Status
 						}
-					case <-time.After(1 * time.Second):
+					case <-time.After(2 * time.Second):
 						So(len(lastStatus), ShouldEqual, 3)
 						So(len(nextSch), ShouldEqual, 3)
 						return
作者	SHA1	备注	提交日期
z4yx	07cb51076e	Bump version to 0.4.1	2020-03-13 17:53:28 +08:00
z4yx	3a2888dd5d	Bye Travis!	2020-03-13 17:24:57 +08:00
z4yx	ada881850a	maybe we don't have to install docker.io	2020-03-13 17:17:02 +08:00
z4yx	6f51188021	create release on tags	2020-03-13 17:14:49 +08:00
z4yx	a517a4bb64	remove docker_test.go	2020-03-13 17:08:39 +08:00
z4yx	b816803eaf	Revert "disable docker_test" This reverts commit `6d17d6b4ca`.	2020-03-13 17:08:19 +08:00
z4yx	6d17d6b4ca	disable docker_test	2020-03-13 17:04:36 +08:00
z4yx	51e7f1d573	add TestRsyncProviderWithOverriddenOptions	2020-03-13 17:02:38 +08:00
z4yx	c99095267e	[docker test] Wait for docker running	2020-03-13 16:57:07 +08:00
z4yx	5c140035ec	[worker teset] Wait for http server starting	2020-03-13 16:41:30 +08:00
z4yx	6ef9ccdfe6	unmask docker	2020-03-13 16:00:55 +08:00
z4yx	8df5e41d5b	systemctl start docker	2020-03-13 15:57:36 +08:00
z4yx	a38a88cf41	run apt update	2020-03-13 15:49:39 +08:00
z4yx	f603aebec9	add test steps	2020-03-13 15:46:56 +08:00
z4yx	80ad3247a0	fix build	2020-03-13 15:16:39 +08:00
z4yx	02468e21c0	add an option "rsync_override"	2020-03-13 15:12:52 +08:00
z4yx	d48815b817	update the condition	2020-03-13 15:06:26 +08:00
z4yx	07cd7b5f1f	update go.mod	2020-03-13 15:04:06 +08:00
z4yx	3f45e8b02b	Merge commit 'ad28e8aacc124ffb751cc77a5e3c3a3ad8d0a97c' into wip-ga	2020-03-13 15:03:39 +08:00
Yuxiang Zhang	ed1f20b1e6	Hello Github Actions	2020-03-13 14:59:56 +08:00
z4yx	ad28e8aacc	using Go Modules	2020-03-13 13:55:57 +08:00
Yuxiang Zhang	230d63e871	Merge pull request #108 from tuna/wip-fail-on-match (close #87 ) [mirror config] job fails on pattern match in log	2020-03-09 21:54:34 +08:00
z4yx	908f098c72	[mirror config] job fails on pattern match in log	2020-03-09 21:48:06 +08:00
Yuxiang Zhang	22cfdfc9c2	[mirror config] extra rsync options (#107 ) * support "rsync_options" array in config * add test for new options * fix tests	2020-03-09 20:48:09 +08:00
z4yx	36010dc33e	Merge branch 'master' of github.com:tuna/tunasync	2019-11-30 00:08:59 +08:00
z4yx	bc416a6088	add tests for post-hooks	2019-11-30 00:08:46 +08:00
z4yx	a065a11b38	change timeout in tests	2019-11-07 12:29:57 +08:00
z4yx	b4fe4db82a	Merge remote-tracking branch 'origin/dev'	2019-11-04 23:11:34 +08:00
z4yx	839363aaaa	reschedule the job if any hook fails	2019-11-04 22:52:03 +08:00
Yuxiang Zhang	08aee8eb42	Merge pull request #98 from ziqin/feature/btrfs-snapshot Reimplement Btrfs snapshots hook	2019-08-31 10:57:44 +08:00
Jeeken Wang	501f77ee41	Merge branch 'master' into feature/btrfs-snapshot	2019-08-15 01:26:28 +08:00
WANG Ziqin	8fd2059013	add doc for setup btrfs snapshots	2019-08-02 13:31:33 +08:00
WANG Ziqin	6b56c4254c	feat(btrfs_snapshot_hook): reimplemented Btrfs snapshots TODO: test coverage	2019-08-02 13:31:33 +08:00