Bump version to 0.6.9

Signed-off-by: Miao Wang <shankerwangmiao@gmail.com>

.github/workflows/release.yml

@@ -25,7 +25,7 @@ jobs:
       run: |
         for i in linux-amd64 linux-arm64; do
           make ARCH=$i all
-          tar -cz --numeric-owner --owner root -f tunasync-$i-bin.tar.gz -C build-$i tunasync tunasynctl 
+          tar -cz --numeric-owner --owner root --group root -f tunasync-$i-bin.tar.gz -C build-$i tunasync tunasynctl 
         done
 
     - name: Create Release

README.md

@@ -1,5 +1,4 @@
-tunasync
-========
+# tunasync
 
 ![Build Status](https://github.com/tuna/tunasync/workflows/tunasync/badge.svg)
 [![Coverage Status](https://coveralls.io/repos/github/tuna/tunasync/badge.svg?branch=master)](https://coveralls.io/github/tuna/tunasync?branch=master)
@@ -12,11 +11,11 @@ tunasync
 
 ## Download
 
-Pre-built binary for Linux x86_64 is available at [Github releases](https://github.com/tuna/tunasync/releases/latest).
+Pre-built binary for Linux x86_64 and ARM64 is available at [Github releases](https://github.com/tuna/tunasync/releases/latest).
 
 ## Design
 
-```
+```text
 # Architecture
 
 - Manager: Central instance for status and job management
@@ -50,13 +49,12 @@ PreSyncing                           Syncing                               Succe
                                 +-----------------+
 ```
 
-
 ## Building
 
 Go version: 1.13
 
-```
-make all
+```shell
+> make all
 ```
 
-Binaries in the `build/`.
+Binaries in the `build-linux-amd64/`.

docs/zh_CN/get_started.md

@@ -1,4 +1,5 @@
 # tunasync 上手指南
+
 date: 2016-10-31 00:50:00
 
 [tunasync](https://github.com/tuna/tunasync) 是[清华大学 TUNA 镜像源](https://mirrors.tuna.tsinghua.edu.cn)目前使用的镜像方案。
@@ -7,32 +8,32 @@ date: 2016-10-31 00:50:00
 
 本例中:
 
- - 只镜像[elvish](https://elvish.io)项目
- - 禁用了https
- - 禁用了cgroup支持
+- 只镜像[elvish](https://elvish.io)项目
+- 禁用了https
+- 禁用了cgroup支持
 
 ## 获得tunasync
 
 ### 二进制包
 
-到 [Github Releases](https://github.com/tuna/tunasync/releases/latest) 下载 `tunasync-linux-bin.tar.gz` 即可。
+到 [Github Releases](https://github.com/tuna/tunasync/releases/latest) 下载 `tunasync-linux-amd64-bin.tar.gz` 即可。
 
 ### 自行编译
 
-```
-$ make
+```shell
+> make
 ```
 
 ## 配置
 
-```
-$ mkdir ~/tunasync_demo
-$ mkdir /tmp/tunasync
+```shell
+> mkdir ~/tunasync_demo
+> mkdir /tmp/tunasync
 ```
 
-`~/tunasync_demo/worker.conf`:
+编辑 `~/tunasync_demo/worker.conf`:
 
-```
+```conf
 [global]
 name = "test_worker"
 log_dir = "/tmp/tunasync/log/tunasync/{{.Name}}"
@@ -64,9 +65,9 @@ upstream = "rsync://rsync.elvish.io/elvish/"
 use_ipv6 = false
 ```
 
-`~/tunasync_demo/manager.conf`:
+编辑 `~/tunasync_demo/manager.conf`:
 
-```
+```conf
 debug = false
 
 [server]
@@ -83,26 +84,26 @@ ca_cert = ""
 
 ### 运行
 
-```
-$ tunasync manager --config ~/tunasync_demo/manager.conf
-$ tunasync worker --config ~/tunasync_demo/worker.conf
+```shell
+> tunasync manager --config ~/tunasync_demo/manager.conf
+> tunasync worker --config ~/tunasync_demo/worker.conf
 ```
 
-本例中，镜像的数据在`/tmp/tunasync/`
+本例中，镜像的数据在 `/tmp/tunasync/`。
 
 ### 控制
 
 查看同步状态
 
-```
-$ tunasynctl list -p 12345 --all
+```shell
+> tunasynctl list -p 12345 --all
 ```
 
 tunasynctl 也支持配置文件。配置文件可以放在 `/etc/tunasync/ctl.conf` 或者 `~/.config/tunasync/ctl.conf` 两个位置，后者可以覆盖前者的配置值。
 
 配置文件内容为：
 
-```
+```conf
 manager_addr = "127.0.0.1"
 manager_port = 12345
 ca_cert = ""
@@ -118,13 +119,13 @@ worker 和 manager 之间用 http(s) 通信，如果你 worker 和 manager 都
 
 可以参看
 
-```
-$ tunasync manager --help
-$ tunasync worker --help
+```shell
+> tunasync manager --help
+> tunasync worker --help
 ```
 
 可以看一下 log 目录
 
-一些 worker 配置文件示例 [workers.conf](workers.conf) 
+一些 worker 配置文件示例 [workers.conf](workers.conf)。
 
-你可能会用到的操作 [tips.md](tips.md) 
+你可能会用到的操作 [tips.md](tips.md)。

internal/msg.go

@@ -1,6 +1,8 @@
 package internal
 
 import (
+	"bytes"
+	"encoding/json"
 	"fmt"
 	"time"
 )
@@ -60,21 +62,45 @@ const (
 )
 
 func (c CmdVerb) String() string {
-	switch c {
-	case CmdStart:
-		return "start"
-	case CmdStop:
-		return "stop"
-	case CmdDisable:
-		return "disable"
-	case CmdRestart:
-		return "restart"
-	case CmdPing:
-		return "ping"
-	case CmdReload:
-		return "reload"
+	mapping := map[CmdVerb]string{
+		CmdStart:   "start",
+		CmdStop:    "stop",
+		CmdDisable: "disable",
+		CmdRestart: "restart",
+		CmdPing:    "ping",
+		CmdReload:  "reload",
 	}
-	return "unknown"
+	return mapping[c]
+}
+
+func NewCmdVerbFromString(s string) CmdVerb {
+	mapping := map[string]CmdVerb{
+		"start":   CmdStart,
+		"stop":    CmdStop,
+		"disable": CmdDisable,
+		"restart": CmdRestart,
+		"ping":    CmdPing,
+		"reload":  CmdReload,
+	}
+	return mapping[s]
+}
+
+// Marshal and Unmarshal for CmdVerb
+func (s CmdVerb) MarshalJSON() ([]byte, error) {
+	buffer := bytes.NewBufferString(`"`)
+	buffer.WriteString(s.String())
+	buffer.WriteString(`"`)
+	return buffer.Bytes(), nil
+}
+
+func (s *CmdVerb) UnmarshalJSON(b []byte) error {
+	var j string
+	err := json.Unmarshal(b, &j)
+	if err != nil {
+		return err
+	}
+	*s = NewCmdVerbFromString(j)
+	return nil
 }
 
 // A WorkerCmd is the command message send from the

internal/version.go

@@ -1,4 +1,4 @@
 package internal
 
 // Version of the program
-const Version string = "0.6.7"
+const Version string = "0.6.9"

manager/server.go

@@ -204,6 +204,8 @@ func (s *Manager) listWorkers(c *gin.Context) {
 		workerInfos = append(workerInfos,
 			WorkerStatus{
 				ID:           w.ID,
+				URL:          w.URL,
+				Token:        "REDACTED",
 				LastOnline:   w.LastOnline,
 				LastRegister: w.LastRegister,
 			})

worker/btrfs_snapshot_hook.go

@@ -1,3 +1,5 @@
+// +build linux
+
 package worker
 
 import (

worker/btrfs_snapshot_hook_nolinux.go

@@ -0,0 +1,30 @@
+// +build !linux
+
+package worker
+
+type btrfsSnapshotHook struct {
+}
+
+func newBtrfsSnapshotHook(provider mirrorProvider, snapshotPath string, mirror mirrorConfig) *btrfsSnapshotHook {
+	return &btrfsSnapshotHook{}
+}
+
+func (h *btrfsSnapshotHook) postExec() error {
+	return nil
+}
+
+func (h *btrfsSnapshotHook) postFail() error {
+	return nil
+}
+
+func (h *btrfsSnapshotHook) postSuccess() error {
+	return nil
+}
+
+func (h *btrfsSnapshotHook) preExec() error {
+	return nil
+}
+
+func (h *btrfsSnapshotHook) preJob() error {
+	return nil
+}

worker/provider_test.go

@@ -91,7 +91,7 @@ exit 0
 					"Done\n",
 				targetDir,
 				fmt.Sprintf(
-					"-aHvh --no-o --no-g --stats --exclude .~tmp~/ "+
+					"-aHvh --no-o --no-g --stats --filter risk .~tmp~/ --exclude .~tmp~/ "+
 						"--delete --delete-after --delay-updates --safe-links "+
 						"--timeout=120 -6 %s %s",
 					provider.upstreamURL, provider.WorkingDir(),
@@ -190,7 +190,7 @@ exit 0
 					"Done\n",
 				targetDir,
 				fmt.Sprintf(
-					"%s %s %s -aHvh --no-o --no-g --stats --exclude .~tmp~/ "+
+					"%s %s %s -aHvh --no-o --no-g --stats --filter risk .~tmp~/ --exclude .~tmp~/ "+
 						"--delete --delete-after --delay-updates --safe-links "+
 						"--timeout=30 -4 --delete-excluded %s %s",
 					provider.username, provider.password, proxyAddr,
@@ -613,14 +613,14 @@ exit 0
 					"Done\n",
 				targetDir,
 				fmt.Sprintf(
-					"-aHvh --no-o --no-g --stats --exclude .~tmp~/ --safe-links "+
-						"--exclude dists/ --timeout=30 -6 "+
+					"-aHvh --no-o --no-g --stats --filter risk .~tmp~/ --exclude .~tmp~/ --safe-links "+
+						"--include=*.diff/ --exclude=*.diff/Index --exclude=Packages* --exclude=Sources* --exclude=Release* --exclude=InRelease --include=i18n/by-hash --exclude=i18n/* --exclude=ls-lR* --timeout=30 -6 "+
 						"--exclude-from %s %s %s",
 					provider.excludeFile, provider.upstreamURL, provider.WorkingDir(),
 				),
 				targetDir,
 				fmt.Sprintf(
-					"-aHvh --no-o --no-g --stats --exclude .~tmp~/ "+
+					"-aHvh --no-o --no-g --stats --filter risk .~tmp~/ --exclude .~tmp~/ "+
 						"--delete --delete-after --delay-updates --safe-links "+
 						"--delete-excluded --cache --timeout=30 -6 --exclude-from %s %s %s",
 					provider.excludeFile, provider.upstreamURL, provider.WorkingDir(),
@@ -655,8 +655,8 @@ exit 0
 			So(err, ShouldBeNil)
 
 			expectedOutput := fmt.Sprintf(
-				"-aHvh --no-o --no-g --stats --exclude .~tmp~/ --safe-links "+
-					"--exclude dists/ --timeout=30 -6 "+
+				"-aHvh --no-o --no-g --stats --filter risk .~tmp~/ --exclude .~tmp~/ --safe-links "+
+					"--include=*.diff/ --exclude=*.diff/Index --exclude=Packages* --exclude=Sources* --exclude=Release* --exclude=InRelease --include=i18n/by-hash --exclude=i18n/* --exclude=ls-lR* --timeout=30 -6 "+
 					"--exclude-from %s %s %s\n",
 				provider.excludeFile, provider.upstreamURL, provider.WorkingDir(),
 			)

worker/rsync_provider.go

@@ -67,7 +67,7 @@ func newRsyncProvider(c rsyncConfig) (*rsyncProvider, error) {
 
 	options := []string{
 		"-aHvh", "--no-o", "--no-g", "--stats",
-		"--exclude", ".~tmp~/",
+		"--filter" , "risk .~tmp~/", "--exclude", ".~tmp~/",
 		"--delete", "--delete-after", "--delay-updates",
 		"--safe-links",
 	}

worker/two_stage_rsync_provider.go

@@ -34,11 +34,12 @@ type twoStageRsyncProvider struct {
 	dataSize      string
 }
 
+// ref: https://salsa.debian.org/mirror-team/archvsync/-/blob/master/bin/ftpsync#L431
 var rsyncStage1Profiles = map[string]([]string){
-	"debian": []string{"dists/"},
+	"debian": []string{"--include=*.diff/", "--exclude=*.diff/Index", "--exclude=Packages*", "--exclude=Sources*", "--exclude=Release*", "--exclude=InRelease", "--include=i18n/by-hash", "--exclude=i18n/*", "--exclude=ls-lR*"},
 	"debian-oldstyle": []string{
-		"Packages*", "Sources*", "Release*",
-		"InRelease", "i18n/*", "ls-lR*", "dep11/*",
+		"--exclude=Packages*", "--exclude=Sources*", "--exclude=Release*",
+		"--exclude=InRelease", "--exclude=i18n/*", "--exclude=ls-lR*", "--exclude=dep11/*",
 	},
 }
 
@@ -62,12 +63,12 @@ func newTwoStageRsyncProvider(c twoStageRsyncConfig) (*twoStageRsyncProvider, er
 		twoStageRsyncConfig: c,
 		stage1Options: []string{
 			"-aHvh", "--no-o", "--no-g", "--stats",
-			"--exclude", ".~tmp~/",
+			"--filter", "risk .~tmp~/", "--exclude", ".~tmp~/",
 			"--safe-links",
 		},
 		stage2Options: []string{
 			"-aHvh", "--no-o", "--no-g", "--stats",
-			"--exclude", ".~tmp~/",
+			"--filter", "risk .~tmp~/", "--exclude", ".~tmp~/",
 			"--delete", "--delete-after", "--delay-updates",
 			"--safe-links",
 		},
@@ -109,12 +110,12 @@ func (p *twoStageRsyncProvider) Options(stage int) ([]string, error) {
 	var options []string
 	if stage == 1 {
 		options = append(options, p.stage1Options...)
-		stage1Excludes, ok := rsyncStage1Profiles[p.stage1Profile]
+		stage1Profile, ok := rsyncStage1Profiles[p.stage1Profile]
 		if !ok {
 			return nil, errors.New("Invalid Stage 1 Profile")
 		}
-		for _, exc := range stage1Excludes {
-			options = append(options, "--exclude", exc)
+		for _, exc := range stage1Profile {
+			options = append(options, exc)
 		}
 
 	} else if stage == 2 {