diff --git a/docs/zh_CN/workers.conf b/docs/zh_CN/workers.conf index 1f2cdda..5f04a6a 100644 --- a/docs/zh_CN/workers.conf +++ b/docs/zh_CN/workers.conf @@ -45,6 +45,7 @@ docker_image = "tunathu/tunasync-scripts:latest" name = "gnu" provider = "rsync" upstream = "rsync://mirrors.ocf.berkeley.edu/gnu/" +rsync_options = [ "--delete-excluded" ] memory_limit = "256M" [[mirrors]] @@ -72,6 +73,7 @@ name = "ubuntu" provider = "two-stage-rsync" stage1_profile = "debian" upstream = "rsync://archive.ubuntu.com/ubuntu/" +rsync_options = [ "--delete-excluded" ] memory_limit = "256M" # vim: ft=toml \ No newline at end of file diff --git a/worker/config.go b/worker/config.go index d33153e..565a11b 100644 --- a/worker/config.go +++ b/worker/config.go @@ -129,13 +129,14 @@ type mirrorConfig struct { ExecOnSuccessExtra []string `toml:"exec_on_success_extra"` ExecOnFailureExtra []string `toml:"exec_on_failure_extra"` - Command string `toml:"command"` - UseIPv6 bool `toml:"use_ipv6"` - UseIPv4 bool `toml:"use_ipv4"` - ExcludeFile string `toml:"exclude_file"` - Username string `toml:"username"` - Password string `toml:"password"` - Stage1Profile string `toml:"stage1_profile"` + Command string `toml:"command"` + UseIPv6 bool `toml:"use_ipv6"` + UseIPv4 bool `toml:"use_ipv4"` + ExcludeFile string `toml:"exclude_file"` + Username string `toml:"username"` + Password string `toml:"password"` + RsyncOptions []string `toml:"rsync_options"` + Stage1Profile string `toml:"stage1_profile"` MemoryLimit string `toml:"memory_limit"` diff --git a/worker/provider.go b/worker/provider.go index 7431ad4..4e3e199 100644 --- a/worker/provider.go +++ b/worker/provider.go @@ -126,19 +126,20 @@ func newMirrorProvider(mirror mirrorConfig, cfg *Config) mirrorProvider { provider = p case provRsync: rc := rsyncConfig{ - name: mirror.Name, - upstreamURL: mirror.Upstream, - rsyncCmd: mirror.Command, - username: mirror.Username, - password: mirror.Password, - excludeFile: mirror.ExcludeFile, - workingDir: mirrorDir, - logDir: logDir, - logFile: filepath.Join(logDir, "latest.log"), - useIPv6: mirror.UseIPv6, - useIPv4: mirror.UseIPv4, - interval: time.Duration(mirror.Interval) * time.Minute, - retry: mirror.Retry, + name: mirror.Name, + upstreamURL: mirror.Upstream, + rsyncCmd: mirror.Command, + username: mirror.Username, + password: mirror.Password, + excludeFile: mirror.ExcludeFile, + extraOptions: mirror.RsyncOptions, + workingDir: mirrorDir, + logDir: logDir, + logFile: filepath.Join(logDir, "latest.log"), + useIPv6: mirror.UseIPv6, + useIPv4: mirror.UseIPv4, + interval: time.Duration(mirror.Interval) * time.Minute, + retry: mirror.Retry, } p, err := newRsyncProvider(rc) if err != nil { @@ -155,6 +156,7 @@ func newMirrorProvider(mirror mirrorConfig, cfg *Config) mirrorProvider { username: mirror.Username, password: mirror.Password, excludeFile: mirror.ExcludeFile, + extraOptions: mirror.RsyncOptions, workingDir: mirrorDir, logDir: logDir, logFile: filepath.Join(logDir, "latest.log"), diff --git a/worker/provider_test.go b/worker/provider_test.go index c704a0c..91b221d 100644 --- a/worker/provider_test.go +++ b/worker/provider_test.go @@ -116,16 +116,17 @@ func TestRsyncProviderWithAuthentication(t *testing.T) { tmpFile := filepath.Join(tmpDir, "log_file") c := rsyncConfig{ - name: "tuna", - upstreamURL: "rsync://rsync.tuna.moe/tuna/", - rsyncCmd: scriptFile, - username: "tunasync", - password: "tunasyncpassword", - workingDir: tmpDir, - logDir: tmpDir, - logFile: tmpFile, - useIPv6: true, - interval: 600 * time.Second, + name: "tuna", + upstreamURL: "rsync://rsync.tuna.moe/tuna/", + rsyncCmd: scriptFile, + username: "tunasync", + password: "tunasyncpassword", + workingDir: tmpDir, + extraOptions: []string{"--delete-excluded"}, + logDir: tmpDir, + logFile: tmpFile, + useIPv4: true, + interval: 600 * time.Second, } provider, err := newRsyncProvider(c) @@ -157,7 +158,7 @@ exit 0 fmt.Sprintf( "%s %s -aHvh --no-o --no-g --stats --exclude .~tmp~/ "+ "--delete --delete-after --delay-updates --safe-links "+ - "--timeout=120 --contimeout=120 -6 %s %s", + "--timeout=120 --contimeout=120 -4 --delete-excluded %s %s", provider.username, provider.password, provider.upstreamURL, provider.WorkingDir(), ), ) @@ -319,6 +320,7 @@ func TestTwoStageRsyncProvider(t *testing.T) { logFile: tmpFile, useIPv6: true, excludeFile: tmpFile, + extraOptions: []string{"--delete-excluded", "--cache"}, username: "hello", password: "world", } @@ -359,14 +361,14 @@ exit 0 fmt.Sprintf( "-aHvh --no-o --no-g --stats --exclude .~tmp~/ --safe-links "+ "--timeout=120 --contimeout=120 --exclude dists/ -6 "+ - "--exclude-from %s %s %s", + "--exclude-from %s --delete-excluded --cache %s %s", provider.excludeFile, provider.upstreamURL, provider.WorkingDir(), ), targetDir, fmt.Sprintf( "-aHvh --no-o --no-g --stats --exclude .~tmp~/ "+ "--delete --delete-after --delay-updates --safe-links "+ - "--timeout=120 --contimeout=120 -6 --exclude-from %s %s %s", + "--timeout=120 --contimeout=120 -6 --exclude-from %s --delete-excluded --cache %s %s", provider.excludeFile, provider.upstreamURL, provider.WorkingDir(), ), ) @@ -398,7 +400,7 @@ exit 0 expectedOutput := fmt.Sprintf( "-aHvh --no-o --no-g --stats --exclude .~tmp~/ --safe-links "+ "--timeout=120 --contimeout=120 --exclude dists/ -6 "+ - "--exclude-from %s %s %s\n", + "--exclude-from %s --delete-excluded --cache %s %s\n", provider.excludeFile, provider.upstreamURL, provider.WorkingDir(), ) diff --git a/worker/rsync_provider.go b/worker/rsync_provider.go index c0ed57f..8905eee 100644 --- a/worker/rsync_provider.go +++ b/worker/rsync_provider.go @@ -13,6 +13,7 @@ type rsyncConfig struct { name string rsyncCmd string upstreamURL, username, password, excludeFile string + extraOptions []string workingDir, logDir, logFile string useIPv6, useIPv4 bool interval time.Duration @@ -65,6 +66,9 @@ func newRsyncProvider(c rsyncConfig) (*rsyncProvider, error) { if c.excludeFile != "" { options = append(options, "--exclude-from", c.excludeFile) } + if c.extraOptions != nil { + options = append(options, c.extraOptions...) + } provider.options = options provider.ctx.Set(_WorkingDirKey, c.workingDir) diff --git a/worker/two_stage_rsync_provider.go b/worker/two_stage_rsync_provider.go index 991e8c8..aa085e3 100644 --- a/worker/two_stage_rsync_provider.go +++ b/worker/two_stage_rsync_provider.go @@ -15,6 +15,7 @@ type twoStageRsyncConfig struct { rsyncCmd string stage1Profile string upstreamURL, username, password, excludeFile string + extraOptions []string workingDir, logDir, logFile string useIPv6 bool interval time.Duration @@ -116,6 +117,9 @@ func (p *twoStageRsyncProvider) Options(stage int) ([]string, error) { if p.excludeFile != "" { options = append(options, "--exclude-from", p.excludeFile) } + if p.extraOptions != nil { + options = append(options, p.extraOptions...) + } return options, nil }