bump version to v0.3.2

Update README.md

.travis.yml

@@ -2,7 +2,7 @@ sudo: required
 
 language: go
 go:
-    - 1.6
+    - 1.8
 
 before_install:
     - sudo apt-get install cgroup-bin

README.md

@@ -19,7 +19,7 @@ Pre-built binary for Linux x86_64 is available at [Github releases](https://gith
 ```
 # Architecture
 
-- Manager: Centural instance on status and job management
+- Manager: Central instance for status and job management
 - Worker: Runs mirror jobs
 
 +------------+ +---+                  +---+
@@ -47,12 +47,12 @@ PreSyncing           Syncing                               Success
 			   |                                   |
 			   |      +-----------------+          | Failed
 			   +------+    post-fail    |<---------+
-					  +-----------------+
+			          +-----------------+
 ```
 
 ## Generate Self-Signed Certificate
 
-Fisrt, create root CA
+First, create root CA
 
 ```
 openssl genrsa -out rootCA.key 2048

cmd/tunasynctl/tunasynctl.go

@@ -140,9 +140,9 @@ func listWorkers(c *cli.Context) error {
 }
 
 func listJobs(c *cli.Context) error {
-	// FIXME: there should be an API on manager server side that return MirrorStatus list to tunasynctl
-	var jobs []tunasync.MirrorStatus
+	var genericJobs interface{}
 	if c.Bool("all") {
+		var jobs []tunasync.WebMirrorStatus
 		_, err := tunasync.GetJSON(baseURL+listJobsPath, &jobs, client)
 		if err != nil {
 			return cli.NewExitError(
@@ -150,8 +150,10 @@ func listJobs(c *cli.Context) error {
 					"of all jobs from manager server: %s", err.Error()),
 				1)
 		}
+		genericJobs = jobs
 
 	} else {
+		var jobs []tunasync.MirrorStatus
 		args := c.Args()
 		if len(args) == 0 {
 			return cli.NewExitError(
@@ -174,9 +176,10 @@ func listJobs(c *cli.Context) error {
 		for range args {
 			jobs = append(jobs, <-ans...)
 		}
+		genericJobs = jobs
 	}
 
-	b, err := json.MarshalIndent(jobs, "", "  ")
+	b, err := json.MarshalIndent(genericJobs, "", "  ")
 	if err != nil {
 		return cli.NewExitError(
 			fmt.Sprintf("Error printing out informations: %s", err.Error()),
@@ -237,6 +240,52 @@ func updateMirrorSize(c *cli.Context) error {
 	return nil
 }
 
+func removeWorker(c *cli.Context) error {
+	args := c.Args()
+	if len(args) != 0 {
+		return cli.NewExitError("Usage: tunasynctl -w <worker-id>", 1)
+	}
+	workerID := c.String("worker")
+	if len(workerID) == 0 {
+		return cli.NewExitError("Please specify the <worker-id>", 1)
+	}
+	url := fmt.Sprintf("%s/workers/%s", baseURL, workerID)
+
+	req, err := http.NewRequest("DELETE", url, nil)
+	if err != nil {
+		logger.Panicf("Invalid HTTP Request: %s", err.Error())
+	}
+	resp, err := client.Do(req)
+
+	if err != nil {
+		return cli.NewExitError(
+			fmt.Sprintf("Failed to send request to manager: %s", err.Error()), 1)
+	}
+	defer resp.Body.Close()
+
+	if resp.StatusCode != http.StatusOK {
+		body, err := ioutil.ReadAll(resp.Body)
+		if err != nil {
+			return cli.NewExitError(
+				fmt.Sprintf("Failed to parse response: %s", err.Error()),
+				1)
+		}
+
+		return cli.NewExitError(fmt.Sprintf("Failed to correctly send"+
+			" command: HTTP status code is not 200: %s", body),
+			1)
+	}
+
+	res := map[string]string{}
+	err = json.NewDecoder(resp.Body).Decode(&res)
+	if res["message"] == "deleted" {
+		logger.Info("Successfully removed the worker")
+	} else {
+		logger.Info("Failed to remove the worker")
+	}
+	return nil
+}
+
 func flushDisabledJobs(c *cli.Context) error {
 	req, err := http.NewRequest("DELETE", baseURL+flushDisabledPath, nil)
 	if err != nil {
@@ -286,11 +335,16 @@ func cmdJob(cmd tunasync.CmdVerb) cli.ActionFunc {
 				"argument WORKER", 1)
 		}
 
+		options := map[string]bool{}
+		if c.Bool("force") {
+			options["force"] = true
+		}
 		cmd := tunasync.ClientCmd{
 			Cmd:      cmd,
 			MirrorID: mirrorID,
 			WorkerID: c.String("worker"),
 			Args:     argsList,
+			Options:  options,
 		}
 		resp, err := tunasync.PostJSON(baseURL+cmdPath, cmd, client)
 		if err != nil {
@@ -411,6 +465,11 @@ func main() {
 		},
 	}
 
+	forceStartFlag := cli.BoolFlag{
+		Name:  "force, f",
+		Usage: "Override the concurrent limit",
+	}
+
 	app.Commands = []cli.Command{
 		{
 			Name:  "list",
@@ -436,6 +495,18 @@ func main() {
 			Flags:  commonFlags,
 			Action: initializeWrapper(listWorkers),
 		},
+		{
+			Name:  "rm-worker",
+			Usage: "Remove a worker",
+			Flags: append(
+				commonFlags,
+				cli.StringFlag{
+					Name:  "worker, w",
+					Usage: "worker-id of the worker to be removed",
+				},
+			),
+			Action: initializeWrapper(removeWorker),
+		},
 		{
 			Name:  "set-size",
 			Usage: "Set mirror size",
@@ -451,7 +522,7 @@ func main() {
 		{
 			Name:   "start",
 			Usage:  "Start a job",
-			Flags:  append(commonFlags, cmdFlags...),
+			Flags:  append(append(commonFlags, cmdFlags...), forceStartFlag),
 			Action: initializeWrapper(cmdJob(tunasync.CmdStart)),
 		},
 		{

internal/msg.go

@@ -13,6 +13,7 @@ type MirrorStatus struct {
 	IsMaster   bool       `json:"is_master"`
 	Status     SyncStatus `json:"status"`
 	LastUpdate time.Time  `json:"last_update"`
+	LastEnded  time.Time  `json:"last_ended"`
 	Upstream   string     `json:"upstream"`
 	Size       string     `json:"size"`
 	ErrorMsg   string     `json:"error_msg"`
@@ -67,9 +68,10 @@ func (c CmdVerb) String() string {
 // A WorkerCmd is the command message send from the
 // manager to a worker
 type WorkerCmd struct {
-	Cmd      CmdVerb  `json:"cmd"`
-	MirrorID string   `json:"mirror_id"`
-	Args     []string `json:"args"`
+	Cmd      CmdVerb         `json:"cmd"`
+	MirrorID string          `json:"mirror_id"`
+	Args     []string        `json:"args"`
+	Options  map[string]bool `json:"options"`
 }
 
 func (c WorkerCmd) String() string {
@@ -82,8 +84,9 @@ func (c WorkerCmd) String() string {
 // A ClientCmd is the command message send from client
 // to the manager
 type ClientCmd struct {
-	Cmd      CmdVerb  `json:"cmd"`
-	MirrorID string   `json:"mirror_id"`
-	WorkerID string   `json:"worker_id"`
-	Args     []string `json:"args"`
+	Cmd      CmdVerb         `json:"cmd"`
+	MirrorID string          `json:"mirror_id"`
+	WorkerID string          `json:"worker_id"`
+	Args     []string        `json:"args"`
+	Options  map[string]bool `json:"options"`
 }

internal/status_web.go

@@ -1,11 +1,9 @@
-package manager
+package internal
 
 import (
 	"encoding/json"
 	"strconv"
 	"time"
-
-	. "github.com/tuna/tunasync/internal"
 )
 
 type textTime struct {
@@ -38,24 +36,28 @@ func (t *stampTime) UnmarshalJSON(b []byte) error {
 	return err
 }
 
-// webMirrorStatus is the mirror status to be shown in the web page
-type webMirrorStatus struct {
+// WebMirrorStatus is the mirror status to be shown in the web page
+type WebMirrorStatus struct {
 	Name         string     `json:"name"`
 	IsMaster     bool       `json:"is_master"`
 	Status       SyncStatus `json:"status"`
 	LastUpdate   textTime   `json:"last_update"`
 	LastUpdateTs stampTime  `json:"last_update_ts"`
+	LastEnded    textTime   `json:"last_ended"`
+	LastEndedTs  stampTime  `json:"last_ended_ts"`
 	Upstream     string     `json:"upstream"`
 	Size         string     `json:"size"` // approximate size
 }
 
-func convertMirrorStatus(m MirrorStatus) webMirrorStatus {
-	return webMirrorStatus{
+func BuildWebMirrorStatus(m MirrorStatus) WebMirrorStatus {
+	return WebMirrorStatus{
 		Name:         m.Name,
 		IsMaster:     m.IsMaster,
 		Status:       m.Status,
 		LastUpdate:   textTime{m.LastUpdate},
 		LastUpdateTs: stampTime{m.LastUpdate},
+		LastEnded:    textTime{m.LastEnded},
+		LastEndedTs:  stampTime{m.LastEnded},
 		Upstream:     m.Upstream,
 		Size:         m.Size,
 	}

internal/status_web_test.go

@@ -0,0 +1,76 @@
+package internal
+
+import (
+	"encoding/json"
+	"testing"
+	"time"
+
+	. "github.com/smartystreets/goconvey/convey"
+)
+
+func TestStatus(t *testing.T) {
+	Convey("status json ser-de should work", t, func() {
+		tz := "Asia/Tokyo"
+		loc, err := time.LoadLocation(tz)
+		So(err, ShouldBeNil)
+		t := time.Date(2016, time.April, 16, 23, 8, 10, 0, loc)
+		m := WebMirrorStatus{
+			Name:         "tunalinux",
+			Status:       Success,
+			LastUpdate:   textTime{t},
+			LastUpdateTs: stampTime{t},
+			LastEnded:    textTime{t},
+			LastEndedTs:  stampTime{t},
+			Size:         "5GB",
+			Upstream:     "rsync://mirrors.tuna.tsinghua.edu.cn/tunalinux/",
+		}
+
+		b, err := json.Marshal(m)
+		So(err, ShouldBeNil)
+		//fmt.Println(string(b))
+		var m2 WebMirrorStatus
+		err = json.Unmarshal(b, &m2)
+		So(err, ShouldBeNil)
+		// fmt.Printf("%#v", m2)
+		So(m2.Name, ShouldEqual, m.Name)
+		So(m2.Status, ShouldEqual, m.Status)
+		So(m2.LastUpdate.Unix(), ShouldEqual, m.LastUpdate.Unix())
+		So(m2.LastUpdateTs.Unix(), ShouldEqual, m.LastUpdate.Unix())
+		So(m2.LastUpdate.UnixNano(), ShouldEqual, m.LastUpdate.UnixNano())
+		So(m2.LastUpdateTs.UnixNano(), ShouldEqual, m.LastUpdate.UnixNano())
+		So(m2.LastEnded.Unix(), ShouldEqual, m.LastEnded.Unix())
+		So(m2.LastEndedTs.Unix(), ShouldEqual, m.LastEnded.Unix())
+		So(m2.LastEnded.UnixNano(), ShouldEqual, m.LastEnded.UnixNano())
+		So(m2.LastEndedTs.UnixNano(), ShouldEqual, m.LastEnded.UnixNano())
+		So(m2.Size, ShouldEqual, m.Size)
+		So(m2.Upstream, ShouldEqual, m.Upstream)
+	})
+	Convey("BuildWebMirrorStatus should work", t, func() {
+		m := MirrorStatus{
+			Name:       "arch-sync3",
+			Worker:     "testWorker",
+			IsMaster:   true,
+			Status:     Failed,
+			LastUpdate: time.Now().Add(-time.Minute * 30),
+			LastEnded:  time.Now(),
+			Upstream:   "mirrors.tuna.tsinghua.edu.cn",
+			Size:       "4GB",
+		}
+
+		var m2 WebMirrorStatus
+		m2 = BuildWebMirrorStatus(m)
+		// fmt.Printf("%#v", m2)
+		So(m2.Name, ShouldEqual, m.Name)
+		So(m2.Status, ShouldEqual, m.Status)
+		So(m2.LastUpdate.Unix(), ShouldEqual, m.LastUpdate.Unix())
+		So(m2.LastUpdateTs.Unix(), ShouldEqual, m.LastUpdate.Unix())
+		So(m2.LastUpdate.UnixNano(), ShouldEqual, m.LastUpdate.UnixNano())
+		So(m2.LastUpdateTs.UnixNano(), ShouldEqual, m.LastUpdate.UnixNano())
+		So(m2.LastEnded.Unix(), ShouldEqual, m.LastEnded.Unix())
+		So(m2.LastEndedTs.Unix(), ShouldEqual, m.LastEnded.Unix())
+		So(m2.LastEnded.UnixNano(), ShouldEqual, m.LastEnded.UnixNano())
+		So(m2.LastEndedTs.UnixNano(), ShouldEqual, m.LastEnded.UnixNano())
+		So(m2.Size, ShouldEqual, m.Size)
+		So(m2.Upstream, ShouldEqual, m.Upstream)
+	})
+}

internal/version.go

@@ -1,3 +1,3 @@
 package internal
 
-const Version string = "0.2-dev"
+const Version string = "0.3.2"

manager/db.go

@@ -14,6 +14,7 @@ type dbAdapter interface {
 	Init() error
 	ListWorkers() ([]WorkerStatus, error)
 	GetWorker(workerID string) (WorkerStatus, error)
+	DeleteWorker(workerID string) error
 	CreateWorker(w WorkerStatus) (WorkerStatus, error)
 	UpdateMirrorStatus(workerID, mirrorID string, status MirrorStatus) (MirrorStatus, error)
 	GetMirrorStatus(workerID, mirrorID string) (MirrorStatus, error)
@@ -95,6 +96,19 @@ func (b *boltAdapter) GetWorker(workerID string) (w WorkerStatus, err error) {
 	return
 }
 
+func (b *boltAdapter) DeleteWorker(workerID string) (err error) {
+	err = b.db.Update(func(tx *bolt.Tx) error {
+		bucket := tx.Bucket([]byte(_workerBucketKey))
+		v := bucket.Get([]byte(workerID))
+		if v == nil {
+			return fmt.Errorf("invalid workerID %s", workerID)
+		}
+		err := bucket.Delete([]byte(workerID))
+		return err
+	})
+	return
+}
+
 func (b *boltAdapter) CreateWorker(w WorkerStatus) (WorkerStatus, error) {
 	err := b.db.Update(func(tx *bolt.Tx) error {
 		bucket := tx.Bucket([]byte(_workerBucketKey))

manager/db_test.go

@@ -40,21 +40,39 @@ func TestBoltAdapter(t *testing.T) {
 				So(err, ShouldBeNil)
 			}
 
-			Convey("get exists worker", func() {
+			Convey("get existent worker", func() {
 				_, err := boltDB.GetWorker(testWorkerIDs[0])
 				So(err, ShouldBeNil)
 			})
 
-			Convey("list exist worker", func() {
+			Convey("list existent workers", func() {
 				ws, err := boltDB.ListWorkers()
 				So(err, ShouldBeNil)
 				So(len(ws), ShouldEqual, 2)
 			})
 
-			Convey("get inexist worker", func() {
+			Convey("get non-existent worker", func() {
 				_, err := boltDB.GetWorker("invalid workerID")
 				So(err, ShouldNotBeNil)
 			})
+
+			Convey("delete existent worker", func() {
+				err := boltDB.DeleteWorker(testWorkerIDs[0])
+				So(err, ShouldBeNil)
+				_, err = boltDB.GetWorker(testWorkerIDs[0])
+				So(err, ShouldNotBeNil)
+				ws, err := boltDB.ListWorkers()
+				So(err, ShouldBeNil)
+				So(len(ws), ShouldEqual, 1)
+			})
+
+			Convey("delete non-existent worker", func() {
+				err := boltDB.DeleteWorker("invalid workerID")
+				So(err, ShouldNotBeNil)
+				ws, err := boltDB.ListWorkers()
+				So(err, ShouldBeNil)
+				So(len(ws), ShouldEqual, 2)
+			})
 		})
 
 		Convey("update mirror status", func() {
@@ -65,6 +83,7 @@ func TestBoltAdapter(t *testing.T) {
 					IsMaster:   true,
 					Status:     Success,
 					LastUpdate: time.Now(),
+					LastEnded:  time.Now(),
 					Upstream:   "mirrors.tuna.tsinghua.edu.cn",
 					Size:       "3GB",
 				},
@@ -73,7 +92,8 @@ func TestBoltAdapter(t *testing.T) {
 					Worker:     testWorkerIDs[1],
 					IsMaster:   true,
 					Status:     Disabled,
-					LastUpdate: time.Now(),
+					LastUpdate: time.Now().Add(-time.Hour),
+					LastEnded:  time.Now(),
 					Upstream:   "mirrors.tuna.tsinghua.edu.cn",
 					Size:       "4GB",
 				},
@@ -82,7 +102,8 @@ func TestBoltAdapter(t *testing.T) {
 					Worker:     testWorkerIDs[1],
 					IsMaster:   true,
 					Status:     Success,
-					LastUpdate: time.Now(),
+					LastUpdate: time.Now().Add(-time.Second),
+					LastEnded:  time.Now(),
 					Upstream:   "mirrors.tuna.tsinghua.edu.cn",
 					Size:       "4GB",
 				},

manager/server.go

@@ -84,6 +84,8 @@ func GetTUNASyncManager(cfg *Config) *Manager {
 	// workerID should be valid in this route group
 	workerValidateGroup := s.engine.Group("/workers", s.workerIDValidator)
 	{
+		// delete specified worker
+		workerValidateGroup.DELETE(":id", s.deleteWorker)
 		// get job list
 		workerValidateGroup.GET(":id/jobs", s.listJobsOfWorker)
 		// post job status
@@ -135,11 +137,11 @@ func (s *Manager) listAllJobs(c *gin.Context) {
 		s.returnErrJSON(c, http.StatusInternalServerError, err)
 		return
 	}
-	webMirStatusList := []webMirrorStatus{}
+	webMirStatusList := []WebMirrorStatus{}
 	for _, m := range mirrorStatusList {
 		webMirStatusList = append(
 			webMirStatusList,
-			convertMirrorStatus(m),
+			BuildWebMirrorStatus(m),
 		)
 	}
 	c.JSON(http.StatusOK, webMirStatusList)
@@ -159,6 +161,22 @@ func (s *Manager) flushDisabledJobs(c *gin.Context) {
 	c.JSON(http.StatusOK, gin.H{_infoKey: "flushed"})
 }
 
+// deleteWorker deletes one worker by id
+func (s *Manager) deleteWorker(c *gin.Context) {
+	workerID := c.Param("id")
+	err := s.adapter.DeleteWorker(workerID)
+	if err != nil {
+		err := fmt.Errorf("failed to delete worker: %s",
+			err.Error(),
+		)
+		c.Error(err)
+		s.returnErrJSON(c, http.StatusInternalServerError, err)
+		return
+	}
+	logger.Noticef("Worker <%s> deleted", workerID)
+	c.JSON(http.StatusOK, gin.H{_infoKey: "deleted"})
+}
+
 // listWrokers respond with informations of all the workers
 func (s *Manager) listWorkers(c *gin.Context) {
 	var workerInfos []WorkerStatus
@@ -242,6 +260,11 @@ func (s *Manager) updateJobOfWorker(c *gin.Context) {
 	} else {
 		status.LastUpdate = curStatus.LastUpdate
 	}
+	if status.Status == Success || status.Status == Failed {
+		status.LastEnded = time.Now()
+	} else {
+		status.LastEnded = curStatus.LastEnded
+	}
 
 	// Only message with meaningful size updates the mirror size
 	if len(curStatus.Size) > 0 && curStatus.Size != "unknown" {
@@ -332,6 +355,7 @@ func (s *Manager) handleClientCmd(c *gin.Context) {
 		Cmd:      clientCmd.Cmd,
 		MirrorID: clientCmd.MirrorID,
 		Args:     clientCmd.Args,
+		Options:  clientCmd.Options,
 	}
 
 	// update job status, even if the job did not disable successfully,

manager/server_test.go

@@ -21,9 +21,16 @@ const (
 )
 
 func TestHTTPServer(t *testing.T) {
+	var listenPort = 5000
 	Convey("HTTP server should work", t, func(ctx C) {
+		listenPort++
+		port := listenPort
+		addr := "127.0.0.1"
+		baseURL := fmt.Sprintf("http://%s:%d", addr, port)
 		InitLogger(true, true, false)
-		s := GetTUNASyncManager(&Config{Debug: false})
+		s := GetTUNASyncManager(&Config{Debug: true})
+		s.cfg.Server.Addr = addr
+		s.cfg.Server.Port = port
 		So(s, ShouldNotBeNil)
 		s.setDBAdapter(&mockDBAdapter{
 			workerStore: map[string]WorkerStatus{
@@ -32,12 +39,8 @@ func TestHTTPServer(t *testing.T) {
 				}},
 			statusStore: make(map[string]MirrorStatus),
 		})
-		port := rand.Intn(10000) + 20000
-		baseURL := fmt.Sprintf("http://127.0.0.1:%d", port)
-		go func() {
-			s.engine.Run(fmt.Sprintf("127.0.0.1:%d", port))
-		}()
-		time.Sleep(50 * time.Microsecond)
+		go s.Run()
+		time.Sleep(50 * time.Millisecond)
 		resp, err := http.Get(baseURL + "/ping")
 		So(err, ShouldBeNil)
 		So(resp.StatusCode, ShouldEqual, http.StatusOK)
@@ -79,6 +82,33 @@ func TestHTTPServer(t *testing.T) {
 				So(len(actualResponseObj), ShouldEqual, 2)
 			})
 
+			Convey("delete an existent worker", func(ctx C) {
+				req, err := http.NewRequest("DELETE", fmt.Sprintf("%s/workers/%s", baseURL, w.ID), nil)
+				So(err, ShouldBeNil)
+				clt := &http.Client{}
+				resp, err := clt.Do(req)
+				So(err, ShouldBeNil)
+				defer resp.Body.Close()
+				res := map[string]string{}
+				err = json.NewDecoder(resp.Body).Decode(&res)
+				So(err, ShouldBeNil)
+				So(res[_infoKey], ShouldEqual, "deleted")
+			})
+
+			Convey("delete non-existent worker", func(ctx C) {
+				invalidWorker := "test_worker233"
+				req, err := http.NewRequest("DELETE", fmt.Sprintf("%s/workers/%s", baseURL, invalidWorker), nil)
+				So(err, ShouldBeNil)
+				clt := &http.Client{}
+				resp, err := clt.Do(req)
+				So(err, ShouldBeNil)
+				defer resp.Body.Close()
+				res := map[string]string{}
+				err = json.NewDecoder(resp.Body).Decode(&res)
+				So(err, ShouldBeNil)
+				So(res[_errorKey], ShouldEqual, "invalid workerID "+invalidWorker)
+			})
+
 			Convey("flush  disabled jobs", func(ctx C) {
 				req, err := http.NewRequest("DELETE", baseURL+"/jobs/disabled", nil)
 				So(err, ShouldBeNil)
@@ -121,11 +151,12 @@ func TestHTTPServer(t *testing.T) {
 					So(m.Size, ShouldEqual, status.Size)
 					So(m.IsMaster, ShouldEqual, status.IsMaster)
 					So(time.Now().Sub(m.LastUpdate), ShouldBeLessThan, 1*time.Second)
+					So(time.Now().Sub(m.LastEnded), ShouldBeLessThan, 1*time.Second)
 
 				})
 
 				Convey("list all job status of all workers", func(ctx C) {
-					var ms []webMirrorStatus
+					var ms []WebMirrorStatus
 					resp, err := GetJSON(baseURL+"/jobs", &ms, nil)
 					So(err, ShouldBeNil)
 					So(resp.StatusCode, ShouldEqual, http.StatusOK)
@@ -137,6 +168,7 @@ func TestHTTPServer(t *testing.T) {
 					So(m.Size, ShouldEqual, status.Size)
 					So(m.IsMaster, ShouldEqual, status.IsMaster)
 					So(time.Now().Sub(m.LastUpdate.Time), ShouldBeLessThan, 1*time.Second)
+					So(time.Now().Sub(m.LastEnded.Time), ShouldBeLessThan, 1*time.Second)
 
 				})
 
@@ -166,6 +198,7 @@ func TestHTTPServer(t *testing.T) {
 						So(m.Size, ShouldEqual, "5GB")
 						So(m.IsMaster, ShouldEqual, status.IsMaster)
 						So(time.Now().Sub(m.LastUpdate), ShouldBeLessThan, 1*time.Second)
+						So(time.Now().Sub(m.LastEnded), ShouldBeLessThan, 1*time.Second)
 					})
 				})
 
@@ -180,6 +213,32 @@ func TestHTTPServer(t *testing.T) {
 					So(err, ShouldBeNil)
 					So(resp.StatusCode, ShouldEqual, http.StatusInternalServerError)
 				})
+
+				// what if status changed to failed
+				status.Status = Failed
+				time.Sleep(3 * time.Second)
+				resp, err = PostJSON(fmt.Sprintf("%s/workers/%s/jobs/%s", baseURL, status.Worker, status.Name), status, nil)
+				defer resp.Body.Close()
+				So(err, ShouldBeNil)
+				So(resp.StatusCode, ShouldEqual, http.StatusOK)
+
+				Convey("What if syncing job failed", func(ctx C) {
+					var ms []MirrorStatus
+					resp, err := GetJSON(baseURL+"/workers/test_worker1/jobs", &ms, nil)
+
+					So(err, ShouldBeNil)
+					So(resp.StatusCode, ShouldEqual, http.StatusOK)
+					// err = json.NewDecoder(resp.Body).Decode(&mirrorStatusList)
+					m := ms[0]
+					So(m.Name, ShouldEqual, status.Name)
+					So(m.Worker, ShouldEqual, status.Worker)
+					So(m.Status, ShouldEqual, status.Status)
+					So(m.Upstream, ShouldEqual, status.Upstream)
+					So(m.Size, ShouldEqual, status.Size)
+					So(m.IsMaster, ShouldEqual, status.IsMaster)
+					So(time.Now().Sub(m.LastUpdate), ShouldBeGreaterThan, 3*time.Second)
+					So(time.Now().Sub(m.LastEnded), ShouldBeLessThan, 1*time.Second)
+				})
 			})
 
 			Convey("update mirror status of an inexisted worker", func(ctx C) {
@@ -190,6 +249,7 @@ func TestHTTPServer(t *testing.T) {
 					IsMaster:   true,
 					Status:     Success,
 					LastUpdate: time.Now(),
+					LastEnded:  time.Now(),
 					Upstream:   "mirrors.tuna.tsinghua.edu.cn",
 					Size:       "4GB",
 				}
@@ -293,6 +353,11 @@ func (b *mockDBAdapter) GetWorker(workerID string) (WorkerStatus, error) {
 	return w, nil
 }
 
+func (b *mockDBAdapter) DeleteWorker(workerID string) error {
+	delete(b.workerStore, workerID)
+	return nil
+}
+
 func (b *mockDBAdapter) CreateWorker(w WorkerStatus) (WorkerStatus, error) {
 	// _, ok := b.workerStore[w.ID]
 	// if ok {

manager/status_test.go

@@ -1,44 +0,0 @@
-package manager
-
-import (
-	"encoding/json"
-	"testing"
-	"time"
-
-	tunasync "github.com/tuna/tunasync/internal"
-
-	. "github.com/smartystreets/goconvey/convey"
-)
-
-func TestStatus(t *testing.T) {
-	Convey("status json ser-de should work", t, func() {
-		tz := "Asia/Tokyo"
-		loc, err := time.LoadLocation(tz)
-		So(err, ShouldBeNil)
-		t := time.Date(2016, time.April, 16, 23, 8, 10, 0, loc)
-		m := webMirrorStatus{
-			Name:         "tunalinux",
-			Status:       tunasync.Success,
-			LastUpdate:   textTime{t},
-			LastUpdateTs: stampTime{t},
-			Size:         "5GB",
-			Upstream:     "rsync://mirrors.tuna.tsinghua.edu.cn/tunalinux/",
-		}
-
-		b, err := json.Marshal(m)
-		So(err, ShouldBeNil)
-		//fmt.Println(string(b))
-		var m2 webMirrorStatus
-		err = json.Unmarshal(b, &m2)
-		So(err, ShouldBeNil)
-		// fmt.Printf("%#v", m2)
-		So(m2.Name, ShouldEqual, m.Name)
-		So(m2.Status, ShouldEqual, m.Status)
-		So(m2.LastUpdate.Unix(), ShouldEqual, m.LastUpdate.Unix())
-		So(m2.LastUpdateTs.Unix(), ShouldEqual, m.LastUpdate.Unix())
-		So(m2.LastUpdate.UnixNano(), ShouldEqual, m.LastUpdate.UnixNano())
-		So(m2.LastUpdateTs.UnixNano(), ShouldEqual, m.LastUpdate.UnixNano())
-		So(m2.Size, ShouldEqual, m.Size)
-		So(m2.Upstream, ShouldEqual, m.Upstream)
-	})
-}

worker/base_provider.go

@@ -20,8 +20,6 @@ type baseProvider struct {
 	cmd       *cmdJob
 	isRunning atomic.Value
 
-	logFile *os.File
-
 	cgroup *cgroupHook
 	zfs    *zfsHook
 	docker *dockerHook
@@ -111,20 +109,21 @@ func (p *baseProvider) Docker() *dockerHook {
 	return p.docker
 }
 
-func (p *baseProvider) prepareLogFile() error {
+func (p *baseProvider) prepareLogFile(append bool) error {
 	if p.LogFile() == "/dev/null" {
 		p.cmd.SetLogFile(nil)
 		return nil
 	}
-	if p.logFile == nil {
-		logFile, err := os.OpenFile(p.LogFile(), os.O_WRONLY|os.O_CREATE, 0644)
-		if err != nil {
-			logger.Errorf("Error opening logfile %s: %s", p.LogFile(), err.Error())
-			return err
-		}
-		p.logFile = logFile
+	appendMode := 0
+	if append {
+		appendMode = os.O_APPEND
 	}
-	p.cmd.SetLogFile(p.logFile)
+	logFile, err := os.OpenFile(p.LogFile(), os.O_WRONLY|os.O_CREATE|appendMode, 0644)
+	if err != nil {
+		logger.Errorf("Error opening logfile %s: %s", p.LogFile(), err.Error())
+		return err
+	}
+	p.cmd.SetLogFile(logFile)
 	return nil
 }
 
@@ -143,32 +142,22 @@ func (p *baseProvider) IsRunning() bool {
 
 func (p *baseProvider) Wait() error {
 	defer func() {
-		p.Lock()
+		logger.Debugf("set isRunning to false: %s", p.Name())
 		p.isRunning.Store(false)
-		if p.logFile != nil {
-			p.logFile.Close()
-			p.logFile = nil
-		}
-		p.Unlock()
 	}()
+	logger.Debugf("calling Wait: %s", p.Name())
 	return p.cmd.Wait()
 }
 
 func (p *baseProvider) Terminate() error {
+	p.Lock()
+	defer p.Unlock()
 	logger.Debugf("terminating provider: %s", p.Name())
 	if !p.IsRunning() {
 		return nil
 	}
 
-	p.Lock()
-	if p.logFile != nil {
-		p.logFile.Close()
-		p.logFile = nil
-	}
-	p.Unlock()
-
 	err := p.cmd.Terminate()
-	p.isRunning.Store(false)
 
 	return err
 }

worker/cmd_provider.go

@@ -1,6 +1,7 @@
 package worker
 
 import (
+	"errors"
 	"time"
 
 	"github.com/anmitsu/go-shlex"
@@ -60,6 +61,13 @@ func (p *cmdProvider) Run() error {
 }
 
 func (p *cmdProvider) Start() error {
+	p.Lock()
+	defer p.Unlock()
+
+	if p.IsRunning() {
+		return errors.New("provider is currently running")
+	}
+
 	env := map[string]string{
 		"TUNASYNC_MIRROR_NAME":  p.Name(),
 		"TUNASYNC_WORKING_DIR":  p.WorkingDir(),
@@ -71,7 +79,7 @@ func (p *cmdProvider) Start() error {
 		env[k] = v
 	}
 	p.cmd = newCmdJob(p, p.command, p.WorkingDir(), env)
-	if err := p.prepareLogFile(); err != nil {
+	if err := p.prepareLogFile(false); err != nil {
 		return err
 	}
 

worker/config.go

@@ -123,6 +123,7 @@ type mirrorConfig struct {
 
 	Command       string `toml:"command"`
 	UseIPv6       bool   `toml:"use_ipv6"`
+	UseIPv4       bool   `toml:"use_ipv4"`
 	ExcludeFile   string `toml:"exclude_file"`
 	Username      string `toml:"username"`
 	Password      string `toml:"password"`

worker/job.go

@@ -5,6 +5,7 @@ import (
 	"fmt"
 	"sync"
 	"sync/atomic"
+	"time"
 
 	tunasync "github.com/tuna/tunasync/internal"
 )
@@ -14,12 +15,13 @@ import (
 type ctrlAction uint8
 
 const (
-	jobStart   ctrlAction = iota
-	jobStop               // stop syncing keep the job
-	jobDisable            // disable the job (stops goroutine)
-	jobRestart            // restart syncing
-	jobPing               // ensure the goroutine is alive
-	jobHalt               // worker halts
+	jobStart      ctrlAction = iota
+	jobStop                  // stop syncing keep the job
+	jobDisable               // disable the job (stops goroutine)
+	jobRestart               // restart syncing
+	jobPing                  // ensure the goroutine is alive
+	jobHalt                  // worker halts
+	jobForceStart            // ignore concurrent limit
 )
 
 type jobMessage struct {
@@ -154,9 +156,7 @@ func (m *mirrorJob) Run(managerChan chan<- jobMessage, semaphore chan empty) err
 			syncDone := make(chan error, 1)
 			go func() {
 				err := provider.Run()
-				if !stopASAP {
-					syncDone <- err
-				}
+				syncDone <- err
 			}()
 
 			select {
@@ -212,22 +212,26 @@ func (m *mirrorJob) Run(managerChan chan<- jobMessage, semaphore chan empty) err
 		return nil
 	}
 
-	runJob := func(kill <-chan empty, jobDone chan<- empty) {
+	runJob := func(kill <-chan empty, jobDone chan<- empty, bypassSemaphore <-chan empty) {
 		select {
 		case semaphore <- empty{}:
 			defer func() { <-semaphore }()
 			runJobWrapper(kill, jobDone)
+		case <-bypassSemaphore:
+			logger.Noticef("Concurrent limit ignored by %s", m.Name())
+			runJobWrapper(kill, jobDone)
 		case <-kill:
 			jobDone <- empty{}
 			return
 		}
 	}
 
+	bypassSemaphore := make(chan empty, 1)
 	for {
 		if m.State() == stateReady {
 			kill := make(chan empty)
 			jobDone := make(chan empty)
-			go runJob(kill, jobDone)
+			go runJob(kill, jobDone, bypassSemaphore)
 
 		_wait_for_job:
 			select {
@@ -248,7 +252,14 @@ func (m *mirrorJob) Run(managerChan chan<- jobMessage, semaphore chan empty) err
 					m.SetState(stateReady)
 					close(kill)
 					<-jobDone
+					time.Sleep(time.Second) // Restart may fail if the process was not exited yet
 					continue
+				case jobForceStart:
+					select { //non-blocking
+					default:
+					case bypassSemaphore <- empty{}:
+					}
+					fallthrough
 				case jobStart:
 					m.SetState(stateReady)
 					goto _wait_for_job
@@ -272,8 +283,14 @@ func (m *mirrorJob) Run(managerChan chan<- jobMessage, semaphore chan empty) err
 		case jobDisable:
 			m.SetState(stateDisabled)
 			return nil
+		case jobForceStart:
+			select { //non-blocking
+			default:
+			case bypassSemaphore <- empty{}:
+			}
+			fallthrough
 		case jobRestart:
-			m.SetState(stateReady)
+			fallthrough
 		case jobStart:
 			m.SetState(stateReady)
 		default:

worker/job_test.go

@@ -135,6 +135,8 @@ echo $TUNASYNC_WORKING_DIR
 				msg = <-managerChan
 				So(msg.status, ShouldEqual, Syncing)
 
+				job.ctrlChan <- jobStart // should be ignored
+
 				job.ctrlChan <- jobStop
 
 				msg = <-managerChan
@@ -170,8 +172,239 @@ echo $TUNASYNC_WORKING_DIR
 				job.ctrlChan <- jobDisable
 				<-job.disabled
 			})
+
+			Convey("If we restart it", func(ctx C) {
+				go job.Run(managerChan, semaphore)
+				job.ctrlChan <- jobStart
+
+				msg := <-managerChan
+				So(msg.status, ShouldEqual, PreSyncing)
+				msg = <-managerChan
+				So(msg.status, ShouldEqual, Syncing)
+
+				job.ctrlChan <- jobRestart
+
+				msg = <-managerChan
+				So(msg.status, ShouldEqual, Failed)
+				So(msg.msg, ShouldEqual, "killed by manager")
+
+				msg = <-managerChan
+				So(msg.status, ShouldEqual, PreSyncing)
+				msg = <-managerChan
+				So(msg.status, ShouldEqual, Syncing)
+				msg = <-managerChan
+				So(msg.status, ShouldEqual, Success)
+
+				expectedOutput := fmt.Sprintf(
+					"%s\n%s\n",
+					provider.WorkingDir(), provider.WorkingDir(),
+				)
+
+				loggedContent, err := ioutil.ReadFile(provider.LogFile())
+				So(err, ShouldBeNil)
+				So(string(loggedContent), ShouldEqual, expectedOutput)
+				job.ctrlChan <- jobDisable
+				<-job.disabled
+			})
+
+			Convey("If we disable it", func(ctx C) {
+				go job.Run(managerChan, semaphore)
+				job.ctrlChan <- jobStart
+
+				msg := <-managerChan
+				So(msg.status, ShouldEqual, PreSyncing)
+				msg = <-managerChan
+				So(msg.status, ShouldEqual, Syncing)
+
+				job.ctrlChan <- jobDisable
+
+				msg = <-managerChan
+				So(msg.status, ShouldEqual, Failed)
+				So(msg.msg, ShouldEqual, "killed by manager")
+
+				<-job.disabled
+			})
+
+			Convey("If we stop it twice, than start it", func(ctx C) {
+				go job.Run(managerChan, semaphore)
+				job.ctrlChan <- jobStart
+
+				msg := <-managerChan
+				So(msg.status, ShouldEqual, PreSyncing)
+				msg = <-managerChan
+				So(msg.status, ShouldEqual, Syncing)
+
+				job.ctrlChan <- jobStop
+
+				msg = <-managerChan
+				So(msg.status, ShouldEqual, Failed)
+				So(msg.msg, ShouldEqual, "killed by manager")
+
+				job.ctrlChan <- jobStop // should be ignored
+
+				job.ctrlChan <- jobStart
+
+				msg = <-managerChan
+				So(msg.status, ShouldEqual, PreSyncing)
+				msg = <-managerChan
+				So(msg.status, ShouldEqual, Syncing)
+				msg = <-managerChan
+				So(msg.status, ShouldEqual, Success)
+
+				expectedOutput := fmt.Sprintf(
+					"%s\n%s\n",
+					provider.WorkingDir(), provider.WorkingDir(),
+				)
+
+				loggedContent, err := ioutil.ReadFile(provider.LogFile())
+				So(err, ShouldBeNil)
+				So(string(loggedContent), ShouldEqual, expectedOutput)
+
+				job.ctrlChan <- jobDisable
+				<-job.disabled
+			})
 		})
 
 	})
 
 }
+
+func TestConcurrentMirrorJobs(t *testing.T) {
+
+	InitLogger(true, true, false)
+
+	Convey("Concurrent MirrorJobs should work", t, func(ctx C) {
+		tmpDir, err := ioutil.TempDir("", "tunasync")
+		defer os.RemoveAll(tmpDir)
+		So(err, ShouldBeNil)
+
+		const CONCURRENT = 5
+
+		var providers [CONCURRENT]*cmdProvider
+		var jobs [CONCURRENT]*mirrorJob
+		for i := 0; i < CONCURRENT; i++ {
+			c := cmdConfig{
+				name:        fmt.Sprintf("job-%d", i),
+				upstreamURL: "http://mirrors.tuna.moe/",
+				command:     "sleep 2",
+				workingDir:  tmpDir,
+				logDir:      tmpDir,
+				logFile:     "/dev/null",
+				interval:    10 * time.Second,
+			}
+
+			var err error
+			providers[i], err = newCmdProvider(c)
+			So(err, ShouldBeNil)
+			jobs[i] = newMirrorJob(providers[i])
+		}
+
+		managerChan := make(chan jobMessage, 10)
+		semaphore := make(chan empty, CONCURRENT-2)
+
+		countingJobs := func(managerChan chan jobMessage, totalJobs, concurrentCheck int) (peakConcurrent, counterFailed int) {
+			counterEnded := 0
+			counterRunning := 0
+			peakConcurrent = 0
+			counterFailed = 0
+			for counterEnded < totalJobs {
+				msg := <-managerChan
+				switch msg.status {
+				case PreSyncing:
+					counterRunning++
+				case Syncing:
+				case Failed:
+					counterFailed++
+					fallthrough
+				case Success:
+					counterEnded++
+					counterRunning--
+				default:
+					So(0, ShouldEqual, 1)
+				}
+				// Test if semaphore works
+				So(counterRunning, ShouldBeLessThanOrEqualTo, concurrentCheck)
+				if counterRunning > peakConcurrent {
+					peakConcurrent = counterRunning
+				}
+			}
+			// select {
+			// case msg := <-managerChan:
+			// 	logger.Errorf("extra message received: %v", msg)
+			// 	So(0, ShouldEqual, 1)
+			// case <-time.After(2 * time.Second):
+			// }
+			return
+		}
+
+		Convey("When we run them all", func(ctx C) {
+			for _, job := range jobs {
+				go job.Run(managerChan, semaphore)
+				job.ctrlChan <- jobStart
+			}
+
+			peakConcurrent, counterFailed := countingJobs(managerChan, CONCURRENT, CONCURRENT-2)
+
+			So(peakConcurrent, ShouldEqual, CONCURRENT-2)
+			So(counterFailed, ShouldEqual, 0)
+
+			for _, job := range jobs {
+				job.ctrlChan <- jobDisable
+				<-job.disabled
+			}
+		})
+		Convey("If we cancel one job", func(ctx C) {
+			for _, job := range jobs {
+				go job.Run(managerChan, semaphore)
+				job.ctrlChan <- jobRestart
+				time.Sleep(200 * time.Millisecond)
+			}
+
+			// Cancel the one waiting for semaphore
+			jobs[len(jobs)-1].ctrlChan <- jobStop
+
+			peakConcurrent, counterFailed := countingJobs(managerChan, CONCURRENT-1, CONCURRENT-2)
+
+			So(peakConcurrent, ShouldEqual, CONCURRENT-2)
+			So(counterFailed, ShouldEqual, 0)
+
+			for _, job := range jobs {
+				job.ctrlChan <- jobDisable
+				<-job.disabled
+			}
+		})
+		Convey("If we override the concurrent limit", func(ctx C) {
+			for _, job := range jobs {
+				go job.Run(managerChan, semaphore)
+				job.ctrlChan <- jobStart
+				time.Sleep(200 * time.Millisecond)
+			}
+
+			jobs[len(jobs)-1].ctrlChan <- jobForceStart
+			jobs[len(jobs)-2].ctrlChan <- jobForceStart
+
+			peakConcurrent, counterFailed := countingJobs(managerChan, CONCURRENT, CONCURRENT)
+
+			So(peakConcurrent, ShouldEqual, CONCURRENT)
+			So(counterFailed, ShouldEqual, 0)
+
+			time.Sleep(1 * time.Second)
+
+			// fmt.Println("Restart them")
+
+			for _, job := range jobs {
+				job.ctrlChan <- jobStart
+			}
+
+			peakConcurrent, counterFailed = countingJobs(managerChan, CONCURRENT, CONCURRENT-2)
+
+			So(peakConcurrent, ShouldEqual, CONCURRENT-2)
+			So(counterFailed, ShouldEqual, 0)
+
+			for _, job := range jobs {
+				job.ctrlChan <- jobDisable
+				<-job.disabled
+			}
+		})
+	})
+}

worker/provider.go

@@ -130,6 +130,7 @@ func newMirrorProvider(mirror mirrorConfig, cfg *Config) mirrorProvider {
 			logDir:      logDir,
 			logFile:     filepath.Join(logDir, "latest.log"),
 			useIPv6:     mirror.UseIPv6,
+			useIPv4:     mirror.UseIPv4,
 			interval:    time.Duration(mirror.Interval) * time.Minute,
 		}
 		p, err := newRsyncProvider(rc)

worker/provider_test.go

@@ -79,11 +79,12 @@ exit 0
 			err = ioutil.WriteFile(scriptFile, []byte(scriptContent), 0755)
 			So(err, ShouldBeNil)
 
+			targetDir, _ := filepath.EvalSymlinks(provider.WorkingDir())
 			expectedOutput := fmt.Sprintf(
 				"syncing to %s\n"+
 					"%s\n"+
 					"Done\n",
-				provider.WorkingDir(),
+				targetDir,
 				fmt.Sprintf(
 					"-aHvh --no-o --no-g --stats --exclude .~tmp~/ "+
 						"--delete --delete-after --delay-updates --safe-links "+
@@ -144,11 +145,12 @@ exit 0
 			err = ioutil.WriteFile(scriptFile, []byte(scriptContent), 0755)
 			So(err, ShouldBeNil)
 
+			targetDir, _ := filepath.EvalSymlinks(provider.WorkingDir())
 			expectedOutput := fmt.Sprintf(
 				"syncing to %s\n"+
 					"%s\n"+
 					"Done\n",
-				provider.WorkingDir(),
+				targetDir,
 				fmt.Sprintf(
 					"%s %s -aHvh --no-o --no-g --stats --exclude .~tmp~/ "+
 						"--delete --delete-after --delay-updates --safe-links "+
@@ -260,6 +262,40 @@ sleep 5
 
 		})
 	})
+	Convey("Command Provider without log file should work", t, func(ctx C) {
+		tmpDir, err := ioutil.TempDir("", "tunasync")
+		defer os.RemoveAll(tmpDir)
+		So(err, ShouldBeNil)
+
+		c := cmdConfig{
+			name:        "run-ls",
+			upstreamURL: "http://mirrors.tuna.moe/",
+			command:     "ls",
+			workingDir:  tmpDir,
+			logDir:      tmpDir,
+			logFile:     "/dev/null",
+			interval:    600 * time.Second,
+		}
+
+		provider, err := newCmdProvider(c)
+		So(err, ShouldBeNil)
+
+		So(provider.IsMaster(), ShouldEqual, false)
+		So(provider.ZFS(), ShouldBeNil)
+		So(provider.Type(), ShouldEqual, provCommand)
+		So(provider.Name(), ShouldEqual, c.name)
+		So(provider.WorkingDir(), ShouldEqual, c.workingDir)
+		So(provider.LogDir(), ShouldEqual, c.logDir)
+		So(provider.LogFile(), ShouldEqual, c.logFile)
+		So(provider.Interval(), ShouldEqual, c.interval)
+
+		Convey("Run the command", func() {
+
+			err = provider.Run()
+			So(err, ShouldBeNil)
+
+		})
+	})
 }
 
 func TestTwoStageRsyncProvider(t *testing.T) {
@@ -280,6 +316,8 @@ func TestTwoStageRsyncProvider(t *testing.T) {
 			logFile:       tmpFile,
 			useIPv6:       true,
 			excludeFile:   tmpFile,
+			username:      "hello",
+			password:      "world",
 		}
 
 		provider, err := newTwoStageRsyncProvider(c)
@@ -306,6 +344,7 @@ exit 0
 			err = provider.Run()
 			So(err, ShouldBeNil)
 
+			targetDir, _ := filepath.EvalSymlinks(provider.WorkingDir())
 			expectedOutput := fmt.Sprintf(
 				"syncing to %s\n"+
 					"%s\n"+
@@ -313,14 +352,14 @@ exit 0
 					"syncing to %s\n"+
 					"%s\n"+
 					"Done\n",
-				provider.WorkingDir(),
+				targetDir,
 				fmt.Sprintf(
 					"-aHvh --no-o --no-g --stats --exclude .~tmp~/ --safe-links "+
 						"--timeout=120 --contimeout=120 --exclude dists/ -6 "+
 						"--exclude-from %s %s %s",
 					provider.excludeFile, provider.upstreamURL, provider.WorkingDir(),
 				),
-				provider.WorkingDir(),
+				targetDir,
 				fmt.Sprintf(
 					"-aHvh --no-o --no-g --stats --exclude .~tmp~/ "+
 						"--delete --delete-after --delay-updates --safe-links "+

worker/rsync_provider.go

@@ -11,7 +11,7 @@ type rsyncConfig struct {
 	rsyncCmd                                     string
 	upstreamURL, username, password, excludeFile string
 	workingDir, logDir, logFile                  string
-	useIPv6                                      bool
+	useIPv6, useIPv4                             bool
 	interval                                     time.Duration
 }
 
@@ -49,6 +49,8 @@ func newRsyncProvider(c rsyncConfig) (*rsyncProvider, error) {
 
 	if c.useIPv6 {
 		options = append(options, "-6")
+	} else if c.useIPv4 {
+		options = append(options, "-4")
 	}
 
 	if c.excludeFile != "" {
@@ -79,6 +81,12 @@ func (p *rsyncProvider) Run() error {
 }
 
 func (p *rsyncProvider) Start() error {
+	p.Lock()
+	defer p.Unlock()
+
+	if p.IsRunning() {
+		return errors.New("provider is currently running")
+	}
 
 	env := map[string]string{}
 	if p.username != "" {
@@ -92,7 +100,7 @@ func (p *rsyncProvider) Start() error {
 	command = append(command, p.upstreamURL, p.WorkingDir())
 
 	p.cmd = newCmdJob(p, command, p.WorkingDir(), env)
-	if err := p.prepareLogFile(); err != nil {
+	if err := p.prepareLogFile(false); err != nil {
 		return err
 	}
 

worker/runner.go

@@ -118,6 +118,9 @@ func (c *cmdJob) Wait() error {
 		return c.retErr
 	default:
 		err := c.cmd.Wait()
+		if c.cmd.Stdout != nil {
+			c.cmd.Stdout.(*os.File).Close()
+		}
 		c.retErr = err
 		close(c.finished)
 		return err

worker/two_stage_rsync_provider.go

@@ -108,7 +108,12 @@ func (p *twoStageRsyncProvider) Options(stage int) ([]string, error) {
 }
 
 func (p *twoStageRsyncProvider) Run() error {
-	defer p.Wait()
+	p.Lock()
+	defer p.Unlock()
+
+	if p.IsRunning() {
+		return errors.New("provider is currently running")
+	}
 
 	env := map[string]string{}
 	if p.username != "" {
@@ -129,7 +134,7 @@ func (p *twoStageRsyncProvider) Run() error {
 		command = append(command, p.upstreamURL, p.WorkingDir())
 
 		p.cmd = newCmdJob(p, command, p.WorkingDir(), env)
-		if err := p.prepareLogFile(); err != nil {
+		if err := p.prepareLogFile(stage > 1); err != nil {
 			return err
 		}
 
@@ -137,9 +142,11 @@ func (p *twoStageRsyncProvider) Run() error {
 			return err
 		}
 		p.isRunning.Store(true)
+		logger.Debugf("set isRunning to true: %s", p.Name())
 
-		err = p.cmd.Wait()
-		p.isRunning.Store(false)
+		p.Unlock()
+		err = p.Wait()
+		p.Lock()
 		if err != nil {
 			return err
 		}

worker/worker.go

@@ -219,7 +219,11 @@ func (w *Worker) makeHTTPServer() {
 		}
 		switch cmd.Cmd {
 		case CmdStart:
-			job.ctrlChan <- jobStart
+			if cmd.Options["force"] {
+				job.ctrlChan <- jobForceStart
+			} else {
+				job.ctrlChan <- jobStart
+			}
 		case CmdRestart:
 			job.ctrlChan <- jobRestart
 		case CmdStop: