bug fix: jobs not being scheduled after timeout

worker/job.go

@@ -180,7 +180,6 @@ func (m *mirrorJob) Run(managerChan chan<- jobMessage, semaphore chan empty) err
 				logger.Debug("syncing done")
 			case <-time.After(timeout):
 				logger.Notice("provider timeout")
-				stopASAP = true
 				termErr = provider.Terminate()
 				syncErr = fmt.Errorf("%s timeout after %v", m.Name(), timeout)
 			case <-kill:

worker/job_test.go

@@ -335,7 +335,6 @@ echo $TUNASYNC_WORKING_DIR
 			})
 		})
 
-
 		Convey("When a job timed out", func(ctx C) {
 			scriptContent := `#!/bin/bash
 echo $TUNASYNC_WORKING_DIR
@@ -371,6 +370,30 @@ echo $TUNASYNC_WORKING_DIR
 				job.ctrlChan <- jobDisable
 				<-job.disabled
 			})
+
+			Convey("It should be retried", func(ctx C) {
+				go job.Run(managerChan, semaphore)
+				job.ctrlChan <- jobStart
+				time.Sleep(1 * time.Second)
+				msg := <-managerChan
+				So(msg.status, ShouldEqual, PreSyncing)
+
+				for i := 0; i < defaultMaxRetry; i++ {
+					msg = <-managerChan
+					So(msg.status, ShouldEqual, Syncing)
+
+					job.ctrlChan <- jobStart // should be ignored
+
+					msg = <-managerChan
+					So(msg.status, ShouldEqual, Failed)
+					So(msg.msg, ShouldContainSubstring, "timeout after")
+					// re-schedule after last try
+					So(msg.schedule, ShouldEqual, i == defaultMaxRetry-1)
+				}
+
+				job.ctrlChan <- jobDisable
+				<-job.disabled
+			})
 		})
 	})