add status manager

tunasync/jobs.py

@@ -28,7 +28,8 @@ def run_job(sema, child_q, manager_q, provider, **settings):
             break
         aquired = True
 
-        status = "unkown"
+        status = "syncing"
+        manager_q.put((provider.name, status))
         try:
             for hook in provider.hooks:
                 hook.before_job(name=provider.name)
@@ -65,6 +66,8 @@ def run_job(sema, child_q, manager_q, provider, **settings):
             provider.name, provider.interval
         ))
 
+        manager_q.put((provider.name, status))
+
         try:
             msg = child_q.get(timeout=provider.interval * 60)
             if msg == "terminate":

tunasync/mirror_config.py

@@ -0,0 +1,109 @@
+#!/usr/bin/env python2
+# -*- coding:utf-8 -*-
+import os
+from .mirror_provider import RsyncProvider, ShellProvider
+from .btrfs_snapshot import BtrfsHook
+
+
+class MirrorConfig(object):
+
+    _valid_providers = set(("rsync", "debmirror", "shell", ))
+
+    def __init__(self, parent, options):
+        self._parent = parent
+        self._popt = self._parent._settings
+        self.options = dict(options.items())  # copy
+        self._validate()
+
+    def _validate(self):
+        provider = self.options.get("provider", None)
+        assert provider in self._valid_providers
+
+        if provider == "rsync":
+            assert "upstream" in self.options
+
+        elif provider == "shell":
+            assert "command" in self.options
+
+        local_dir_tmpl = self.options.get(
+            "local_dir", self._popt["global"]["local_dir"])
+
+        self.options["local_dir"] = local_dir_tmpl.format(
+            mirror_root=self._popt["global"]["mirror_root"],
+            mirror_name=self.name,
+        )
+
+        if "interval" not in self.options:
+            self.options["interval"] = self._popt["global"]["interval"]
+
+        assert isinstance(self.options["interval"], int)
+
+        log_dir = self._popt["global"]["log_dir"]
+        if "log_file" not in self.options:
+            self.options["log_file"] = os.path.join(
+                log_dir, self.name, "{date}.log")
+
+        if "use_btrfs" not in self.options:
+            self.options["use_btrfs"] = self._parent.use_btrfs
+        assert self.options["use_btrfs"] in (True, False)
+
+    def __getattr__(self, key):
+        if key in self.__dict__:
+            return self.__dict__[key]
+        else:
+            return self.__dict__["options"].get(key, None)
+
+    def to_provider(self, hooks=[]):
+        if self.provider == "rsync":
+            provider = RsyncProvider(
+                self.name,
+                self.upstream,
+                self.local_dir,
+                self.use_ipv6,
+                self.exclude_file,
+                self.log_file,
+                self.interval,
+                hooks,
+            )
+        elif self.options["provider"] == "shell":
+            provider = ShellProvider(
+                self.name,
+                self.command,
+                self.local_dir,
+                self.log_file,
+                self.interval,
+                hooks
+            )
+
+        return provider
+
+    def compare(self, other):
+        assert self.name == other.name
+
+        for key, val in self.options.iteritems():
+            if other.options.get(key, None) != val:
+                return False
+
+        return True
+
+    def hooks(self):
+        hooks = []
+        parent = self._parent
+        if self.options["use_btrfs"]:
+            working_dir = parent.btrfs_working_dir_tmpl.format(
+                mirror_root=parent.mirror_root,
+                mirror_name=self.name
+            )
+            service_dir = parent.btrfs_service_dir_tmpl.format(
+                mirror_root=parent.mirror_root,
+                mirror_name=self.name
+            )
+            gc_dir = parent.btrfs_gc_dir_tmpl.format(
+                mirror_root=parent.mirror_root,
+                mirror_name=self.name
+            )
+            hooks.append(BtrfsHook(service_dir, working_dir, gc_dir))
+
+        return hooks
+
+# vim: ts=4 sw=4 sts=4 expandtab

tunasync/status_manager.py

@@ -0,0 +1,61 @@
+#!/usr/bin/env python2
+# -*- coding:utf-8 -*-
+import json
+from datetime import datetime
+
+
+class StatusManager(object):
+
+    def __init__(self, parent, dbfile):
+        self.parent = parent
+        self.dbfile = dbfile
+        self.init_mirrors()
+
+    def init_mirrors(self):
+        mirrors = {}
+        try:
+            with open(self.dbfile) as f:
+                _mirrors = json.load(f)
+                for m in _mirrors:
+                    mirrors[m["name"]] = m
+        except:
+            for name, _ in self.parent.mirrors.iteritems():
+                mirrors[name] = {
+                    'name': name,
+                    'last_update': '-',
+                    'status': 'unknown',
+                }
+        self.mirrors = mirrors
+
+    def update_status(self, name, status):
+
+        _m = self.mirrors.get(name, {
+            'name': name,
+            'last_update': '-',
+            'status': '-',
+        })
+
+        if status in ("syncing", "fail"):
+            update_time = _m["last_update"]
+        elif status == "success":
+            update_time = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
+        else:
+            print("Invalid status: {}, from {}".format(status, name))
+
+        self.mirrors[name] = {
+            'name': name,
+            'last_update': update_time,
+            'status': status,
+        }
+
+        with open(self.dbfile, 'wb') as f:
+            _mirrors = sorted(
+                [m for _, m in self.mirrors.items()],
+                key=lambda x: x['name']
+            )
+
+            print("Updated status file, {}:{}".format(name, status))
+            json.dump(_mirrors, f)
+
+
+# vim: ts=4 sw=4 sts=4 expandtab

tunasync/tunasync.py

@@ -1,117 +1,14 @@
 #!/usr/bin/env python2
 # -*- coding:utf-8 -*-
-import os.path
 import signal
 import sys
 import toml
 
 from multiprocessing import Process, Semaphore, Queue
 from . import jobs
-from .mirror_provider import RsyncProvider, ShellProvider
-from .btrfs_snapshot import BtrfsHook
 from .hook import JobHook
-
-
-class MirrorConfig(object):
-
-    _valid_providers = set(("rsync", "debmirror", "shell", ))
-
-    def __init__(self, parent, options):
-        self._parent = parent
-        self._popt = self._parent._settings
-        self.options = dict(options.items())  # copy
-        self._validate()
-
-    def _validate(self):
-        provider = self.options.get("provider", None)
-        assert provider in self._valid_providers
-
-        if provider == "rsync":
-            assert "upstream" in self.options
-
-        elif provider == "shell":
-            assert "command" in self.options
-
-        local_dir_tmpl = self.options.get(
-            "local_dir", self._popt["global"]["local_dir"])
-
-        self.options["local_dir"] = local_dir_tmpl.format(
-            mirror_root=self._popt["global"]["mirror_root"],
-            mirror_name=self.name,
-        )
-
-        if "interval" not in self.options:
-            self.options["interval"] = self._popt["global"]["interval"]
-
-        assert isinstance(self.options["interval"], int)
-
-        log_dir = self._popt["global"]["log_dir"]
-        if "log_file" not in self.options:
-            self.options["log_file"] = os.path.join(
-                log_dir, self.name, "{date}.log")
-
-        if "use_btrfs" not in self.options:
-            self.options["use_btrfs"] = self._parent.use_btrfs
-        assert self.options["use_btrfs"] in (True, False)
-
-    def __getattr__(self, key):
-        if key in self.__dict__:
-            return self.__dict__[key]
-        else:
-            return self.__dict__["options"].get(key, None)
-
-    def to_provider(self, hooks=[]):
-        if self.provider == "rsync":
-            provider = RsyncProvider(
-                self.name,
-                self.upstream,
-                self.local_dir,
-                self.use_ipv6,
-                self.exclude_file,
-                self.log_file,
-                self.interval,
-                hooks,
-            )
-        elif self.options["provider"] == "shell":
-            provider = ShellProvider(
-                self.name,
-                self.command,
-                self.local_dir,
-                self.log_file,
-                self.interval,
-                hooks
-            )
-
-        return provider
-
-    def compare(self, other):
-        assert self.name == other.name
-
-        for key, val in self.options.iteritems():
-            if other.options.get(key, None) != val:
-                return False
-
-        return True
-
-    def hooks(self):
-        hooks = []
-        parent = self._parent
-        if self.options["use_btrfs"]:
-            working_dir = parent.btrfs_working_dir_tmpl.format(
-                mirror_root=parent.mirror_root,
-                mirror_name=self.name
-            )
-            service_dir = parent.btrfs_service_dir_tmpl.format(
-                mirror_root=parent.mirror_root,
-                mirror_name=self.name
-            )
-            gc_dir = parent.btrfs_gc_dir_tmpl.format(
-                mirror_root=parent.mirror_root,
-                mirror_name=self.name
-            )
-            hooks.append(BtrfsHook(service_dir, working_dir, gc_dir))
-
-        return hooks
+from .mirror_config import MirrorConfig
+from .status_manager import StatusManager
 
 
 class TUNASync(object):
@@ -146,6 +43,9 @@ class TUNASync(object):
         self.btrfs_working_dir_tmpl = self._settings["btrfs"]["working_dir"]
         self.btrfs_gc_dir_tmpl = self._settings["btrfs"]["gc_dir"]
 
+        self.status_file = self._settings["global"]["status_file"]
+        self.status_manager = StatusManager(self, self.status_file)
+
     def add_hook(self, h):
         assert isinstance(h, JobHook)
         self._hooks.append(h)
@@ -203,6 +103,11 @@ class TUNASync(object):
             if status == "QUIT":
                 print("New configuration applied to {}".format(name))
                 self.run_provider(name)
+            else:
+                try:
+                    self.status_manager.update_status(name, status)
+                except Exception as e:
+                    print(e)
 
     def run_provider(self, name):
         if name not in self.providers: