commit 7f6cdad769ce2d3eb19f4a2a14f1b5bca158241e
Author: Bùi Thành Nhân <hi@imnhan.com>
Date:   Sat Aug 29 15:43:06 2020 +0700

    protect scheduler against exceptions
    
    A failed worker is now put back into the queue, for a maximum of 3
    retries before its current turn is skipped.
    
    Also, in the update title worker: just skip title if source site is
    currently acting up.

diff --git a/pyproject.toml b/pyproject.toml
index 54f5d77..f71d4ea 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "pytaku"
-version = "0.3.9"
+version = "0.3.10"
 description = "Self-hostable web-based manga reader"
 authors = ["Bùi Thành Nhân <hi@imnhan.com>"]
 license = "AGPL-3.0-only"
diff --git a/src/pytaku/scheduler.py b/src/pytaku/scheduler.py
index 1617c8d..c64cb21 100644
--- a/src/pytaku/scheduler.py
+++ b/src/pytaku/scheduler.py
@@ -1,8 +1,13 @@
 import time
+import traceback
 from abc import ABC, abstractmethod
 from datetime import datetime, timedelta
 from pathlib import Path
 
+from requests.exceptions import ReadTimeout
+
+from mangoapi.exceptions import SourceSite5xxError
+
 from .conf import config
 from .persistence import delete_expired_tokens, find_outdated_titles, save_title
 from .source_sites import get_title
@@ -17,8 +22,13 @@ def main_loop():
         for worker in workers:
             if worker.should_run():
                 print("Running", worker.__class__.__name__)
-                worker.run()
-                worker.after_run()
+                try:
+                    worker.run()
+                    worker.after_run()
+                except Exception:
+                    stacktrace = traceback.format_exc()
+                    print(stacktrace)
+                    worker.after_error(stacktrace)
         time.sleep(5)
 
 
@@ -27,6 +37,7 @@ class Worker(ABC):
 
     def __init__(self):
         self.last_run = datetime(1, 1, 1)
+        self.error_count = 0
 
     def should_run(self):
         return now() - self.last_run >= self.interval
@@ -38,6 +49,15 @@ def run(self):
     def after_run(self):
         self.last_run = now()
 
+    def after_error(self, stacktrace):
+        # TODO: email or send stacktrace to an ops chat channel or something
+        self.error_count += 1
+
+        # If failed repeatedly: give up this run
+        if self.error_count > 3:
+            self.last_run = now()
+            self.error_count = 0
+
 
 class UpdateOutdatedTitles(Worker):
     interval = timedelta(hours=2)
@@ -45,9 +65,12 @@ class UpdateOutdatedTitles(Worker):
     def run(self):
         for title in find_outdated_titles():
             print(f"Updating title {title['id']} from {title['site']}...", end="")
-            updated_title = get_title(title["site"], title["id"])
-            save_title(updated_title)
-            print(" done")
+            try:
+                updated_title = get_title(title["site"], title["id"])
+                save_title(updated_title)
+                print(" done")
+            except (SourceSite5xxError, ReadTimeout) as e:
+                print(" skipped because of server error:", str(e))
 
 
 class DeleteExpiredTokens(Worker):