Repos / pytaku / 8195b99666
commit 8195b996664f6a5e27e83316188d267b421fff48
Author: Bùi Thành Nhân <hi@imnhan.com>
Date:   Sat Aug 29 16:24:28 2020 +0700

    concurrent & crash-proof search
    
    I still dislike the catch-all though, so gotta fix that once I've sorted
    out user-facing error reporting.
    
    Also increased mangasee search cache lifespan.

diff --git a/pyproject.toml b/pyproject.toml
index f71d4ea..e6764bf 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "pytaku"
-version = "0.3.10"
+version = "0.3.11"
 description = "Self-hostable web-based manga reader"
 authors = ["Bùi Thành Nhân <hi@imnhan.com>"]
 license = "AGPL-3.0-only"
diff --git a/src/mangoapi/base_site.py b/src/mangoapi/base_site.py
index 65990c1..6f1884d 100644
--- a/src/mangoapi/base_site.py
+++ b/src/mangoapi/base_site.py
@@ -48,6 +48,10 @@ def login(self, username, password):
 
     def _http_request(self, method, *args, **kwargs):
         request_func = getattr(self._session, method)
+
+        if "timeout" not in kwargs:
+            kwargs["timeout"] = 5
+
         resp = request_func(*args, **kwargs)
 
         if 500 <= resp.status_code <= 599:
diff --git a/src/mangoapi/mangasee.py b/src/mangoapi/mangasee.py
index 8ffa305..3b07bf9 100644
--- a/src/mangoapi/mangasee.py
+++ b/src/mangoapi/mangasee.py
@@ -25,7 +25,7 @@ def __init__(self, keyval_store=None):
         self.keyval_store = keyval_store
 
     def get_title(self, title_id):
-        resp = self.http_get(f"https://mangasee123.com/manga/{title_id}", timeout=3)
+        resp = self.http_get(f"https://mangasee123.com/manga/{title_id}")
         html = resp.text
         name = regexes["title_name"].search(html).group(1).strip()
         desc = regexes["title_desc"].search(html).group(1).strip()
@@ -102,7 +102,7 @@ def search_title(self, query):
             titles = None
             if self.keyval_store:
                 titles = json.loads(
-                    self.keyval_store.get("mangasee_titles", "null", since="-1 day")
+                    self.keyval_store.get("mangasee_titles", "null", since="-20 day")
                 )
             if not titles:
                 print("Fetching mangasee title list...", end="")
diff --git a/src/pytaku/source_sites.py b/src/pytaku/source_sites.py
index ebfd3ba..62f0438 100644
--- a/src/pytaku/source_sites.py
+++ b/src/pytaku/source_sites.py
@@ -1,3 +1,6 @@
+import traceback
+from concurrent.futures import ThreadPoolExecutor, as_completed
+
 from mangoapi import get_site_class
 
 from .conf import config
@@ -55,7 +58,27 @@ def search_title_all_sites(query):
     Returns dict in the form of {site_name: List[Title]}
     I should really look into proper type annotations huh.
     """
-    return {
-        site_name: search_title(site_name, query)
-        for site_name in ("mangasee", "mangadex")
-    }
+    site_names = ("mangasee", "mangadex")
+    results = {}
+
+    def safe_search(site_name, query):
+        try:
+            return search_title(site_name, query)
+        except Exception:
+            print(f"{site_name}'s search function shat the bed:")
+            traceback.print_exc()
+            return []
+
+    # Concurrently search from multiple sites.
+    # Obviously network I/O bound so a thread pool is appropriate enough.
+    with ThreadPoolExecutor(max_workers=5) as executor:
+        future_to_site_name = {
+            executor.submit(safe_search, site_name, query): site_name
+            for site_name in site_names
+        }
+        for future in as_completed(future_to_site_name):
+            site_results = future.result()
+            site_name = future_to_site_name[future]
+            results[site_name] = site_results
+
+    return results