Repos / pytaku / 8195b99666
commit 8195b996664f6a5e27e83316188d267b421fff48
Author: Bùi Thành Nhân <hi@imnhan.com>
Date: Sat Aug 29 16:24:28 2020 +0700
concurrent & crash-proof search
I still dislike the catch-all though, so gotta fix that once I've sorted
out user-facing error reporting.
Also increased mangasee search cache lifespan.
diff --git a/pyproject.toml b/pyproject.toml
index f71d4ea..e6764bf 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -1,6 +1,6 @@
[tool.poetry]
name = "pytaku"
-version = "0.3.10"
+version = "0.3.11"
description = "Self-hostable web-based manga reader"
authors = ["Bùi Thành Nhân <hi@imnhan.com>"]
license = "AGPL-3.0-only"
diff --git a/src/mangoapi/base_site.py b/src/mangoapi/base_site.py
index 65990c1..6f1884d 100644
--- a/src/mangoapi/base_site.py
+++ b/src/mangoapi/base_site.py
@@ -48,6 +48,10 @@ def login(self, username, password):
def _http_request(self, method, *args, **kwargs):
request_func = getattr(self._session, method)
+
+ if "timeout" not in kwargs:
+ kwargs["timeout"] = 5
+
resp = request_func(*args, **kwargs)
if 500 <= resp.status_code <= 599:
diff --git a/src/mangoapi/mangasee.py b/src/mangoapi/mangasee.py
index 8ffa305..3b07bf9 100644
--- a/src/mangoapi/mangasee.py
+++ b/src/mangoapi/mangasee.py
@@ -25,7 +25,7 @@ def __init__(self, keyval_store=None):
self.keyval_store = keyval_store
def get_title(self, title_id):
- resp = self.http_get(f"https://mangasee123.com/manga/{title_id}", timeout=3)
+ resp = self.http_get(f"https://mangasee123.com/manga/{title_id}")
html = resp.text
name = regexes["title_name"].search(html).group(1).strip()
desc = regexes["title_desc"].search(html).group(1).strip()
@@ -102,7 +102,7 @@ def search_title(self, query):
titles = None
if self.keyval_store:
titles = json.loads(
- self.keyval_store.get("mangasee_titles", "null", since="-1 day")
+ self.keyval_store.get("mangasee_titles", "null", since="-20 day")
)
if not titles:
print("Fetching mangasee title list...", end="")
diff --git a/src/pytaku/source_sites.py b/src/pytaku/source_sites.py
index ebfd3ba..62f0438 100644
--- a/src/pytaku/source_sites.py
+++ b/src/pytaku/source_sites.py
@@ -1,3 +1,6 @@
+import traceback
+from concurrent.futures import ThreadPoolExecutor, as_completed
+
from mangoapi import get_site_class
from .conf import config
@@ -55,7 +58,27 @@ def search_title_all_sites(query):
Returns dict in the form of {site_name: List[Title]}
I should really look into proper type annotations huh.
"""
- return {
- site_name: search_title(site_name, query)
- for site_name in ("mangasee", "mangadex")
- }
+ site_names = ("mangasee", "mangadex")
+ results = {}
+
+ def safe_search(site_name, query):
+ try:
+ return search_title(site_name, query)
+ except Exception:
+ print(f"{site_name}'s search function shat the bed:")
+ traceback.print_exc()
+ return []
+
+ # Concurrently search from multiple sites.
+ # Obviously network I/O bound so a thread pool is appropriate enough.
+ with ThreadPoolExecutor(max_workers=5) as executor:
+ future_to_site_name = {
+ executor.submit(safe_search, site_name, query): site_name
+ for site_name in site_names
+ }
+ for future in as_completed(future_to_site_name):
+ site_results = future.result()
+ site_name = future_to_site_name[future]
+ results[site_name] = site_results
+
+ return results