Repos / pytaku / d892366fd6
commit d892366fd6e3a0185ec253d3dc1c7da83467067d
Author: Bùi Thành Nhân <hi@imnhan.com>
Date: Sun Aug 9 15:26:57 2020 +0700
implement mangasee search
diff --git a/src/mangoapi/mangadex.py b/src/mangoapi/mangadex.py
index c78103b..cb5c07b 100644
--- a/src/mangoapi/mangadex.py
+++ b/src/mangoapi/mangadex.py
@@ -67,7 +67,12 @@ def search_title(self, query):
matches = TITLES_PATTERN.findall(md_resp.text)
titles = [
- {"id": int(id), "name": name.strip(), "site": "mangadex"}
+ {
+ "id": id,
+ "name": name.strip(),
+ "site": "mangadex",
+ "thumbnail": f"https://mangadex.org/images/manga/{id}.large.jpg",
+ }
for id, name in matches
]
return titles
diff --git a/src/mangoapi/mangasee.py b/src/mangoapi/mangasee.py
index 0cfd85c..9fd25fe 100644
--- a/src/mangoapi/mangasee.py
+++ b/src/mangoapi/mangasee.py
@@ -1,7 +1,17 @@
+import json
+
+import apsw
+import requests
+
from mangoapi.base_site import Site
class Mangasee(Site):
+ search_table = None
+
+ def __init__(self, keyval_store=None):
+ self.keyval_store = keyval_store
+
def get_title(self, title_id):
pass
@@ -9,4 +19,68 @@ def get_chapter(self, chapter_id):
pass
def search_title(self, query):
- return []
+ """
+ Json blob of all mangasee titles: https://mangasee123.com/_search.php
+ The structure is something like this:
+ [
+ {
+ 'i': '',
+ 's': '',
+ 'a': ['', '']
+ }
+ ]
+ Where `i` is id, `s` is name, and `a` is a list of alternative names.
+ So we can just read that once and build an in-memory sqlite db for offline full
+ text search.
+
+ Additionally, if we have a local key-value store, try that before actually
+ sending an http request to mangasee.
+ """
+ if not self.search_table:
+ titles = None
+ if self.keyval_store:
+ titles = json.loads(
+ self.keyval_store.get("mangasee_titles", "null", since="-1 day")
+ )
+ if not titles:
+ print("Fetching mangasee title list...", end="")
+ resp = requests.get("https://mangasee123.com/_search.php")
+ print(" done")
+ titles = resp.json()
+ self.keyval_store.set("mangasee_titles", resp.text)
+ self.search_table = SearchTable(titles)
+
+ return [
+ {
+ "id": row[0],
+ "name": row[1],
+ "site": "mangasee",
+ "thumbnail": f"https://cover.mangabeast01.com/cover/{row[0]}.jpg",
+ }
+ for row in self.search_table.search(query)
+ ]
+
+
+class SearchTable:
+ def __init__(self, titles: list):
+ self.db = apsw.Connection(":memory:")
+ cursor = self.db.cursor()
+ cursor.execute(
+ "CREATE VIRTUAL TABLE titles USING FTS5(id UNINDEXED, name, alt_names);"
+ )
+
+ rows = []
+ for t in titles:
+ id = t["i"]
+ name = t["s"]
+ alt_names = t["a"]
+ rows.append((id, name, " ".join(alt_names)))
+
+ cursor.executemany(
+ "INSERT INTO titles(id, name, alt_names) VALUES(?,?,?);", rows,
+ )
+
+ def search(self, query):
+ return self.db.cursor().execute(
+ "SELECT id, name FROM titles(?) ORDER BY rank;", (query,)
+ )
diff --git a/src/pytaku/database/migrations/latest_schema.sql b/src/pytaku/database/migrations/latest_schema.sql
index 2d88116..21255b4 100644
--- a/src/pytaku/database/migrations/latest_schema.sql
+++ b/src/pytaku/database/migrations/latest_schema.sql
@@ -55,3 +55,8 @@ CREATE TABLE read (
foreign key (chapter_id, site) references chapter (id, site),
unique(user_id, chapter_id, site)
);
+CREATE TABLE keyval_store (
+ key text primary key,
+ value text not null,
+ updated_at text default (datetime('now'))
+);
diff --git a/src/pytaku/database/migrations/m0002.sql b/src/pytaku/database/migrations/m0002.sql
new file mode 100644
index 0000000..35d184e
--- /dev/null
+++ b/src/pytaku/database/migrations/m0002.sql
@@ -0,0 +1,5 @@
+create table keyval_store (
+ key text primary key,
+ value text not null,
+ updated_at text default (datetime('now'))
+);
diff --git a/src/pytaku/persistence.py b/src/pytaku/persistence.py
index f5ed0b3..a97ae51 100644
--- a/src/pytaku/persistence.py
+++ b/src/pytaku/persistence.py
@@ -282,3 +282,32 @@ def find_outdated_titles(since="-6 hours"):
return run_sql(
"SELECT id, site FROM title WHERE updated_at <= datetime('now', ?);", (since,)
)
+
+
+class KeyvalStore:
+ @staticmethod
+ def get(key: str, default=None, since=None) -> str:
+ if since is None:
+ result = run_sql("SELECT value FROM keyval_store WHERE key=?;", (key,))
+ else:
+ result = run_sql(
+ """
+ SELECT value FROM keyval_store WHERE key=?
+ AND updated_at >= datetime('now', ?);
+ """,
+ (key, since),
+ )
+ return result[0] if result else default
+
+ @staticmethod
+ def set(key: str, value: str):
+ # let's not allow crap in by accident
+ assert isinstance(key, str)
+ assert isinstance(value, str)
+ run_sql(
+ """
+ INSERT INTO keyval_store (key, value) VALUES (?,?)
+ ON CONFLICT (key) DO UPDATE SET value=excluded.value, updated_at=datetime('now');
+ """,
+ (key, value),
+ )
diff --git a/src/pytaku/source_sites.py b/src/pytaku/source_sites.py
index a3cc247..d97f43b 100644
--- a/src/pytaku/source_sites.py
+++ b/src/pytaku/source_sites.py
@@ -1,6 +1,7 @@
from mangoapi import get_site_class
from .conf import config
+from .persistence import KeyvalStore
"""
This module adapts mangoapi's API to a more convenient one for app-wide use.
@@ -20,6 +21,8 @@ def _get_site(name):
if name == "mangadex":
site.username = config.MANGADEX_USERNAME
site.password = config.MANGADEX_PASSWORD
+ elif name == "mangasee":
+ site.keyval_store = KeyvalStore
return site
diff --git a/src/pytaku/templates/search.html b/src/pytaku/templates/search.html
index c38f548..1e9b23a 100644
--- a/src/pytaku/templates/search.html
+++ b/src/pytaku/templates/search.html
@@ -26,6 +26,7 @@
background-color: var(--bg-black);
color: white;
text-decoration: none;
+ max-width: 150px;
}
.result span {
padding: .5rem;
@@ -59,7 +60,7 @@ <h2 class="result-text">No results for "{{ query }}".</h2>
{% for title in titles %}
<a class="result" href="{{ url_for('title_view', title_id=title['id'], site=title['site']) }}"
title="{{ title['name'] }}">
- <img src="https://mangadex.org/images/manga/{{ title['id'] }}.large.jpg" alt="{{ title['name'] }}">
+ <img src="{{ title['thumbnail'] }}" alt="{{ title['name'] }}">
<span>{{ title['name'] | truncate(50) }}</span>
</a>
{% endfor %}