Repos / pytaku / 0e1fda117e
commit 0e1fda117e4a5190f3fa916f2de66fe6d180a556
Author: Bùi Thành Nhân <hi@imnhan.com>
Date: Wed Aug 12 23:46:40 2020 +0700
WIP import from tachiyomi
todo: ensure latest chapters, mark them as read
diff --git a/src/pytaku/database/common.py b/src/pytaku/database/common.py
index f9a36d8..dcb443f 100644
--- a/src/pytaku/database/common.py
+++ b/src/pytaku/database/common.py
@@ -34,3 +34,7 @@ def run_sql(*args, **kwargs):
def run_sql_on_demand(*args, **kwargs):
return get_conn().cursor().execute(*args, **kwargs)
+
+
+def run_sql_many(*args, **kwargs):
+ return get_conn().cursor().executemany(*args, **kwargs)
diff --git a/src/pytaku/main.py b/src/pytaku/main.py
index 5c49f17..e5339d8 100644
--- a/src/pytaku/main.py
+++ b/src/pytaku/main.py
@@ -1,10 +1,14 @@
import base64
+import json
import re
+from concurrent.futures import ThreadPoolExecutor, as_completed
from datetime import timedelta
+from typing import List, Tuple
import requests
from flask import (
Flask,
+ flash,
make_response,
redirect,
render_template,
@@ -19,6 +23,7 @@
follow,
get_followed_titles,
get_prev_next_chapters,
+ import_follows,
load_chapter,
load_title,
register_user,
@@ -40,7 +45,9 @@
app = Flask(__name__)
app.config.update(
- SECRET_KEY=config.FLASK_SECRET_KEY, PERMANENT_SESSION_LIFETIME=timedelta(days=365),
+ SECRET_KEY=config.FLASK_SECRET_KEY,
+ PERMANENT_SESSION_LIFETIME=timedelta(days=365),
+ MAX_CONTENT_LENGTH=10 * 1024 * 1024, # max 10MiB payload
)
@@ -263,3 +270,95 @@ def _is_manga_img_url(
),
):
return pattern.match(url)
+
+
+@app.route("/import", methods=["GET", "POST"])
+@require_login
+def import_view():
+ if request.method == "POST":
+
+ # check if the post request has the file part
+ if "tachiyomi" not in request.files:
+ flash("No file part")
+ return redirect(request.url)
+ file = request.files["tachiyomi"]
+
+ # if user does not select file, browser also
+ # submit an empty part without filename
+ if file.filename == "":
+ flash("No selected file")
+ return redirect(request.url)
+
+ if file:
+ text = file.read()
+ site_title_pairs = read_tachiyomi_follows(text)
+ if site_title_pairs is None:
+ flash("Malformed input file.")
+ return redirect(request.url)
+
+ # First fetch & save titles if they're not already in db
+ ensure_titles(site_title_pairs)
+
+ # Then follow them all
+ for site, title_id in site_title_pairs:
+ follow(session["user"]["id"], site, title_id)
+
+ # Mark them all as "read" too.
+ print("TODO")
+
+ flash(f"Added {len(site_title_pairs)} follows.")
+
+ return render_template("import.html")
+
+
+def read_tachiyomi_follows(text: str) -> List[Tuple[str, str]]:
+ try:
+ data = json.loads(text)
+ mangadex_id = None
+ mangasee_id = None
+ for extension in data["extensions"]:
+ id, name = extension.split(":")
+ if name == "MangaDex":
+ mangadex_id = int(id)
+ elif name == "Mangasee":
+ mangasee_id = int(id)
+ assert mangadex_id and mangasee_id
+
+ results = []
+ for manga in data["mangas"]:
+ path = manga["manga"][0]
+ site_id = manga["manga"][2]
+ if site_id == mangadex_id:
+ site = "mangadex"
+ title_id = path[len("/manga/") : -1]
+ elif site_id == mangasee_id:
+ site = "mangasee"
+ title_id = path[len("/manga/") :]
+ else:
+ continue
+ results.append((site, title_id))
+
+ return results
+
+ except Exception: # yikes
+ import traceback
+
+ traceback.print_exc()
+ return None
+
+
+def ensure_titles(site_title_pairs: List[Tuple[str, str]]):
+ new_titles = [
+ (site, title_id)
+ for site, title_id in site_title_pairs
+ if load_title(site, title_id) is None # again, n+1 queries are fine in sqlite
+ ]
+ print(f"Fetching {len(new_titles)} new titles out of {len(site_title_pairs)}.")
+ with ThreadPoolExecutor(max_workers=3) as executor:
+ futures = [
+ executor.submit(get_title, site, title_id) for site, title_id in new_titles
+ ]
+ for future in as_completed(futures):
+ title = future.result()
+ save_title(title)
+ print(f"Saved {title['site']}: {title['name']}")
diff --git a/src/pytaku/persistence.py b/src/pytaku/persistence.py
index 5158069..27e9d36 100644
--- a/src/pytaku/persistence.py
+++ b/src/pytaku/persistence.py
@@ -1,9 +1,10 @@
import json
+from typing import List, Tuple
import apsw
import argon2
-from .database.common import run_sql, run_sql_on_demand
+from .database.common import run_sql, run_sql_many, run_sql_on_demand
def save_title(title):
@@ -214,7 +215,10 @@ def verify_username_password(username, password):
def follow(user_id, site, title_id):
run_sql(
- "INSERT INTO follow (user_id, site, title_id) VALUES (?, ?, ?);",
+ """
+ INSERT INTO follow (user_id, site, title_id) VALUES (?, ?, ?)
+ ON CONFLICT DO NOTHING;
+ """,
(user_id, site, title_id),
)
@@ -321,3 +325,12 @@ def set(key: str, value: str):
""",
(key, value),
)
+
+
+def import_follows(user_id: int, site_title_pairs: List[Tuple[str, str]]):
+ run_sql_many(
+ """
+ INSERT INTO follow (user_id, site, title_id) VALUES (?, ?, ?);
+ """,
+ [(user_id, site, title_id) for site, title_id in site_title_pairs],
+ )
diff --git a/src/pytaku/templates/import.html b/src/pytaku/templates/import.html
new file mode 100644
index 0000000..8734d98
--- /dev/null
+++ b/src/pytaku/templates/import.html
@@ -0,0 +1,36 @@
+{% extends 'base.html' %}
+
+{% block title %}
+Import followed titles
+{% endblock %}
+
+{% block head %}
+<style>
+input[type=file] {
+ border: 2px solid black;
+ border-radius: var(--border-radius);
+ margin: 1rem 0;
+}
+
+.message {
+ color: red;
+}
+</style>
+{% endblock %}
+
+{% block content %}
+<h1>Importing from Tachiyomi</h1>
+
+<form method="POST" enctype="multipart/form-data" class="upload-form">
+ <p>Go to <b>Settings > Backup > Create backup</b>, then upload the generated json file here:</p>
+ <input type="file" name="tachiyomi"><br>
+ {{ ibutton(text='Submit') }}
+</form>
+
+{% with messages = get_flashed_messages() %}
+ {% for message in messages %}
+ <p class="message">{{ message }}</p>
+ {% endfor %}
+{% endwith %}
+
+{% endblock %}
diff --git a/tests/pytaku/test_import_follows.py b/tests/pytaku/test_import_follows.py
new file mode 100644
index 0000000..309fd95
--- /dev/null
+++ b/tests/pytaku/test_import_follows.py
@@ -0,0 +1,41 @@
+from pytaku.main import read_tachiyomi_follows
+
+
+def test_read_tachiyomi_follows():
+ data = """
+{
+ "version": 2,
+ "mangas": [
+ { "manga": ["/manga/Ajin", "Ajin", 9, 0, 0] },
+ {
+ "manga": [
+ "/manga/28664/",
+ "Ano Hito no I ni wa Boku ga Tarinai",
+ 2499283573021220255,
+ 0,
+ 0
+ ]
+ },
+ { "manga": ["/manga/Chainsaw-Man", "Chainsaw Man", 9, 0, 0] },
+ { "manga": ["/manga/Chi-No-Wadachi", "Chi no Wadachi", 9, 0, 0] },
+ { "manga": ["/manga/13318/", "Dagashi Kashi", 2499283573021220255, 0, 0] },
+ { "manga": ["/manga/31688/", "Dai Dark", 2499283573021220255, 0, 0] }
+ ],
+ "categories": [],
+ "extensions": [
+ "1998944621602463790:MANGA Plus by SHUEISHA",
+ "2499283573021220255:MangaDex",
+ "4637971935551651734:Guya",
+ "9064882169246918586:Jaimini's Box",
+ "9:Mangasee"
+ ]
+}
+ """
+ assert read_tachiyomi_follows(data) == [
+ ("mangasee", "Ajin"),
+ ("mangadex", "28664"),
+ ("mangasee", "Chainsaw-Man"),
+ ("mangasee", "Chi-No-Wadachi"),
+ ("mangadex", "13318"),
+ ("mangadex", "31688"),
+ ]