Repos / pytaku / e0389c9fe5
commit e0389c9fe51d7d031040fbaa70ea330587aea75d
Author: Bùi Thành Nhân <hi@imnhan.com>
Date: Fri Oct 30 14:09:58 2020 +0700
save both md & md@h links; fallback logic on FE
Preload logic is still wonky though: if fallback happens while
preloading, the alt urls (md@h ones) are cached but when user actually
navigates to next chapter, browser will try the default urls first, fail
again, then try alt urls again whose cache has expired (at least on
firefox when I tested it).
diff --git a/README.md b/README.md
index 28a5153..fb4b0e2 100644
--- a/README.md
+++ b/README.md
@@ -44,11 +44,11 @@ # run 2 processes:
## Frontend ##
-sudo pacman -S entr # to watch source files
+doas pacman -S entr # to watch source files
npm install -g --prefix ~/.node_modules esbuild # to bundle js
# Listen for changes in js-src dir, automatically build minified bundle:
-find src/pytaku/js-src -name '*.js' | entr -r \
+find src/pytaku/js-src -name '*.js' | entr -rc \
esbuild src/pytaku/js-src/main.js \
--bundle --sourcemap --minify \
--outfile=src/pytaku/static/js/main.min.js
@@ -63,6 +63,11 @@ ## Code QA tools
- Python: black, isort, flake8 without mccabe
- JavaScript: jshint, prettier
+```sh
+doas pacman python-black python-isort flake8 prettier
+npm install -g --prefix ~/.node_modules jshint
+```
+
# Production
```sh
diff --git a/pyproject.toml b/pyproject.toml
index 763ce36..6880fc8 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -1,6 +1,6 @@
[tool.poetry]
name = "pytaku"
-version = "0.3.25"
+version = "0.3.26"
description = "Self-hostable web-based manga reader"
authors = ["Bùi Thành Nhân <hi@imnhan.com>"]
license = "AGPL-3.0-only"
diff --git a/src/mangoapi/mangadex.py b/src/mangoapi/mangadex.py
index 044ed4a..da2f705 100644
--- a/src/mangoapi/mangadex.py
+++ b/src/mangoapi/mangadex.py
@@ -52,24 +52,37 @@ def get_chapter(self, title_id, chapter_id):
md_json = md_resp.json()
assert md_json["status"] == "OK"
- # 'server' value points to a likely temporary MangaDex@Home instance, while
- # 'server_fallback' would be MD's own server e.g. s5.mangadex.org...
- # The latter may be down (like, literally at the time of writing), so for now
- # let's prioritize the MD@Home server.
- # I don't know how stable MD@Home links are, but it probably won't matter,
- # since `persistence.load_chapter()` will re-fetch if existing db record is more
- # than 1 day old anyway.
- # TODO: A more robust solution is to save both links to db, but I'm not in the
- # mood for it atm.
- server = md_json["server"] or md_json.get("server_fallback")
- img_path = f"{server}{md_json['hash']}"
+ # 2 cases:
+ # - If 'server_fallback' is absent, it means 'server' points to MD's own server
+ # e.g. s5.mangadex.org...
+ # - Otherwise, 'server' points to a likely ephemeral MD@H node, while
+ # 'server_fallback' now points to MD's own server.
+ #
+ # MD's own links apparently go dead sometimes, but MD@H links seem to expire
+ # quickly all the time, so it's probably a good idea to store both anyway.
+
+ server_fallback = md_json.get("server_fallback")
+ if server_fallback:
+ md_server = server_fallback
+ mdah_server = md_json["server"]
+ else:
+ md_server = md_json["server"]
+ mdah_server = None
chapter = {
"id": chapter_id,
"title_id": str(md_json["manga_id"]),
"site": "mangadex",
"name": md_json["title"],
- "pages": [f"{img_path}/{page}" for page in md_json["page_array"]],
+ "pages": [
+ f"{md_server}{md_json['hash']}/{page}" for page in md_json["page_array"]
+ ],
+ "pages_alt": [
+ f"{mdah_server}{md_json['hash']}/{page}"
+ for page in md_json["page_array"]
+ ]
+ if mdah_server
+ else [],
"groups": _extract_groups(md_json),
"is_webtoon": md_json["long_strip"] == 1,
**_parse_chapter_number(md_json["chapter"]),
diff --git a/src/mangoapi/mangasee.py b/src/mangoapi/mangasee.py
index 03d852d..a34fe54 100644
--- a/src/mangoapi/mangasee.py
+++ b/src/mangoapi/mangasee.py
@@ -74,6 +74,7 @@ def get_chapter(self, title_id, chapter_id):
_generate_img_src(img_server, title_id, chapter_data["Chapter"], p)
for p in range(1, num_pages + 1)
],
+ "pages_alt": [],
"groups": [],
"is_webtoon": False,
**numbers,
diff --git a/src/pytaku/database/migrations/latest_schema.sql b/src/pytaku/database/migrations/latest_schema.sql
index 894207d..307e5a7 100644
--- a/src/pytaku/database/migrations/latest_schema.sql
+++ b/src/pytaku/database/migrations/latest_schema.sql
@@ -63,7 +63,7 @@ CREATE TABLE IF NOT EXISTS "chapter"(
pages text,
groups text,
updated_at text default (datetime('now')),
- is_webtoon boolean,
+ is_webtoon boolean, pages_alt text not null default '[]',
foreign key (title_id, site) references title (id, site),
unique(site, title_id, id)
diff --git a/src/pytaku/database/migrations/m0007.sql b/src/pytaku/database/migrations/m0007.sql
new file mode 100644
index 0000000..a90530f
--- /dev/null
+++ b/src/pytaku/database/migrations/m0007.sql
@@ -0,0 +1,6 @@
+-- Add alternative page urls as backup because mangadex is flaky.
+begin transaction;
+
+alter table chapter add column pages_alt text not null default '[]';
+
+commit;
diff --git a/src/pytaku/js-src/routes/chapter.js b/src/pytaku/js-src/routes/chapter.js
index f2489d8..ea4d77a 100644
--- a/src/pytaku/js-src/routes/chapter.js
+++ b/src/pytaku/js-src/routes/chapter.js
@@ -35,6 +35,29 @@ const ImgStatus = {
FAILED: "failed",
};
+function FallbackableImg(initialVNode) {
+ let currentSrc;
+ return {
+ oninit: (vnode) => {
+ currentSrc = vnode.attrs.src;
+ },
+ view: (vnode) => {
+ return m("img", {
+ src: currentSrc,
+ style: vnode.attrs.style,
+ onload: vnode.attrs.onload,
+ onerror: (ev) => {
+ if (currentSrc === vnode.attrs.src && vnode.attrs.altsrc !== null) {
+ currentSrc = vnode.attrs.altsrc;
+ } else {
+ vnode.attrs.onerror(ev);
+ }
+ },
+ });
+ },
+ };
+}
+
function Chapter(initialVNode) {
let isLoading = false;
let chapter = {};
@@ -48,9 +71,11 @@ function Chapter(initialVNode) {
function loadNextPage() {
if (pendingPages.length > 0) {
+ let [src, altsrc] = pendingPages.splice(0, 1)[0];
loadedPages.push({
status: ImgStatus.LOADING,
- src: pendingPages.splice(0, 1)[0],
+ src,
+ altsrc,
});
} else if (chapter.next_chapter && nextChapterPromise === null) {
/* Once all pages of this chapter have been loaded,
@@ -62,7 +87,15 @@ function Chapter(initialVNode) {
chapterId: chapter.next_chapter.id,
}).then((nextChapter) => {
console.log("Preloading next chapter:", fullChapterName(nextChapter));
- nextChapterPendingPages = nextChapter.pages.slice();
+ if (nextChapter.pages_alt.length > 0) {
+ nextChapterPendingPages = nextChapter.pages.map((page, i) => {
+ return [page, nextChapter.pages_alt[i]];
+ });
+ } else {
+ nextChapterPendingPages = nextChapter.pages.map((page) => {
+ return [page, null];
+ });
+ }
// Apparently preloading one at a time was too slow so let's go with 2.
preloadNextChapterPage();
preloadNextChapterPage();
@@ -73,7 +106,8 @@ function Chapter(initialVNode) {
function preloadNextChapterPage() {
if (nextChapterPendingPages !== null) {
if (nextChapterPendingPages.length > 0) {
- nextChapterLoadedPages.push(nextChapterPendingPages.splice(0, 1)[0]);
+ const [src, altsrc] = nextChapterPendingPages.splice(0, 1)[0];
+ nextChapterLoadedPages.push({ src, altsrc });
}
}
}
@@ -96,8 +130,16 @@ function Chapter(initialVNode) {
chapter = resp;
document.title = fullChapterName(chapter);
- // Clone array here to avoid mutating the model
- pendingPages = chapter.pages.slice();
+ // "zip" pages & pages_alt into pendingPages
+ if (chapter.pages_alt.length > 0) {
+ pendingPages = chapter.pages.map((page, i) => {
+ return [page, chapter.pages_alt[i]];
+ });
+ } else {
+ pendingPages = chapter.pages.map((page) => {
+ return [page, null];
+ });
+ }
// start loading pages, 3 at a time:
loadNextPage();
@@ -184,8 +226,9 @@ function Chapter(initialVNode) {
[
loadedPages.map((page, pageIndex) =>
m("div", { key: page.src }, [
- m("img", {
+ m(FallbackableImg, {
src: page.src,
+ altsrc: page.altsrc,
style: {
display:
page.status === ImgStatus.SUCCEEDED ? "block" : "none",
@@ -211,16 +254,17 @@ function Chapter(initialVNode) {
: null,
])
),
- pendingPages.map((page) => m(PendingPlaceholder)),
+ pendingPages.map(() => m(PendingPlaceholder)),
]
),
buttons,
nextChapterLoadedPages.map((page) =>
- m("img.chapter--preloader", {
+ m(FallbackableImg, {
style: { display: "none" },
onload: preloadNextChapterPage,
onerror: preloadNextChapterPage,
- src: page,
+ src: page.src,
+ altsrc: page.altsrc,
})
),
]);
diff --git a/src/pytaku/main.py b/src/pytaku/main.py
index 68c7e83..0ec696a 100644
--- a/src/pytaku/main.py
+++ b/src/pytaku/main.py
@@ -283,6 +283,7 @@ def api_chapter(site, title_id, chapter_id):
if site in ("mangadex", "mangasee"):
chapter["pages"] = [proxied(p) for p in chapter["pages"]]
+ chapter["pages_alt"] = [proxied(p) for p in chapter["pages_alt"]]
# YIIIIKES
title = load_title(site, title_id)
diff --git a/src/pytaku/persistence.py b/src/pytaku/persistence.py
index 4152a89..bb2a9af 100644
--- a/src/pytaku/persistence.py
+++ b/src/pytaku/persistence.py
@@ -105,6 +105,7 @@ def save_chapter(chapter):
num_minor,
name,
pages,
+ pages_alt,
groups,
is_webtoon
) VALUES (
@@ -115,6 +116,7 @@ def save_chapter(chapter):
:num_minor,
:name,
:pages,
+ :pages_alt,
:groups,
:is_webtoon
) ON CONFLICT (id, title_id, site) DO UPDATE SET
@@ -122,6 +124,7 @@ def save_chapter(chapter):
num_minor=excluded.num_minor,
name=excluded.name,
pages=excluded.pages,
+ pages_alt=excluded.pages_alt,
groups=excluded.groups,
is_webtoon=excluded.is_webtoon,
updated_at=datetime('now')
@@ -135,6 +138,7 @@ def save_chapter(chapter):
"num_minor": chapter.get("num_minor"),
"name": chapter["name"],
"pages": json.dumps(chapter["pages"]),
+ "pages_alt": json.dumps(chapter["pages_alt"]),
"groups": json.dumps(chapter["groups"]),
"is_webtoon": chapter["is_webtoon"],
},
@@ -145,7 +149,7 @@ def load_chapter(site, title_id, chapter_id, ignore_old=True):
updated_at = "datetime('now', '-1 days')" if ignore_old else "'1980-01-01'"
result = run_sql(
f"""
- SELECT id, title_id, site, num_major, num_minor, name, pages, groups, is_webtoon
+ SELECT id, title_id, site, num_major, num_minor, name, pages, pages_alt, groups, is_webtoon
FROM chapter
WHERE site=? AND title_id=? AND id=? AND updated_at > {updated_at};
""",
@@ -158,6 +162,7 @@ def load_chapter(site, title_id, chapter_id, ignore_old=True):
else:
chapter = result[0]
chapter["pages"] = json.loads(chapter["pages"])
+ chapter["pages_alt"] = json.loads(chapter["pages_alt"])
chapter["groups"] = json.loads(chapter["groups"])
return chapter
diff --git a/tests/mangoapi/test_mangadex.py b/tests/mangoapi/test_mangadex.py
index 7bc8883..1dec5a5 100644
--- a/tests/mangoapi/test_mangadex.py
+++ b/tests/mangoapi/test_mangadex.py
@@ -85,6 +85,7 @@ def test_get_title():
def test_get_chapter():
chap = Mangadex().get_chapter("doesn't matter", "696882")
pages = chap.pop("pages")
+ pages_alt = chap.pop("pages_alt")
assert chap == {
"id": "696882",
"title_id": "12088",
@@ -97,6 +98,7 @@ def test_get_chapter():
"num_minor": 5,
}
assert len(pages) == 16
+ assert len(pages_alt) == 16
def test_search():
diff --git a/tests/mangoapi/test_mangasee.py b/tests/mangoapi/test_mangasee.py
index e4cb3b9..fb943ec 100644
--- a/tests/mangoapi/test_mangasee.py
+++ b/tests/mangoapi/test_mangasee.py
@@ -29,6 +29,7 @@ def test_get_title():
def test_get_chapter():
chapter = Mangasee().get_chapter("Yu-Yu-Hakusho", "63.5")
pages = chapter.pop("pages")
+ pages_alt = chapter.pop("pages_alt")
assert chapter == {
"groups": [],
"id": "63.5",
@@ -42,6 +43,7 @@ def test_get_chapter():
}
assert pages[0] == "https://s1.mangabeast01.com/manga/Yu-Yu-Hakusho/0063.5-001.png"
assert pages[-1] == "https://s1.mangabeast01.com/manga/Yu-Yu-Hakusho/0063.5-031.png"
+ assert pages_alt == []
def test_search_title():