Repos / pytaku / 036177fcf6
commit 036177fcf6f6545aaa32d049fef1242bf308f198
Author: Bùi Thành Nhân <hi@imnhan.com>
Date: Fri Sep 3 12:54:43 2021 +0700
use mangadex's new API
diff --git a/src/mangoapi/mangadex.py b/src/mangoapi/mangadex.py
index 85e06e4..7340e19 100644
--- a/src/mangoapi/mangadex.py
+++ b/src/mangoapi/mangadex.py
@@ -1,13 +1,12 @@
import html
import re
-import time
import bbcode
-from mangoapi.base_site import Site, requires_login
+from mangoapi.base_site import Site
MANGAPLUS_GROUP_ID = 9097
-LONG_STRIP_TAG_ID = 36
+WEB_COMIC_TAG_ID = "e197df38-d0e7-43b5-9b09-2842d0c326dd"
_bbparser = bbcode.Parser()
_bbparser.add_simple_formatter(
@@ -17,129 +16,136 @@
class Mangadex(Site):
def get_title(self, title_id):
- url = f"https://mangadex.org/api/v2/manga/{title_id}?include=chapters"
- md_resp = self.http_get(url)
+ md_resp = self.http_get(
+ f"https://api.mangadex.org/manga/{title_id}",
+ params={"includes[]": "cover_art"},
+ )
+ assert md_resp.status_code == 200
md_json = md_resp.json()
- assert md_json["status"] == "OK"
- manga = md_json["data"]["manga"]
- chapters = md_json["data"]["chapters"]
- groups = md_json["data"]["groups"]
- groups_dict = {group["id"]: group["name"] for group in groups}
+ attrs = md_json["data"]["attributes"]
- cover = manga["mainCover"].split("/")[-1]
- ext_start_index = cover.find(".") + 1
- url_params_index = cover.rfind("?")
- ext_end_index = url_params_index if url_params_index != -1 else None
- cover_ext = cover[ext_start_index:ext_end_index]
+ is_web_comic = False
+ for tag in attrs["tags"]:
+ if tag["id"] == WEB_COMIC_TAG_ID:
+ is_web_comic = True
+ break
- current_timestamp = time.time()
+ cover = None
+ for rel in md_json["relationships"]:
+ if rel["type"] == "cover_art":
+ cover = rel["attributes"]["fileName"]
title = {
"id": title_id,
- "name": manga["title"],
+ "name": attrs["title"]["en"],
"site": "mangadex",
- "cover_ext": cover_ext,
- "alt_names": manga["altTitles"],
+ "cover_ext": cover,
+ "alt_names": [alt["en"] for alt in attrs["altTitles"]],
"descriptions": [
- _bbparser.format(html.unescape(manga["description"]).strip())
+ _bbparser.format(html.unescape(attrs["description"]["en"]).strip())
],
"descriptions_format": "html",
- "is_webtoon": LONG_STRIP_TAG_ID in manga["tags"],
- "chapters": [
+ "is_webtoon": is_web_comic,
+ "chapters": self.get_chapters_list(title_id),
+ }
+ return title
+
+ def get_chapters_list(self, title_id):
+ resp = self.http_get(
+ f"https://api.mangadex.org/manga/{title_id}/aggregate",
+ params={"translatedLanguage[]": "en"},
+ )
+ assert resp.status_code == 200
+ volumes: dict = resp.json()["volumes"]
+ chapters = []
+
+ # Counting on python's spanking new key-order-preserving dicts here.
+ # But WHY THE ACTUAL FUCK would you (mangadex) depend on JSON's key-value pairs ordering?
+ # A JSON object's keys is supposed to be unordered FFS.
+ # If it actually becomes a problem I'll do chapter sorting later. Soon. Ish.
+ for vol in volumes.values():
+ chapters += [
{
- "id": str(chap["id"]),
- "name": chap["title"],
- "volume": int(chap["volume"]) if chap["volume"] else None,
- "groups": [
- html.unescape(groups_dict[group_id])
- for group_id in chap["groups"]
- ],
+ "id": chap["id"],
+ "name": "",
+ "groups": [], # TODO
+ "volume": None if vol["volume"] == "none" else int(vol["volume"]),
**_parse_chapter_number(chap["chapter"]),
}
- for chap in chapters
- if chap["language"] == "gb"
- and MANGAPLUS_GROUP_ID not in chap["groups"]
- and chap["timestamp"] <= current_timestamp
- # ^ Chapter may be listed but with access delayed for a certain amount
- # of time set by uploader, in which case we just filter it out. God I
- # hate this generation of Patreon "scanlators".
- ],
- }
- return title
+ for chap in vol["chapters"].values() # again, fucking yikes
+ ]
+
+ return chapters
def get_chapter(self, title_id, chapter_id):
- md_resp = self.http_get(
- f"https://mangadex.org/api/v2/chapter/{chapter_id}?saver=0"
- )
+ md_resp = self.http_get(f"https://api.mangadex.org/chapter/{chapter_id}")
+ assert md_resp.status_code == 200
md_json = md_resp.json()
- assert md_json["status"] == "OK"
data = md_json["data"]
- # 2 cases:
- # - If 'serverFallback' is absent, it means 'server' points to MD's own server
- # e.g. s5.mangadex.org...
- # - Otherwise, 'server' points to a likely ephemeral MD@H node, while
- # 'serverFallback' now points to MD's own server.
- #
- # MD's own links apparently go dead sometimes, but MD@H links seem to expire
- # quickly all the time, so it's probably a good idea to store both anyway.
-
- server_fallback = data.get("serverFallback")
- if server_fallback:
- md_server = server_fallback
- mdah_server = data["server"]
- else:
- md_server = data["server"]
- mdah_server = None
+ title_id = None
+ for rel in data["relationships"]:
+ if rel["type"] == "manga":
+ title_id = rel["id"]
+ break
+
+ chapter_hash = data["attributes"]["hash"]
+ filenames = data["attributes"]["data"]
+ md_server = "https://uploads.mangadex.org"
+
+ mdah_server = self._get_md_at_home_server(chapter_id)
+ if mdah_server != md_server:
+ print(">> MDAH-server:", mdah_server)
chapter = {
"id": chapter_id,
- "title_id": str(data["mangaId"]),
+ "title_id": title_id,
"site": "mangadex",
- "name": data["title"],
- "pages": [f"{md_server}{data['hash']}/{page}" for page in data["pages"]],
+ "name": data["attributes"]["title"],
+ "pages": [
+ f"{md_server}/data/{chapter_hash}/{filename}" for filename in filenames
+ ],
"pages_alt": [
- f"{mdah_server}{data['hash']}/{page}" for page in data["pages"]
+ f"{mdah_server}/data/{chapter_hash}/{filename}"
+ for filename in filenames
]
- if mdah_server
+ if mdah_server is not None and mdah_server != md_server
else [],
- "groups": [html.unescape(group["name"]) for group in data["groups"]],
- **_parse_chapter_number(data["chapter"]),
+ "groups": [], # TODO
+ **_parse_chapter_number(data["attributes"]["chapter"]),
}
return chapter
- @requires_login
+ def _get_md_at_home_server(self, chapter_id):
+ resp = self.http_get(f"https://api.mangadex.org/at-home/server/{chapter_id}")
+ return resp.json()["baseUrl"] if resp.status_code == 200 else None
+
def search_title(self, query):
- md_resp = self.http_get(f"https://mangadex.org/quick_search/{query}")
-
- matches = TITLES_PATTERN.findall(md_resp.text)
- titles = [
- {
- "id": id,
- "name": name.strip(),
- "site": "mangadex",
- "thumbnail": f"https://mangadex.org/images/manga/{id}.large.jpg",
- }
- for id, name in matches
- ]
- return titles
+ params = {"limit": 100, "title": query, "includes[]": "cover_art"}
+ md_resp = self.http_get("https://api.mangadex.org/manga", params=params)
+ assert md_resp.status_code == 200
+ results = md_resp.json()["results"]
+
+ titles = []
+ for result in results:
+ data = result["data"]
+ cover = None
+ for rel in result["relationships"]:
+ if rel["type"] == "cover_art":
+ cover = rel["attributes"]["fileName"]
+ titles.append(
+ {
+ "id": data["id"],
+ "name": data["attributes"]["title"]["en"],
+ "site": "mangadex",
+ "thumbnail": f"https://uploads.mangadex.org/covers/{data['id']}/{cover}.256.jpg",
+ }
+ )
- def login(self, username, password):
- form_data = {
- "login_username": username,
- "login_password": password,
- "two_factor": "",
- "remember_me": "1",
- }
- self.http_post(
- "https://mangadex.org/ajax/actions.ajax.php?function=login",
- data=form_data,
- headers={"X-Requested-With": "XMLHttpRequest"},
- )
- self.is_logged_in = True
+ return titles
def title_cover(self, title_id, cover_ext):
- return f"https://mangadex.org/images/manga/{title_id}.{cover_ext}"
+ return f"https://uploads.mangadex.org/covers/{title_id}/{cover_ext}.256.jpg"
def title_thumbnail(self, title_id):
return f"https://mangadex.org/images/manga/{title_id}.large.jpg"
@@ -156,7 +162,7 @@ def title_source_url(self, title_id):
def _parse_chapter_number(string):
- if string == "":
+ if string == "none":
# most likely a oneshot
return {"number": ""}
nums = string.split(".")
diff --git a/src/pytaku/main.py b/src/pytaku/main.py
index 88e51d1..8096c12 100644
--- a/src/pytaku/main.py
+++ b/src/pytaku/main.py
@@ -74,7 +74,7 @@ def _decode_proxy_url(b64_url):
def _is_manga_img_url(
url,
- cover_pattern=re.compile(r"^https://([\w-]+\.)?mangadex\.org/images"),
+ cover_pattern=re.compile(r"^https://([\w-]+\.)?mangadex\.org/(images|covers)"),
):
"""
Check if either a cover or page img url.
diff --git a/src/pytaku/static/spa.css b/src/pytaku/static/spa.css
index 43efd93..ea8d676 100644
--- a/src/pytaku/static/spa.css
+++ b/src/pytaku/static/spa.css
@@ -335,7 +335,7 @@ .search--result-text {
/* Title route */
.title--cover {
- width: 400px;
+ max-width: 400px;
border: 1px solid black;
}
diff --git a/tests/mangoapi/test_mangadex.py b/tests/mangoapi/test_mangadex.py
index 74f3c30..886a20e 100644
--- a/tests/mangoapi/test_mangadex.py
+++ b/tests/mangoapi/test_mangadex.py
@@ -1,127 +1,108 @@
from mangoapi.mangadex import Mangadex
-from pytaku.conf import config
def test_get_title():
- title = Mangadex().get_title("2597")
+ title = Mangadex().get_title("8af3ad21-3e7e-4fb5-b344-d0044ec154fc")
+ chapters = title.pop("chapters")
assert title == {
- "id": "2597",
- "name": "Sayonara Football",
+ "id": "8af3ad21-3e7e-4fb5-b344-d0044ec154fc",
+ "name": "Beelzebub",
"site": "mangadex",
- "cover_ext": "jpg",
- "alt_names": ["Adiós al fútbol", "さよならフットボール", "再见足球"],
- "is_webtoon": False,
- "descriptions": [
- "Nozomi wants to enter the newcomer's competition. But the coach is against it, because their club is a boy's football club and she's... a she. Will she be able to enter the match she wants to play in?"
+ "cover_ext": "bab3ccbf-7479-4117-ad92-4dedced54ceb.jpg",
+ "alt_names": [
+ "Beruzebabu",
+ "Вельзевул",
+ "เด็กพันธุ์นรกสั่งลุย",
+ "べるぜバブ",
+ "恶魔奶爸",
+ "惡魔奶爸",
+ "바알세불",
],
- "chapters": [
- {
- "id": "84598",
- "name": "Epilogue",
- "volume": 2,
- "groups": ["Shoujo Crusade"],
- "number": "8",
- "num_major": 8,
- },
- {
- "id": "84596",
- "name": "Football Under the Blue Sky",
- "volume": 2,
- "groups": ["Shoujo Crusade"],
- "number": "7",
- "num_major": 7,
- },
- {
- "id": "84594",
- "name": "Everyone in a Crisis",
- "volume": 2,
- "groups": ["Shoujo Crusade"],
- "number": "6",
- "num_major": 6,
- },
- {
- "id": "84592",
- "name": "Clash and Decide",
- "volume": 2,
- "groups": ["Shoujo Crusade"],
- "number": "5",
- "num_major": 5,
- },
- {
- "id": "84590",
- "name": "And There's the Whistle",
- "volume": 1,
- "groups": ["Shoujo Crusade"],
- "number": "4",
- "num_major": 4,
- },
- {
- "id": "84589",
- "name": "A Plan to Become a Regular",
- "volume": 1,
- "groups": ["Shoujo Crusade"],
- "number": "3",
- "num_major": 3,
- },
- {
- "id": "84587",
- "name": "Her Determination at That Time",
- "volume": 1,
- "groups": ["Shoujo Crusade"],
- "number": "2",
- "num_major": 2,
- },
- {
- "id": "84585",
- "name": "The Entry of an Unmanageable Woman",
- "volume": 1,
- "groups": ["Shoujo Crusade"],
- "number": "1",
- "num_major": 1,
- },
+ "descriptions": [
+ "Ishiyama High is a school populated entirely by "
+ "delinquents, where nonstop violence and lawlessness are the "
+ "norm. However, there is one universally acknowledged "
+ "rule—don't cross first year student Tatsumi Oga, "
+ "Ishiyama's most vicious fighter.<br /><br />One day, "
+ "Oga is by a riverbed when he encounters a man floating down "
+ "the river. After being retrieved by Oga, the man splits "
+ "down the middle to reveal a baby, which crawls onto "
+ "Oga's back and immediately forms an attachment to him. "
+ "Though he doesn't know it yet, this baby is named "
+ "Kaiser de Emperana Beelzebub IV, or "Baby Beel" "
+ "for short—the son of the Demon Lord!<br /><br />As if "
+ "finding the future Lord of the Underworld isn't enough, "
+ "Oga is also confronted by Hildegard, Beel's demon maid "
+ "who insists he take responsibility as Beel's guardian. "
+ "Together they attempt to raise Baby Beel—although "
+ "surrounded by juvenile delinquents and demonic powers, the "
+ "two of them may be in for more of a challenge than they can "
+ "imagine.<br /><hr />"
],
+ "descriptions_format": "html",
+ "is_webtoon": False,
+ }
+
+ assert len(chapters) == 252
+ assert chapters[1] == {
+ "id": "dfd3fd5a-a20f-460f-b726-e9cb168bfe3d",
+ "name": "",
+ "volume": 28,
+ "groups": [],
+ "number": "245",
+ "num_major": 245,
}
def test_get_title_webtoon():
- title = Mangadex().get_title("1")
+ title = Mangadex().get_title("6e3553b9-ddb5-4d37-b7a3-99998044774e")
assert title["is_webtoon"] is True
-def test_get_title_no_url_params():
- title = Mangadex().get_title("23801")
- assert title["cover_ext"] == "jpg"
-
-
def test_get_chapter():
- chap = Mangadex().get_chapter("doesn't matter", "696882")
+ chap = Mangadex().get_chapter("_", "7f49d795-d525-4b65-9fd8-ddb58425683e")
pages = chap.pop("pages")
pages_alt = chap.pop("pages_alt")
assert chap == {
- "id": "696882",
- "title_id": "12088",
+ "id": "7f49d795-d525-4b65-9fd8-ddb58425683e",
+ "title_id": "6e3553b9-ddb5-4d37-b7a3-99998044774e",
"site": "mangadex",
- "name": "Extras",
- "groups": ["Träumerei Scans", "GlassChair"],
- "number": "81.5",
- "num_major": 81,
+ "name": "Volume 15 Extras (Epilogue & Prologue)",
+ "groups": [],
+ "number": "222.5",
+ "num_major": 222,
"num_minor": 5,
}
- assert len(pages) == 16
- assert len(pages_alt) == 16
+ assert len(pages) == 53
+ assert len(pages_alt) == 0
def test_search():
md = Mangadex()
- md.username = config.MANGADEX_USERNAME
- md.password = config.MANGADEX_PASSWORD
-
- results = md.search_title("sayonara football")
+ results = md.search_title("beelzebub")
assert results == [
{
- "id": "2597",
- "name": "Sayonara Football",
+ "id": "8af3ad21-3e7e-4fb5-b344-d0044ec154fc",
+ "name": "Beelzebub",
+ "site": "mangadex",
+ "thumbnail": "https://uploads.mangadex.org/covers/8af3ad21-3e7e-4fb5-b344-d0044ec154fc/bab3ccbf-7479-4117-ad92-4dedced54ceb.jpg.256.jpg",
+ },
+ {
+ "id": "b4320039-9b91-44a7-a60d-c7ba8c0684e7",
+ "name": "Beelzebub-jou no Oki ni Mesu mama.",
+ "site": "mangadex",
+ "thumbnail": "https://uploads.mangadex.org/covers/b4320039-9b91-44a7-a60d-c7ba8c0684e7/ed566a45-c9f2-4f7e-84f8-ae7fc328ab15.jpg.256.jpg",
+ },
+ {
+ "id": "a453af66-0dac-4966-b246-b37c96b27245",
+ "name": "Makai kara Kita Maid-san",
+ "site": "mangadex",
+ "thumbnail": "https://uploads.mangadex.org/covers/a453af66-0dac-4966-b246-b37c96b27245/7e50b22d-b027-4ee1-bd15-94f05fa6cecb.jpg.256.jpg",
+ },
+ {
+ "id": "72378871-9afc-47bd-902f-0d8116adb390",
+ "name": "Beelzebub - Digital Colored Comics",
"site": "mangadex",
- "thumbnail": "https://mangadex.org/images/manga/2597.large.jpg",
- }
+ "thumbnail": "https://uploads.mangadex.org/covers/72378871-9afc-47bd-902f-0d8116adb390/c8b9b385-b7b9-4101-bf71-4f0d66fc35ff.jpg.256.jpg",
+ },
]