Repos / pytaku / 3cd65b5032
commit 3cd65b503233e118b6cdc21588666df1af3a2971
Author: Bùi Thành Nhân <hi@imnhan.com>
Date: Tue Jun 15 21:18:06 2021 +0700
bypass mangasee's cloudflare protection
diff --git a/README.md b/README.md
index fb4b0e2..69c7bde 100644
--- a/README.md
+++ b/README.md
@@ -24,6 +24,7 @@ # Development
```sh
## Backend ##
+doas pacman -S nodejs # used by cloudscraper to bypass Cloudflare
poetry install
pip install --upgrade pip
@@ -71,6 +72,7 @@ ## Code QA tools
# Production
```sh
+sudo apt install nodejs # used by cloudscraper to bypass Cloudflare
pip install --user --upgrade pip
pip install --user pytaku
pip install https://github.com/rogerbinns/apsw/releases/download/3.32.2-r1/apsw-3.32.2-r1.zip \
diff --git a/poetry.lock b/poetry.lock
index 92ef442..361dd90 100644
--- a/poetry.lock
+++ b/poetry.lock
@@ -5,7 +5,6 @@ description = "Disable App Nap on OS X 10.9"
category = "dev"
optional = false
python-versions = "*"
-marker = "python_version >= \"3.4\" and sys_platform == \"darwin\""
[[package]]
name = "argon2-cffi"
@@ -15,15 +14,15 @@ category = "main"
optional = false
python-versions = "*"
-[package.extras]
-dev = ["coverage (>=5.0.2)", "hypothesis", "pytest", "sphinx", "wheel", "pre-commit"]
-docs = ["sphinx"]
-tests = ["coverage (>=5.0.2)", "hypothesis", "pytest"]
-
[package.dependencies]
cffi = ">=1.0.0"
six = "*"
+[package.extras]
+dev = ["coverage[toml] (>=5.0.2)", "hypothesis", "pytest", "sphinx", "wheel", "pre-commit"]
+docs = ["sphinx"]
+tests = ["coverage[toml] (>=5.0.2)", "hypothesis", "pytest"]
+
[[package]]
name = "atomicwrites"
version = "1.4.0"
@@ -31,7 +30,6 @@ description = "Atomic file writes."
category = "dev"
optional = false
python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*"
-marker = "sys_platform == \"win32\""
[[package]]
name = "attrs"
@@ -54,7 +52,6 @@ description = "Specifications for callback functions passed in to an API"
category = "dev"
optional = false
python-versions = "*"
-marker = "python_version >= \"3.4\""
[[package]]
name = "bbcode"
@@ -99,6 +96,19 @@ category = "main"
optional = false
python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*"
+[[package]]
+name = "cloudscraper"
+version = "1.2.58"
+description = "A Python module to bypass Cloudflare's anti-bot page."
+category = "main"
+optional = false
+python-versions = "*"
+
+[package.dependencies]
+pyparsing = ">=2.4.7"
+requests = ">=2.9.2"
+requests-toolbelt = ">=0.9.1"
+
[[package]]
name = "colorama"
version = "0.4.3"
@@ -106,7 +116,6 @@ description = "Cross-platform colored terminal text."
category = "dev"
optional = false
python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*"
-marker = "python_version >= \"3.4\" and sys_platform == \"win32\" or sys_platform == \"win32\""
[[package]]
name = "decorator"
@@ -115,7 +124,6 @@ description = "Decorators for Humans"
category = "dev"
optional = false
python-versions = ">=2.6, !=3.0.*, !=3.1.*"
-marker = "python_version >= \"3.4\""
[[package]]
name = "flask"
@@ -125,17 +133,17 @@ category = "main"
optional = false
python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*"
-[package.extras]
-dev = ["pytest", "coverage", "tox", "sphinx", "pallets-sphinx-themes", "sphinxcontrib-log-cabinet", "sphinx-issues"]
-docs = ["sphinx", "pallets-sphinx-themes", "sphinxcontrib-log-cabinet", "sphinx-issues"]
-dotenv = ["python-dotenv"]
-
[package.dependencies]
click = ">=5.1"
itsdangerous = ">=0.24"
Jinja2 = ">=2.10.1"
Werkzeug = ">=0.15"
+[package.extras]
+dev = ["pytest", "coverage", "tox", "sphinx", "pallets-sphinx-themes", "sphinxcontrib-log-cabinet", "sphinx-issues"]
+docs = ["sphinx", "pallets-sphinx-themes", "sphinxcontrib-log-cabinet", "sphinx-issues"]
+dotenv = ["python-dotenv"]
+
[[package]]
name = "goodconf"
version = "1.0.0"
@@ -146,7 +154,7 @@ python-versions = "*"
[package.extras]
maintainer = ["zest.releaser"]
-tests = ["django (<2.1)", "ruamel.yaml", "pytest (3.5.0)", "pytest-cov (2.5.1)", "pytest-mock (1.7.1)"]
+tests = ["django (<2.1)", "ruamel.yaml", "pytest (==3.5.0)", "pytest-cov (==2.5.1)", "pytest-mock (==1.7.1)"]
yaml = ["ruamel.yaml"]
[[package]]
@@ -163,9 +171,6 @@ gevent = ["gevent (>=0.13)"]
setproctitle = ["setproctitle"]
tornado = ["tornado (>=0.2)"]
-[package.dependencies]
-setuptools = ">=3.0"
-
[[package]]
name = "idna"
version = "2.10"
@@ -181,15 +186,14 @@ description = "Read metadata from Python packages"
category = "dev"
optional = false
python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,>=2.7"
-marker = "python_version < \"3.8\""
+
+[package.dependencies]
+zipp = ">=0.5"
[package.extras]
docs = ["sphinx", "rst.linker"]
testing = ["packaging", "pep517", "importlib-resources (>=1.3)"]
-[package.dependencies]
-zipp = ">=0.5"
-
[[package]]
name = "iniconfig"
version = "1.0.1"
@@ -207,11 +211,7 @@ optional = false
python-versions = ">=2.7"
[package.dependencies]
-setuptools = "*"
-
-[package.dependencies.ipython]
-version = ">=5.1.0"
-python = ">=3.4"
+ipython = {version = ">=5.1.0", markers = "python_version >= \"3.4\""}
[[package]]
name = "ipython"
@@ -220,7 +220,18 @@ description = "IPython: Productive Interactive Computing"
category = "dev"
optional = false
python-versions = ">=3.7"
-marker = "python_version >= \"3.4\""
+
+[package.dependencies]
+appnope = {version = "*", markers = "sys_platform == \"darwin\""}
+backcall = "*"
+colorama = {version = "*", markers = "sys_platform == \"win32\""}
+decorator = "*"
+jedi = ">=0.10"
+pexpect = {version = "*", markers = "sys_platform != \"win32\""}
+pickleshare = "*"
+prompt-toolkit = ">=2.0.0,<3.0.0 || >3.0.0,<3.0.1 || >3.0.1,<3.1.0"
+pygments = "*"
+traitlets = ">=4.2"
[package.extras]
all = ["Sphinx (>=1.3)", "ipykernel", "ipyparallel", "ipywidgets", "nbconvert", "nbformat", "nose (>=0.10.1)", "notebook", "numpy (>=1.14)", "pygments", "qtconsole", "requests", "testpath"]
@@ -233,19 +244,6 @@ parallel = ["ipyparallel"]
qtconsole = ["qtconsole"]
test = ["nose (>=0.10.1)", "requests", "testpath", "pygments", "nbformat", "ipykernel", "numpy (>=1.14)"]
-[package.dependencies]
-appnope = "*"
-backcall = "*"
-colorama = "*"
-decorator = "*"
-jedi = ">=0.10"
-pexpect = "*"
-pickleshare = "*"
-prompt-toolkit = ">=2.0.0,<3.0.0 || >3.0.0,<3.0.1 || >3.0.1,<3.1.0"
-pygments = "*"
-setuptools = ">=18.5"
-traitlets = ">=4.2"
-
[[package]]
name = "ipython-genutils"
version = "0.2.0"
@@ -253,7 +251,6 @@ description = "Vestigial utilities from IPython"
category = "dev"
optional = false
python-versions = "*"
-marker = "python_version >= \"3.4\""
[[package]]
name = "itsdangerous"
@@ -270,15 +267,14 @@ description = "An autocompletion tool for Python that can be used for text edito
category = "dev"
optional = false
python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*"
-marker = "python_version >= \"3.4\""
-
-[package.extras]
-qa = ["flake8 (3.7.9)"]
-testing = ["Django (<3.1)", "colorama", "docopt", "pytest (>=3.9.0,<5.0.0)"]
[package.dependencies]
parso = ">=0.7.0,<0.8.0"
+[package.extras]
+qa = ["flake8 (==3.7.9)"]
+testing = ["Django (<3.1)", "colorama", "docopt", "pytest (>=3.9.0,<5.0.0)"]
+
[[package]]
name = "jinja2"
version = "2.11.2"
@@ -287,12 +283,12 @@ category = "main"
optional = false
python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*"
-[package.extras]
-i18n = ["Babel (>=0.8)"]
-
[package.dependencies]
MarkupSafe = ">=0.23"
+[package.extras]
+i18n = ["Babel (>=0.8)"]
+
[[package]]
name = "markupsafe"
version = "1.1.1"
@@ -328,7 +324,6 @@ description = "A Python Parser"
category = "dev"
optional = false
python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*"
-marker = "python_version >= \"3.4\""
[package.extras]
testing = ["docopt", "pytest (>=3.0.7)"]
@@ -340,7 +335,6 @@ description = "Pexpect allows easy control of interactive console applications."
category = "dev"
optional = false
python-versions = "*"
-marker = "python_version >= \"3.4\" and sys_platform != \"win32\""
[package.dependencies]
ptyprocess = ">=0.5"
@@ -352,7 +346,6 @@ description = "Tiny 'shelve'-like database with concurrency support"
category = "dev"
optional = false
python-versions = "*"
-marker = "python_version >= \"3.4\""
[[package]]
name = "pluggy"
@@ -362,14 +355,12 @@ category = "dev"
optional = false
python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*"
+[package.dependencies]
+importlib-metadata = {version = ">=0.12", markers = "python_version < \"3.8\""}
+
[package.extras]
dev = ["pre-commit", "tox"]
-[package.dependencies]
-[package.dependencies.importlib-metadata]
-version = ">=0.12"
-python = "<3.8"
-
[[package]]
name = "prompt-toolkit"
version = "3.0.6"
@@ -377,7 +368,6 @@ description = "Library for building powerful interactive command lines in Python
category = "dev"
optional = false
python-versions = ">=3.6.1"
-marker = "python_version >= \"3.4\""
[package.dependencies]
wcwidth = "*"
@@ -389,7 +379,6 @@ description = "Run a subprocess in a pseudo terminal"
category = "dev"
optional = false
python-versions = "*"
-marker = "python_version >= \"3.4\" and sys_platform != \"win32\""
[[package]]
name = "py"
@@ -414,13 +403,12 @@ description = "Pygments is a syntax highlighting package written in Python."
category = "dev"
optional = false
python-versions = ">=3.5"
-marker = "python_version >= \"3.4\""
[[package]]
name = "pyparsing"
version = "2.4.7"
description = "Python parsing module"
-category = "dev"
+category = "main"
optional = false
python-versions = ">=2.6, !=3.0.*, !=3.1.*, !=3.2.*"
@@ -432,14 +420,11 @@ category = "dev"
optional = false
python-versions = ">=3.5"
-[package.extras]
-checkqa_mypy = ["mypy (0.780)"]
-testing = ["argcomplete", "hypothesis (>=3.56)", "mock", "nose", "requests", "xmlschema"]
-
[package.dependencies]
-atomicwrites = ">=1.0"
+atomicwrites = {version = ">=1.0", markers = "sys_platform == \"win32\""}
attrs = ">=17.4.0"
-colorama = "*"
+colorama = {version = "*", markers = "sys_platform == \"win32\""}
+importlib-metadata = {version = ">=0.12", markers = "python_version < \"3.8\""}
iniconfig = "*"
more-itertools = ">=4.0.0"
packaging = "*"
@@ -447,9 +432,9 @@ pluggy = ">=0.12,<1.0"
py = ">=1.8.2"
toml = "*"
-[package.dependencies.importlib-metadata]
-version = ">=0.12"
-python = "<3.8"
+[package.extras]
+checkqa_mypy = ["mypy (==0.780)"]
+testing = ["argcomplete", "hypothesis (>=3.56)", "mock", "nose", "requests", "xmlschema"]
[[package]]
name = "requests"
@@ -459,16 +444,27 @@ category = "main"
optional = false
python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*"
-[package.extras]
-security = ["pyOpenSSL (>=0.14)", "cryptography (>=1.3.4)"]
-socks = ["PySocks (>=1.5.6,<1.5.7 || >1.5.7)", "win-inet-pton"]
-
[package.dependencies]
certifi = ">=2017.4.17"
chardet = ">=3.0.2,<4"
idna = ">=2.5,<3"
urllib3 = ">=1.21.1,<1.25.0 || >1.25.0,<1.25.1 || >1.25.1,<1.26"
+[package.extras]
+security = ["pyOpenSSL (>=0.14)", "cryptography (>=1.3.4)"]
+socks = ["PySocks (>=1.5.6,!=1.5.7)", "win-inet-pton"]
+
+[[package]]
+name = "requests-toolbelt"
+version = "0.9.1"
+description = "A utility belt for advanced users of python-requests"
+category = "main"
+optional = false
+python-versions = "*"
+
+[package.dependencies]
+requests = ">=2.0.1,<3.0.0"
+
[[package]]
name = "six"
version = "1.15.0"
@@ -492,16 +488,15 @@ description = "Traitlets Python config system"
category = "dev"
optional = false
python-versions = "*"
-marker = "python_version >= \"3.4\""
-
-[package.extras]
-test = ["pytest", "mock"]
[package.dependencies]
decorator = "*"
ipython-genutils = "*"
six = "*"
+[package.extras]
+test = ["pytest", "mock"]
+
[[package]]
name = "urllib3"
version = "1.25.10"
@@ -513,7 +508,7 @@ python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, <4"
[package.extras]
brotli = ["brotlipy (>=0.6.0)"]
secure = ["certifi", "cryptography (>=1.3.4)", "idna (>=2.0.0)", "pyOpenSSL (>=0.14)", "ipaddress"]
-socks = ["PySocks (>=1.5.6,<1.5.7 || >1.5.7,<2.0)"]
+socks = ["PySocks (>=1.5.6,!=1.5.7,<2.0)"]
[[package]]
name = "wcwidth"
@@ -522,7 +517,6 @@ description = "Measures the displayed width of unicode strings in a terminal"
category = "dev"
optional = false
python-versions = "*"
-marker = "python_version >= \"3.4\""
[[package]]
name = "werkzeug"
@@ -543,16 +537,15 @@ description = "Backport of pathlib-compatible object wrapper for zip files"
category = "dev"
optional = false
python-versions = ">=3.6"
-marker = "python_version < \"3.8\""
[package.extras]
docs = ["sphinx", "jaraco.packaging (>=3.2)", "rst.linker (>=1.9)"]
testing = ["jaraco.itertools", "func-timeout"]
[metadata]
-lock-version = "1.0"
+lock-version = "1.1"
python-versions = "^3.7"
-content-hash = "1753bef74f04a35a21914127fef75ab7bfc5c14e04c0c48613c0d637bcc6df91"
+content-hash = "6ab6afb2d27321e79013421b33c1fa4011bdc5b33b04ffd20906b4a4122c0900"
[metadata.files]
appnope = [
@@ -578,6 +571,10 @@ argon2-cffi = [
{file = "argon2_cffi-20.1.0-cp38-cp38-win_amd64.whl", hash = "sha256:9dfd5197852530294ecb5795c97a823839258dfd5eb9420233c7cfedec2058f2"},
{file = "argon2_cffi-20.1.0-cp39-cp39-win32.whl", hash = "sha256:e2db6e85c057c16d0bd3b4d2b04f270a7467c147381e8fd73cbbe5bc719832be"},
{file = "argon2_cffi-20.1.0-cp39-cp39-win_amd64.whl", hash = "sha256:8a84934bd818e14a17943de8099d41160da4a336bcc699bb4c394bbb9b94bd32"},
+ {file = "argon2_cffi-20.1.0-pp36-pypy36_pp73-macosx_10_7_x86_64.whl", hash = "sha256:b94042e5dcaa5d08cf104a54bfae614be502c6f44c9c89ad1535b2ebdaacbd4c"},
+ {file = "argon2_cffi-20.1.0-pp36-pypy36_pp73-win32.whl", hash = "sha256:8282b84ceb46b5b75c3a882b28856b8cd7e647ac71995e71b6705ec06fc232c3"},
+ {file = "argon2_cffi-20.1.0-pp37-pypy37_pp73-macosx_10_7_x86_64.whl", hash = "sha256:3aa804c0e52f208973845e8b10c70d8957c9e5a666f702793256242e9167c4e0"},
+ {file = "argon2_cffi-20.1.0-pp37-pypy37_pp73-win_amd64.whl", hash = "sha256:36320372133a003374ef4275fbfce78b7ab581440dfca9f9471be3dd9a522428"},
]
atomicwrites = [
{file = "atomicwrites-1.4.0-py2.py3-none-any.whl", hash = "sha256:6d1784dea7c0c8d4a5172b6c620f40b6e4cbfdf96d783691f2e1302a7b88e197"},
@@ -637,6 +634,10 @@ click = [
{file = "click-7.1.2-py2.py3-none-any.whl", hash = "sha256:dacca89f4bfadd5de3d7489b7c8a566eee0d3676333fbb50030263894c38c0dc"},
{file = "click-7.1.2.tar.gz", hash = "sha256:d2b5255c7c6349bc1bd1e59e08cd12acbbd63ce649f2588755783aa94dfb6b1a"},
]
+cloudscraper = [
+ {file = "cloudscraper-1.2.58-py2.py3-none-any.whl", hash = "sha256:674fd739f9412188aae8d6614e3e6316939fc0670ef5646abd3d316f1a59d3c2"},
+ {file = "cloudscraper-1.2.58.tar.gz", hash = "sha256:dda29028c5628b5ba3e4dc43816ed38fd46bd945ef938c420f185586a6d8dff2"},
+]
colorama = [
{file = "colorama-0.4.3-py2.py3-none-any.whl", hash = "sha256:7d73d2a99753107a36ac6b455ee49046802e59d9d076ef8e47b61499fa29afff"},
{file = "colorama-0.4.3.tar.gz", hash = "sha256:e96da0d330793e2cb9485e9ddfd918d456036c7149416295932478192f4436a1"},
@@ -783,6 +784,10 @@ requests = [
{file = "requests-2.24.0-py2.py3-none-any.whl", hash = "sha256:fe75cc94a9443b9246fc7049224f75604b113c36acb93f87b80ed42c44cbb898"},
{file = "requests-2.24.0.tar.gz", hash = "sha256:b3559a131db72c33ee969480840fff4bb6dd111de7dd27c8ee1f820f4f00231b"},
]
+requests-toolbelt = [
+ {file = "requests-toolbelt-0.9.1.tar.gz", hash = "sha256:968089d4584ad4ad7c171454f0a5c6dac23971e9472521ea3b6d49d610aa6fc0"},
+ {file = "requests_toolbelt-0.9.1-py2.py3-none-any.whl", hash = "sha256:380606e1d10dc85c3bd47bf5a6095f815ec007be7a8b69c878507068df059e6f"},
+]
six = [
{file = "six-1.15.0-py2.py3-none-any.whl", hash = "sha256:8b74bedcbbbaca38ff6d7491d76f2b06b3592611af620f8426e82dddb04a5ced"},
{file = "six-1.15.0.tar.gz", hash = "sha256:30639c035cdb23534cd4aa2dd52c3bf48f06e5f4a941509c8bafd8ce11080259"},
diff --git a/pyproject.toml b/pyproject.toml
index db670a0..c61cc32 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -27,6 +27,7 @@ requests = "^2.24.0"
goodconf = "^1.0.0"
argon2-cffi = "^20.1.0"
bbcode = "^1.1.0"
+cloudscraper = "^1.2.58"
[tool.poetry.dev-dependencies]
pytest = "^6.0.1"
diff --git a/src/mangoapi/base_site.py b/src/mangoapi/base_site.py
index 9bb4dde..b3a6479 100644
--- a/src/mangoapi/base_site.py
+++ b/src/mangoapi/base_site.py
@@ -1,6 +1,7 @@
import functools
from abc import ABC, abstractmethod
+import cloudscraper
import requests
from .exceptions import (
@@ -15,12 +16,7 @@ def __init__(self):
self.username = None
self.password = None
self.is_logged_in = False
- self._session = requests.Session()
- self._session.headers.update(
- {
- "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/78.0.3904.108 Safari/537.36"
- }
- )
+ self._session = cloudscraper.create_scraper(browser="chrome")
@abstractmethod
def get_title(self, title_id):