Repos / pytaku / baaf53e72f
commit baaf53e72fadb66323860ddcdcbadf4e635d5092
Author: Bùi Thành Nhân <hi@imnhan.com>
Date: Sat Jun 19 16:20:52 2021 +0700
not-so-secret sauce proxy
Until they're on to us again I guess
diff --git a/src/mangoapi/base_site.py b/src/mangoapi/base_site.py
index 08f4332..9189367 100644
--- a/src/mangoapi/base_site.py
+++ b/src/mangoapi/base_site.py
@@ -1,9 +1,12 @@
import functools
from abc import ABC, abstractmethod
+from urllib.parse import urlparse
import cloudscraper
import requests
+from pytaku.conf import config
+
from .exceptions import (
SourceSite5xxError,
SourceSiteTimeoutError,
@@ -55,9 +58,21 @@ def login(self, username, password):
raise NotImplementedError()
def _http_request(self, method, url, *args, **kwargs):
+ headers = kwargs.get("headers", {})
if "timeout" not in kwargs:
kwargs["timeout"] = 5
+ # Proxy shit
+ parsed_url = urlparse(url)
+ url = parsed_url._replace(
+ netloc=config.FAASPROXY_NETLOC,
+ scheme="https",
+ path=config.FAASPROXY_PATH + parsed_url.path,
+ ).geturl()
+ headers["Faasproxy-Target-Host"] = parsed_url.netloc
+ headers["Faasproxy-Key"] = config.FAASPROXY_KEY
+ kwargs["headers"] = headers
+
request_func = getattr(self._session, method)
try:
resp = request_func(url, *args, **kwargs)
diff --git a/src/pytaku/__init__.py b/src/pytaku/__init__.py
index f320f42..c3c9882 100644
--- a/src/pytaku/__init__.py
+++ b/src/pytaku/__init__.py
@@ -56,6 +56,7 @@ def generate_config():
def scheduler():
+ config.load()
from .scheduler import main_loop
main_loop()
diff --git a/src/pytaku/conf.py b/src/pytaku/conf.py
index 5fe3d0e..ff1a9cc 100644
--- a/src/pytaku/conf.py
+++ b/src/pytaku/conf.py
@@ -14,5 +14,9 @@ class Config(GoodConf):
PROXY_CACHE_MAX_SIZE = Value(default=1024 * 1024 * 1024 * 5) # 5GiB in bytes
PROXY_CACHE_MAX_AGE = Value(default=3600 * 24 * 2) # 2 days in seconds
+ FAASPROXY_NETLOC = Value()
+ FAASPROXY_PATH = Value()
+ FAASPROXY_KEY = Value()
+
config = Config(default_files=["pytaku.conf.json"])
diff --git a/src/pytaku/scheduler.py b/src/pytaku/scheduler.py
index c8dff33..389228e 100644
--- a/src/pytaku/scheduler.py
+++ b/src/pytaku/scheduler.py
@@ -76,7 +76,7 @@ def run(self):
save_title(updated_title)
print(" done")
if title["site"] == "mangasee":
- time.sleep(random.randint(5, 10))
+ time.sleep(2)
except (SourceSite5xxError, ReadTimeout, JSONDecodeError) as e:
print(" skipped because of server error:", e.__class__.__name__, str(e))