from http.server import BaseHTTPRequestHandler, HTTPServer
from urllib.request import Request, urlopen
from urllib.error import URLError, HTTPError
import re
TARGET_URL = "https://m.slrclub.com/l/free"
class ProxyHandler(BaseHTTPRequestHandler):
def do_GET(self):
requested_path = self.path
if requested_path == "/" or requested_path == "":
requested_path = "/l/free"
target_url = f"https://m.slrclub.com{requested_path}"
try:
req = Request(
target_url,
headers={
"User-Agent": "Mozilla/5.0 (compatible; Python crawler/1.0)",
"Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
},
)
with urlopen(req, timeout=15) as response:
content_type = response.headers.get_content_type()
charset = response.headers.get_content_charset(failobj="utf-8")
raw_html = response.read().decode(charset, errors="replace")
# Rewrite relative links to use this local proxy.
if "
]*>)",
rf"\1",
raw_html,
count=1,
flags=re.IGNORECASE,
)
self.send_response(200)
self.send_header("Content-Type", f"{content_type}; charset={charset}")
self.end_headers()
self.wfile.write(raw_html.encode(charset, errors="replace"))
except HTTPError as e:
self.send_response(e.code)
self.send_header("Content-Type", "text/plain; charset=utf-8")
self.end_headers()
self.wfile.write(f"HTTP Error: {e.code} {e.reason}".encode("utf-8"))
except URLError as e:
self.send_response(502)
self.send_header("Content-Type", "text/plain; charset=utf-8")
self.end_headers()
self.wfile.write(f"Failed to fetch target page: {e.reason}".encode("utf-8"))
except Exception as e:
self.send_response(500)
self.send_header("Content-Type", "text/plain; charset=utf-8")
self.end_headers()
self.wfile.write(f"Server error: {e}".encode("utf-8"))
def run(server_class=HTTPServer, handler_class=ProxyHandler, port=8000):
server_address = ("", port)
try:
httpd = server_class(server_address, handler_class)
except OSError as exc:
if exc.errno == 98:
raise SystemExit(f"Port {port} is already in use. Run with a different port using --port.") from exc
raise
print(f"Serving crawler homepage at http://localhost:{port}/")
httpd.serve_forever()
if __name__ == "__main__":
import argparse
parser = argparse.ArgumentParser(description="Proxy crawler for https://m.slrclub.com/l/free")
parser.add_argument("--port", type=int, default=8000, help="Port to listen on")
args = parser.parse_args()
run(port=args.port)