154 lines
6.6 KiB
Python
154 lines
6.6 KiB
Python
from http.server import BaseHTTPRequestHandler, HTTPServer
|
|
from urllib.request import Request, urlopen
|
|
from urllib.error import URLError, HTTPError
|
|
import re
|
|
from html import unescape
|
|
|
|
TARGET_URL = "https://m.slrclub.com/l/free"
|
|
|
|
class ProxyHandler(BaseHTTPRequestHandler):
|
|
def do_GET(self):
|
|
requested_path = self.path
|
|
if requested_path == "/" or requested_path == "":
|
|
requested_path = "/l/free"
|
|
|
|
target_url = f"https://m.slrclub.com{requested_path}"
|
|
|
|
try:
|
|
req = Request(
|
|
target_url,
|
|
headers={
|
|
"User-Agent": "Mozilla/5.0 (compatible; Python crawler/1.0)",
|
|
"Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
|
|
},
|
|
)
|
|
with urlopen(req, timeout=15) as response:
|
|
content_type = response.headers.get_content_type()
|
|
charset = response.headers.get_content_charset(failobj="utf-8")
|
|
raw_html = response.read().decode(charset, errors="replace")
|
|
|
|
# Extract up to 10 post entries (제목 + 링크) and render a simple list.
|
|
items = []
|
|
|
|
# First try: find anchors inside list items (common for post lists)
|
|
li_blocks = re.findall(r"<li[^>]*>(.*?)</li>", raw_html, flags=re.DOTALL | re.IGNORECASE)
|
|
for block in li_blocks:
|
|
m = re.search(r"<a[^>]+href=[\"']([^\"']+)[\"'][^>]*>(.*?)</a>", block, flags=re.DOTALL | re.IGNORECASE)
|
|
if m:
|
|
href = m.group(1).strip()
|
|
anchor_html = m.group(0)
|
|
inner_html = m.group(2)
|
|
txt = re.sub(r"<[^>]+>", "", inner_html).strip()
|
|
txt = unescape(txt)
|
|
|
|
# If inner text is empty, try anchor title or image alt attributes
|
|
if not txt:
|
|
t = re.search(r'title=["\']([^"\']+)["\']', anchor_html, flags=re.IGNORECASE)
|
|
if t:
|
|
txt = unescape(t.group(1).strip())
|
|
|
|
if not txt:
|
|
img_alt = re.search(r'<img[^>]+alt=["\']([^"\']+)["\']', anchor_html, flags=re.IGNORECASE)
|
|
if img_alt:
|
|
txt = unescape(img_alt.group(1).strip())
|
|
|
|
# Only append if we have a non-empty title
|
|
if txt:
|
|
items.append((href, txt))
|
|
|
|
# Fallback: collect anchors from whole page if no list-items found
|
|
if not items:
|
|
anchors = re.findall(r"<a[^>]+href=[\"']([^\"']+)[\"'][^>]*>(.*?)</a>", raw_html, flags=re.DOTALL | re.IGNORECASE)
|
|
for href, txt_html in anchors:
|
|
href = href.strip()
|
|
txt = re.sub(r"<[^>]+>", "", txt_html).strip()
|
|
txt = unescape(txt)
|
|
if txt and not href.lower().startswith("javascript:"):
|
|
items.append((href, txt))
|
|
|
|
# Deduplicate and filter out obvious navigation/header links
|
|
seen = set()
|
|
filtered = []
|
|
for href, txt in items:
|
|
key = (href, txt)
|
|
if key in seen:
|
|
continue
|
|
seen.add(key)
|
|
lowtxt = (txt or "").strip().lower()
|
|
if href in ("#", "/", "", "/l/free"):
|
|
continue
|
|
if lowtxt in ("홈", "로그인", "검색", "공지", "more", "더보기"):
|
|
continue
|
|
filtered.append((href, txt))
|
|
|
|
# Skip the top 5 posts and show the next 10
|
|
posts = filtered[5:5+10]
|
|
|
|
# Build simple HTML list for posts only
|
|
host = self.headers.get("Host", f"localhost:{self.server.server_port}")
|
|
out_lines = [
|
|
'<!doctype html>',
|
|
'<html><head><meta charset="utf-8"><title>SLRClub - 게시글 목록</title></head><body>',
|
|
'<h1>게시글 (최대 10개)</h1>',
|
|
'<ul>'
|
|
]
|
|
for href, txt in posts:
|
|
# Build absolute URL to original site so clicks open the real page
|
|
abs_url = href
|
|
if href.startswith("/"):
|
|
abs_url = "https://m.slrclub.com" + href
|
|
elif href.startswith("//"):
|
|
abs_url = "https:" + href
|
|
elif not href.startswith("http"):
|
|
# relative path without leading slash
|
|
abs_url = "https://m.slrclub.com/" + href.lstrip("/")
|
|
|
|
# Escape minimal characters in title
|
|
safe_txt = txt.replace("<", "<").replace(">", ">")
|
|
out_lines.append(f'<li><a href="{abs_url}" target="_blank" rel="noopener noreferrer">{safe_txt}</a></li>')
|
|
|
|
out_lines.extend(["</ul>", "</body></html>"])
|
|
out = "\n".join(out_lines)
|
|
|
|
self.send_response(200)
|
|
self.send_header("Content-Type", "text/html; charset=utf-8")
|
|
self.end_headers()
|
|
self.wfile.write(out.encode("utf-8", errors="replace"))
|
|
|
|
except HTTPError as e:
|
|
self.send_response(e.code)
|
|
self.send_header("Content-Type", "text/plain; charset=utf-8")
|
|
self.end_headers()
|
|
self.wfile.write(f"HTTP Error: {e.code} {e.reason}".encode("utf-8"))
|
|
except URLError as e:
|
|
self.send_response(502)
|
|
self.send_header("Content-Type", "text/plain; charset=utf-8")
|
|
self.end_headers()
|
|
self.wfile.write(f"Failed to fetch target page: {e.reason}".encode("utf-8"))
|
|
except Exception as e:
|
|
self.send_response(500)
|
|
self.send_header("Content-Type", "text/plain; charset=utf-8")
|
|
self.end_headers()
|
|
self.wfile.write(f"Server error: {e}".encode("utf-8"))
|
|
|
|
|
|
def run(server_class=HTTPServer, handler_class=ProxyHandler, port=8000):
|
|
server_address = ("", port)
|
|
try:
|
|
httpd = server_class(server_address, handler_class)
|
|
except OSError as exc:
|
|
if exc.errno == 98:
|
|
raise SystemExit(f"Port {port} is already in use. Run with a different port using --port.") from exc
|
|
raise
|
|
print(f"Serving crawler homepage at http://localhost:{port}/")
|
|
httpd.serve_forever()
|
|
|
|
|
|
if __name__ == "__main__":
|
|
import argparse
|
|
|
|
parser = argparse.ArgumentParser(description="Proxy crawler for https://m.slrclub.com/l/free")
|
|
parser.add_argument("--port", type=int, default=8000, help="Port to listen on")
|
|
args = parser.parse_args()
|
|
run(port=args.port)
|