commit e20e6e36dde438907db7b0aa607ea755ddfe9cf3 Author: Longtail Date: Sat Oct 31 14:33:07 2026 +0800 first commit diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..06059ce --- /dev/null +++ b/.gitignore @@ -0,0 +1,5 @@ +__pycache__/ +*.pyc +.DS_Store +downloads/ +.venv/ diff --git a/README.md b/README.md new file mode 100644 index 0000000..bb0daa7 --- /dev/null +++ b/README.md @@ -0,0 +1,57 @@ +# 探数新闻下载器 + +在本机浏览器中填写 API Key、获取频道、选择数量，下载新闻正文和图片。所有项目文件与下载结果都位于本目录，不修改微信下载器。 + +## 启动 + +安装 `uv` 后，macOS 可以双击 `start.command`。也可以在本目录运行： + +```sh +uv run --no-project --python 3.12 --with-requirements requirements.txt python app.py +``` + +启动后自动打开 http://127.0.0.1:8765 。保留启动窗口，退出时按 Ctrl+C。端口被占用时加 `--port 8766`；不自动打开浏览器时加 `--no-browser`。 + +1. 填写探数 API Key，点击「获取新闻频道」。频道来自真实接口，不使用写死的列表。 +2. 选择频道、输入目标数量，点击「开始下载」。 +3. 页面显示成功保存的数量、百分比、重复数、失败数和最近 300 条日志。日志固定高度，可滚动；向上查看时不会强制滚回底部。 +4. 下载结果保存在本目录的 `downloads` 文件夹。点击「停止下载」后，等待当前请求或文章保存结束。 +5. 「调试终端」默认折叠，支持频道查询以及列表查询的 `type`、`num`、`start` 参数。测试请求只显示响应，不保存文章；同样消耗 API 调用额度。输出最多保留 100000 字符。 + +密钥仅在页面和服务运行内存中使用，不保存到配置、历史数据库或浏览器存储。服务仅监听本机地址。刷新页面可查看当前任务进度，开始新任务需重新填写密钥。单个服务同时只运行一个下载任务，请勿同时启动多个实例操作同一下载目录。 + +## 保存格式 + +沿用 `wechat_official_acc_article_downloader` 的格式和图片处理规则： + +```text +downloads/ +├── .history.sqlite3 +└── 新闻标题/ + ├── 新闻标题.md + ├── image001.jpg + └── image002.png +``` + +Markdown 开头是一级标题，空行后直接接由接口 `content` 转换的正文，不额外添加接口 `time` 对应的发布时间或分隔线。末尾为 `来源：{src}，如有侵权，请联系删除。`，来源为空时使用「未知作者」。不在开头添加原文链接或作者。正文中的图片下载到同一目录、使用相对引用、按真实文件类型命名，同一图片地址只下载一次。接口的 `pic` 封面不单独插入正文。 + +标题中的不合法字符会替换；不同文章碰巧同名时添加 ` (2)` 等后缀，目录和 Markdown 文件保持同名。任何正文图片下载失败，该篇不计入完成数，并清理半成品，再继续其他新闻；下次任务可以重试。 + +## 去重与分页 + +- 下载图片前，检查原文 URL / PC URL（移除常见跟踪参数和片段）及规范化正文指纹。任一匹配即跳过，标题相同但正文和网址不同不会单凭标题去重。 +- SQLite 保存成功文章的指纹，跨任务、跨频道和服务重启后继续去重。被删除的文章目录不会阻止重新下载。不要删除 `.history.sqlite3`，否则会失去对应历史记录。 +- 每次任务从最新位置 `start=0` 开始，按返回数量推进，每页最多 40 条；重复和失败不计入成功数量，并继续后面的新闻。 +- 接口限制 `start ≤ 400`，一次任务最多覆盖约 440 条，具体还受返回数量和实时更新影响。达到分页边界或收到空列表时停止，未达目标就显示「未达到目标」，不会伪装成 100%。 +- 列表接口不自动重试，防止隐蔽消耗计费额度。API 业务错误、网络错误或响应格式错误会中断任务，已成功保存的文章保留。图片请求沿用原下载器的重试策略。 +- 新闻列表会实时变化，无法保证服务器在不同请求中不返回同一新闻；本工具在保存前去重，尽量避免重复图片下载与文件保存。 + +接口依据：[探数官方新闻 API 文档](https://www.tanshuapi.com/market/detail-85)。 + +## 验证 + +```sh +uv run --no-project --python 3.12 --with-requirements requirements.txt python -m unittest discover -s tests -v +``` + +测试使用模拟新闻响应与临时目录，不消耗真实 API 额度。覆盖格式与图片引用、图片失败清理、历史去重、分页边界、失败重试、取消、日志上限和本地 HTTP 接口。真实账号权限、余额、返回内容与远程图片可达性，需要在页面填入有效 Key 后验证。 diff --git a/app.py b/app.py new file mode 100644 index 0000000..a9e1273 --- /dev/null +++ b/app.py @@ -0,0 +1,286 @@ +#!/usr/bin/env python3 +"""仅在本机运行的探数新闻下载器。""" +from collections import deque +from datetime import datetime +from hashlib import sha256 +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +from pathlib import Path +from urllib.parse import urlsplit, urlunsplit, parse_qsl, urlencode +import argparse +import json +import secrets +import sqlite3 +import threading +import time +import unicodedata +import webbrowser + +import requests +from bs4 import BeautifulSoup +from article_writer import save_article + +ROOT = Path(__file__).resolve().parent +API = 'https://api.tanshuapi.com/api/toutiao/v1/' + + +def api_request(key, endpoint, params=None): + if endpoint not in ('channel', 'index'): + raise ValueError('未知接口') + try: + # 不自动重试计费接口，避免一次操作产生不可见的重复调用。 + response = requests.get(API + endpoint, params={'key': key, **(params or {})}, timeout=(10, 40)) + response.raise_for_status() + payload = response.json() + except requests.RequestException: + raise ValueError('接口网络请求失败或超时，请在调试终端检查后重试') from None + except ValueError: + raise ValueError('接口未返回有效 JSON') from None + if not isinstance(payload, dict): + raise ValueError('接口响应格式错误') + return payload + + +def result_list(payload): + if payload.get('code') != 1: + raise ValueError(f"探数接口错误 {payload.get('code')}：{payload.get('msg', '未知错误')}") + data = payload.get('data') + if not isinstance(data, dict) or not isinstance(data.get('list'), list): + raise ValueError('接口响应缺少 data.list') + return data['list'] + + +def fingerprints(article): + values = [] + for field in ('url', 'weburl'): + value = str(article.get(field) or '').strip() + if value: + u = urlsplit(value) + query = [(k, v) for k, v in parse_qsl(u.query) if not k.lower().startswith('utm_') and k.lower() not in ('spm', 'from', 'pos', 'vt')] + values.append('url:' + urlunsplit(('', u.netloc.lower(), u.path, urlencode(sorted(query)), ''))) + body = BeautifulSoup(str(article.get('content') or ''), 'html.parser').get_text(' ', strip=True) + body = ''.join(unicodedata.normalize('NFKC', body).split()) + if body: + values.append('body:' + body) + return [sha256(value.encode()).hexdigest() for value in values] + + +class Downloader: + def __init__(self, root=ROOT / 'downloads', requester=api_request, writer=save_article): + self.root = Path(root).resolve() + self.root.mkdir(parents=True, exist_ok=True) + self.requester, self.writer = requester, writer + self.lock = threading.RLock() + self.stop = threading.Event() + self.thread = None + self.logs = deque(maxlen=300) + self.seq = 0 + self.state = dict(status='idle', saved=0, target=0, skipped=0, failed=0, scanned=0, requests=0, channel='', message='等待开始') + with self.db() as db: + db.execute('CREATE TABLE IF NOT EXISTS seen (fingerprint TEXT PRIMARY KEY, directory TEXT NOT NULL)') + + def db(self): + return sqlite3.connect(self.root / '.history.sqlite3') + + def log(self, message): + with self.lock: + self.seq += 1 + self.logs.append(dict(id=self.seq, time=datetime.now().strftime('%H:%M:%S'), message=message)) + + def update(self, **values): + with self.lock: + self.state.update(values) + + def snapshot(self): + with self.lock: + return {**self.state, 'logs': list(self.logs), 'output': str(self.root)} + + def start(self, key, channel, target): + with self.lock: + if self.thread and self.thread.is_alive(): + raise ValueError('已有任务正在运行') + self.stop.clear() + self.logs.clear() + self.state = dict(status='running', saved=0, target=target, skipped=0, failed=0, scanned=0, requests=0, channel=channel, message='正在下载') + self.thread = threading.Thread(target=self.run, args=(key, channel, target), daemon=True) + self.thread.start() + + def run(self, key, channel, target): + saved = skipped = failed = scanned = calls = 0 + seen_in_run = set() + self.log(f'开始下载频道「{channel}」，目标 {target} 条；历史重复新闻将跳过') + try: + with self.db() as db: + offset = 0 + while offset <= 400 and saved < target and not self.stop.is_set(): + count = min(40, target - saved) + calls += 1 + self.update(requests=calls) + self.log(f'请求第 {calls} 页：start={offset}，num={count}') + items = result_list(self.requester(key, 'index', {'type': channel, 'num': count, 'start': offset})) + if not items: + self.log('接口返回空列表，已停止') + break + for article in items: + if saved >= target or self.stop.is_set(): + break + scanned += 1 + try: + if not isinstance(article, dict): + raise ValueError('新闻条目格式错误') + keys = fingerprints(article) + existing = any(k in seen_in_run for k in keys) + for k in keys: + row = db.execute('SELECT directory FROM seen WHERE fingerprint=?', (k,)).fetchone() + if row and Path(row[0]).is_dir(): + existing = True + seen_in_run.update(keys) + if existing: + skipped += 1 + self.log('跳过重复新闻：' + str(article.get('title', '无标题'))[:160]) + continue + result = self.writer(article, self.root) + try: + db.executemany('INSERT OR REPLACE INTO seen VALUES (?, ?)', [(k, result.directory) for k in keys]) + db.commit() + except Exception: + # 保存目录尚未登记成功，回滚本次产物，避免下次重复保存。 + import shutil + shutil.rmtree(result.directory, ignore_errors=True) + raise + saved += 1 + self.log(f'下载新闻 {saved}/{target} 条：{result.title}') + except Exception as exc: + failed += 1 + self.log(f'保存失败：{str(exc).replace(key, "[已隐藏]")[:400]}') + finally: + self.update(saved=saved, skipped=skipped, failed=failed, scanned=scanned) + # 避免越过 start=400；已消费的条目决定下一起始位置。 + offset += len(items) + if offset > 400: + self.log('已到达接口允许的分页范围') + if saved < target and offset <= 400: + self.stop.wait(0.3) + status = 'completed' if saved >= target else 'cancelled' if self.stop.is_set() else 'partial' + message = f'已保存 {saved}/{target} 条，跳过重复 {skipped} 条，失败 {failed} 条' + if status == 'partial': + message += '；当前可获取新闻不足，未达到目标数量' + if status == 'cancelled': + message += '；任务已停止' + self.update(status=status, message=message) + self.log(message) + except Exception as exc: + message = str(exc).replace(key, '[已隐藏]')[:500] + self.update(status='error', message=message) + self.log('任务中断：' + message) + + +def make_handler(manager, token): + class Handler(BaseHTTPRequestHandler): + def log_message(self, *args): + pass + + def send(self, status, payload, mime='application/json; charset=utf-8'): + data = json.dumps(payload, ensure_ascii=False).encode() if isinstance(payload, dict) else payload + self.send_response(status) + self.send_header('Content-Type', mime) + self.send_header('Content-Length', str(len(data))) + self.send_header('Cache-Control', 'no-store') + self.send_header('X-Content-Type-Options', 'nosniff') + self.end_headers() + self.wfile.write(data) + + def local(self): + return self.headers.get('Host') in (f'127.0.0.1:{self.server.server_port}', f'localhost:{self.server.server_port}') + + def do_GET(self): + if not self.local(): + return self.send(403, {'error': '仅允许本机访问'}) + path = urlsplit(self.path).path + if path == '/api/state': + return self.send(200, manager.snapshot()) + if path == '/api/session': + return self.send(200, {'token': token}) + files = {'/': ('index.html', 'text/html; charset=utf-8'), '/app.js': ('app.js', 'text/javascript; charset=utf-8'), '/style.css': ('style.css', 'text/css; charset=utf-8')} + if path not in files: + return self.send(404, {'error': '未找到页面'}) + name, mime = files[path] + self.send(200, (ROOT / 'static' / name).read_bytes(), mime) + + def do_POST(self): + if not self.local() or self.headers.get('X-Session-Token') != token: + return self.send(403, {'error': '页面会话失效，请刷新页面'}) + key = '' + try: + length = int(self.headers.get('Content-Length', 0)) + if not 0 < length <= 16384: + raise ValueError('请求大小无效') + body = json.loads(self.rfile.read(length)) + if not isinstance(body, dict): + raise ValueError('请求必须是 JSON 对象') + if self.path == '/api/stop': + manager.stop.set() + return self.send(200, {'ok': True}) + key = str(body.get('key') or '').strip() + if not key or len(key) > 512: + raise ValueError('请填写有效 API Key') + if self.path == '/api/channels': + channels = result_list(manager.requester(key, 'channel')) + if not channels or not all(isinstance(c, str) and c for c in channels): + raise ValueError('频道列表为空或格式错误') + return self.send(200, {'channels': channels}) + if self.path == '/api/start': + target = body.get('target') + channel = body.get('channel') + if type(target) is not int or not 1 <= target <= 100000: + raise ValueError('下载数量须为 1 至 100000 的整数') + if not isinstance(channel, str) or not channel.strip() or len(channel) > 100: + raise ValueError('请选择新闻频道') + manager.start(key, channel, target) + return self.send(200, {'ok': True}) + if self.path == '/api/debug': + endpoint = body.get('endpoint') + params = {} + if endpoint == 'index': + for field, low, high in [('num', 1, 40), ('start', 0, 400)]: + value = body.get(field) + if type(value) is not int or not low <= value <= high: + raise ValueError(f'{field} 必须在 {low}–{high} 之间') + params[field] = value + params['type'] = str(body.get('channel') or '') + before = time.monotonic() + payload = manager.requester(key, endpoint, params) + # 服务端响应同样隐藏 key，且只作为文本展示，不渲染上游 HTML。 + clean = json.loads(json.dumps(payload, ensure_ascii=False).replace(key, '[已隐藏]')) + return self.send(200, {'elapsed': round(time.monotonic() - before, 2), 'request': API + str(endpoint) + '?' + urlencode({'key': '[已隐藏]', **params}), 'response': clean}) + self.send(404, {'error': '未知操作'}) + except Exception as exc: + message = str(exc) + if key: + message = message.replace(key, '[已隐藏]') + self.send(400, {'error': message[:500]}) + return Handler + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('--port', type=int, default=8765) + parser.add_argument('--no-browser', action='store_true') + args = parser.parse_args() + manager = Downloader() + server = ThreadingHTTPServer(('127.0.0.1', args.port), make_handler(manager, secrets.token_urlsafe(32))) + url = f'http://127.0.0.1:{server.server_port}' + print(f'新闻下载器：{url}\n保存目录：{manager.root}\n按 Ctrl+C 退出', flush=True) + if not args.no_browser: + webbrowser.open(url) + try: + server.serve_forever() + except KeyboardInterrupt: + manager.stop.set() + if manager.thread: + manager.thread.join(timeout=45) + finally: + server.server_close() + + +if __name__ == '__main__': + main() diff --git a/article_writer.py b/article_writer.py new file mode 100644 index 0000000..04a6ca0 --- /dev/null +++ b/article_writer.py @@ -0,0 +1,212 @@ +#!/usr/bin/env python3 +"""将探数新闻正文保存为与微信下载器相同的 Markdown 目录格式。""" + +from __future__ import annotations + +import base64 +from dataclasses import asdict, dataclass +from pathlib import Path +import re +import shutil +import tempfile +from urllib.parse import unquote, urljoin, urlsplit + +import requests +from bs4 import BeautifulSoup +from markdownify import markdownify +from requests.adapters import HTTPAdapter +from urllib3.util.retry import Retry + + +class ArticleDownloadError(RuntimeError): + """页面不是文章、图片下载失败等可向调用方报告的错误。""" + + +@dataclass +class ArticleResult: + title: str + source_url: str + directory: str + markdown: str + images: list[str] + warnings: list[str] + + def to_dict(self) -> dict: + return asdict(self) + + +def safe_name(title: str) -> str: + """跨平台目录/文件名；目录名与其中 Markdown 的文件名保持一致。""" + name = re.sub(r'[<>:"/\\|?*\x00-\x1f\x7f]', '_', title) + name = re.sub(r'\s+', ' ', name).strip(' .') + # 同时限制 UTF-8 字节数，给 .md 和重名后缀保留空间。 + name = name.encode('utf-8')[:180].decode('utf-8', errors='ignore').rstrip(' .') + if not name: + name = '未命名文章' + if re.fullmatch(r'(?i:CON|PRN|AUX|NUL|COM[1-9]|LPT[1-9])(?:\..*)?', name): + name = '_' + name + return name + + +def _session() -> requests.Session: + session = requests.Session() + session.headers.update({ + 'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) ' + 'AppleWebKit/537.36 (KHTML, like Gecko) Chrome/130.0.0.0 Safari/537.36', + 'Accept-Language': 'zh-CN,zh;q=0.9,en;q=0.8', + }) + retry = Retry(total=3, backoff_factor=0.6, + status_forcelist=[429, 500, 502, 503, 504], allowed_methods=['GET']) + session.mount('https://', HTTPAdapter(max_retries=retry)) + session.mount('http://', HTTPAdapter(max_retries=retry)) + return session + + +def _get(session: requests.Session, url: str, timeout: float, limit: int, + referer: str | None = None) -> tuple[bytes, str, str]: + if urlsplit(url).scheme not in ('http', 'https'): + raise ArticleDownloadError(f'不支持的下载地址：{url[:200]}') + headers = {'Referer': referer} if referer else {} + with session.get(url, timeout=(min(timeout, 15), timeout), + headers=headers, stream=True) as response: + response.raise_for_status() + chunks, size = [], 0 + for chunk in response.iter_content(65536): + size += len(chunk) + if size > limit: + raise ArticleDownloadError(f'文件超过大小限制（{limit // 1024 // 1024} MB）：{url[:200]}') + chunks.append(chunk) + return b''.join(chunks), response.headers.get('Content-Type', ''), response.url + + +def _image_ext(data: bytes, mime: str) -> str: + if data.startswith(b'\xff\xd8\xff'): + return '.jpg' + if data.startswith(b'\x89PNG\r\n\x1a\n'): + return '.png' + if data.startswith((b'GIF87a', b'GIF89a')): + return '.gif' + if data[:4] == b'RIFF' and data[8:12] == b'WEBP': + return '.webp' + if len(data) >= 12 and data[4:8] == b'ftyp' and data[8:12] in (b'avif', b'avis'): + return '.avif' + if b' str: + for attr in ('data-src', 'data-original', 'data-actualsrc', 'data-lazy-src', 'src'): + value = img.get(attr) + if value and str(value).strip(): + return str(value).strip() + srcset = img.get('data-srcset') or img.get('srcset') + if srcset: + return str(srcset).split(',')[-1].strip().split()[0] + return '' + + +def save_article(article: dict, output_root: str | Path = '.', *, timeout: float = 30, + allow_missing_images: bool = False) -> ArticleResult: + """从 API 正文保存文章；图片失败时清理半成品，不覆盖已有同名目录。""" + url = str(article.get('weburl') or article.get('url') or '') + title = str(article.get('title') or '').strip() + raw_content = str(article.get('content') or '').strip() + if not title or not raw_content: + raise ArticleDownloadError('新闻标题或正文为空') + if timeout <= 0: + raise ValueError('timeout 必须大于 0') + with _session() as session: + final_url = url + soup = BeautifulSoup('
' + raw_content + '
', 'html.parser') + content = soup.article + for tag in content.select('script, style, noscript, iframe, form, button'): + tag.decompose() + # 部分微信排版把插图放在行内 CSS 背景中，也转为普通图片参与下载。 + for tag in list(content.find_all(style=True)): + for match in re.finditer(r'url\(\s*[\'\"]?(.*?)[\'\"]?\s*\)', str(tag['style'])): + source = match.group(1).strip() + if source and not source.startswith('#'): + background = soup.new_tag('img', src=source, alt='') + tag.insert(0, background) + if not content.get_text(strip=True) and not content.find('img'): + raise ArticleDownloadError('文章正文为空，可能需要浏览器执行 JavaScript') + for link in content.find_all('a', href=True): + href = str(link['href']).strip() + if href and not href.startswith('#'): + absolute = urljoin(final_url, href) + if urlsplit(absolute).scheme in ('http', 'https', 'mailto', 'tel'): + link['href'] = absolute + else: + del link['href'] + + root = Path(output_root).expanduser().resolve() + root.mkdir(parents=True, exist_ok=True) + stage = Path(tempfile.mkdtemp(prefix='.article-', dir=root)) + images, warnings, downloaded = [], [], {} + try: + for img in list(content.find_all('img')): + source = _image_source(img) + if not source: + img.replace_with(soup.new_string('[图片缺少下载地址]')) + if not allow_missing_images: + raise ArticleDownloadError('正文中存在没有下载地址的图片') + warnings.append('正文中存在没有下载地址的图片') + continue + image_url = urljoin(final_url, source) + try: + if image_url not in downloaded: + if source.startswith('data:image/'): + header, payload = source.split(',', 1) + data = base64.b64decode(payload, validate=True) if ';base64' in header else unquote(payload).encode() + mime = header[5:].split(';')[0] + if len(data) > 50 * 1024 * 1024: + raise ArticleDownloadError('内嵌图片超过 50 MB') + else: + data, mime, _ = _get(session, image_url, timeout, 50 * 1024 * 1024, final_url) + ext = _image_ext(data, mime) + filename = f'image{len(images) + 1:03d}{ext}' + (stage / filename).write_bytes(data) + images.append(filename) + downloaded[image_url] = filename + # 清除懒加载/响应式属性，使转换器只引用本地文件。 + alt = str(img.get('alt') or '') + img.attrs = {'src': downloaded[image_url], 'alt': alt} + except (requests.RequestException, ArticleDownloadError, ValueError) as exc: + if not allow_missing_images: + raise ArticleDownloadError(f'图片下载失败：{image_url[:200]}；{exc}') from exc + warnings.append(f'图片下载失败：{image_url[:200]}；{exc}') + img.replace_with(soup.new_string(f'[图片下载失败：{img.get("alt") or "未命名图片"}]')) + + # 新闻图片后常带空的图注标题，避免生成单独的 ## 和多余空行。 + for heading in content.select('h1, h2, h3, h4, h5, h6'): + if not heading.get_text(strip=True) and not heading.find('img'): + heading.decompose() + body = markdownify(str(content), heading_style='ATX', bullets='-', + strip=['picture', 'source']).strip() + # 微信富文本中各段常没有标准 p 标签；markdownify 仍会保留文字和图片顺序。 + author = str(article.get('src') or '未知作者').strip() or '未知作者' + text = f'# {title}\n\n' + text += body + f'\n\n来源：{author}，如有侵权，请联系删除。\n' + basename = safe_name(title) + # 原子认领新目录，支持多个进程并发保存同名文章。 + index = 1 + while True: + name = basename if index == 1 else f'{basename} ({index})' + destination = root / name + try: + destination.mkdir() + break + except FileExistsError: + index += 1 + try: + (stage / f'{name}.md').write_text(text, encoding='utf-8') + for item in stage.iterdir(): + shutil.move(str(item), destination / item.name) + except BaseException: + shutil.rmtree(destination, ignore_errors=True) + raise + return ArticleResult(title, url, str(destination), str(destination / f'{name}.md'), + [str(destination / x) for x in images], warnings) + finally: + shutil.rmtree(stage, ignore_errors=True) diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..4d42bf4 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,3 @@ +requests>=2.32,<3 +beautifulsoup4>=4.12,<5 +markdownify>=1.1,<2 diff --git a/start.command b/start.command new file mode 100755 index 0000000..8e43a78 --- /dev/null +++ b/start.command @@ -0,0 +1,8 @@ +#!/bin/zsh +cd "${0:A:h}" +if ! command -v uv >/dev/null 2>&1; then + print '未找到 uv，请先安装 uv 后再启动。' + read '?按回车关闭…' + exit 1 +fi +uv run --no-project --python 3.12 --with-requirements requirements.txt python app.py diff --git a/static/app.js b/static/app.js new file mode 100644 index 0000000..528fcd5 --- /dev/null +++ b/static/app.js @@ -0,0 +1,79 @@ +'use strict'; +const $ = id => document.getElementById(id); +let token = '', running = false, starting = false, loadingChannels = false, loadedKey = '', lastLog = -1; +const labels = {idle:'等待开始',running:'下载中',completed:'已完成',partial:'未达到目标',cancelled:'已停止',error:'任务中断'}; +const key = () => $('key').value.trim(); +function controls() { + const busy = running || starting; + $('start').disabled = busy || !loadedKey || loadedKey !== key() || !$('channel').value; + $('stop').disabled = !running; + $('channels').disabled = busy || loadingChannels; + for (const id of ['key','channel','target']) $(id).disabled = busy || (id === 'channel' && !loadedKey); +} +async function post(path, body) { + const response = await fetch('/api/' + path, {method:'POST',headers:{'Content-Type':'application/json','X-Session-Token':token},body:JSON.stringify(body)}); + const data = await response.json(); + if (!response.ok) throw new Error(data.error || '请求失败'); + return data; +} +$('key').addEventListener('input', () => {loadedKey=''; $('channel').replaceChildren(new Option('请重新获取频道','')); controls();}); +$('channels').addEventListener('click', async () => { + loadingChannels=true; controls(); $('notice').textContent='正在获取频道…'; + const submittedKey = key(); + try { + const data = await post('channels',{key:submittedKey}); + if (key() !== submittedKey) return; + $('channel').replaceChildren(...data.channels.map(c => new Option(c,c))); + loadedKey=submittedKey; $('notice').textContent=`已获取 ${data.channels.length} 个频道。`; + $('debug-channel').value=$('channel').value; + } catch(error) {$('notice').textContent=error.message;} + finally {loadingChannels=false; controls();} +}); +$('channel').addEventListener('change', () => {$('debug-channel').value=$('channel').value; controls();}); +$('download-form').addEventListener('submit', async event => { + event.preventDefault(); starting=true; controls(); $('notice').textContent=''; + try {await post('start',{key:key(),channel:$('channel').value,target:Number($('target').value)}); await refresh();} + catch(error) {$('notice').textContent=error.message;} + finally {starting=false; controls();} +}); +$('stop').addEventListener('click', async () => { + $('stop').disabled=true; + try {await post('stop',{}); $('notice').textContent='正在停止，等待当前请求或文章保存结束…';} + catch(error) {$('notice').textContent=error.message;} +}); +$('debug-form').addEventListener('submit', async event => { + event.preventDefault(); $('debug-submit').disabled=true; $('debug-output').textContent='正在请求…'; + try { + const data=await post('debug',{key:key(),endpoint:$('endpoint').value,channel:$('debug-channel').value,num:Number($('debug-num').value),start:Number($('debug-start').value)}); + const output=JSON.stringify(data,null,2); + $('debug-output').textContent=output.slice(0,100000)+(output.length>100000?'\n…输出超过 100000 字符，已截断。':''); + } catch(error) {$('debug-output').textContent=error.message;} + finally {$('debug-submit').disabled=false;} +}); +async function refresh() { + const response=await fetch('/api/state'); + if (!response.ok) throw new Error('无法读取下载状态'); + const data=await response.json(); + running=data.status==='running'; controls(); + const percent=data.target ? Math.floor(data.saved/data.target*100) : 0; + $('status').textContent=labels[data.status] || data.status; + $('progress-text').textContent=`下载新闻 ${data.saved}/${data.target} 条`; + $('percent').textContent=`已完成 ${percent}%`; $('progress').value=percent; + for (const id of ['saved','skipped','failed','requests','message','output']) $(id).textContent=data[id]; + const latest=data.logs.length ? data.logs[data.logs.length-1].id : 0; + if (latest!==lastLog) { + const logs=$('logs'); const bottom=logs.scrollHeight-logs.scrollTop-logs.clientHeight<40; + logs.replaceChildren(...data.logs.map(log => {const row=document.createElement('div'); const stamp=document.createElement('time'); stamp.textContent=log.time; row.append(stamp,document.createTextNode(log.message)); return row;})); + if (bottom) logs.scrollTop=logs.scrollHeight; + lastLog=latest; + } +} +async function poll() { + try {await refresh();} + catch(error) {$('notice').textContent='与本地服务连接中断，请检查启动窗口。';} + setTimeout(poll,700); +} +(async () => { + try {const response=await fetch('/api/session'); token=(await response.json()).token; await poll();} + catch(error) {$('notice').textContent='初始化失败，请刷新页面。';} +})(); diff --git a/static/index.html b/static/index.html new file mode 100644 index 0000000..68d77fd --- /dev/null +++ b/static/index.html @@ -0,0 +1,14 @@ + + +探数 · 新闻下载器 +
+
TANSHU / NEWS ARCHIVE

把新闻，保存到本地。

选择频道，收集新闻。正文与图片一起保存，自动跳过已下载的内容。

+

下载设置

本地运行
+

密钥仅用于本次页面会话，不写入文件或浏览器存储。

+
+

接口每次最多 40 条，起始位置最多 400。遇到重复会继续向后查找；可用新闻不足时会说明原因。

+

下载进度

等待开始
下载新闻 0/0 条已完成 0%
已保存 0重复跳过 0保存失败 0接口请求 0

等待开始

保存位置：加载中…

+

运行输出

保留最近 300 条
运行情况将在这里显示。
+
调试终端 手动请求接口

使用上方 API Key。请求会消耗接口调用额度，但不会保存新闻。

尚未发送请求。
+
Markdown 正文 · 本地图片 · 历史去重
+
diff --git a/static/style.css b/static/style.css new file mode 100644 index 0000000..803f19b --- /dev/null +++ b/static/style.css @@ -0,0 +1 @@ +:root{font-family:Inter,-apple-system,BlinkMacSystemFont,"Segoe UI","PingFang SC",sans-serif;color:#20312e;background:#f3f6f3;font-size:15px}*{box-sizing:border-box}body{margin:0}main{max-width:960px;margin:0 auto;padding:48px 24px 24px}header{margin-bottom:32px}.brand{font-size:12px;letter-spacing:2px;font-weight:700;color:#437b69}h1{font-size:36px;letter-spacing:-1px;margin:14px 0}header p{color:#687a74;line-height:1.8}h2{font-size:17px;margin:0}.card{padding:26px 28px;background:#fff;border:1px solid #dfe7e1;border-radius:16px;margin-bottom:20px;box-shadow:0 4px 16px #243c2904}.section-heading{display:flex;align-items:center;justify-content:space-between;gap:10px;margin-bottom:24px}.badge{background:#edf4ef;color:#4c7564;padding:6px 10px;border-radius:20px;font-size:12px}.key-row{display:flex;gap:12px}.key-row input{flex:1;min-width:0}label{display:block;font-weight:600;margin-bottom:9px;font-size:13px}input,select,button{font:inherit;border-radius:8px}input,select{background:#fbfcfb;border:1px solid #d7e0d9;width:100%;padding:12px;color:inherit}input:focus,select:focus,button:focus-visible,summary:focus-visible{outline:3px solid #8dcab5;outline-offset:2px}button{cursor:pointer;border:1px solid #286850;background:#286850;color:#fff;padding:12px 22px;font-weight:600;white-space:nowrap}.secondary{background:#fff;color:#345e4f;border-color:#cedbd3}button:disabled{opacity:.45;cursor:not-allowed}.fields{display:grid;grid-template-columns:1fr 1fr;gap:20px;margin:24px 0 16px}.hint{font-size:12px;color:#79867f;line-height:1.8;margin:10px 0}.actions{display:flex;gap:12px;margin-top:22px}#notice{color:#ac4c32;font-size:13px;white-space:pre-wrap}#notice:empty{display:none}.progress-heading{display:flex;justify-content:space-between;gap:16px;font-size:14px}#percent{color:#3f7b60}progress{display:block;width:100%;height:12px;border:0;border-radius:10px;overflow:hidden;margin:18px 0 20px;background:#edf2ef}progress::-webkit-progress-bar{background:#edf2ef}progress::-webkit-progress-value{background:#3c8b68;transition:width .25s}progress::-moz-progress-bar{background:#3c8b68}.stats{display:flex;gap:28px;flex-wrap:wrap;color:#7a8880;font-size:13px}.stats b{color:#304e40;margin-left:8px;font-size:17px}#message{font-size:13px;color:#52685b;line-height:1.8}.path{overflow-wrap:anywhere}.console{height:260px;overflow:auto;background:#15251f;color:#c5dacf;border-radius:10px;padding:18px;font:12px/1.9 ui-monospace,SFMono-Regular,Consolas,monospace;white-space:pre-wrap;overflow-wrap:anywhere}.console time{color:#7eac94;margin-right:14px}.empty{color:#7e9889}summary{cursor:pointer;font-weight:600}summary span{font-size:12px;font-weight:400;color:#79867f;margin-left:12px}#debug-output{margin-bottom:0;height:300px}footer{text-align:center;color:#89958e;font-size:12px;margin:32px 0}@media(max-width:600px){main{padding:26px 14px}h1{font-size:28px}.card{padding:22px 18px}.key-row{flex-direction:column}.fields{grid-template-columns:1fr}.stats{gap:14px}.progress-heading{font-size:12px}} diff --git a/tests/test_downloader.py b/tests/test_downloader.py new file mode 100644 index 0000000..106d0cb --- /dev/null +++ b/tests/test_downloader.py @@ -0,0 +1,164 @@ +import base64 +import json +from pathlib import Path +import sys +import tempfile +import threading +import unittest +from http.server import ThreadingHTTPServer + +import requests +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) +from app import Downloader, make_handler, fingerprints +from article_writer import save_article + + +def article(i): + return dict(title=f'新闻{i}', content=f'

第{i}篇新闻内容

', url=f'https://example.com/{i}', src='测试来源', time='2026-10-31') + + +class DownloadTests(unittest.TestCase): + def setUp(self): + self.temp = tempfile.TemporaryDirectory() + self.root = Path(self.temp.name) + + def tearDown(self): + self.temp.cleanup() + + def run_job(self, manager, target): + manager.start('test-key', '头条', target) + manager.thread.join(10) + self.assertFalse(manager.thread.is_alive()) + return manager.snapshot() + + def test_format_and_local_image(self): + data = article(1) + image = 'data:image/png;base64,' + base64.b64encode(b'\x89PNG\r\n\x1a\nexample').decode() + data['content'] += f'' + result = save_article(data, self.root) + body = Path(result.markdown).read_text() + self.assertTrue(body.startswith('# 新闻1\n\n第1篇新闻内容')) + self.assertNotIn('发布时间：2026-10-31', body) + self.assertNotIn('\n---\n', body) + self.assertTrue(body.endswith('来源：测试来源，如有侵权，请联系删除。\n')) + self.assertEqual(len(result.images), 1) + self.assertEqual(body.count('image001.png'), 2) + self.assertNotIn('data:image', body) + second = save_article(data, self.root) + self.assertEqual(Path(second.markdown).name, '新闻1 (2).md') + + def test_bad_image_cleans_partial(self): + data = article(1) + data['content'] += '' + with self.assertRaises(Exception): + save_article(data, self.root) + self.assertEqual(list(self.root.iterdir()), []) + + def test_duplicate_then_fill_and_restart(self): + items = [article(1), article(1), article(2), article(3)] + calls = [] + def request(key, endpoint, params): + calls.append(params.copy()) + return {'code': 1, 'data': {'list': items[params['start']:params['start']+params['num']]}} + manager = Downloader(self.root, request) + state = self.run_job(manager, 3) + self.assertEqual((state['status'], state['saved'], state['skipped']), ('completed', 3, 1)) + self.assertEqual([c['start'] for c in calls], [0, 3]) + restarted = Downloader(self.root, request) + state = self.run_job(restarted, 3) + self.assertEqual((state['status'], state['saved'], state['skipped']), ('partial', 0, 4)) + self.assertEqual(len(list(self.root.glob('*/*.md'))), 3) + + def test_offset_limit_and_partial_progress(self): + offsets = [] + def request(key, endpoint, params): + offsets.append(params['start']) + return {'code': 1, 'data': {'list': [article(i) for i in range(params['start'], params['start']+params['num'])]}} + state = self.run_job(Downloader(self.root, request), 500) + self.assertEqual(state['status'], 'partial') + self.assertEqual(state['saved'], 440) + self.assertEqual(max(offsets), 400) + + def test_error_and_key_redaction(self): + def request(*args): + return {'code': 10001, 'msg': 'invalid test-key'} + manager = Downloader(self.root, request) + state = self.run_job(manager, 10) + self.assertEqual(state['status'], 'error') + self.assertNotIn('test-key', json.dumps(state)) + + def test_stop_and_concurrent_start(self): + entered, release = threading.Event(), threading.Event() + def request(*args): + entered.set() + release.wait(2) + return {'code': 1, 'data': {'list': [article(1)]}} + manager = Downloader(self.root, request) + manager.start('key', '头条', 10) + self.assertTrue(entered.wait(2)) + with self.assertRaises(ValueError): + manager.start('key', '头条', 10) + manager.stop.set() + release.set() + manager.thread.join(3) + self.assertEqual(manager.snapshot()['status'], 'cancelled') + self.assertEqual(manager.snapshot()['saved'], 0) + + def test_failed_save_can_retry_next_job(self): + attempts = [] + def writer(data, root): + attempts.append(1) + if len(attempts) == 1: + raise ValueError('模拟图片失败') + return save_article(data, root) + def request(key, endpoint, params): + return {'code': 1, 'data': {'list': [article(1)] if params['start'] == 0 else []}} + manager = Downloader(self.root, request, writer) + self.assertEqual(self.run_job(manager, 1)['failed'], 1) + self.assertEqual(self.run_job(manager, 1)['saved'], 1) + + def test_fingerprint_content_and_tracking(self): + a, b = article(1), article(2) + b['content'] = a['content'].replace('

', '

').replace('

', '
') + self.assertTrue(set(fingerprints(a)) & set(fingerprints(b))) + b['content'] = '其他内容' + b['url'] = a['url'] + '?utm_source=abc#fragment' + self.assertTrue(set(fingerprints(a)) & set(fingerprints(b))) + + def test_log_cap(self): + manager = Downloader(self.root) + for i in range(400): + manager.log(str(i)) + self.assertEqual(len(manager.snapshot()['logs']), 300) + self.assertEqual(manager.snapshot()['logs'][0]['message'], '100') + + def test_http_routes_and_access(self): + def request(key, endpoint, params=None): + return {'code': 1, 'data': {'list': ['头条', '科技']}} + manager = Downloader(self.root, request) + server = ThreadingHTTPServer(('127.0.0.1', 0), make_handler(manager, 'token')) + worker = threading.Thread(target=server.serve_forever, daemon=True) + worker.start() + url = f'http://127.0.0.1:{server.server_port}' + try: + self.assertIn('新闻下载器', requests.get(url).text) + self.assertEqual(requests.get(url+'/api/state').json()['status'], 'idle') + self.assertEqual(requests.post(url+'/api/channels', json={'key': 'test'}).status_code, 403) + self.assertEqual(requests.get(url, headers={'Host':'evil.example'}).status_code, 403) + headers = {'X-Session-Token': 'token'} + self.assertEqual(requests.post(url+'/api/channels', headers=headers, json={'key':'test'}).json()['channels'], ['头条','科技']) + invalid = requests.post(url+'/api/start', headers=headers, json={'key':'test', 'target':0,'channel':'头条'}) + self.assertEqual(invalid.status_code, 400) + invalid = requests.post(url+'/api/debug', headers=headers, json={'key':'test','endpoint':'index','num':41,'start':0}) + self.assertEqual(invalid.status_code, 400) + debug = requests.post(url+'/api/debug', headers=headers, json={'key':'test','endpoint':'channel'}) + self.assertEqual(debug.status_code, 200) + self.assertNotIn('key=test', debug.text) + finally: + server.shutdown() + server.server_close() + worker.join() + + +if __name__ == '__main__': + unittest.main()