first commit

.gitignore

@@ -0,0 +1,5 @@
+__pycache__/
+*.pyc
+.DS_Store
+downloads/
+.venv/

README.md

@@ -0,0 +1,57 @@
+# 探数新闻下载器
+
+在本机浏览器中填写 API Key、获取频道、选择数量，下载新闻正文和图片。所有项目文件与下载结果都位于本目录，不修改微信下载器。
+
+## 启动
+
+安装 `uv` 后，macOS 可以双击 `start.command`。也可以在本目录运行：
+
+```sh
+uv run --no-project --python 3.12 --with-requirements requirements.txt python app.py
+```
+
+启动后自动打开 http://127.0.0.1:8765 。保留启动窗口，退出时按 Ctrl+C。端口被占用时加 `--port 8766`；不自动打开浏览器时加 `--no-browser`。
+
+1. 填写探数 API Key，点击「获取新闻频道」。频道来自真实接口，不使用写死的列表。
+2. 选择频道、输入目标数量，点击「开始下载」。
+3. 页面显示成功保存的数量、百分比、重复数、失败数和最近 300 条日志。日志固定高度，可滚动；向上查看时不会强制滚回底部。
+4. 下载结果保存在本目录的 `downloads` 文件夹。点击「停止下载」后，等待当前请求或文章保存结束。
+5. 「调试终端」默认折叠，支持频道查询以及列表查询的 `type`、`num`、`start` 参数。测试请求只显示响应，不保存文章；同样消耗 API 调用额度。输出最多保留 100000 字符。
+
+密钥仅在页面和服务运行内存中使用，不保存到配置、历史数据库或浏览器存储。服务仅监听本机地址。刷新页面可查看当前任务进度，开始新任务需重新填写密钥。单个服务同时只运行一个下载任务，请勿同时启动多个实例操作同一下载目录。
+
+## 保存格式
+
+沿用 `wechat_official_acc_article_downloader` 的格式和图片处理规则：
+
+```text
+downloads/
+├── .history.sqlite3
+└── 新闻标题/
+    ├── 新闻标题.md
+    ├── image001.jpg
+    └── image002.png
+```
+
+Markdown 开头是一级标题，空行后直接接由接口 `content` 转换的正文，不额外添加接口 `time` 对应的发布时间或分隔线。末尾为 `来源：{src}，如有侵权，请联系删除。`，来源为空时使用「未知作者」。不在开头添加原文链接或作者。正文中的图片下载到同一目录、使用相对引用、按真实文件类型命名，同一图片地址只下载一次。接口的 `pic` 封面不单独插入正文。
+
+标题中的不合法字符会替换；不同文章碰巧同名时添加 ` (2)` 等后缀，目录和 Markdown 文件保持同名。任何正文图片下载失败，该篇不计入完成数，并清理半成品，再继续其他新闻；下次任务可以重试。
+
+## 去重与分页
+
+- 下载图片前，检查原文 URL / PC URL（移除常见跟踪参数和片段）及规范化正文指纹。任一匹配即跳过，标题相同但正文和网址不同不会单凭标题去重。
+- SQLite 保存成功文章的指纹，跨任务、跨频道和服务重启后继续去重。被删除的文章目录不会阻止重新下载。不要删除 `.history.sqlite3`，否则会失去对应历史记录。
+- 每次任务从最新位置 `start=0` 开始，按返回数量推进，每页最多 40 条；重复和失败不计入成功数量，并继续后面的新闻。
+- 接口限制 `start ≤ 400`，一次任务最多覆盖约 440 条，具体还受返回数量和实时更新影响。达到分页边界或收到空列表时停止，未达目标就显示「未达到目标」，不会伪装成 100%。
+- 列表接口不自动重试，防止隐蔽消耗计费额度。API 业务错误、网络错误或响应格式错误会中断任务，已成功保存的文章保留。图片请求沿用原下载器的重试策略。
+- 新闻列表会实时变化，无法保证服务器在不同请求中不返回同一新闻；本工具在保存前去重，尽量避免重复图片下载与文件保存。
+
+接口依据：[探数官方新闻 API 文档](https://www.tanshuapi.com/market/detail-85)。
+
+## 验证
+
+```sh
+uv run --no-project --python 3.12 --with-requirements requirements.txt python -m unittest discover -s tests -v
+```
+
+测试使用模拟新闻响应与临时目录，不消耗真实 API 额度。覆盖格式与图片引用、图片失败清理、历史去重、分页边界、失败重试、取消、日志上限和本地 HTTP 接口。真实账号权限、余额、返回内容与远程图片可达性，需要在页面填入有效 Key 后验证。

app.py

@@ -0,0 +1,286 @@
+#!/usr/bin/env python3
+"""仅在本机运行的探数新闻下载器。"""
+from collections import deque
+from datetime import datetime
+from hashlib import sha256
+from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
+from pathlib import Path
+from urllib.parse import urlsplit, urlunsplit, parse_qsl, urlencode
+import argparse
+import json
+import secrets
+import sqlite3
+import threading
+import time
+import unicodedata
+import webbrowser
+
+import requests
+from bs4 import BeautifulSoup
+from article_writer import save_article
+
+ROOT = Path(__file__).resolve().parent
+API = 'https://api.tanshuapi.com/api/toutiao/v1/'
+
+
+def api_request(key, endpoint, params=None):
+    if endpoint not in ('channel', 'index'):
+        raise ValueError('未知接口')
+    try:
+        # 不自动重试计费接口，避免一次操作产生不可见的重复调用。
+        response = requests.get(API + endpoint, params={'key': key, **(params or {})}, timeout=(10, 40))
+        response.raise_for_status()
+        payload = response.json()
+    except requests.RequestException:
+        raise ValueError('接口网络请求失败或超时，请在调试终端检查后重试') from None
+    except ValueError:
+        raise ValueError('接口未返回有效 JSON') from None
+    if not isinstance(payload, dict):
+        raise ValueError('接口响应格式错误')
+    return payload
+
+
+def result_list(payload):
+    if payload.get('code') != 1:
+        raise ValueError(f"探数接口错误 {payload.get('code')}：{payload.get('msg', '未知错误')}")
+    data = payload.get('data')
+    if not isinstance(data, dict) or not isinstance(data.get('list'), list):
+        raise ValueError('接口响应缺少 data.list')
+    return data['list']
+
+
+def fingerprints(article):
+    values = []
+    for field in ('url', 'weburl'):
+        value = str(article.get(field) or '').strip()
+        if value:
+            u = urlsplit(value)
+            query = [(k, v) for k, v in parse_qsl(u.query) if not k.lower().startswith('utm_') and k.lower() not in ('spm', 'from', 'pos', 'vt')]
+            values.append('url:' + urlunsplit(('', u.netloc.lower(), u.path, urlencode(sorted(query)), '')))
+    body = BeautifulSoup(str(article.get('content') or ''), 'html.parser').get_text(' ', strip=True)
+    body = ''.join(unicodedata.normalize('NFKC', body).split())
+    if body:
+        values.append('body:' + body)
+    return [sha256(value.encode()).hexdigest() for value in values]
+
+
+class Downloader:
+    def __init__(self, root=ROOT / 'downloads', requester=api_request, writer=save_article):
+        self.root = Path(root).resolve()
+        self.root.mkdir(parents=True, exist_ok=True)
+        self.requester, self.writer = requester, writer
+        self.lock = threading.RLock()
+        self.stop = threading.Event()
+        self.thread = None
+        self.logs = deque(maxlen=300)
+        self.seq = 0
+        self.state = dict(status='idle', saved=0, target=0, skipped=0, failed=0, scanned=0, requests=0, channel='', message='等待开始')
+        with self.db() as db:
+            db.execute('CREATE TABLE IF NOT EXISTS seen (fingerprint TEXT PRIMARY KEY, directory TEXT NOT NULL)')
+
+    def db(self):
+        return sqlite3.connect(self.root / '.history.sqlite3')
+
+    def log(self, message):
+        with self.lock:
+            self.seq += 1
+            self.logs.append(dict(id=self.seq, time=datetime.now().strftime('%H:%M:%S'), message=message))
+
+    def update(self, **values):
+        with self.lock:
+            self.state.update(values)
+
+    def snapshot(self):
+        with self.lock:
+            return {**self.state, 'logs': list(self.logs), 'output': str(self.root)}
+
+    def start(self, key, channel, target):
+        with self.lock:
+            if self.thread and self.thread.is_alive():
+                raise ValueError('已有任务正在运行')
+            self.stop.clear()
+            self.logs.clear()
+            self.state = dict(status='running', saved=0, target=target, skipped=0, failed=0, scanned=0, requests=0, channel=channel, message='正在下载')
+            self.thread = threading.Thread(target=self.run, args=(key, channel, target), daemon=True)
+            self.thread.start()
+
+    def run(self, key, channel, target):
+        saved = skipped = failed = scanned = calls = 0
+        seen_in_run = set()
+        self.log(f'开始下载频道「{channel}」，目标 {target} 条；历史重复新闻将跳过')
+        try:
+            with self.db() as db:
+                offset = 0
+                while offset <= 400 and saved < target and not self.stop.is_set():
+                    count = min(40, target - saved)
+                    calls += 1
+                    self.update(requests=calls)
+                    self.log(f'请求第 {calls} 页：start={offset}，num={count}')
+                    items = result_list(self.requester(key, 'index', {'type': channel, 'num': count, 'start': offset}))
+                    if not items:
+                        self.log('接口返回空列表，已停止')
+                        break
+                    for article in items:
+                        if saved >= target or self.stop.is_set():
+                            break
+                        scanned += 1
+                        try:
+                            if not isinstance(article, dict):
+                                raise ValueError('新闻条目格式错误')
+                            keys = fingerprints(article)
+                            existing = any(k in seen_in_run for k in keys)
+                            for k in keys:
+                                row = db.execute('SELECT directory FROM seen WHERE fingerprint=?', (k,)).fetchone()
+                                if row and Path(row[0]).is_dir():
+                                    existing = True
+                            seen_in_run.update(keys)
+                            if existing:
+                                skipped += 1
+                                self.log('跳过重复新闻：' + str(article.get('title', '无标题'))[:160])
+                                continue
+                            result = self.writer(article, self.root)
+                            try:
+                                db.executemany('INSERT OR REPLACE INTO seen VALUES (?, ?)', [(k, result.directory) for k in keys])
+                                db.commit()
+                            except Exception:
+                                # 保存目录尚未登记成功，回滚本次产物，避免下次重复保存。
+                                import shutil
+                                shutil.rmtree(result.directory, ignore_errors=True)
+                                raise
+                            saved += 1
+                            self.log(f'下载新闻 {saved}/{target} 条：{result.title}')
+                        except Exception as exc:
+                            failed += 1
+                            self.log(f'保存失败：{str(exc).replace(key, "[已隐藏]")[:400]}')
+                        finally:
+                            self.update(saved=saved, skipped=skipped, failed=failed, scanned=scanned)
+                    # 避免越过 start=400；已消费的条目决定下一起始位置。
+                    offset += len(items)
+                    if offset > 400:
+                        self.log('已到达接口允许的分页范围')
+                    if saved < target and offset <= 400:
+                        self.stop.wait(0.3)
+            status = 'completed' if saved >= target else 'cancelled' if self.stop.is_set() else 'partial'
+            message = f'已保存 {saved}/{target} 条，跳过重复 {skipped} 条，失败 {failed} 条'
+            if status == 'partial':
+                message += '；当前可获取新闻不足，未达到目标数量'
+            if status == 'cancelled':
+                message += '；任务已停止'
+            self.update(status=status, message=message)
+            self.log(message)
+        except Exception as exc:
+            message = str(exc).replace(key, '[已隐藏]')[:500]
+            self.update(status='error', message=message)
+            self.log('任务中断：' + message)
+
+
+def make_handler(manager, token):
+    class Handler(BaseHTTPRequestHandler):
+        def log_message(self, *args):
+            pass
+
+        def send(self, status, payload, mime='application/json; charset=utf-8'):
+            data = json.dumps(payload, ensure_ascii=False).encode() if isinstance(payload, dict) else payload
+            self.send_response(status)
+            self.send_header('Content-Type', mime)
+            self.send_header('Content-Length', str(len(data)))
+            self.send_header('Cache-Control', 'no-store')
+            self.send_header('X-Content-Type-Options', 'nosniff')
+            self.end_headers()
+            self.wfile.write(data)
+
+        def local(self):
+            return self.headers.get('Host') in (f'127.0.0.1:{self.server.server_port}', f'localhost:{self.server.server_port}')
+
+        def do_GET(self):
+            if not self.local():
+                return self.send(403, {'error': '仅允许本机访问'})
+            path = urlsplit(self.path).path
+            if path == '/api/state':
+                return self.send(200, manager.snapshot())
+            if path == '/api/session':
+                return self.send(200, {'token': token})
+            files = {'/': ('index.html', 'text/html; charset=utf-8'), '/app.js': ('app.js', 'text/javascript; charset=utf-8'), '/style.css': ('style.css', 'text/css; charset=utf-8')}
+            if path not in files:
+                return self.send(404, {'error': '未找到页面'})
+            name, mime = files[path]
+            self.send(200, (ROOT / 'static' / name).read_bytes(), mime)
+
+        def do_POST(self):
+            if not self.local() or self.headers.get('X-Session-Token') != token:
+                return self.send(403, {'error': '页面会话失效，请刷新页面'})
+            key = ''
+            try:
+                length = int(self.headers.get('Content-Length', 0))
+                if not 0 < length <= 16384:
+                    raise ValueError('请求大小无效')
+                body = json.loads(self.rfile.read(length))
+                if not isinstance(body, dict):
+                    raise ValueError('请求必须是 JSON 对象')
+                if self.path == '/api/stop':
+                    manager.stop.set()
+                    return self.send(200, {'ok': True})
+                key = str(body.get('key') or '').strip()
+                if not key or len(key) > 512:
+                    raise ValueError('请填写有效 API Key')
+                if self.path == '/api/channels':
+                    channels = result_list(manager.requester(key, 'channel'))
+                    if not channels or not all(isinstance(c, str) and c for c in channels):
+                        raise ValueError('频道列表为空或格式错误')
+                    return self.send(200, {'channels': channels})
+                if self.path == '/api/start':
+                    target = body.get('target')
+                    channel = body.get('channel')
+                    if type(target) is not int or not 1 <= target <= 100000:
+                        raise ValueError('下载数量须为 1 至 100000 的整数')
+                    if not isinstance(channel, str) or not channel.strip() or len(channel) > 100:
+                        raise ValueError('请选择新闻频道')
+                    manager.start(key, channel, target)
+                    return self.send(200, {'ok': True})
+                if self.path == '/api/debug':
+                    endpoint = body.get('endpoint')
+                    params = {}
+                    if endpoint == 'index':
+                        for field, low, high in [('num', 1, 40), ('start', 0, 400)]:
+                            value = body.get(field)
+                            if type(value) is not int or not low <= value <= high:
+                                raise ValueError(f'{field} 必须在 {low}–{high} 之间')
+                            params[field] = value
+                        params['type'] = str(body.get('channel') or '')
+                    before = time.monotonic()
+                    payload = manager.requester(key, endpoint, params)
+                    # 服务端响应同样隐藏 key，且只作为文本展示，不渲染上游 HTML。
+                    clean = json.loads(json.dumps(payload, ensure_ascii=False).replace(key, '[已隐藏]'))
+                    return self.send(200, {'elapsed': round(time.monotonic() - before, 2), 'request': API + str(endpoint) + '?' + urlencode({'key': '[已隐藏]', **params}), 'response': clean})
+                self.send(404, {'error': '未知操作'})
+            except Exception as exc:
+                message = str(exc)
+                if key:
+                    message = message.replace(key, '[已隐藏]')
+                self.send(400, {'error': message[:500]})
+    return Handler
+
+
+def main():
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument('--port', type=int, default=8765)
+    parser.add_argument('--no-browser', action='store_true')
+    args = parser.parse_args()
+    manager = Downloader()
+    server = ThreadingHTTPServer(('127.0.0.1', args.port), make_handler(manager, secrets.token_urlsafe(32)))
+    url = f'http://127.0.0.1:{server.server_port}'
+    print(f'新闻下载器：{url}\n保存目录：{manager.root}\n按 Ctrl+C 退出', flush=True)
+    if not args.no_browser:
+        webbrowser.open(url)
+    try:
+        server.serve_forever()
+    except KeyboardInterrupt:
+        manager.stop.set()
+        if manager.thread:
+            manager.thread.join(timeout=45)
+    finally:
+        server.server_close()
+
+
+if __name__ == '__main__':
+    main()

article_writer.py

@@ -0,0 +1,212 @@
+#!/usr/bin/env python3
+"""将探数新闻正文保存为与微信下载器相同的 Markdown 目录格式。"""
+
+from __future__ import annotations
+
+import base64
+from dataclasses import asdict, dataclass
+from pathlib import Path
+import re
+import shutil
+import tempfile
+from urllib.parse import unquote, urljoin, urlsplit
+
+import requests
+from bs4 import BeautifulSoup
+from markdownify import markdownify
+from requests.adapters import HTTPAdapter
+from urllib3.util.retry import Retry
+
+
+class ArticleDownloadError(RuntimeError):
+    """页面不是文章、图片下载失败等可向调用方报告的错误。"""
+
+
+@dataclass
+class ArticleResult:
+    title: str
+    source_url: str
+    directory: str
+    markdown: str
+    images: list[str]
+    warnings: list[str]
+
+    def to_dict(self) -> dict:
+        return asdict(self)
+
+
+def safe_name(title: str) -> str:
+    """跨平台目录/文件名；目录名与其中 Markdown 的文件名保持一致。"""
+    name = re.sub(r'[<>:"/\\|?*\x00-\x1f\x7f]', '_', title)
+    name = re.sub(r'\s+', ' ', name).strip(' .')
+    # 同时限制 UTF-8 字节数，给 .md 和重名后缀保留空间。
+    name = name.encode('utf-8')[:180].decode('utf-8', errors='ignore').rstrip(' .')
+    if not name:
+        name = '未命名文章'
+    if re.fullmatch(r'(?i:CON|PRN|AUX|NUL|COM[1-9]|LPT[1-9])(?:\..*)?', name):
+        name = '_' + name
+    return name
+
+
+def _session() -> requests.Session:
+    session = requests.Session()
+    session.headers.update({
+        'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) '
+                      'AppleWebKit/537.36 (KHTML, like Gecko) Chrome/130.0.0.0 Safari/537.36',
+        'Accept-Language': 'zh-CN,zh;q=0.9,en;q=0.8',
+    })
+    retry = Retry(total=3, backoff_factor=0.6,
+                  status_forcelist=[429, 500, 502, 503, 504], allowed_methods=['GET'])
+    session.mount('https://', HTTPAdapter(max_retries=retry))
+    session.mount('http://', HTTPAdapter(max_retries=retry))
+    return session
+
+
+def _get(session: requests.Session, url: str, timeout: float, limit: int,
+         referer: str | None = None) -> tuple[bytes, str, str]:
+    if urlsplit(url).scheme not in ('http', 'https'):
+        raise ArticleDownloadError(f'不支持的下载地址：{url[:200]}')
+    headers = {'Referer': referer} if referer else {}
+    with session.get(url, timeout=(min(timeout, 15), timeout),
+                     headers=headers, stream=True) as response:
+        response.raise_for_status()
+        chunks, size = [], 0
+        for chunk in response.iter_content(65536):
+            size += len(chunk)
+            if size > limit:
+                raise ArticleDownloadError(f'文件超过大小限制（{limit // 1024 // 1024} MB）：{url[:200]}')
+            chunks.append(chunk)
+        return b''.join(chunks), response.headers.get('Content-Type', ''), response.url
+
+
+def _image_ext(data: bytes, mime: str) -> str:
+    if data.startswith(b'\xff\xd8\xff'):
+        return '.jpg'
+    if data.startswith(b'\x89PNG\r\n\x1a\n'):
+        return '.png'
+    if data.startswith((b'GIF87a', b'GIF89a')):
+        return '.gif'
+    if data[:4] == b'RIFF' and data[8:12] == b'WEBP':
+        return '.webp'
+    if len(data) >= 12 and data[4:8] == b'ftyp' and data[8:12] in (b'avif', b'avis'):
+        return '.avif'
+    if b'<svg' in data[:1024].lower() and mime.split(';')[0] == 'image/svg+xml':
+        return '.svg'
+    raise ArticleDownloadError(f'返回内容不是支持的图片（Content-Type: {mime or "未知"}）')
+
+
+def _image_source(img) -> str:
+    for attr in ('data-src', 'data-original', 'data-actualsrc', 'data-lazy-src', 'src'):
+        value = img.get(attr)
+        if value and str(value).strip():
+            return str(value).strip()
+    srcset = img.get('data-srcset') or img.get('srcset')
+    if srcset:
+        return str(srcset).split(',')[-1].strip().split()[0]
+    return ''
+
+
+def save_article(article: dict, output_root: str | Path = '.', *, timeout: float = 30,
+                 allow_missing_images: bool = False) -> ArticleResult:
+    """从 API 正文保存文章；图片失败时清理半成品，不覆盖已有同名目录。"""
+    url = str(article.get('weburl') or article.get('url') or '')
+    title = str(article.get('title') or '').strip()
+    raw_content = str(article.get('content') or '').strip()
+    if not title or not raw_content:
+        raise ArticleDownloadError('新闻标题或正文为空')
+    if timeout <= 0:
+        raise ValueError('timeout 必须大于 0')
+    with _session() as session:
+        final_url = url
+        soup = BeautifulSoup('<article>' + raw_content + '</article>', 'html.parser')
+        content = soup.article
+        for tag in content.select('script, style, noscript, iframe, form, button'):
+            tag.decompose()
+        # 部分微信排版把插图放在行内 CSS 背景中，也转为普通图片参与下载。
+        for tag in list(content.find_all(style=True)):
+            for match in re.finditer(r'url\(\s*[\'\"]?(.*?)[\'\"]?\s*\)', str(tag['style'])):
+                source = match.group(1).strip()
+                if source and not source.startswith('#'):
+                    background = soup.new_tag('img', src=source, alt='')
+                    tag.insert(0, background)
+        if not content.get_text(strip=True) and not content.find('img'):
+            raise ArticleDownloadError('文章正文为空，可能需要浏览器执行 JavaScript')
+        for link in content.find_all('a', href=True):
+            href = str(link['href']).strip()
+            if href and not href.startswith('#'):
+                absolute = urljoin(final_url, href)
+                if urlsplit(absolute).scheme in ('http', 'https', 'mailto', 'tel'):
+                    link['href'] = absolute
+                else:
+                    del link['href']
+
+        root = Path(output_root).expanduser().resolve()
+        root.mkdir(parents=True, exist_ok=True)
+        stage = Path(tempfile.mkdtemp(prefix='.article-', dir=root))
+        images, warnings, downloaded = [], [], {}
+        try:
+            for img in list(content.find_all('img')):
+                source = _image_source(img)
+                if not source:
+                    img.replace_with(soup.new_string('[图片缺少下载地址]'))
+                    if not allow_missing_images:
+                        raise ArticleDownloadError('正文中存在没有下载地址的图片')
+                    warnings.append('正文中存在没有下载地址的图片')
+                    continue
+                image_url = urljoin(final_url, source)
+                try:
+                    if image_url not in downloaded:
+                        if source.startswith('data:image/'):
+                            header, payload = source.split(',', 1)
+                            data = base64.b64decode(payload, validate=True) if ';base64' in header else unquote(payload).encode()
+                            mime = header[5:].split(';')[0]
+                            if len(data) > 50 * 1024 * 1024:
+                                raise ArticleDownloadError('内嵌图片超过 50 MB')
+                        else:
+                            data, mime, _ = _get(session, image_url, timeout, 50 * 1024 * 1024, final_url)
+                        ext = _image_ext(data, mime)
+                        filename = f'image{len(images) + 1:03d}{ext}'
+                        (stage / filename).write_bytes(data)
+                        images.append(filename)
+                        downloaded[image_url] = filename
+                    # 清除懒加载/响应式属性，使转换器只引用本地文件。
+                    alt = str(img.get('alt') or '')
+                    img.attrs = {'src': downloaded[image_url], 'alt': alt}
+                except (requests.RequestException, ArticleDownloadError, ValueError) as exc:
+                    if not allow_missing_images:
+                        raise ArticleDownloadError(f'图片下载失败：{image_url[:200]}；{exc}') from exc
+                    warnings.append(f'图片下载失败：{image_url[:200]}；{exc}')
+                    img.replace_with(soup.new_string(f'[图片下载失败：{img.get("alt") or "未命名图片"}]'))
+
+            # 新闻图片后常带空的图注标题，避免生成单独的 ## 和多余空行。
+            for heading in content.select('h1, h2, h3, h4, h5, h6'):
+                if not heading.get_text(strip=True) and not heading.find('img'):
+                    heading.decompose()
+            body = markdownify(str(content), heading_style='ATX', bullets='-',
+                               strip=['picture', 'source']).strip()
+            # 微信富文本中各段常没有标准 p 标签；markdownify 仍会保留文字和图片顺序。
+            author = str(article.get('src') or '未知作者').strip() or '未知作者'
+            text = f'# {title}\n\n'
+            text += body + f'\n\n来源：{author}，如有侵权，请联系删除。\n'
+            basename = safe_name(title)
+            # 原子认领新目录，支持多个进程并发保存同名文章。
+            index = 1
+            while True:
+                name = basename if index == 1 else f'{basename} ({index})'
+                destination = root / name
+                try:
+                    destination.mkdir()
+                    break
+                except FileExistsError:
+                    index += 1
+            try:
+                (stage / f'{name}.md').write_text(text, encoding='utf-8')
+                for item in stage.iterdir():
+                    shutil.move(str(item), destination / item.name)
+            except BaseException:
+                shutil.rmtree(destination, ignore_errors=True)
+                raise
+            return ArticleResult(title, url, str(destination), str(destination / f'{name}.md'),
+                                 [str(destination / x) for x in images], warnings)
+        finally:
+            shutil.rmtree(stage, ignore_errors=True)

requirements.txt

@@ -0,0 +1,3 @@
+requests>=2.32,<3
+beautifulsoup4>=4.12,<5
+markdownify>=1.1,<2

start.command

@@ -0,0 +1,8 @@
+#!/bin/zsh
+cd "${0:A:h}"
+if ! command -v uv >/dev/null 2>&1; then
+  print '未找到 uv，请先安装 uv 后再启动。'
+  read '?按回车关闭…'
+  exit 1
+fi
+uv run --no-project --python 3.12 --with-requirements requirements.txt python app.py

static/app.js

@@ -0,0 +1,79 @@
+'use strict';
+const $ = id => document.getElementById(id);
+let token = '', running = false, starting = false, loadingChannels = false, loadedKey = '', lastLog = -1;
+const labels = {idle:'等待开始',running:'下载中',completed:'已完成',partial:'未达到目标',cancelled:'已停止',error:'任务中断'};
+const key = () => $('key').value.trim();
+function controls() {
+  const busy = running || starting;
+  $('start').disabled = busy || !loadedKey || loadedKey !== key() || !$('channel').value;
+  $('stop').disabled = !running;
+  $('channels').disabled = busy || loadingChannels;
+  for (const id of ['key','channel','target']) $(id).disabled = busy || (id === 'channel' && !loadedKey);
+}
+async function post(path, body) {
+  const response = await fetch('/api/' + path, {method:'POST',headers:{'Content-Type':'application/json','X-Session-Token':token},body:JSON.stringify(body)});
+  const data = await response.json();
+  if (!response.ok) throw new Error(data.error || '请求失败');
+  return data;
+}
+$('key').addEventListener('input', () => {loadedKey=''; $('channel').replaceChildren(new Option('请重新获取频道','')); controls();});
+$('channels').addEventListener('click', async () => {
+  loadingChannels=true; controls(); $('notice').textContent='正在获取频道…';
+  const submittedKey = key();
+  try {
+    const data = await post('channels',{key:submittedKey});
+    if (key() !== submittedKey) return;
+    $('channel').replaceChildren(...data.channels.map(c => new Option(c,c)));
+    loadedKey=submittedKey; $('notice').textContent=`已获取 ${data.channels.length} 个频道。`;
+    $('debug-channel').value=$('channel').value;
+  } catch(error) {$('notice').textContent=error.message;}
+  finally {loadingChannels=false; controls();}
+});
+$('channel').addEventListener('change', () => {$('debug-channel').value=$('channel').value; controls();});
+$('download-form').addEventListener('submit', async event => {
+  event.preventDefault(); starting=true; controls(); $('notice').textContent='';
+  try {await post('start',{key:key(),channel:$('channel').value,target:Number($('target').value)}); await refresh();}
+  catch(error) {$('notice').textContent=error.message;}
+  finally {starting=false; controls();}
+});
+$('stop').addEventListener('click', async () => {
+  $('stop').disabled=true;
+  try {await post('stop',{}); $('notice').textContent='正在停止，等待当前请求或文章保存结束…';}
+  catch(error) {$('notice').textContent=error.message;}
+});
+$('debug-form').addEventListener('submit', async event => {
+  event.preventDefault(); $('debug-submit').disabled=true; $('debug-output').textContent='正在请求…';
+  try {
+    const data=await post('debug',{key:key(),endpoint:$('endpoint').value,channel:$('debug-channel').value,num:Number($('debug-num').value),start:Number($('debug-start').value)});
+    const output=JSON.stringify(data,null,2);
+    $('debug-output').textContent=output.slice(0,100000)+(output.length>100000?'\n…输出超过 100000 字符，已截断。':'');
+  } catch(error) {$('debug-output').textContent=error.message;}
+  finally {$('debug-submit').disabled=false;}
+});
+async function refresh() {
+  const response=await fetch('/api/state');
+  if (!response.ok) throw new Error('无法读取下载状态');
+  const data=await response.json();
+  running=data.status==='running'; controls();
+  const percent=data.target ? Math.floor(data.saved/data.target*100) : 0;
+  $('status').textContent=labels[data.status] || data.status;
+  $('progress-text').textContent=`下载新闻 ${data.saved}/${data.target} 条`;
+  $('percent').textContent=`已完成 ${percent}%`; $('progress').value=percent;
+  for (const id of ['saved','skipped','failed','requests','message','output']) $(id).textContent=data[id];
+  const latest=data.logs.length ? data.logs[data.logs.length-1].id : 0;
+  if (latest!==lastLog) {
+    const logs=$('logs'); const bottom=logs.scrollHeight-logs.scrollTop-logs.clientHeight<40;
+    logs.replaceChildren(...data.logs.map(log => {const row=document.createElement('div'); const stamp=document.createElement('time'); stamp.textContent=log.time; row.append(stamp,document.createTextNode(log.message)); return row;}));
+    if (bottom) logs.scrollTop=logs.scrollHeight;
+    lastLog=latest;
+  }
+}
+async function poll() {
+  try {await refresh();}
+  catch(error) {$('notice').textContent='与本地服务连接中断，请检查启动窗口。';}
+  setTimeout(poll,700);
+}
+(async () => {
+  try {const response=await fetch('/api/session'); token=(await response.json()).token; await poll();}
+  catch(error) {$('notice').textContent='初始化失败，请刷新页面。';}
+})();

static/index.html

@@ -0,0 +1,14 @@
+<!doctype html>
+<html lang="zh-CN">
+<head><meta charset="UTF-8"><meta name="viewport" content="width=device-width, initial-scale=1"><title>探数 · 新闻下载器</title><link rel="stylesheet" href="/style.css"><script src="/app.js" defer></script></head>
+<body><main>
+<header><div class="brand">TANSHU / NEWS ARCHIVE</div><h1>把新闻，保存到本地。</h1><p>选择频道，收集新闻。正文与图片一起保存，自动跳过已下载的内容。</p></header>
+<section class="card"><div class="section-heading"><h2>下载设置</h2><span class="badge">本地运行</span></div>
+<form id="download-form"><label for="key">API Key</label><div class="key-row"><input id="key" type="password" placeholder="填写探数 API Key" autocomplete="off" required maxlength="512"><button id="channels" type="button" class="secondary">获取新闻频道</button></div><p class="hint">密钥仅用于本次页面会话，不写入文件或浏览器存储。</p>
+<div class="fields"><div><label for="channel">新闻频道</label><select id="channel" required disabled><option value="">先填写密钥并获取频道</option></select></div><div><label for="target">下载数量</label><input id="target" type="number" min="1" max="100000" step="1" value="10" required></div></div>
+<p class="hint">接口每次最多 40 条，起始位置最多 400。遇到重复会继续向后查找；可用新闻不足时会说明原因。</p><div class="actions"><button id="start" disabled>开始下载</button><button id="stop" type="button" class="secondary" disabled>停止下载</button></div><p id="notice" role="alert"></p></form></section>
+<section class="card"><div class="section-heading"><h2>下载进度</h2><span id="status" class="badge">等待开始</span></div><div class="progress-heading"><strong id="progress-text">下载新闻 0/0 条</strong><span id="percent">已完成 0%</span></div><progress id="progress" max="100" value="0" aria-label="下载进度"></progress><div class="stats"><span>已保存 <b id="saved">0</b></span><span>重复跳过 <b id="skipped">0</b></span><span>保存失败 <b id="failed">0</b></span><span>接口请求 <b id="requests">0</b></span></div><p id="message">等待开始</p><p class="hint path">保存位置：<span id="output">加载中…</span></p></section>
+<section class="card"><div class="section-heading"><h2>运行输出</h2><span class="hint">保留最近 300 条</span></div><div id="logs" class="console" role="log" aria-label="运行输出"><div class="empty">运行情况将在这里显示。</div></div></section>
+<details class="card"><summary>调试终端 <span>手动请求接口</span></summary><p class="hint">使用上方 API Key。请求会消耗接口调用额度，但不会保存新闻。</p><form id="debug-form"><div class="fields"><div><label for="endpoint">请求接口</label><select id="endpoint"><option value="channel">新闻频道查询 /channel</option><option value="index">新闻列表查询 /index</option></select></div><div><label for="debug-channel">type · 频道</label><input id="debug-channel" value="头条"></div><div><label for="debug-num">num · 数量（1–40）</label><input id="debug-num" type="number" min="1" max="40" value="1" required></div><div><label for="debug-start">start · 起始位置（0–400）</label><input id="debug-start" type="number" min="0" max="400" value="0" required></div></div><button id="debug-submit" class="secondary">发送测试请求</button></form><pre id="debug-output" class="console">尚未发送请求。</pre></details>
+<footer>Markdown 正文 · 本地图片 · 历史去重</footer>
+</main></body></html>

static/style.css

@@ -0,0 +1 @@
+:root{font-family:Inter,-apple-system,BlinkMacSystemFont,"Segoe UI","PingFang SC",sans-serif;color:#20312e;background:#f3f6f3;font-size:15px}*{box-sizing:border-box}body{margin:0}main{max-width:960px;margin:0 auto;padding:48px 24px 24px}header{margin-bottom:32px}.brand{font-size:12px;letter-spacing:2px;font-weight:700;color:#437b69}h1{font-size:36px;letter-spacing:-1px;margin:14px 0}header p{color:#687a74;line-height:1.8}h2{font-size:17px;margin:0}.card{padding:26px 28px;background:#fff;border:1px solid #dfe7e1;border-radius:16px;margin-bottom:20px;box-shadow:0 4px 16px #243c2904}.section-heading{display:flex;align-items:center;justify-content:space-between;gap:10px;margin-bottom:24px}.badge{background:#edf4ef;color:#4c7564;padding:6px 10px;border-radius:20px;font-size:12px}.key-row{display:flex;gap:12px}.key-row input{flex:1;min-width:0}label{display:block;font-weight:600;margin-bottom:9px;font-size:13px}input,select,button{font:inherit;border-radius:8px}input,select{background:#fbfcfb;border:1px solid #d7e0d9;width:100%;padding:12px;color:inherit}input:focus,select:focus,button:focus-visible,summary:focus-visible{outline:3px solid #8dcab5;outline-offset:2px}button{cursor:pointer;border:1px solid #286850;background:#286850;color:#fff;padding:12px 22px;font-weight:600;white-space:nowrap}.secondary{background:#fff;color:#345e4f;border-color:#cedbd3}button:disabled{opacity:.45;cursor:not-allowed}.fields{display:grid;grid-template-columns:1fr 1fr;gap:20px;margin:24px 0 16px}.hint{font-size:12px;color:#79867f;line-height:1.8;margin:10px 0}.actions{display:flex;gap:12px;margin-top:22px}#notice{color:#ac4c32;font-size:13px;white-space:pre-wrap}#notice:empty{display:none}.progress-heading{display:flex;justify-content:space-between;gap:16px;font-size:14px}#percent{color:#3f7b60}progress{display:block;width:100%;height:12px;border:0;border-radius:10px;overflow:hidden;margin:18px 0 20px;background:#edf2ef}progress::-webkit-progress-bar{background:#edf2ef}progress::-webkit-progress-value{background:#3c8b68;transition:width .25s}progress::-moz-progress-bar{background:#3c8b68}.stats{display:flex;gap:28px;flex-wrap:wrap;color:#7a8880;font-size:13px}.stats b{color:#304e40;margin-left:8px;font-size:17px}#message{font-size:13px;color:#52685b;line-height:1.8}.path{overflow-wrap:anywhere}.console{height:260px;overflow:auto;background:#15251f;color:#c5dacf;border-radius:10px;padding:18px;font:12px/1.9 ui-monospace,SFMono-Regular,Consolas,monospace;white-space:pre-wrap;overflow-wrap:anywhere}.console time{color:#7eac94;margin-right:14px}.empty{color:#7e9889}summary{cursor:pointer;font-weight:600}summary span{font-size:12px;font-weight:400;color:#79867f;margin-left:12px}#debug-output{margin-bottom:0;height:300px}footer{text-align:center;color:#89958e;font-size:12px;margin:32px 0}@media(max-width:600px){main{padding:26px 14px}h1{font-size:28px}.card{padding:22px 18px}.key-row{flex-direction:column}.fields{grid-template-columns:1fr}.stats{gap:14px}.progress-heading{font-size:12px}}

tests/test_downloader.py

@@ -0,0 +1,164 @@
+import base64
+import json
+from pathlib import Path
+import sys
+import tempfile
+import threading
+import unittest
+from http.server import ThreadingHTTPServer
+
+import requests
+sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
+from app import Downloader, make_handler, fingerprints
+from article_writer import save_article
+
+
+def article(i):
+    return dict(title=f'新闻{i}', content=f'<p>第{i}篇新闻内容</p>', url=f'https://example.com/{i}', src='测试来源', time='2026-10-31')
+
+
+class DownloadTests(unittest.TestCase):
+    def setUp(self):
+        self.temp = tempfile.TemporaryDirectory()
+        self.root = Path(self.temp.name)
+
+    def tearDown(self):
+        self.temp.cleanup()
+
+    def run_job(self, manager, target):
+        manager.start('test-key', '头条', target)
+        manager.thread.join(10)
+        self.assertFalse(manager.thread.is_alive())
+        return manager.snapshot()
+
+    def test_format_and_local_image(self):
+        data = article(1)
+        image = 'data:image/png;base64,' + base64.b64encode(b'\x89PNG\r\n\x1a\nexample').decode()
+        data['content'] += f'<img src="{image}"><img src="{image}">'
+        result = save_article(data, self.root)
+        body = Path(result.markdown).read_text()
+        self.assertTrue(body.startswith('# 新闻1\n\n第1篇新闻内容'))
+        self.assertNotIn('发布时间：2026-10-31', body)
+        self.assertNotIn('\n---\n', body)
+        self.assertTrue(body.endswith('来源：测试来源，如有侵权，请联系删除。\n'))
+        self.assertEqual(len(result.images), 1)
+        self.assertEqual(body.count('image001.png'), 2)
+        self.assertNotIn('data:image', body)
+        second = save_article(data, self.root)
+        self.assertEqual(Path(second.markdown).name, '新闻1 (2).md')
+
+    def test_bad_image_cleans_partial(self):
+        data = article(1)
+        data['content'] += '<img src="data:image/png;base64,aW52YWxpZA==">'
+        with self.assertRaises(Exception):
+            save_article(data, self.root)
+        self.assertEqual(list(self.root.iterdir()), [])
+
+    def test_duplicate_then_fill_and_restart(self):
+        items = [article(1), article(1), article(2), article(3)]
+        calls = []
+        def request(key, endpoint, params):
+            calls.append(params.copy())
+            return {'code': 1, 'data': {'list': items[params['start']:params['start']+params['num']]}}
+        manager = Downloader(self.root, request)
+        state = self.run_job(manager, 3)
+        self.assertEqual((state['status'], state['saved'], state['skipped']), ('completed', 3, 1))
+        self.assertEqual([c['start'] for c in calls], [0, 3])
+        restarted = Downloader(self.root, request)
+        state = self.run_job(restarted, 3)
+        self.assertEqual((state['status'], state['saved'], state['skipped']), ('partial', 0, 4))
+        self.assertEqual(len(list(self.root.glob('*/*.md'))), 3)
+
+    def test_offset_limit_and_partial_progress(self):
+        offsets = []
+        def request(key, endpoint, params):
+            offsets.append(params['start'])
+            return {'code': 1, 'data': {'list': [article(i) for i in range(params['start'], params['start']+params['num'])]}}
+        state = self.run_job(Downloader(self.root, request), 500)
+        self.assertEqual(state['status'], 'partial')
+        self.assertEqual(state['saved'], 440)
+        self.assertEqual(max(offsets), 400)
+
+    def test_error_and_key_redaction(self):
+        def request(*args):
+            return {'code': 10001, 'msg': 'invalid test-key'}
+        manager = Downloader(self.root, request)
+        state = self.run_job(manager, 10)
+        self.assertEqual(state['status'], 'error')
+        self.assertNotIn('test-key', json.dumps(state))
+
+    def test_stop_and_concurrent_start(self):
+        entered, release = threading.Event(), threading.Event()
+        def request(*args):
+            entered.set()
+            release.wait(2)
+            return {'code': 1, 'data': {'list': [article(1)]}}
+        manager = Downloader(self.root, request)
+        manager.start('key', '头条', 10)
+        self.assertTrue(entered.wait(2))
+        with self.assertRaises(ValueError):
+            manager.start('key', '头条', 10)
+        manager.stop.set()
+        release.set()
+        manager.thread.join(3)
+        self.assertEqual(manager.snapshot()['status'], 'cancelled')
+        self.assertEqual(manager.snapshot()['saved'], 0)
+
+    def test_failed_save_can_retry_next_job(self):
+        attempts = []
+        def writer(data, root):
+            attempts.append(1)
+            if len(attempts) == 1:
+                raise ValueError('模拟图片失败')
+            return save_article(data, root)
+        def request(key, endpoint, params):
+            return {'code': 1, 'data': {'list': [article(1)] if params['start'] == 0 else []}}
+        manager = Downloader(self.root, request, writer)
+        self.assertEqual(self.run_job(manager, 1)['failed'], 1)
+        self.assertEqual(self.run_job(manager, 1)['saved'], 1)
+
+    def test_fingerprint_content_and_tracking(self):
+        a, b = article(1), article(2)
+        b['content'] = a['content'].replace('<p>', '<div>').replace('</p>', '</div>')
+        self.assertTrue(set(fingerprints(a)) & set(fingerprints(b)))
+        b['content'] = '其他内容'
+        b['url'] = a['url'] + '?utm_source=abc#fragment'
+        self.assertTrue(set(fingerprints(a)) & set(fingerprints(b)))
+
+    def test_log_cap(self):
+        manager = Downloader(self.root)
+        for i in range(400):
+            manager.log(str(i))
+        self.assertEqual(len(manager.snapshot()['logs']), 300)
+        self.assertEqual(manager.snapshot()['logs'][0]['message'], '100')
+
+    def test_http_routes_and_access(self):
+        def request(key, endpoint, params=None):
+            return {'code': 1, 'data': {'list': ['头条', '科技']}}
+        manager = Downloader(self.root, request)
+        server = ThreadingHTTPServer(('127.0.0.1', 0), make_handler(manager, 'token'))
+        worker = threading.Thread(target=server.serve_forever, daemon=True)
+        worker.start()
+        url = f'http://127.0.0.1:{server.server_port}'
+        try:
+            self.assertIn('新闻下载器', requests.get(url).text)
+            self.assertEqual(requests.get(url+'/api/state').json()['status'], 'idle')
+            self.assertEqual(requests.post(url+'/api/channels', json={'key': 'test'}).status_code, 403)
+            self.assertEqual(requests.get(url, headers={'Host':'evil.example'}).status_code, 403)
+            headers = {'X-Session-Token': 'token'}
+            self.assertEqual(requests.post(url+'/api/channels', headers=headers, json={'key':'test'}).json()['channels'], ['头条','科技'])
+            invalid = requests.post(url+'/api/start', headers=headers, json={'key':'test', 'target':0,'channel':'头条'})
+            self.assertEqual(invalid.status_code, 400)
+            invalid = requests.post(url+'/api/debug', headers=headers, json={'key':'test','endpoint':'index','num':41,'start':0})
+            self.assertEqual(invalid.status_code, 400)
+            debug = requests.post(url+'/api/debug', headers=headers, json={'key':'test','endpoint':'channel'})
+            self.assertEqual(debug.status_code, 200)
+            self.assertNotIn('key=test', debug.text)
+        finally:
+            server.shutdown()
+            server.server_close()
+            worker.join()
+
+
+if __name__ == '__main__':
+    unittest.main()