#!/usr/bin/env python3
"""仅在本机运行的探数新闻下载器。"""
from contextlib import contextmanager
from collections import deque
from datetime import datetime
from hashlib import sha256
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
from pathlib import Path
from urllib.parse import urlsplit, urlunsplit, parse_qsl, urlencode
import argparse
import json
import secrets
import sqlite3
import threading
import time
import unicodedata
import webbrowser

import requests
from bs4 import BeautifulSoup
from article_writer import save_article

ROOT = Path(__file__).resolve().parent
API = 'https://api.tanshuapi.com/api/toutiao/v1/'


def api_request(key, endpoint, params=None):
    if endpoint not in ('channel', 'index'):
        raise ValueError('未知接口')
    try:
        # 不自动重试计费接口，避免一次操作产生不可见的重复调用。
        response = requests.get(API + endpoint, params={'key': key, **(params or {})}, timeout=(10, 40))
        response.raise_for_status()
        payload = response.json()
    except requests.RequestException:
        raise ValueError('接口网络请求失败或超时，请在调试终端检查后重试') from None
    except ValueError:
        raise ValueError('接口未返回有效 JSON') from None
    if not isinstance(payload, dict):
        raise ValueError('接口响应格式错误')
    return payload


def result_list(payload):
    if payload.get('code') != 1:
        raise ValueError(f"探数接口错误 {payload.get('code')}：{payload.get('msg', '未知错误')}")
    data = payload.get('data')
    if not isinstance(data, dict) or not isinstance(data.get('list'), list):
        raise ValueError('接口响应缺少 data.list')
    return data['list']


def fingerprints(article):
    values = []
    for field in ('url', 'weburl'):
        value = str(article.get(field) or '').strip()
        if value:
            u = urlsplit(value)
            query = [(k, v) for k, v in parse_qsl(u.query) if not k.lower().startswith('utm_') and k.lower() not in ('spm', 'from', 'pos', 'vt')]
            values.append('url:' + urlunsplit(('', u.netloc.lower(), u.path, urlencode(sorted(query)), '')))
    body = BeautifulSoup(str(article.get('content') or ''), 'html.parser').get_text(' ', strip=True)
    body = ''.join(unicodedata.normalize('NFKC', body).split())
    if body:
        values.append('body:' + body)
    return [sha256(value.encode()).hexdigest() for value in values]


class Downloader:
    def __init__(self, root=ROOT / 'downloads', requester=api_request, writer=save_article, on_saved=None):
        self.root = Path(root).resolve()
        self.root.mkdir(parents=True, exist_ok=True)
        self.requester, self.writer = requester, writer
        self.on_saved = on_saved
        self.lock = threading.RLock()
        self.stop = threading.Event()
        self.thread = None
        self.logs = deque(maxlen=300)
        self.seq = 0
        self.state = dict(status='idle', saved=0, target=0, skipped=0, failed=0, scanned=0, requests=0, channel='', message='等待开始')
        with self.db() as db:
            db.execute('CREATE TABLE IF NOT EXISTS seen (fingerprint TEXT PRIMARY KEY, directory TEXT NOT NULL)')

    @contextmanager
    def db(self):
        connection = sqlite3.connect(self.root / '.history.sqlite3')
        try:
            with connection:
                yield connection
        finally:
            connection.close()

    def log(self, message):
        with self.lock:
            self.seq += 1
            self.logs.append(dict(id=self.seq, time=datetime.now().strftime('%H:%M:%S'), message=message))

    def update(self, **values):
        with self.lock:
            self.state.update(values)

    def snapshot(self):
        with self.lock:
            return {**self.state, 'logs': list(self.logs), 'output': str(self.root)}

    def start(self, key, channel, target):
        with self.lock:
            if self.thread and self.thread.is_alive():
                raise ValueError('已有任务正在运行')
            self.stop.clear()
            self.logs.clear()
            self.state = dict(status='running', saved=0, target=target, skipped=0, failed=0, scanned=0, requests=0, channel=channel, message='正在下载')
            self.thread = threading.Thread(target=self.run, args=(key, channel, target), daemon=True)
            self.thread.start()

    def run(self, key, channel, target):
        saved = skipped = failed = scanned = calls = 0
        seen_in_run = set()
        self.log(f'开始下载频道「{channel}」，目标 {target} 条；历史重复新闻将跳过')
        try:
            with self.db() as db:
                offset = 0
                while offset <= 400 and saved < target and not self.stop.is_set():
                    count = min(40, target - saved)
                    calls += 1
                    self.update(requests=calls)
                    self.log(f'请求第 {calls} 页：start={offset}，num={count}')
                    items = result_list(self.requester(key, 'index', {'type': channel, 'num': count, 'start': offset}))
                    if not items:
                        self.log('接口返回空列表，已停止')
                        break
                    for article in items:
                        if saved >= target or self.stop.is_set():
                            break
                        scanned += 1
                        try:
                            if not isinstance(article, dict):
                                raise ValueError('新闻条目格式错误')
                            keys = fingerprints(article)
                            existing = any(k in seen_in_run for k in keys)
                            for k in keys:
                                row = db.execute('SELECT directory FROM seen WHERE fingerprint=?', (k,)).fetchone()
                                if row and Path(row[0]).is_dir():
                                    existing = True
                            seen_in_run.update(keys)
                            if existing:
                                skipped += 1
                                self.log('跳过重复新闻：' + str(article.get('title', '无标题'))[:160])
                                continue
                            result = self.writer(article, self.root)
                            try:
                                db.executemany('INSERT OR REPLACE INTO seen VALUES (?, ?)', [(k, result.directory) for k in keys])
                                db.commit()
                            except Exception:
                                # 保存目录尚未登记成功，回滚本次产物，避免下次重复保存。
                                import shutil
                                shutil.rmtree(result.directory, ignore_errors=True)
                                raise
                            saved += 1
                            if self.on_saved:
                                try:
                                    self.on_saved(article, result, keys)
                                except Exception as error:
                                    self.log('文章已保存，入队需处理：' + str(error).replace(key, '[已隐藏]')[:300])
                            self.log(f'下载新闻 {saved}/{target} 条：{result.title}')
                        except Exception as exc:
                            failed += 1
                            self.log(f'保存失败：{str(exc).replace(key, "[已隐藏]")[:400]}')
                        finally:
                            self.update(saved=saved, skipped=skipped, failed=failed, scanned=scanned)
                    # 避免越过 start=400；已消费的条目决定下一起始位置。
                    offset += len(items)
                    if offset > 400:
                        self.log('已到达接口允许的分页范围')
                    if saved < target and offset <= 400:
                        self.stop.wait(0.3)
            status = 'completed' if saved >= target else 'cancelled' if self.stop.is_set() else 'partial'
            message = f'已保存 {saved}/{target} 条，跳过重复 {skipped} 条，失败 {failed} 条'
            if status == 'partial':
                message += '；当前可获取新闻不足，未达到目标数量'
            if status == 'cancelled':
                message += '；任务已停止'
            self.update(status=status, message=message)
            self.log(message)
        except Exception as exc:
            message = str(exc).replace(key, '[已隐藏]')[:500]
            self.update(status='error', message=message)
            self.log('任务中断：' + message)


def make_handler(manager, token):
    class Handler(BaseHTTPRequestHandler):
        def log_message(self, *args):
            pass

        def send(self, status, payload, mime='application/json; charset=utf-8'):
            data = json.dumps(payload, ensure_ascii=False).encode() if isinstance(payload, dict) else payload
            self.send_response(status)
            self.send_header('Content-Type', mime)
            self.send_header('Content-Length', str(len(data)))
            self.send_header('Cache-Control', 'no-store')
            self.send_header('X-Content-Type-Options', 'nosniff')
            self.end_headers()
            self.wfile.write(data)

        def local(self):
            return self.headers.get('Host') in (f'127.0.0.1:{self.server.server_port}', f'localhost:{self.server.server_port}')

        def do_GET(self):
            if not self.local():
                return self.send(403, {'error': '仅允许本机访问'})
            path = urlsplit(self.path).path
            if path == '/api/state':
                return self.send(200, manager.snapshot())
            if path == '/api/session':
                return self.send(200, {'token': token})
            files = {'/': ('index.html', 'text/html; charset=utf-8'), '/app.js': ('app.js', 'text/javascript; charset=utf-8'), '/style.css': ('style.css', 'text/css; charset=utf-8')}
            if path not in files:
                return self.send(404, {'error': '未找到页面'})
            name, mime = files[path]
            self.send(200, (ROOT / 'static' / name).read_bytes(), mime)

        def do_POST(self):
            if not self.local() or self.headers.get('X-Session-Token') != token:
                return self.send(403, {'error': '页面会话失效，请刷新页面'})
            key = ''
            try:
                length = int(self.headers.get('Content-Length', 0))
                if not 0 < length <= 16384:
                    raise ValueError('请求大小无效')
                body = json.loads(self.rfile.read(length))
                if not isinstance(body, dict):
                    raise ValueError('请求必须是 JSON 对象')
                if self.path == '/api/stop':
                    manager.stop.set()
                    return self.send(200, {'ok': True})
                key = str(body.get('key') or '').strip()
                if not key or len(key) > 512:
                    raise ValueError('请填写有效 API Key')
                if self.path == '/api/channels':
                    channels = result_list(manager.requester(key, 'channel'))
                    if not channels or not all(isinstance(c, str) and c for c in channels):
                        raise ValueError('频道列表为空或格式错误')
                    return self.send(200, {'channels': channels})
                if self.path == '/api/start':
                    target = body.get('target')
                    channel = body.get('channel')
                    if type(target) is not int or not 1 <= target <= 100000:
                        raise ValueError('下载数量须为 1 至 100000 的整数')
                    if not isinstance(channel, str) or not channel.strip() or len(channel) > 100:
                        raise ValueError('请选择新闻频道')
                    manager.start(key, channel, target)
                    return self.send(200, {'ok': True})
                if self.path == '/api/debug':
                    endpoint = body.get('endpoint')
                    params = {}
                    if endpoint == 'index':
                        for field, low, high in [('num', 1, 40), ('start', 0, 400)]:
                            value = body.get(field)
                            if type(value) is not int or not low <= value <= high:
                                raise ValueError(f'{field} 必须在 {low}–{high} 之间')
                            params[field] = value
                        params['type'] = str(body.get('channel') or '')
                    before = time.monotonic()
                    payload = manager.requester(key, endpoint, params)
                    # 服务端响应同样隐藏 key，且只作为文本展示，不渲染上游 HTML。
                    clean = json.loads(json.dumps(payload, ensure_ascii=False).replace(key, '[已隐藏]'))
                    return self.send(200, {'elapsed': round(time.monotonic() - before, 2), 'request': API + str(endpoint) + '?' + urlencode({'key': '[已隐藏]', **params}), 'response': clean})
                self.send(404, {'error': '未知操作'})
            except Exception as exc:
                message = str(exc)
                if key:
                    message = message.replace(key, '[已隐藏]')
                self.send(400, {'error': message[:500]})
    return Handler


def main():
    parser = argparse.ArgumentParser(description=__doc__)
    parser.add_argument('--port', type=int, default=8765)
    parser.add_argument('--no-browser', action='store_true')
    args = parser.parse_args()
    manager = Downloader()
    server = ThreadingHTTPServer(('127.0.0.1', args.port), make_handler(manager, secrets.token_urlsafe(32)))
    url = f'http://127.0.0.1:{server.server_port}'
    print(f'新闻下载器：{url}\n保存目录：{manager.root}\n按 Ctrl+C 退出', flush=True)
    if not args.no_browser:
        webbrowser.open(url)
    try:
        server.serve_forever()
    except KeyboardInterrupt:
        manager.stop.set()
        if manager.thread:
            manager.thread.join(timeout=45)
    finally:
        server.server_close()


if __name__ == '__main__':
    main()