diff --git a/README.md b/README.md index bdad4c2..6be0d60 100644 --- a/README.md +++ b/README.md @@ -69,7 +69,7 @@ downloads/ 本工具下载结果（本地） - `test/history/` 保存从本目录移入的两张演示截图与演示任务 JSON。根目录其他测试文件未搬动。 - `发布去重记录.json` 是实际受理证据，继续留在原处，并在服务启动时迁入去重库。它不代表完整历史已对账。原文件可找到时还建立内容指纹；原文件丢失时仍保留原 Markdown 摘要。 -- 原始下载文件不改写。长标题生成保守 UTF-16 长度不超过 80 的派生标题并加 `...`，原题仍展示在队列；网站真实计数规则尚需实测。 +- 原始下载文件不改写。标题按 UTF-8 字节数计长：ASCII 为 1，中文和中文标点为 3，emoji 通常为 4。超过 75 时保留不超过 75 的完整前缀，再加 `...`，总长最多 78；原题仍保留。 - 排版仅处理与文章来源一致的末尾标准尾注，保留正文中的其他“来源”文字。图片顺序、标识改变会回滚排版。 - WebP/AVIF/SVG 当前明确拦截，需用户另存支持格式并更新 Markdown；没有静默丢图或自动转码。单图须小于 20 MiB，图片 base64 总大小不超过 48 MiB。 - 本地库含文章内容快照和账号标识，不含账号密码、Cookie 或上传凭据。运行数据和历史截图已加入忽略规则。 @@ -122,3 +122,9 @@ node test/browser.test.cjs ## 0.2.1 临时审核提示规则 更新油猴中的 `browser/auto-news.user.js` 至 0.2.1，并重启本地服务、重新配对后刷新论坛页面。连接脚本无需更新。结果适配器和执行器中的旧详情页核对代码以行注释保留；后端旧核对接口继续保留供人工历史补核使用。自动受理使用当前任务的人工提交批次校验，审核提示可不含帖子链接，不创建虚构帖子 ID。 + +## 标题自动截断 + +自动队列、手动目录和文章包导入均应用上述规则；已截断的标题不会重复添加省略号。已有队列在领取未开始的任务时更新标题，已导入、待审核或已提交任务保持现场标题。 + +应用更新：重启本地服务，并在油猴重新安装 `browser/auto-news.user.js`。原始 Markdown 不改写。 diff --git a/articles.py b/articles.py index 53d5023..8feba2e 100644 --- a/articles.py +++ b/articles.py @@ -56,15 +56,18 @@ def validate_images(body, directory): raise ValueError('配图为空或达到 20 MiB：' + target) -def short_title(title, budget=80): - # Conservative UTF-16 budget: do not split astral characters / surrogate pairs. - if len(title.encode('utf-16-le')) // 2 <= budget: +def short_title(title, budget=75): + """Keep a UTF-8 prefix within budget, adding three ASCII dots if shortened.""" + if len(title.encode('utf-8')) <= budget or (title.endswith('...') and len(title.encode('utf-8')) <= budget + 3): return title out = '' + used = 0 for ch in title: - if len((out + ch).encode('utf-16-le')) // 2 > budget - 3: + size = len(ch.encode('utf-8')) + if used + size > budget: break out += ch + used += size return out + '...' diff --git a/browser/auto-news.user.js b/browser/auto-news.user.js index bdf58f3..0246558 100644 --- a/browser/auto-news.user.js +++ b/browser/auto-news.user.js @@ -153,12 +153,31 @@ return output.join('\n').trim(); } + function shortTitle(title, budget = 75) { + const size = ch => { + const code = ch.codePointAt(0); + return code <= 0x7f ? 1 : code <= 0x7ff ? 2 : code <= 0xffff ? 3 : 4; + }; + // A generated prefix plus "..." (up to 78 bytes) is already safe. + const total = Array.from(title).reduce((sum, ch) => sum + size(ch), 0); + if (total <= budget || (title.endsWith('...') && total <= budget + 3)) return title; + let prefix = '', used = 0; + for (const ch of title) { + const bytes = size(ch); + if (used + bytes > budget) break; + prefix += ch; + used += bytes; + } + return prefix + '...'; + } + function compile(markdown, mdPath, fileMap, titleOverride) { let body = String(markdown).replace(/^\uFEFF/, '').replace(/\r\n?/g, '\n'); const firstHeading = body.match(/^\s*#\s+([^\n]+)\n?/); - const title = String(titleOverride || firstHeading?.[1] || mdPath.split('/').pop().replace(/\.md$/i, '')).trim(); + const originalTitle = String(titleOverride || firstHeading?.[1] || mdPath.split('/').pop().replace(/\.md$/i, '')).trim(); + const title = shortTitle(originalTitle); if (firstHeading) body = body.slice(firstHeading[0].length).trim(); - if (!title || Array.from(title).length > 80) throw new Error('标题为空或超过 80 字，请修改 Markdown 标题后重试。'); + if (!title) throw new Error('标题为空，请修改 Markdown 标题后重试。'); if (!body.trim()) throw new Error('正文为空。'); const warnings = []; if (/<\/?[a-z][^>]*>/i.test(body)) warnings.push('存在 HTML，将作为文字保留；请检查排版。'); @@ -531,7 +550,7 @@ $('diagnostics').onclick = () => downloadJSON({ ...summary(), logs }, 'dww11-import-diagnostics.json'); win.addEventListener('pagehide', releasePreviews, { once: true }); } - return { VERSION, normalizePath, imagePath, readLink, markdownToBBCode, compile, prepareFiles, preparePackage, parseAid, attachmentId, verifyEditorImages, mount }; + return { VERSION, shortTitle, normalizePath, imagePath, readLink, markdownToBBCode, compile, prepareFiles, preparePackage, parseAid, attachmentId, verifyEditorImages, mount }; }); // Extracted from dww11-wechat-oneclick.user.js v1.0.9; package source is authoritative. diff --git a/browser/importer.js b/browser/importer.js index 2b9a4b0..7582b9e 100644 --- a/browser/importer.js +++ b/browser/importer.js @@ -153,12 +153,31 @@ return output.join('\n').trim(); } + function shortTitle(title, budget = 75) { + const size = ch => { + const code = ch.codePointAt(0); + return code <= 0x7f ? 1 : code <= 0x7ff ? 2 : code <= 0xffff ? 3 : 4; + }; + // A generated prefix plus "..." (up to 78 bytes) is already safe. + const total = Array.from(title).reduce((sum, ch) => sum + size(ch), 0); + if (total <= budget || (title.endsWith('...') && total <= budget + 3)) return title; + let prefix = '', used = 0; + for (const ch of title) { + const bytes = size(ch); + if (used + bytes > budget) break; + prefix += ch; + used += bytes; + } + return prefix + '...'; + } + function compile(markdown, mdPath, fileMap, titleOverride) { let body = String(markdown).replace(/^\uFEFF/, '').replace(/\r\n?/g, '\n'); const firstHeading = body.match(/^\s*#\s+([^\n]+)\n?/); - const title = String(titleOverride || firstHeading?.[1] || mdPath.split('/').pop().replace(/\.md$/i, '')).trim(); + const originalTitle = String(titleOverride || firstHeading?.[1] || mdPath.split('/').pop().replace(/\.md$/i, '')).trim(); + const title = shortTitle(originalTitle); if (firstHeading) body = body.slice(firstHeading[0].length).trim(); - if (!title || Array.from(title).length > 80) throw new Error('标题为空或超过 80 字，请修改 Markdown 标题后重试。'); + if (!title) throw new Error('标题为空，请修改 Markdown 标题后重试。'); if (!body.trim()) throw new Error('正文为空。'); const warnings = []; if (/<\/?[a-z][^>]*>/i.test(body)) warnings.push('存在 HTML，将作为文字保留；请检查排版。'); @@ -531,5 +550,5 @@ $('diagnostics').onclick = () => downloadJSON({ ...summary(), logs }, 'dww11-import-diagnostics.json'); win.addEventListener('pagehide', releasePreviews, { once: true }); } - return { VERSION, normalizePath, imagePath, readLink, markdownToBBCode, compile, prepareFiles, preparePackage, parseAid, attachmentId, verifyEditorImages, mount }; + return { VERSION, shortTitle, normalizePath, imagePath, readLink, markdownToBBCode, compile, prepareFiles, preparePackage, parseAid, attachmentId, verifyEditorImages, mount }; }); diff --git a/store.py b/store.py index 6f6046b..b1d2261 100644 --- a/store.py +++ b/store.py @@ -10,7 +10,7 @@ import threading import time from urllib.parse import urlsplit, parse_qs -from articles import inspect_article +from articles import inspect_article, short_title ACCEPTED = {'published', 'accepted_pending_moderation'} SUBMITTED = {'user_submitting', 'submit_unknown'} @@ -204,6 +204,7 @@ class Store: self.run['switch'] = dict(state='pending', **{'from': account['uid']}, created_at=time.time()) self._persist(db) return dict(action='switch', run=self.run) + job['title'] = short_title(job.get('original_title', job['title'])) job.update(state='preparing', account=account, fid=self.run['config']['fid'], error='') self.run['active'] = job['id'] self._save(db, job) diff --git a/test/importer.test.cjs b/test/importer.test.cjs index a306026..2ddcdcf 100644 --- a/test/importer.test.cjs +++ b/test/importer.test.cjs @@ -29,12 +29,11 @@ test('测试文章：标题、3 张配图、插入顺序、来源与段落完整 assert.equal(output, originalBody); }); -test('缺图、远程图片、超出目录、超大图片与超长标题在上传前被拒绝', () => { +test('缺图、远程图片、超出目录与超大图片在上传前被拒绝', () => { assert.throws(() => core.compile('# 标题\n\n![](missing.jpg)', 'a.md', new Map()), /找不到配图/); assert.throws(() => core.compile('# 标题\n\n![](https://x/a.jpg)', 'a.md', new Map()), /目录内/); assert.throws(() => core.imagePath('a.md', '../secret.jpg'), /超出/); assert.throws(() => core.imagePath('a.md', '%2fetc/passwd'), /相对路径/); - assert.throws(() => core.compile('# ' + '长'.repeat(81) + '\n正文', 'a.md', new Map()), /80/); assert.throws(() => core.compile('# 标题\n\n![](big.png)', 'a.md', new Map([['big.png', {size:20*1024*1024}]])), /20 MB/); }); @@ -209,3 +208,39 @@ test('正文图片尚在加载时等待 load；终止后释放等待', async () const cancelled=core.verifyEditorImages({querySelectorAll:()=>[pending]},['8'],controller.signal); controller.abort();await assert.rejects(cancelled,/终止/);assert.equal(pending.events.size,0); }); + + +test('标题按 UTF-8 预算截断，目录和文章包保持一致且重复处理稳定', async () => { + const cases = [ + ['A'.repeat(75), 'A'.repeat(75)], + ['A'.repeat(76), 'A'.repeat(75) + '...'], + ['中'.repeat(25), '中'.repeat(25)], + ['中'.repeat(26), '中'.repeat(25) + '...'], + ['A中，'.repeat(12), 'A中，'.repeat(10) + 'A中...'], + ['😀'.repeat(20), '😀'.repeat(18) + '...'], + ]; + for (const [title, expected] of cases) { + assert.equal(core.shortTitle(title), expected); + assert.equal(core.shortTitle(expected), expected); + assert.ok(Buffer.byteLength(expected) <= 78); + const markdown = '# ' + title + '\n正文'; + const direct = core.compile(markdown, 'a.md', new Map()); + const packaged = await core.preparePackage({schemaVersion:1, markdown, title, images:[]}); + assert.equal(direct.title, expected); + assert.equal(packaged.title, expected); + const python = execFileSync('python3', ['-c', 'import sys; from articles import short_title; print(short_title(sys.argv[1]), end="")', title], {cwd:path.join(__dirname, '..'), encoding:'utf8'}); + assert.equal(python, expected); + } +}); + +test('长标题预览、实际填写和返回结果一致', async () => { + const h = harness(); + const title = '新闻，'.repeat(20); + await h.api.stagePackage({schemaVersion:1, markdown:'# ' + title + '\n正文', images:[]}); + const expected = '新闻，'.repeat(8) + '新...'; + assert.equal(h.api.getState().title, expected); + const result = await h.api.importPrepared(); + assert.equal(h.ids.get('subject').value, expected); + assert.equal(result.title, expected); + assert.equal(h.requests.length, 0); +}); diff --git a/test/test_queue.py b/test/test_queue.py index beb411a..5cfad54 100644 --- a/test/test_queue.py +++ b/test/test_queue.py @@ -104,9 +104,26 @@ class QueueTests(unittest.TestCase): def test_title_does_not_break_emoji(self): short = short_title('😀' * 50) self.assertTrue(short.endswith('...')) - self.assertLessEqual(len(short.encode('utf-16-le')) // 2, 80) + self.assertLessEqual(len(short.encode('utf-8')), 78) self.assertEqual(short_title('短标题'), '短标题') + def test_existing_queued_title_updates_before_import_only(self): + original = '中' * 30 + self.md.write_text('# ' + original + '\n\n正文。\n\n来源：测试，如有侵权，请联系删除。') + self.store.enqueue(self.article) + with self.store.db() as db: + job = self.store.state()['jobs'][0] + job['title'] = original # Simulate a queue created by the old version. + self.store._save(db, job) + self.store.control('start') + job = self.store.next(self.owner)['job'] + self.assertEqual(job['title'], '中' * 25 + '...') + self.assertEqual(self.store.package(dict(owner='tab-a', id=job['id']))['title'], job['title']) + self.submit(job) + restarted = Store(self.root / 'queue.sqlite3') + self.assertEqual(restarted.state()['jobs'][0]['snapshot']['title'], job['title']) + self.assertEqual(restarted.state()['jobs'][0]['original_title'], original) + def test_unsupported_images_stop_before_queue(self): (self.article / 'a.webp').write_bytes(b'webp') with self.assertRaisesRegex(ValueError, '暂不支持'):