diff --git a/src/defuddle.ts b/src/defuddle.ts index a513b30d0..96a7656b3 100644 --- a/src/defuddle.ts +++ b/src/defuddle.ts @@ -288,31 +288,35 @@ export class Defuddle { } } - // Also deduplicate adjacent images outside figures (same alt, different src) - const imgs = Array.from(body.querySelectorAll('img')); + // Also deduplicate images outside figures that are the SAME underlying + // image appearing twice (e.g. lazy-load / reader-mode hydration emitting + // a second copy at a different size). Two images are duplicates only when + // their normalized src (host + path, ignoring the query string) matches + // but the raw src differs. Distinct images are NOT merged just because + // they share alt text: galleries — and WeChat 公众号 articles in + // particular — give every image the same generic alt such as "Image" (or + // none at all), and keying dedup on alt collapsed them all to one image. + // (kepano/defuddle#286) + const imgs = Array.from(body.querySelectorAll('img')) + .filter(img => !img.closest('noscript') && !img.closest('figure') && img.parentElement); for (let i = 0; i < imgs.length; i++) { const img = imgs[i]; - if (img.closest('noscript') || img.closest('figure')) continue; - if (!img.parentElement) continue; - - const alt = (img.getAttribute('alt') || '').trim(); - if (!alt) continue; + if (!img.parentElement) continue; // removed earlier as a duplicate loser const src = img.getAttribute('src') || ''; if (!src || src.startsWith('data:')) continue; + const norm = this._normalizeSrc(src); for (let j = i + 1; j < imgs.length; j++) { const other = imgs[j]; - if (other.closest('noscript') || other.closest('figure')) continue; if (!other.parentElement) continue; - - const otherAlt = (other.getAttribute('alt') || '').trim(); - if (otherAlt !== alt) break; const otherSrc = other.getAttribute('src') || ''; if (!otherSrc || otherSrc.startsWith('data:')) continue; - if (otherSrc === src) break; // legitimate repeat + if (otherSrc === src) continue; // identical src → legitimate repeat, keep both + if (this._normalizeSrc(otherSrc) !== norm) continue; // different image → keep both + // Same asset, different size/query variant → keep the best one this._keepBestImage([img, other]); - if (!img.parentElement) break; // img was the loser + if (!img.parentElement) break; // img was the loser; advance to next i } } diff --git a/tests/image-dedup-generic-alt.test.ts b/tests/image-dedup-generic-alt.test.ts new file mode 100644 index 000000000..72d3267d2 --- /dev/null +++ b/tests/image-dedup-generic-alt.test.ts @@ -0,0 +1,75 @@ +import { describe, test, expect } from 'vitest'; +import { Defuddle } from '../src/node'; +import { parseDocument } from './helpers'; + +const URL = 'https://example.com'; + +function countImages(md: string): number { + return (md.match(/!\[/g) || []).length; +} + +describe('image dedup: generic/identical alt across distinct images', () => { + // Regression for kepano/defuddle#286 and the WeChat 公众号 "only first image" + // symptom: many DISTINCT images sharing a generic identical alt (e.g. alt="Image") + // were collapsed to one by the alt-based dedup heuristic. + test('keeps all distinct images that share an identical generic alt (bare imgs)', async () => { + const imgs = Array.from({ length: 6 }, (_, i) => + `

Image

` + ).join('\n'); + const html = `T
+

Gallery

+

Intro paragraph with enough words for the content scorer to keep this article body.

+ ${imgs} +

Closing paragraph with sufficient text so the parser treats this as real content.

+
`; + + const result = await Defuddle(parseDocument(html, URL), URL, { separateMarkdown: true }); + expect(countImages(result.contentMarkdown || '')).toBe(6); + }); + + test('keeps all distinct images that share an empty alt (bare imgs)', async () => { + const imgs = Array.from({ length: 5 }, (_, i) => + `

` + ).join('\n'); + const html = `T
+

No-alt gallery

+

Intro paragraph with enough words for the content scorer to keep this article body.

+ ${imgs} +

Closing paragraph with sufficient text so the parser treats this as real content.

+
`; + + const result = await Defuddle(parseDocument(html, URL), URL, { separateMarkdown: true }); + expect(countImages(result.contentMarkdown || '')).toBe(5); + }); + + test('WeChat-style mmbiz images (data-src + alt="Image") all survive', async () => { + // Mimics post-scroll WeChat DOM: real src + data-src, all alt="Image", + // each a distinct asset (different hash path, identical /640 size suffix). + const imgs = Array.from({ length: 8 }, (_, i) => + `

Image

` + ).join('\n'); + const html = `WeChat article +
+

正文开头，需要足够的中文内容让正文打分器把这段识别为主体内容而不是噪音。

+ ${imgs} +

正文结尾，同样需要足够的文字让解析器保留整篇文章主体。

+
`; + + const result = await Defuddle(parseDocument(html, 'https://mp.weixin.qq.com/s/test'), + 'https://mp.weixin.qq.com/s/test', { separateMarkdown: true }); + expect(countImages(result.contentMarkdown || '')).toBe(8); + }); + + test('still keeps a legitimately repeated image (identical src, same alt)', async () => { + const html = `T
+

Repeat

+

Here is a formula E=mc2 inline with more words here.

+

The same formula E=mc2 appears again with more words.

+
`; + + const result = await Defuddle(parseDocument(html, URL), URL, { separateMarkdown: true }); + expect(countImages(result.contentMarkdown || '')).toBe(2); + }); +});