Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
30 changes: 17 additions & 13 deletions src/defuddle.ts
Original file line number Diff line number Diff line change
Expand Up @@ -288,31 +288,35 @@ export class Defuddle {
}
}

// Also deduplicate adjacent images outside figures (same alt, different src)
const imgs = Array.from(body.querySelectorAll('img'));
// Also deduplicate images outside figures that are the SAME underlying
// image appearing twice (e.g. lazy-load / reader-mode hydration emitting
// a second copy at a different size). Two images are duplicates only when
// their normalized src (host + path, ignoring the query string) matches
// but the raw src differs. Distinct images are NOT merged just because
// they share alt text: galleries — and WeChat 公众号 articles in
// particular — give every image the same generic alt such as "Image" (or
// none at all), and keying dedup on alt collapsed them all to one image.
// (kepano/defuddle#286)
const imgs = Array.from(body.querySelectorAll('img'))
.filter(img => !img.closest('noscript') && !img.closest('figure') && img.parentElement);
for (let i = 0; i < imgs.length; i++) {
const img = imgs[i];
if (img.closest('noscript') || img.closest('figure')) continue;
if (!img.parentElement) continue;

const alt = (img.getAttribute('alt') || '').trim();
if (!alt) continue;
if (!img.parentElement) continue; // removed earlier as a duplicate loser
const src = img.getAttribute('src') || '';
if (!src || src.startsWith('data:')) continue;
const norm = this._normalizeSrc(src);

for (let j = i + 1; j < imgs.length; j++) {
const other = imgs[j];
if (other.closest('noscript') || other.closest('figure')) continue;
if (!other.parentElement) continue;

const otherAlt = (other.getAttribute('alt') || '').trim();
if (otherAlt !== alt) break;
const otherSrc = other.getAttribute('src') || '';
if (!otherSrc || otherSrc.startsWith('data:')) continue;
if (otherSrc === src) break; // legitimate repeat
if (otherSrc === src) continue; // identical src → legitimate repeat, keep both
if (this._normalizeSrc(otherSrc) !== norm) continue; // different image → keep both

// Same asset, different size/query variant → keep the best one
this._keepBestImage([img, other]);
if (!img.parentElement) break; // img was the loser
if (!img.parentElement) break; // img was the loser; advance to next i
}
}

Expand Down
75 changes: 75 additions & 0 deletions tests/image-dedup-generic-alt.test.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,75 @@
import { describe, test, expect } from 'vitest';
import { Defuddle } from '../src/node';
import { parseDocument } from './helpers';

const URL = 'https://example.com';

function countImages(md: string): number {
return (md.match(/!\[/g) || []).length;
}

describe('image dedup: generic/identical alt across distinct images', () => {
// Regression for kepano/defuddle#286 and the WeChat 公众号 "only first image"
// symptom: many DISTINCT images sharing a generic identical alt (e.g. alt="Image")
// were collapsed to one by the alt-based dedup heuristic.
test('keeps all distinct images that share an identical generic alt (bare imgs)', async () => {
const imgs = Array.from({ length: 6 }, (_, i) =>
`<p><img src="https://cdn.example.com/photo-${i}.jpg" alt="Image"></p>`
).join('\n');
const html = `<html><head><title>T</title></head><body><article>
<h1>Gallery</h1>
<p>Intro paragraph with enough words for the content scorer to keep this article body.</p>
${imgs}
<p>Closing paragraph with sufficient text so the parser treats this as real content.</p>
</article></body></html>`;

const result = await Defuddle(parseDocument(html, URL), URL, { separateMarkdown: true });
expect(countImages(result.contentMarkdown || '')).toBe(6);
});

test('keeps all distinct images that share an empty alt (bare imgs)', async () => {
const imgs = Array.from({ length: 5 }, (_, i) =>
`<p><img src="https://cdn.example.com/pic-${i}.png" alt=""></p>`
).join('\n');
const html = `<html><head><title>T</title></head><body><article>
<h1>No-alt gallery</h1>
<p>Intro paragraph with enough words for the content scorer to keep this article body.</p>
${imgs}
<p>Closing paragraph with sufficient text so the parser treats this as real content.</p>
</article></body></html>`;

const result = await Defuddle(parseDocument(html, URL), URL, { separateMarkdown: true });
expect(countImages(result.contentMarkdown || '')).toBe(5);
});

test('WeChat-style mmbiz images (data-src + alt="Image") all survive', async () => {
// Mimics post-scroll WeChat DOM: real src + data-src, all alt="Image",
// each a distinct asset (different hash path, identical /640 size suffix).
const imgs = Array.from({ length: 8 }, (_, i) =>
`<p style="text-align:center;"><img class="rich_pages wxw-img" ` +
`data-src="https://mmbiz.qpic.cn/mmbiz_png/hashAAA${i}BBB/640?wx_fmt=png" ` +
`src="https://mmbiz.qpic.cn/mmbiz_png/hashAAA${i}BBB/640?wx_fmt=png" alt="Image"></p>`
).join('\n');
const html = `<html><head><title>WeChat article</title></head><body>
<div id="js_content"><article>
<p>正文开头,需要足够的中文内容让正文打分器把这段识别为主体内容而不是噪音。</p>
${imgs}
<p>正文结尾,同样需要足够的文字让解析器保留整篇文章主体。</p>
</article></div></body></html>`;

const result = await Defuddle(parseDocument(html, 'https://mp.weixin.qq.com/s/test'),
'https://mp.weixin.qq.com/s/test', { separateMarkdown: true });
expect(countImages(result.contentMarkdown || '')).toBe(8);
});

test('still keeps a legitimately repeated image (identical src, same alt)', async () => {
const html = `<html><head><title>T</title></head><body><article>
<h1>Repeat</h1>
<p>Here is a formula <img src="https://example.com/f.png" alt="E=mc2"> inline with more words here.</p>
<p>The same formula <img src="https://example.com/f.png" alt="E=mc2"> appears again with more words.</p>
</article></body></html>`;

const result = await Defuddle(parseDocument(html, URL), URL, { separateMarkdown: true });
expect(countImages(result.contentMarkdown || '')).toBe(2);
});
});