diff --git a/src/removals/selectors.ts b/src/removals/selectors.ts index 20e62c7ce..eaea7fa6e 100644 --- a/src/removals/selectors.ts +++ b/src/removals/selectors.ts @@ -12,6 +12,26 @@ import { DebugRemoval } from '../types'; import { textPreview, logDebug } from '../utils'; import { getClassName, hasResponsiveShowClass } from '../utils/dom'; +function hasSubstantiveTagsContent(el: Element): boolean { + if ((el.id || '').toLowerCase() !== 'tags') return false; + if (!el.querySelector('h1, h2, h3, h4, h5, h6')) return false; + + const text = el.textContent?.trim() || ''; + if (!text) return false; + + const linkText = Array.from(el.querySelectorAll('a')) + .map(a => a.textContent || '') + .join(' ') + .trim(); + if (linkText && linkText.length / text.length > 0.7) return false; + + return Array.from(el.querySelectorAll('p')).some(p => { + const paragraph = p.textContent?.trim() || ''; + const words = paragraph.split(/\s+/).filter(Boolean).length; + return words >= 2 && /[.!?。！？]/.test(paragraph); + }); +} + export function removeBySelector(doc: Document, debug: boolean, removeExact: boolean = true, removePartial: boolean = true, mainContent?: Element | null, debugRemovals?: DebugRemoval[], skipHiddenExactSelectors: boolean = false) { const startTime = Date.now(); let exactSelectorCount = 0; @@ -43,6 +63,9 @@ export function removeBySelector(doc: Document, debug: boolean, removeExact: boo if (el.matches(HIDDEN_EXACT_SELECTOR) && hasResponsiveShowClass(getClassName(el))) { return; } + if (hasSubstantiveTagsContent(el)) { + return; + } elementsToRemove.set(el, { type: 'exact' }); exactSelectorCount++; } diff --git a/tests/issues-321-tags-section.test.ts b/tests/issues-321-tags-section.test.ts new file mode 100644 index 000000000..bbeff6876 --- /dev/null +++ b/tests/issues-321-tags-section.test.ts @@ -0,0 +1,42 @@ +import { describe, test, expect } from 'vitest'; +import { Defuddle } from '../src/node'; +import { parseDocument } from './helpers'; + +const html = ` + + +Tags section test + +
+

Article title

+

Intro heading

+

Intro paragraph with enough text to be treated as content.

+ +
+

Content heading

+

Content paragraph that should remain in the extracted output.

+
+ + +
+ + +`; + +describe('issue #321', () => { + test('keeps substantive content sections with id="tags"', async () => { + const result = await Defuddle(parseDocument(html, 'https://example.com/tags-section'), 'https://example.com/tags-section', { + separateMarkdown: true, + }); + + expect(result.contentMarkdown).toContain('## Intro heading'); + expect(result.contentMarkdown).toContain('Intro paragraph with enough text'); + expect(result.contentMarkdown).toContain('## Content heading'); + expect(result.contentMarkdown).toContain('Content paragraph that should remain'); + expect(result.contentMarkdown).not.toContain('/tags/alpha'); + expect(result.contentMarkdown).not.toContain('/tags/beta'); + }); +});