From 4db9e91575fdec421564430b76ec2e4821510435 Mon Sep 17 00:00:00 2001 From: Simon Date: Wed, 22 Apr 2026 03:10:07 +0200 Subject: [PATCH] Add DuckDuckGo Lite extractor --- src/extractor-registry.ts | 8 + src/extractors/duckduckgo-lite.ts | 228 ++++++++++++++++++++++++++++ tests/expected/duckduckgo-lite.md | 19 +++ tests/fixtures/duckduckgo-lite.html | 96 ++++++++++++ 4 files changed, 351 insertions(+) create mode 100644 src/extractors/duckduckgo-lite.ts create mode 100644 tests/expected/duckduckgo-lite.md create mode 100644 tests/fixtures/duckduckgo-lite.html diff --git a/src/extractor-registry.ts b/src/extractor-registry.ts index bff057c96..e0f3174c1 100644 --- a/src/extractor-registry.ts +++ b/src/extractor-registry.ts @@ -22,6 +22,7 @@ import { ThreadsExtractor } from './extractors/threads'; import { BlueskyExtractor } from './extractors/bluesky'; import { DiscourseExtractor } from './extractors/discourse'; import { MediumExtractor } from './extractors/medium'; +import { DuckDuckGoLiteExtractor } from './extractors/duckduckgo-lite'; type ExtractorConstructor = new (document: Document, url: string, schemaOrgData?: any, options?: ExtractorOptions) => BaseExtractor; @@ -191,6 +192,13 @@ export class ExtractorRegistry { extractor: DiscourseExtractor }); + this.register({ + patterns: [ + 'lite.duckduckgo.com', + ], + extractor: DuckDuckGoLiteExtractor + }); + this.register({ patterns: [/.*/], extractor: BbcodeDataExtractor diff --git a/src/extractors/duckduckgo-lite.ts b/src/extractors/duckduckgo-lite.ts new file mode 100644 index 000000000..ebe8b0a98 --- /dev/null +++ b/src/extractors/duckduckgo-lite.ts @@ -0,0 +1,228 @@ +import { BaseExtractor, ExtractorOptions } from './_base'; +import { ExtractorResult } from '../types/extractors'; +import { serializeHTML, escapeHtml } from '../utils/dom'; + +interface SearchResult { + title: string; + url: string; + snippet: string; +} + +interface ZeroClickInfo { + header: string; + body: string; + textContent: string; +} + +export class DuckDuckGoLiteExtractor extends BaseExtractor { + private isSearchPage: boolean; + + constructor(document: Document, url: string, schemaOrgData?: any, options?: ExtractorOptions) { + super(document, url, schemaOrgData, options); + this.isSearchPage = this.detectSearchPage(); + } + + private detectSearchPage(): boolean { + return !!( + this.document.querySelector('input.query[name="q"]') && + this.document.querySelector('a.result-link') && + this.document.querySelector('table[border="0"]') + ); + } + + canExtract(): boolean { + return this.isSearchPage; + } + + extract(): ExtractorResult { + const zeroClick = this.extractZeroClickInfo(); + const results = this.extractResults(); + const contentHtml = this.buildListingHtml(results, zeroClick); + const searchQuery = this.getSearchQuery(); + + return { + content: contentHtml, + contentHtml: contentHtml, + extractedContent: { + query: searchQuery, + resultCount: String(results.length), + }, + variables: { + title: searchQuery || 'DuckDuckGo Search', + site: 'DuckDuckGo Lite', + description: zeroClick?.textContent || (searchQuery + ? `Search results for "${searchQuery}"` + : 'DuckDuckGo Lite search results'), + } + }; + } + + // Reads the current query string from the search input + private getSearchQuery(): string { + const queryInput = this.document.querySelector('input.query[name="q"]'); + return queryInput?.getAttribute('value') || ''; + } + + // DuckDuckGo wraps external URLs in a redirect so it can track clicks + // Unwraps the real destination from the `uddg` query parameter + private unwrapRedirectUrl(href: string): string { + try { + const url = new URL(href, this.url); + const actual = url.searchParams.get('uddg'); + return actual ? decodeURIComponent(actual) : href; + } catch { + return href; + } + } + + // Zero-click info appears in a `table[border="0"]` that does NOT contain + // `.result-link` elements. The structure is: + // Row 1: "Zero-click info: Title" + // Row 2: Description text + "More at Source" link + // Row 3: Spacer + // The extracted HTML is wrapped in a `
` + private extractZeroClickInfo(): ZeroClickInfo | null { + const tables = Array.from(this.document.querySelectorAll('table[border="0"]')); + + for (const table of tables) { + // Skip result tables — zero-click tables have no .result-link + if (table.querySelector('a.result-link')) continue; + + const rows = Array.from(table.querySelectorAll('tr')); + if (rows.length < 2) continue; + + const firstCell = rows[0].querySelector('td'); + if (!firstCell) continue; + + const firstText = firstCell.textContent || ''; + if (!firstText.includes('Zero-click info:')) continue; + + // Header: "Zero-click info: Title" + const headerHtml = this.unwrapLinksInHtml(firstCell); + + // Body: description text + "More at Source" link + const bodyCell = rows[1].querySelector('td'); + if (!bodyCell) continue; + + // Clone so we can safely remove the "More at" link + const cleanBody = bodyCell.cloneNode(true) as Element; + const links = Array.from(cleanBody.querySelectorAll('a')); + for (const link of links) { + if (link.textContent?.includes('More at')) { + link.remove(); + break; + } + } + + const bodyHtml = this.unwrapLinksInHtml(cleanBody); + + // Text-only version for the description variable + const textContent = cleanBody.textContent?.trim() || ''; + + return { header: headerHtml, body: bodyHtml, textContent }; + } + + return null; + } + + // Unwraps DuckDuckGo redirect URLs in all tags within the given element. + // Operates on the element's inner HTML string. + private unwrapLinksInHtml(el: Element): string { + const clone = el.cloneNode(true) as Element; + clone.querySelectorAll('a').forEach(a => { + const href = a.getAttribute('href') || ''; + const unwrapped = this.unwrapRedirectUrl(href); + if (unwrapped !== href) { + a.setAttribute('href', unwrapped); + } + }); + return serializeHTML(clone); + } + + // Iterates over the top-level result tables and collects each + // title / URL / snippet triplet into a structured array. + // + // The lite layout uses a flat table where each result is + // three consecutive rows: + // 1. title link + // 2. snippet + // 3. blank spacer + private extractResults(): SearchResult[] { + const results: SearchResult[] = []; + const tables = Array.from(this.document.querySelectorAll('table[border="0"]')); + + for (const table of tables) { + // Skip zero-click info tables (no .result-link) + if (!table.querySelector('a.result-link')) continue; + + const rows = Array.from(table.querySelectorAll('tr')); + let i = 0; + + while (i < rows.length) { + const row = rows[i]; + const linkEl = row.querySelector('a.result-link'); + + if (linkEl) { + const title = linkEl.textContent?.trim() || ''; + const href = linkEl.getAttribute('href') || ''; + const url = this.unwrapRedirectUrl(href); + + i++; + const snippetRow = rows[i]; + const snippetEl = snippetRow?.querySelector('.result-snippet'); + const snippet = snippetEl ? serializeHTML(snippetEl) : ''; + + // Skip the blank spacer row + i++; + i++; + + if (title && url) { + results.push({ title, url, snippet }); + } + } else { + i++; + } + } + } + + return results; + } + + + // Builds the final ordered list HTML. When there are no results a + // short "No results" message is returned instead of an empty string. + // Zero-click info, when present, is prepended as a blockquote. + private buildListingHtml(results: SearchResult[], zeroClick?: ZeroClickInfo | null): string { + let html = ''; + + if (zeroClick) { + html += '
'; + html += `

${zeroClick.header}

`; + if (zeroClick.body) { + html += `

${zeroClick.body}

`; + } + html += '
'; + } + + if (results.length === 0) { + html += '

No results found.

'; + return html; + } + + const items = results.map(result => { + let itemHtml = '
  • '; + itemHtml += `${escapeHtml(result.title)}`; + + if (result.snippet) { + itemHtml += `

    ${result.snippet}

    `; + } + + itemHtml += '
    • '; + return itemHtml; + }); + + html += `
      ${items.join('')}
    `; + + return html; + } +} diff --git a/tests/expected/duckduckgo-lite.md b/tests/expected/duckduckgo-lite.md new file mode 100644 index 000000000..b77177aad --- /dev/null +++ b/tests/expected/duckduckgo-lite.md @@ -0,0 +1,19 @@ +```json +{ + "title": "test query", + "author": "", + "site": "DuckDuckGo Lite", + "published": "" +} +``` + +> Zero-click info: [Gasoline](https://en.wikipedia.org/wiki/Gasoline) +> +> Gasoline or petrol is a petrochemical product characterized as a transparent, yellowish and flammable liquid normally used as a fuel for spark-ignited internal combustion engines. + +1. [Example Result Title](https://example.com/test-page) + This is a **test** snippet describing the **query** result with some highlighted terms. +2. [Another Example Page](https://example.org/another-page?foo=bar) + More content about **test** and **query** functionality. +3. [Foo Example Dot Net](https://foo.example.net/page/) + A third result with different **query** content here. \ No newline at end of file diff --git a/tests/fixtures/duckduckgo-lite.html b/tests/fixtures/duckduckgo-lite.html new file mode 100644 index 000000000..0b93885b7 --- /dev/null +++ b/tests/fixtures/duckduckgo-lite.html @@ -0,0 +1,96 @@ + + + + + + + test query at DuckDuckGo + + +

     

    +
    + DuckDuckGo +
    +

     

    + +
    + + +
    + +

     

    + + + + + + + + + + + +
    Zero-click info: Gasoline
    + Gasoline or petrol is a petrochemical product characterized as a transparent, yellowish and flammable liquid normally used as a fuel for spark-ignited internal combustion engines. + More at Wikipedia +
     
    + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
    + 1.  + + Example Result Title +
       This is a test snippet describing the query result with some highlighted terms.
      
    + 2.  + + Another Example Page +
        + More content about test and query functionality. +
      
    + 3.  + + Foo Example Dot Net +
        + A third result with different query content here. +
      
    + +

     

    + + +