Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 8 additions & 0 deletions src/extractor-registry.ts
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@ import { ThreadsExtractor } from './extractors/threads';
import { BlueskyExtractor } from './extractors/bluesky';
import { DiscourseExtractor } from './extractors/discourse';
import { MediumExtractor } from './extractors/medium';
import { DuckDuckGoLiteExtractor } from './extractors/duckduckgo-lite';

type ExtractorConstructor = new (document: Document, url: string, schemaOrgData?: any, options?: ExtractorOptions) => BaseExtractor;

Expand Down Expand Up @@ -191,6 +192,13 @@ export class ExtractorRegistry {
extractor: DiscourseExtractor
});

this.register({
patterns: [
'lite.duckduckgo.com',
],
extractor: DuckDuckGoLiteExtractor
});

this.register({
patterns: [/.*/],
extractor: BbcodeDataExtractor
Expand Down
228 changes: 228 additions & 0 deletions src/extractors/duckduckgo-lite.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,228 @@
import { BaseExtractor, ExtractorOptions } from './_base';
import { ExtractorResult } from '../types/extractors';
import { serializeHTML, escapeHtml } from '../utils/dom';

interface SearchResult {
title: string;
url: string;
snippet: string;
}

interface ZeroClickInfo {
header: string;
body: string;
textContent: string;
}

export class DuckDuckGoLiteExtractor extends BaseExtractor {
private isSearchPage: boolean;

constructor(document: Document, url: string, schemaOrgData?: any, options?: ExtractorOptions) {
super(document, url, schemaOrgData, options);
this.isSearchPage = this.detectSearchPage();
}

private detectSearchPage(): boolean {
return !!(
this.document.querySelector('input.query[name="q"]') &&
this.document.querySelector('a.result-link') &&
this.document.querySelector('table[border="0"]')
);
}

canExtract(): boolean {
return this.isSearchPage;
}

extract(): ExtractorResult {
const zeroClick = this.extractZeroClickInfo();
const results = this.extractResults();
const contentHtml = this.buildListingHtml(results, zeroClick);
const searchQuery = this.getSearchQuery();

return {
content: contentHtml,
contentHtml: contentHtml,
extractedContent: {
query: searchQuery,
resultCount: String(results.length),
},
variables: {
title: searchQuery || 'DuckDuckGo Search',
site: 'DuckDuckGo Lite',
description: zeroClick?.textContent || (searchQuery
? `Search results for "${searchQuery}"`
: 'DuckDuckGo Lite search results'),
}
};
}

// Reads the current query string from the search input
private getSearchQuery(): string {
const queryInput = this.document.querySelector('input.query[name="q"]');
return queryInput?.getAttribute('value') || '';
}

// DuckDuckGo wraps external URLs in a redirect so it can track clicks
// Unwraps the real destination from the `uddg` query parameter
private unwrapRedirectUrl(href: string): string {
try {
const url = new URL(href, this.url);
const actual = url.searchParams.get('uddg');
return actual ? decodeURIComponent(actual) : href;
} catch {
return href;
}
}

// Zero-click info appears in a `table[border="0"]` that does NOT contain
// `.result-link` elements. The structure is:
// Row 1: "Zero-click info: <a href="...">Title</a>"
// Row 2: Description text + "More at <q>Source</q>" link
// Row 3: Spacer
// The extracted HTML is wrapped in a `<blockquote data-callout="info">`
private extractZeroClickInfo(): ZeroClickInfo | null {
const tables = Array.from(this.document.querySelectorAll('table[border="0"]'));

for (const table of tables) {
// Skip result tables — zero-click tables have no .result-link
if (table.querySelector('a.result-link')) continue;

const rows = Array.from(table.querySelectorAll('tr'));
if (rows.length < 2) continue;

const firstCell = rows[0].querySelector('td');
if (!firstCell) continue;

const firstText = firstCell.textContent || '';
if (!firstText.includes('Zero-click info:')) continue;

// Header: "Zero-click info: <a>Title</a>"
const headerHtml = this.unwrapLinksInHtml(firstCell);

// Body: description text + "More at <q>Source</q>" link
const bodyCell = rows[1].querySelector('td');
if (!bodyCell) continue;

// Clone so we can safely remove the "More at" link
const cleanBody = bodyCell.cloneNode(true) as Element;
const links = Array.from(cleanBody.querySelectorAll('a'));
for (const link of links) {
if (link.textContent?.includes('More at')) {
link.remove();
break;
}
}

const bodyHtml = this.unwrapLinksInHtml(cleanBody);

// Text-only version for the description variable
const textContent = cleanBody.textContent?.trim() || '';

return { header: headerHtml, body: bodyHtml, textContent };
}

return null;
}

// Unwraps DuckDuckGo redirect URLs in all <a> tags within the given element.
// Operates on the element's inner HTML string.
private unwrapLinksInHtml(el: Element): string {
const clone = el.cloneNode(true) as Element;
clone.querySelectorAll('a').forEach(a => {
const href = a.getAttribute('href') || '';
const unwrapped = this.unwrapRedirectUrl(href);
if (unwrapped !== href) {
a.setAttribute('href', unwrapped);
}
});
return serializeHTML(clone);
}

// Iterates over the top-level result tables and collects each
// title / URL / snippet triplet into a structured array.
//
// The lite layout uses a flat table where each result is
// three consecutive rows:
// 1. title link
// 2. snippet
// 3. blank spacer
private extractResults(): SearchResult[] {
const results: SearchResult[] = [];
const tables = Array.from(this.document.querySelectorAll('table[border="0"]'));

for (const table of tables) {
// Skip zero-click info tables (no .result-link)
if (!table.querySelector('a.result-link')) continue;

const rows = Array.from(table.querySelectorAll('tr'));
let i = 0;

while (i < rows.length) {
const row = rows[i];
const linkEl = row.querySelector('a.result-link');

if (linkEl) {
const title = linkEl.textContent?.trim() || '';
const href = linkEl.getAttribute('href') || '';
const url = this.unwrapRedirectUrl(href);

i++;
const snippetRow = rows[i];
const snippetEl = snippetRow?.querySelector('.result-snippet');
const snippet = snippetEl ? serializeHTML(snippetEl) : '';

// Skip the blank spacer row
i++;
i++;

if (title && url) {
results.push({ title, url, snippet });
}
} else {
i++;
}
}
}

return results;
}


// Builds the final ordered list HTML. When there are no results a
// short "No results" message is returned instead of an empty string.
// Zero-click info, when present, is prepended as a blockquote.
private buildListingHtml(results: SearchResult[], zeroClick?: ZeroClickInfo | null): string {
let html = '';

if (zeroClick) {
html += '<blockquote data-callout="info">';
html += `<p>${zeroClick.header}</p>`;
if (zeroClick.body) {
html += `<p>${zeroClick.body}</p>`;
}
html += '</blockquote>';
}

if (results.length === 0) {
html += '<p>No results found.</p>';
return html;
}

const items = results.map(result => {
let itemHtml = '<li>';
itemHtml += `<a href="${escapeHtml(result.url)}">${escapeHtml(result.title)}</a>`;

if (result.snippet) {
itemHtml += `<p>${result.snippet}</p>`;
}

itemHtml += '</li>';
return itemHtml;
});

html += `<ol>${items.join('')}</ol>`;

return html;
}
}
19 changes: 19 additions & 0 deletions tests/expected/duckduckgo-lite.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,19 @@
```json
{
"title": "test query",
"author": "",
"site": "DuckDuckGo Lite",
"published": ""
}
```

> Zero-click info: [Gasoline](https://en.wikipedia.org/wiki/Gasoline)
>
> Gasoline or petrol is a petrochemical product characterized as a transparent, yellowish and flammable liquid normally used as a fuel for spark-ignited internal combustion engines.

1. [Example Result Title](https://example.com/test-page)
This is a **test** snippet describing the **query** result with some highlighted terms.
2. [Another Example Page](https://example.org/another-page?foo=bar)
More content about **test** and **query** functionality.
3. [Foo Example Dot Net](https://foo.example.net/page/)
A third result with different **query** content here.
96 changes: 96 additions & 0 deletions tests/fixtures/duckduckgo-lite.html
Original file line number Diff line number Diff line change
@@ -0,0 +1,96 @@
<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.01 Transitional//EN" "http://www.w3.org/TR/html4/loose.dtd">
<html>
<!--{"url": "https://lite.duckduckgo.com/lite/?q=test+query"}-->
<head>
<meta http-equiv="content-type" content="text/html; charset=UTF-8">
<meta name="viewport" content="width=device-width, initial-scale=1.0, maximum-scale=3.0, user-scalable=1;">
<title>test query at DuckDuckGo</title>
</head>
<body>
<p class='extra'>&nbsp;</p>
<div class="header">
DuckDuckGo
</div>
<p class='extra'>&nbsp;</p>

<form action="/lite/" method="post">
<input class='query' type="text" size="40" name="q" value="test query">
<input class='submit' type="submit" value="Search">
</form>

<p class='extra'>&nbsp;</p>

<table border="0">
<tr>
<td>Zero-click info: <a rel="nofollow" href="https://lite.duckduckgo.com/lite/?q=test+query&amp;uddg=https%3A%2F%2Fen.wikipedia.org%2Fwiki%2FGasoline">Gasoline</a></td>
</tr>
<tr>
<td>
Gasoline or petrol is a petrochemical product characterized as a transparent, yellowish and flammable liquid normally used as a fuel for spark-ignited internal combustion engines.
<a rel="nofollow" href="https://lite.duckduckgo.com/lite/?q=test+query&amp;uddg=https%3A%2F%2Fen.wikipedia.org%2Fwiki%2FGasoline">More at <q>Wikipedia</q></a>
</td>
</tr>
<tr>
<td>&nbsp;</td>
</tr>
</table>

<table border="0">
<tr>
<td valign="top">
1.&nbsp;
</td>
<td>
<a rel="nofollow" href="https://example.com/test-page" class='result-link'>Example Result Title</a>
</td>
</tr>
<tr>
<td>&nbsp;&nbsp;&nbsp;</td>
<td class='result-snippet'>This is a <b>test</b> snippet describing the <b>query</b> result with some highlighted terms.</td>
</tr>
<tr>
<td>&nbsp;</td>
<td>&nbsp;</td>
</tr>
<tr>
<td valign="top">
2.&nbsp;
</td>
<td>
<a rel="nofollow" href="https://example.org/another-page?foo=bar" class='result-link'>Another Example Page</a>
</td>
</tr>
<tr>
<td>&nbsp;&nbsp;&nbsp;</td>
<td class='result-snippet'>
More content about <b>test</b> and <b>query</b> functionality.
</td>
</tr>
<tr>
<td>&nbsp;</td>
<td>&nbsp;</td>
</tr>
<tr>
<td valign="top">
3.&nbsp;
</td>
<td>
<a rel="nofollow" href="https://foo.example.net/page/" class='result-link'>Foo Example Dot Net</a>
</td>
</tr>
<tr>
<td>&nbsp;&nbsp;&nbsp;</td>
<td class='result-snippet'>
A third result with different <b>query</b> content here.
</td>
</tr>
<tr>
<td>&nbsp;</td>
<td>&nbsp;</td>
</tr>
</table>

<p class='extra'>&nbsp;</p>

</body>
</html>