Work/better detect code lang

.gitignore

@@ -1,9 +1,12 @@
 .DS_Store
 .cursorrules
 .context
+/.*
 
 node_modules/
 
 dist/
 .wrangler/
 .dev.vars
+
+AGENTS.md

PR_EXPLANATIONS.md

@@ -0,0 +1,56 @@
+## Summary
+Improve automatic language detection for fenced code blocks in markdown output.
+
+The main goal is to emit better fence labels (for example ` ```javascript `, ` ```typescript `, ` ```bash `) instead of empty or incorrect labels when Defuddle extracts code from documentation pages.
+
+## Main implementation
+### New file added first
+- `src/elements/detect-code-lang.ts`
+
+This file centralizes language guessing logic.
+
+## Detection strategy
+Language detection now uses ordered heuristics:
+
+1. Strong signature rules first (high confidence)
+	- `php`, `html`, `xml`
+	- shebang-based detection (`bash`, `python`, `javascript`)
+	- `dockerfile`, `cpp`, `java`, `go`, `rust`, `ruby`, `python`, `sql`, `latex`
+
+2. Context/shape-based rules
+	- `typescript` and `tsx` patterns
+	- `css` and `yaml`
+	- markdown content heuristics (`isMarkdown`)
+	- `json`
+
+3. Final fallback
+	- common CLI patterns => `bash`
+	- JS fallback detection via declarations/import/export patterns
+
+Rule order is intentional to reduce false positives (for example HTML before CSS, markdown before generic fallbacks, JSON after YAML checks).
+
+## Other files updated in the original flow
+- `src/elements/code.ts`
+  - Uses the new detector in the code normalization/serialization pipeline.
+- `src/extractors/github.ts`
+  - Keeps GitHub extraction aligned with improved language labels.
+
+## Why this improves output
+- More code fences include a reliable language tag.
+- Syntax highlighting quality is better in markdown consumers (including Obsidian).
+- Less manual cleanup after scraping technical docs.
+
+## Backward compatibility
+- If no rule matches confidently, behavior still falls back safely.
+- The change is additive: existing extraction flow remains intact.
+
+## Manual validation
+Tested against a mix of docs and code-heavy pages, including:
+- https://github.com/kepano/defuddle
+- https://kieran.casa/aws-sdk-waiters-ts/
+- https://expressjs.com/en/guide/routing/
+
+Also verified on pages containing shell snippets, TypeScript/JavaScript examples, and markdown documentation blocks.
+
+## Notes
+Branch: `work/better-detect-code-lang`.

README.md

@@ -292,7 +292,22 @@ To build the package, you'll need Node.js and npm installed. Then run:
 npm install
 
 # Clean and build
-npm run build
+bun run build
+```
+
+### Local testing
+
+Smoke-test the CLI against a real URL (requires `bun run build` first):
+
+```bash
+# Use `--` to let your script receive the flags
+bun run start:dev -- https://en.wikipedia.org/wiki/2026_Venezuela_earthquakes -m -f
+```
+
+Or with any URL:
+
+```bash
+bun run start:dev <url> [-m] [-f]
 ```
 
 ## Third-party services

defuddle.code-workspace

@@ -0,0 +1,11 @@
+{
+	"folders": [
+		{
+			"path": "../defuddle"
+		},
+		{
+			"path": "../npx-crawly-template"
+		}
+	],
+	"settings": {}
+}

package.json

@@ -38,6 +38,7 @@
 		}
 	},
 	"scripts": {
+		"start:dev": "node dist/cli.js parse",
 		"clean": "rm -rf dist",
 		"build:types": "tsc --project tsconfig.declarations.json",
 		"build:js": "webpack",

skills-lock.json

@@ -0,0 +1,11 @@
+{
+  "version": 1,
+  "skills": {
+    "git-workflow": {
+      "source": "netresearch/git-workflow-skill",
+      "sourceType": "github",
+      "skillPath": "skills/git-workflow/SKILL.md",
+      "computedHash": "11a80581aa8ab63122316ed4c64386dd65c860f46d61b99d8dd6044787b12ec1"
+    }
+  }
+}

src/elements/code.ts

@@ -1,18 +1,23 @@
 import { isTextNode, isElement, countWords } from '../utils';
+import { guessCodeLanguage } from './detect-code-lang';
 
 // Language patterns
 const HIGHLIGHTER_PATTERNS = [
-	/^language-(\w+)$/,          // language-javascript
-	/^lang-(\w+)$/,              // lang-javascript
-	/^(\w+)-code$/,              // javascript-code
-	/^code-(\w+)$/,              // code-javascript
-	/^syntax-(\w+)$/,            // syntax-javascript
-	/^code-snippet__(\w+)$/,     // code-snippet__javascript
-	/^highlight-(\w+)$/,         // highlight-javascript
-	/^(\w+)-snippet$/,           // javascript-snippet
+	/^language-([a-z0-9_+-]+)$/i,          // language-javascript
+	/^lang-([a-z0-9_+-]+)$/i,              // lang-javascript
+	/^(?:[\w-]+-)?lang-([a-z0-9_+-]+)$/i,  // mw-highlight-lang-javascript (Wikipedia)
+	/^([a-z0-9_+-]+)-code$/i,              // javascript-code
+	/^code-([a-z0-9_+-]+)$/i,              // code-javascript
+	/^syntax-([a-z0-9_+-]+)$/i,            // syntax-javascript
+	/^code-snippet__([a-z0-9_+-]+)$/i,     // code-snippet__javascript
+	/^highlight-([a-z0-9_+-]+)$/i,         // highlight-javascript
+	/^highlight-source-([a-z0-9_+-]+)$/i,  // highlight-source-javascript (GitHub)
+	/^highlight-text-([a-z0-9_+-]+)$/i,    // highlight-text-md (GitHub markdown)
+	/^highlight-text-([a-z0-9_+-]+)/i,     // highlight-text-html-basic -> html (compound names)
+	/^([a-z0-9_+-]+)-snippet$/i,           // javascript-snippet
 
 	// fallback
-	/(?:^|\s)(?:language|lang|brush|syntax)-(\w+)(?:\s|$)/i
+	/(?:^|\s)(?:language|lang|brush|syntax|source)-(\w+)(?:\s|$)/i
 ];
 
 // Languages to detect in code blocks
@@ -133,6 +138,7 @@ export const codeBlockRules = [
 			'.syntaxhighlighter',
 			'.highlight',
 			'.highlight-source',
+			'[class*="highlight-text-"]',
 			'.wp-block-syntaxhighlighter-code',
 			'.wp-block-code',
 			'div[class*="language-"]',
@@ -180,18 +186,24 @@ export const codeBlockRules = [
 				}
 
 				// Check class names for patterns and supported languages
-				const classNames = Array.from(element.classList || []);
+				const classNames = new Set<string>();
+				const classAttr = element.getAttribute('class') || '';
+				classAttr.split(/\s+/).filter(Boolean).forEach(name => classNames.add(name));
+				for (const name of Array.from(element.classList || [])) {
+					if (name) classNames.add(name);
+				}
+				const classList = Array.from(classNames);
 
 				// Check for syntax highlighter specific format
 				if (element.classList?.contains('syntaxhighlighter')) {
-					const langClass = classNames.find(c => !['syntaxhighlighter', 'nogutter'].includes(c));
+					const langClass = classList.find(c => !['syntaxhighlighter', 'nogutter'].includes(c));
 					if (langClass && CODE_LANGUAGES.has(langClass.toLowerCase())) {
 						return langClass.toLowerCase();
 					}
 				}
 
 				// Check patterns
-				for (const className of classNames) {
+				for (const className of classList) {
 					for (const pattern of HIGHLIGHTER_PATTERNS) {
 						const match = className.toLowerCase().match(pattern);
 						if (match && match[1] && CODE_LANGUAGES.has(match[1].toLowerCase())) {
@@ -201,7 +213,7 @@ export const codeBlockRules = [
 				}
 
 				// If all else fails, check for bare language names
-				for (const className of classNames) {
+				for (const className of classList) {
 					if (CODE_LANGUAGES.has(className.toLowerCase())) {
 						return className.toLowerCase();
 					}
@@ -462,6 +474,29 @@ export const codeBlockRules = [
 					.replace(/\n+$/, '');           // Remove extra newlines at end
 			}
 
+			// Fallback: guess language from code content when no HTML hint exists
+			if (!language) {
+				language = guessCodeLanguage(codeContent) || '';
+			}
+
+			// Normalize language aliases to canonical names
+			if (language) {
+				const aliasMap: Record<string, string> = {
+					md: 'markdown',
+					shell: 'bash',
+					sh: 'bash',
+					js: 'javascript',
+					ts: 'typescript',
+					py: 'python',
+					rb: 'ruby',
+					hs: 'haskell',
+					rs: 'rust',
+					cs: 'csharp',
+					yml: 'yaml',
+				};
+				language = aliasMap[language] || language;
+			}
+
 			// Remove code block header/toolbar siblings (e.g. filename labels, copy buttons)
 			// before replacing, so they don't leak into content when wrappers are flattened.
 			// Only remove non-semantic divs/spans, not headings, paragraphs, etc.

src/elements/detect-code-lang.ts

@@ -0,0 +1,107 @@
+type LangRule = { test: (s: string) => boolean; lang: string };
+
+function isMarkdown(s: string): boolean {
+	const lines = s.split('\n');
+	let headingCount = 0;
+	let blockquoteCount = 0;
+	for (const line of lines) {
+		if (/^#{1,6}\s+/.test(line)) headingCount++;
+		if (/^>\s+/.test(line)) blockquoteCount++;
+	}
+	const linkCount = (s.match(/\[.+?\]\(.*?\)/g) || []).length;
+	const fenceCount = (s.match(/^(`{3,}|~{3,})/gm) || []).length;
+	const boldItalicCount = (s.match(/(\*\*|__).*?\1/g) || []).length;
+
+	let features = 0;
+	if (headingCount >= 2) features++;
+	if (linkCount >= 2) features++;
+	if (fenceCount >= 1) features++;
+	if (boldItalicCount >= 1) features++;
+	if (blockquoteCount >= 1) features++;
+	if (headingCount >= 1 && linkCount >= 1) features++;
+	if (headingCount >= 1 && fenceCount >= 1) features++;
+	// Blockquote content (like callout examples) is almost certainly markdown
+	if (blockquoteCount >= 2) features += 2;
+	if (blockquoteCount >= 1 && linkCount >= 1) features += 1;
+
+	return features >= 2;
+}
+
+const LANG_RULES: LangRule[] = [
+	// Specific markers first
+	{ test: s => /^\s*<\?php\b/i.test(s), lang: 'php' },
+	{ test: s => /^\s*<!DOCTYPE\b/i.test(s) || /^\s*<html[\s>]/i.test(s) || /^<!DOCTYPE/i.test(s), lang: 'html' },
+	{ test: s => /^\s*<\?xml\b/.test(s), lang: 'xml' },
+
+	// Shebang
+	{ test: s => /^#!\/.*(?:bash|sh|zsh)/.test(s), lang: 'bash' },
+	{ test: s => /^#!\/.*(?:python|python3)/.test(s), lang: 'python' },
+	{ test: s => /^#!\/.*node/.test(s), lang: 'javascript' },
+
+	// Dockerfile
+	{ test: s => /^(?:FROM|RUN|CMD|ENTRYPOINT|COPY|ADD|WORKDIR|ENV|EXPOSE)\s+/im.test(s), lang: 'dockerfile' },
+
+	// C / C++
+	{ test: s => /^#include\s+[<"].*[>"]/m.test(s) && /\bint\s+main\s*\(/.test(s), lang: 'cpp' },
+
+	// Java
+	{ test: s => /\bpublic\s+(?:class|interface|enum)\s+\w+/.test(s) || /\bpublic\s+static\s+void\s+main\s*\(/.test(s), lang: 'java' },
+
+	// Go
+	{ test: s => /\bpackage\s+\w+/.test(s) && /\bfunc\s+/.test(s), lang: 'go' },
+
+	// Rust
+	{ test: s => /\bfn\s+\w+/.test(s) && /\blet\s+mut\b/.test(s), lang: 'rust' },
+
+	// Ruby
+	{ test: s => /\bdef\s+\w+/.test(s) && /\bend\b/.test(s), lang: 'ruby' },
+	{ test: s => /\brequire\s+['"]/.test(s) && /\bmodule\s+\w+/.test(s), lang: 'ruby' },
+
+	// Python
+	{ test: s => /^\s*(?:def|class)\s+\w+.*:$/m.test(s), lang: 'python' },
+	{ test: s => /\bimport\s+\w+/.test(s) && /\bprint\s*\(/.test(s), lang: 'python' },
+
+	// TypeScript / TSX
+	{ test: s => /\b(?:interface|type|enum)\s+\w+/.test(s) && /:\s*\w+[\s;,\)]/.test(s), lang: 'typescript' },
+	{ test: s => /\bimport\b[\s\S]*?\bfrom\s+['"]/.test(s) && /<[A-Z]\w+[\s/>]/.test(s), lang: 'tsx' },
+
+	// SQL
+	{ test: s => /\b(?:SELECT|INSERT|CREATE TABLE|ALTER TABLE|DELETE FROM|UPDATE|DROP TABLE)\b/im.test(s) && /\b(?:FROM|INTO|SET|WHERE)\b/im.test(s), lang: 'sql' },
+
+	// LaTeX
+	{ test: s => /\\documentclass\b/.test(s) || /\\(?:section|subsection|begin|end)\{/.test(s), lang: 'latex' },
+
+	// CSS (must come after HTML)
+	{ test: s => /[.#@][a-z][\w-]*\s*\{/i.test(s) && /:\s*[^;]+;/.test(s), lang: 'css' },
+
+	// YAML (key: value patterns, no braces/brackets)
+	{ test: s => /^\w[\w-]*:\s+/m.test(s) && !/[{};\[\]]/.test(s), lang: 'yaml' },
+
+	// Markdown (heuristic — requires multiple features)
+	{ test: isMarkdown, lang: 'markdown' },
+
+	// JSON (must come after YAML to avoid matching `key: "value"` as JSON)
+	{ test: s => /^\s*[\[{]/.test(s) && /"[^"]+"\s*:/.test(s) && /[\}\]]\s*$/.test(s), lang: 'json' },
+
+	// Bash fallback: common CLI patterns
+	{ test: s => /^\s*\$\s+\w/.test(s) || /\b(?:sudo|apt|brew|npm|yarn|bun|pip|curl|wget|git)\b/.test(s), lang: 'bash' },
+];
+
+function detectJavaScript(sample: string): string | null {
+	const hasDeclarations = /\b(?:const|let|var)\s+\w+\s*=/.test(sample) || /\bfunction\s+\w+\s*\(/.test(sample);
+	if (hasDeclarations) {
+		return /<[A-Z]\w+[\s/>]/.test(sample) ? 'jsx' : 'javascript';
+	}
+	if (/\bimport\b[\s\S]*?\bfrom\s+['"]/.test(sample)) return 'javascript';
+	if (/\bexport\s+default\s*\{/.test(sample)) return 'javascript';
+	if (/\bdata\s*\(\)/.test(sample) && /\bcomputed\s*:\s*\{/.test(sample)) return 'javascript';
+	return null;
+}
+
+export function guessCodeLanguage(code: string): string | null {
+	const sample = code.split('\n').slice(0, 30).join('\n');
+	for (const rule of LANG_RULES) {
+		if (rule.test(sample)) return rule.lang;
+	}
+	return detectJavaScript(sample);
+}

src/extractors/github.ts

@@ -233,8 +233,8 @@ export class GitHubExtractor extends BaseExtractor {
 			const wrapper = pre.parentElement;
 			if (!wrapper) return;
 
-			// Extract language from wrapper class (e.g. "highlight-source-ts")
-			const langMatch = wrapper.className.match(/highlight-source-(\w+)/);
+			// Extract language from wrapper class (e.g. "highlight-source-ts", "highlight-text-html-basic")
+			const langMatch = wrapper.className.match(/highlight-(?:source|text)-(\w+)/);
 			const lang = langMatch?.[1] || '';
 
 			// Use data-snippet-clipboard-copy-content if available (clean text),

src/markdown.ts

@@ -629,7 +629,7 @@ export function createMarkdownContent(content: string, url: string) {
 
 			const language = codeElement.getAttribute('data-lang')
 				|| codeElement.getAttribute('data-language')
-				|| codeElement.getAttribute('class')?.match(/language-(\w+)/)?.[1]
+				|| codeElement.getAttribute('class')?.match(/language-([a-z0-9_+-]+)/i)?.[1]
 				|| node.getAttribute('data-language')
 				|| '';
 			const code = codeElement.textContent || '';

tests/markdown.test.ts

@@ -108,6 +108,23 @@ describe('Markdown conversion', () => {
 		});
 	});
 
+	describe('github code language detection', () => {
+		test('should detect shell language from highlight-source-shell wrapper on non-GitHub pages', async () => {
+			const html = `<html><head><title>Test</title></head><body><article>
+				<div class="highlight highlight-source-shell notranslate position-relative overflow-auto" dir="auto"><pre>git clone https://github.com/laiso/site2pdf.git
+cd site2pdf
+npm install
+npm run build
+npm link</pre></div>
+			</article></body></html>`;
+
+			const result = await Defuddle(parseDocument(html, 'https://example.com/docs/code'), 'https://example.com/docs/code', { separateMarkdown: true });
+
+			expect(result.contentMarkdown).toContain('```bash');
+			expect(result.contentMarkdown).toContain('npm link');
+		});
+	});
+
 	describe('ragged table columns', () => {
 		test('should preserve trailing columns when body rows are wider than header', async () => {
 			const html = `<html><head><title>Test</title></head><body><article>

tsconfig.json

@@ -13,4 +13,4 @@
 	},
 	"include": ["src/**/*"],
 	"exclude": ["node_modules", "dist"]
-} 
+}