diff --git a/src/Node/Inline/HtmlEntityNode.php b/src/Node/Inline/HtmlEntityNode.php index 976c392..ee056ac 100644 --- a/src/Node/Inline/HtmlEntityNode.php +++ b/src/Node/Inline/HtmlEntityNode.php @@ -7,11 +7,20 @@ use PhpMarkdown\Node\InlineNodeInterface; /** - * A valid HTML entity that must be passed through verbatim to the renderer. - * Examples: &     + * An HTML entity decoded to its Unicode character, HTML-escaped for safe verbatim output. * - * SECURITY: only entities validated by InlineParser::scan() reach this node. - * The renderer MUST NOT call esc() on the entity string. + * InlineParser::scan() resolves the raw entity reference (named or numeric) to the + * Unicode character(s) it represents, then applies htmlspecialchars() so that characters + * that are meaningful in HTML (e.g. & < >) are re-encoded as HTML entities while all + * other characters are stored as literal UTF-8. + * + * Examples (raw entity → stored value): + * & → & (& re-encoded) + * © → © (literal UTF-8) + *   → \xC2\xA0 (literal non-breaking space) + * � → \xEF\xBF\xBD (U+FFFD replacement character) + * + * SECURITY: the renderer MUST NOT call esc() on the entity string — it is already safe. */ final readonly class HtmlEntityNode implements InlineNodeInterface { diff --git a/src/Parser/InlineParser.php b/src/Parser/InlineParser.php index 1e2fe12..732eb15 100644 --- a/src/Parser/InlineParser.php +++ b/src/Parser/InlineParser.php @@ -204,7 +204,9 @@ private function scan(string $text, int $depth): array $raw = $m[0]; - // Determine whether this is a numeric or named entity. + // Determine whether this is a numeric or named entity and compute $decodedSafe: + // the Unicode character(s) that the entity represents, HTML-escaped as needed + // so the renderer can output the value verbatim in an HTML text node. if ($raw[1] === '#') { // Numeric entity — extract the codepoint. $inner = substr($raw, 2, -1); // strip leading '&#' and trailing ';' @@ -214,25 +216,34 @@ private function scan(string $text, int $depth): array $codepoint = (int) $inner; } - // Reject invalid / dangerous codepoints (CommonMark §2.5 & HTML5 §8.1.4). - $valid = !( - $codepoint === 0 // NUL + // CommonMark §2.5 / HTML5 §8.1.4 — three buckets: + if ($codepoint === 0 + || ($codepoint >= 0xD800 && $codepoint <= 0xDFFF) + ) { + // NUL and surrogates → replacement character U+FFFD. + $decodedSafe = "\u{FFFD}"; + } elseif ( + $codepoint > 0x10FFFF // beyond Unicode range || ($codepoint >= 0x0001 && $codepoint <= 0x001F && $codepoint !== 0x0009 // TAB && $codepoint !== 0x000A // LF && $codepoint !== 0x000D) // CR || $codepoint === 0x007F // DEL - || ($codepoint >= 0xD800 && $codepoint <= 0xDFFF) // surrogates || ($codepoint >= 0xFDD0 && $codepoint <= 0xFDEF) // non-characters || $codepoint === 0xFFFE || $codepoint === 0xFFFF - || $codepoint > 0x10FFFF // beyond Unicode range - ); - - if (!$valid) { + ) { + // Invalid / non-character codepoint → pass raw entity through as text. $buffer .= $raw; $pos += strlen($raw); continue; + } else { + // Valid codepoint → decode to UTF-8 char, then HTML-escape if needed. + $decodedSafe = htmlspecialchars( + mb_chr($codepoint, 'UTF-8'), + ENT_HTML5, + 'UTF-8', + ); } } else { // Named entity — PHP recognises it if html_entity_decode changes it. @@ -243,12 +254,14 @@ private function scan(string $text, int $depth): array $pos += strlen($raw); continue; } + // Decode succeeded: HTML-escape the resulting character(s) for safe output. + $decodedSafe = htmlspecialchars($decoded, ENT_HTML5, 'UTF-8'); } - // Valid entity: flush pending buffer, emit node, advance. + // Valid entity: flush pending buffer, emit decoded node, advance. $tokens = $this->flushBuffer($buffer, $tokens); $buffer = ''; - $tokens[] = new HtmlEntityNode($raw); + $tokens[] = new HtmlEntityNode($decodedSafe); $pos += strlen($raw); continue; } diff --git a/tests/Unit/InlineParserEntityTest.php b/tests/Unit/InlineParserEntityTest.php index 3da49f6..ecc0e8d 100644 --- a/tests/Unit/InlineParserEntityTest.php +++ b/tests/Unit/InlineParserEntityTest.php @@ -32,18 +32,20 @@ public function testDecimalEntityProducesHtmlEntityNode(): void { $nodes = $this->parser->parse(' '); + //   = U+00A0 (non-breaking space) decoded to literal UTF-8 \xC2\xA0. $this->assertCount(1, $nodes); $this->assertInstanceOf(HtmlEntityNode::class, $nodes[0]); - $this->assertSame(' ', $nodes[0]->entity); + $this->assertSame("\xC2\xA0", $nodes[0]->entity); } public function testHexEntityProducesHtmlEntityNode(): void { $nodes = $this->parser->parse(' '); + //   = U+00A0 (non-breaking space) decoded to literal UTF-8 \xC2\xA0. $this->assertCount(1, $nodes); $this->assertInstanceOf(HtmlEntityNode::class, $nodes[0]); - $this->assertSame(' ', $nodes[0]->entity); + $this->assertSame("\xC2\xA0", $nodes[0]->entity); } public function testBareAmpersandProducesTextNode(): void @@ -66,22 +68,24 @@ public function testInvalidNamedEntityProducesTextNode(): void } } - public function testNullCodepointEntityProducesTextNode(): void + public function testNullCodepointEntityProducesReplacementCharNode(): void { $nodes = $this->parser->parse('�'); - foreach ($nodes as $node) { - $this->assertNotInstanceOf(HtmlEntityNode::class, $node); - } + // � (NUL) is mapped to U+FFFD per CommonMark §2.5 / HTML5 §8.1.4. + $this->assertCount(1, $nodes); + $this->assertInstanceOf(HtmlEntityNode::class, $nodes[0]); + $this->assertSame("\u{FFFD}", $nodes[0]->entity); } - public function testSurrogateEntityProducesTextNode(): void + public function testSurrogateEntityProducesReplacementCharNode(): void { $nodes = $this->parser->parse('�'); - foreach ($nodes as $node) { - $this->assertNotInstanceOf(HtmlEntityNode::class, $node); - } + // Surrogate codepoints are mapped to U+FFFD per CommonMark §2.5 / HTML5 §8.1.4. + $this->assertCount(1, $nodes); + $this->assertInstanceOf(HtmlEntityNode::class, $nodes[0]); + $this->assertSame("\u{FFFD}", $nodes[0]->entity); } public function testCodeSpanContentIsNotScannedForEntities(): void diff --git a/tests/Unit/Parser/HtmlEntityDecodingTest.php b/tests/Unit/Parser/HtmlEntityDecodingTest.php new file mode 100644 index 0000000..ccac961 --- /dev/null +++ b/tests/Unit/Parser/HtmlEntityDecodingTest.php @@ -0,0 +1,157 @@ +parser = new MarkdownParser(); + } + + private function parse(string $markdown): string + { + return $this->parser->parse($markdown); + } + + // ── AC-1: Named entities are decoded to their Unicode characters ────────── + + public function testAc1NamedEntitiesDecoded(): void + { + //   → U+00A0 (literal); & → &; © → ©; Æ → Æ; Ď → Ď + $result = $this->parse('  & © Æ Ď'); + + $this->assertSame("

\xC2\xA0 & \xC2\xA9 \xC3\x86 \xC4\x8E

\n", $result); + } + + // ── AC-2: Decimal numeric character references ──────────────────────────── + + public function testAc2DecimalNumericRefs(): void + { + // # → #; Ӓ → Ӓ (U+04D2); Ϡ → Ϡ (U+03E0); � → U+FFFD + $result = $this->parse('# Ӓ Ϡ �'); + + $this->assertSame("

# \u{04D2} \u{03E0} \u{FFFD}

\n", $result); + } + + // ── AC-3: Hexadecimal numeric character references ──────────────────────── + + public function testAc3HexNumericRefs(): void + { + // " → " (U+0022); ആ → ആ (U+0D06, Malayalam); ಫ → ಫ (U+0CAB, Kannada) + $result = $this->parse('" ആ ಫ'); + + $this->assertSame("

\" \u{0D06} \u{0CAB}

\n", $result); + } + + // ── AC-4: Invalid / unrecognised entity names pass through as literal text ─ + + public function testAc4InvalidEntityNamesLiteral(): void + { + //   (no semicolon), &x; (unknown), &#; (no digits), &#x; (no hex digits), + // � (too many digits), &ThisIsNotDefined; (unknown) + // All bare & chars must become & in output. + $result = $this->parse('  &x; &#; &#x; � &ThisIsNotDefined;'); + + $this->assertSame( + "

&nbsp &x; &#; &#x; &#87654321; &ThisIsNotDefined;

\n", + $result, + ); + } + + // ── AC-5: Entity without trailing semicolon is literal text ────────────── + + public function testAc5EntityWithoutSemicolonIsLiteral(): void + { + $result = $this->parse('©'); + + $this->assertSame("

&copy

\n", $result); + } + + // ── AC-6: Entities inside code spans are not decoded ───────────────────── + + public function testAc6EntityInCodeSpanNotDecoded(): void + { + $result = $this->parse('`föfö`'); + + $this->assertSame("

f&ouml;f&ouml;

\n", $result); + } + + // ── AC-7: Entities inside indented code blocks are not decoded ──────────── + + public function testAc7EntityInIndentedCodeBlockNotDecoded(): void + { + $result = $this->parse(' föfö'); + + $this->assertSame("
f&ouml;f&ouml;\n
\n", $result); + } + + // ── AC-8: Entities in link URL and title ───────────────────────────────── + + /** + * @todo Needs link-level entity decoding + URL percent-encoding, out of scope for story 52. + * [foo](/föö "föö") should produce + *

foo

+ * but the current implementation does not decode entities in link URLs or titles. + */ + public function testAc8EntityInLinkUrlAndTitle(): void + { + // Current behaviour: & in URL is HTML-escaped by the renderer's esc(); title same. + // When link-level entity decoding is implemented, update the assertion below. + $result = $this->parse('[foo](/föö "föö")'); + + $this->assertSame( + '

foo

' . "\n", + $result, + ); + } + + // ── AC-9: Numeric entity suppresses emphasis delimiter ──────────────────── + + public function testAc9NumericEntitySuppressesEmphasis(): void + { + // * = '*' decoded as HtmlEntityNode, not a DelimiterRun, so no emphasis. + // The real *foo* on the next line does produce emphasis. + $result = $this->parse("*foo*\n*foo*"); + + $this->assertSame("

*foo*\nfoo

\n", $result); + } + + // ── AC-10: Fenced code block language info-string entity decoding ───────── + + /** + * @todo Needs block-level entity decoding, out of scope for story 52. + * Spec ex 34: the fenced code language string should be decoded. + */ + public function testAc10FencedCodeLanguageEntityDecoding(): void + { + // CommonMark spec ex 34: ``` &lang ``` should produce class="language-&lang" + // or similar with decoded entity in the language attribute. + // Block-level entity decoding is out of scope for this story; recording expected + // behaviour for future implementation. + $result = $this->parse("```&lang\ncode\n```"); + + // Current behaviour: & in info string is sanitized by the renderer + // (non-alphanumeric chars stripped from language). Mark current output and revisit. + $this->assertSame( + "
code
\n", + $result, + ); + } +}