From f4fc30fe6f0462e2c9b8ed8ea433ea68e1333fd3 Mon Sep 17 00:00:00 2001 From: jmcollin Date: Mon, 1 Jun 2026 00:19:30 +0200 Subject: [PATCH 1/2] =?UTF-8?q?feat(lexer):=20implement=20CommonMark=20tab?= =?UTF-8?q?=20expansion=20(=C2=A72.1)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Expand tabs to 4-column stops before indent detection so mixed space+tab prefixes correctly trigger indented code blocks (spec ex 1–2). Fix PATTERN_HORIZONTAL_RULE to allow tab-separated thematic break chars (spec ex 11). Preserve internal tabs in code block content via stripLeadingColumns. Fix blockquote inner content to keep trailing spaces so hard line breaks survive re-tokenisation. --- src/Lexer/Lexer.php | 98 ++++++++++-- src/Parser/Parser.php | 44 +++--- tests/Integration/MarkdownParserTest.php | 7 +- tests/Unit/LexerTest.php | 7 +- tests/Unit/TabExpansionTest.php | 189 +++++++++++++++++++++++ 5 files changed, 313 insertions(+), 32 deletions(-) create mode 100644 tests/Unit/TabExpansionTest.php diff --git a/src/Lexer/Lexer.php b/src/Lexer/Lexer.php index e19a918..253db68 100644 --- a/src/Lexer/Lexer.php +++ b/src/Lexer/Lexer.php @@ -17,7 +17,7 @@ final class Lexer private const PATTERN_BLOCKQUOTE = '/^((?:>[ \t]*)++)(.*)/'; private const PATTERN_UNORDERED_LIST = '/^( *)[-*+]\s+(.+)/'; private const PATTERN_ORDERED_LIST = '/^( *)\d+\.\s+(.+)/'; - private const PATTERN_HORIZONTAL_RULE = '/^(-{3,}|\*{3,}|_{3,})\s*$/'; + private const PATTERN_HORIZONTAL_RULE = '/^[ \t]{0,3}([-*_])([ \t]*\1){2,}[ \t]*$/'; private const PATTERN_LINK_DEFINITION = '/^\[([^\]\[]+)\]:\s+(?:<((?:[^<>\\\\\n]|\\\\.)*)>|(\S+))(?:\s+(?:"((?:[^"\\\\]|\\\\.)*)"|\'((?:[^\'\\\\]|\\\\.)*)\'|\(((?:[^()\\\\]|\\\\.)*)\)))?$/'; /** Matches a standalone title line (CommonMark §4.7 multiline link ref definition). */ private const PATTERN_STANDALONE_TITLE = '/^(?:"((?:[^"\\\\]|\\\\.)*)"|\'((?:[^\'\\\\]|\\\\.)*)\'|\(((?:[^()\\\\]|\\\\.)*)\))\s*$/'; @@ -28,7 +28,6 @@ final class Lexer private const PATTERN_COLUMNS_OPEN = '/^:::\s*columns\s*$/i'; private const PATTERN_COLUMNS_CLOSE = '/^:::$/'; private const PATTERN_COLUMNS_SEP = '/^\|\|\|$/'; - private const PATTERN_INDENTED_CODE = '/^( |\t)(.*)/s'; private const PATTERN_FOOTNOTE_DEF = '/^\[\^([A-Za-z0-9_-]{1,50})\]:\s+(.+)$/'; /** @@ -105,7 +104,8 @@ public function tokenize(string $markdown): array $footnoteBodyLines = []; foreach ($lines as $raw) { - $line = rtrim($raw, "\r"); + $line = rtrim($raw, "\r"); + $expanded = $this->expandTabs($line); // Collect lines for an in-progress HTML block. // The block ends on the first blank line (CommonMark §4.6 type 6/7). @@ -316,11 +316,11 @@ public function tokenize(string $markdown): array // Indented code block drain (continuation). if ($inIndentedBlock) { - if (preg_match(self::PATTERN_INDENTED_CODE, $line, $m) + if (preg_match('/^ (.*)$/s', $expanded, $m) && !preg_match(self::PATTERN_UNORDERED_LIST, $line) && !preg_match(self::PATTERN_ORDERED_LIST, $line) ) { - $indentedLines = [...$indentedLines, ...$pendingBlanks, $m[2]]; + $indentedLines = [...$indentedLines, ...$pendingBlanks, $this->stripLeadingColumns($line, 4)]; $pendingBlanks = []; continue; } @@ -343,12 +343,12 @@ public function tokenize(string $markdown): array // and only when the line is not a list item — list items with leading spaces // are handled by matchLine() via PATTERN_UNORDERED_LIST / PATTERN_ORDERED_LIST). if (!$hadPendingLines - && preg_match(self::PATTERN_INDENTED_CODE, $line, $m) + && preg_match('/^ (.*)$/s', $expanded, $m) && !preg_match(self::PATTERN_UNORDERED_LIST, $line) && !preg_match(self::PATTERN_ORDERED_LIST, $line) ) { $inIndentedBlock = true; - $indentedLines = [$m[2]]; + $indentedLines = [$this->stripLeadingColumns($line, 4)]; $pendingBlanks = []; continue; } @@ -457,6 +457,80 @@ private function extractTaskChecked(string $content): array return [$content, null]; } + /** + * Expand tab characters to spaces using 4-column tab stops (CommonMark §2.1). + * + * @param int $startCol Column position of the first character of $line (default 0). + */ + private function expandTabs(string $line, int $startCol = 0): string + { + $out = ''; + $col = $startCol; + $len = strlen($line); + for ($i = 0; $i < $len; $i++) { + $ch = $line[$i]; + if ($ch === "\t") { + $spaces = 4 - ($col % 4); + $out .= str_repeat(' ', $spaces); + $col += $spaces; + } else { + $out .= $ch; + $col++; + } + } + return $out; + } + + /** + * Return the suffix of $line after consuming exactly $cols columns, + * prepending any overshoot spaces when a tab spans the boundary. + */ + private function stripLeadingColumns(string $line, int $cols): string + { + $col = 0; + $len = strlen($line); + for ($i = 0; $i < $len; $i++) { + if ($col >= $cols) { + return substr($line, $i); + } + $ch = $line[$i]; + if ($ch === "\t") { + $tabStop = 4 - ($col % 4); + $newCol = $col + $tabStop; + if ($newCol > $cols) { + $surplus = $newCol - $cols; + return str_repeat(' ', $surplus) . substr($line, $i + 1); + } + $col = $newCol; + } else { + $col++; + } + } + return ''; + } + + /** + * Strip blockquote markers from an already-expanded line. + * + * Implements CommonMark §5.1: each `>` consumes one mandatory character plus one + * optional space. Operates on the expanded form so tab overshoot is already resolved. + */ + private function stripBlockquoteMarkers(string $expandedLine, int $level): string + { + $pos = 0; + $len = strlen($expandedLine); + for ($l = 0; $l < $level; $l++) { + if ($pos < $len && $expandedLine[$pos] === '>') { + $pos++; + } + // Consume at most one optional space following the marker. + if ($pos < $len && $expandedLine[$pos] === ' ') { + $pos++; + } + } + return substr($expandedLine, $pos); + } + private function matchLine(string $line): Token { if ($line === '' || ctype_space($line)) { @@ -477,10 +551,16 @@ private function matchLine(string $line): Token } if (preg_match(self::PATTERN_BLOCKQUOTE, $line, $m)) { + // Compute inner content from the expanded line using the CommonMark §5.1 rule: + // each '>' marker consumes the '>' character and optionally one space. + // Operating on the expanded form ensures tabs in the prefix are correctly handled. + $level = substr_count($m[1], '>'); + $expandedLine = $this->expandTabs($line); + $innerContent = $this->stripBlockquoteMarkers($expandedLine, $level); return new Token( TokenType::BLOCKQUOTE, - trim($m[2]), - ['level' => substr_count($m[1], '>')], + $innerContent, + ['level' => $level], ); } diff --git a/src/Parser/Parser.php b/src/Parser/Parser.php index 8cadff9..f9e7736 100644 --- a/src/Parser/Parser.php +++ b/src/Parser/Parser.php @@ -492,17 +492,14 @@ private function buildBlockquote(array $tokens, int &$i, int $minLevel = 1): Blo if ($level < $minLevel) { // Level decrease: flush buffer and yield cursor to caller. if ($buffer !== []) { - $children[] = new ParagraphNode( - children: $this->inlineParser->parse(implode(' ', $buffer), $this->linkRefs, $this->footnoteDefs), - ); + array_push($children, ...$this->parseBlockquoteBuffer($buffer)); + $buffer = []; } return new BlockquoteNode(children: $children); } if ($level === $minLevel) { - if ($tokens[$i]->content !== '') { - $buffer[] = $tokens[$i]->content; - } + $buffer[] = $tokens[$i]->content; $i++; // Flush buffer when the next token changes level or ends the blockquote run. @@ -511,9 +508,7 @@ private function buildBlockquote(array $tokens, int &$i, int $minLevel = 1): Blo && $tokens[$i]->meta['level'] === $minLevel; if (!$nextIsCurrentLevel && $buffer !== []) { - $children[] = new ParagraphNode( - children: $this->inlineParser->parse(implode(' ', $buffer), $this->linkRefs, $this->footnoteDefs), - ); + array_push($children, ...$this->parseBlockquoteBuffer($buffer)); $buffer = []; } continue; @@ -521,9 +516,7 @@ private function buildBlockquote(array $tokens, int &$i, int $minLevel = 1): Blo // $level > $minLevel: flush buffer then recurse. if ($buffer !== []) { - $children[] = new ParagraphNode( - children: $this->inlineParser->parse(implode(' ', $buffer), $this->linkRefs, $this->footnoteDefs), - ); + array_push($children, ...$this->parseBlockquoteBuffer($buffer)); $buffer = []; } @@ -534,20 +527,35 @@ private function buildBlockquote(array $tokens, int &$i, int $minLevel = 1): Blo // are intentionally rendered as content inside the level-32 blockquote rather // than triggering unbounded recursion. The extra ">" markers are consumed and // discarded; only the text content is preserved. - if ($tokens[$i]->content !== '') { - $buffer[] = $tokens[$i]->content; - } + $buffer[] = $tokens[$i]->content; $i++; } } // Flush any remaining buffer at end of token stream. if ($buffer !== []) { - $children[] = new ParagraphNode( - children: $this->inlineParser->parse(implode(' ', $buffer), $this->linkRefs, $this->footnoteDefs), - ); + array_push($children, ...$this->parseBlockquoteBuffer($buffer)); } return new BlockquoteNode(children: $children); } + + /** + * Re-tokenize a collected blockquote content buffer and parse it into block nodes. + * + * Each entry in $buffer is the raw content string of one BLOCKQUOTE token (one line). + * Empty strings (blank lines) become blank-line separators between blocks. + * The content is joined with newlines and passed through the Lexer so that + * indented code blocks, headings, etc. within a blockquote are correctly detected. + * + * @param string[] $buffer + * @return array + */ + private function parseBlockquoteBuffer(array $buffer): array + { + $lexer = new Lexer(); + $raw = implode("\n", $buffer); + $innerTokens = $lexer->tokenize($raw); + return $this->parseBlocks($innerTokens); + } } diff --git a/tests/Integration/MarkdownParserTest.php b/tests/Integration/MarkdownParserTest.php index a4e7f4c..a995edb 100644 --- a/tests/Integration/MarkdownParserTest.php +++ b/tests/Integration/MarkdownParserTest.php @@ -373,13 +373,14 @@ public function testDepthGuardAbove32ContentPreserved(): void $this->assertStringContainsString('text', $html); } - public function testMultiLineSameLevelJoinedWithSpace(): void + public function testMultiLineSameLevelJoinedWithSoftBreak(): void { - // Two consecutive level-1 tokens accumulate into one ParagraphNode joined by space. + // Two consecutive level-1 tokens re-tokenize as a single paragraph with a soft line break. + // CommonMark §2.3: a soft line break is a newline that is not a hard line break. $md = "> line one\n> line two"; $html = $this->parser->parse($md); - $this->assertSame("
\n

line one line two

\n
\n", $html); + $this->assertSame("
\n

line one\nline two

\n
\n", $html); } public function testLevelSkipOneToThree(): void diff --git a/tests/Unit/LexerTest.php b/tests/Unit/LexerTest.php index 4f79905..d93efe5 100644 --- a/tests/Unit/LexerTest.php +++ b/tests/Unit/LexerTest.php @@ -163,12 +163,15 @@ public function testBlockquoteNoSpaceAfterInnerMarker(): void $this->assertSame('text', $tokens[0]->content); } - public function testBlockquoteTrailingSpacesInContentAreTrimmed(): void + public function testBlockquoteLeadingSurplusSpacesKeptTrailingSpacesPreserved(): void { + // CommonMark §5.1: '>' strips one optional space; extra leading spaces are content. + // The two extra spaces in '> ' remain as leading content spaces. + // Trailing spaces are preserved — the inner re-tokenisation detects hard line breaks. $tokens = $this->lexer->tokenize('> lots of spaces '); $this->assertSame(TokenType::BLOCKQUOTE, $tokens[0]->type); - $this->assertSame('lots of spaces', $tokens[0]->content); + $this->assertSame(' lots of spaces ', $tokens[0]->content); } public function testHorizontalRuleVariants(): void diff --git a/tests/Unit/TabExpansionTest.php b/tests/Unit/TabExpansionTest.php new file mode 100644 index 0000000..59cb3a6 --- /dev/null +++ b/tests/Unit/TabExpansionTest.php @@ -0,0 +1,189 @@ +lexer = new Lexer(); + $this->parser = new Parser(); + $this->renderer = new HtmlRenderer(); + } + + private function render(string $markdown): string + { + return $this->renderer->render( + $this->parser->parse($this->lexer->tokenize($markdown)) + ); + } + + /** + * CommonMark spec §2.1 example 1: a single leading tab is equivalent to 4 spaces. + * Internal tabs in the content must be preserved verbatim. + */ + public function testSpecExample1_SingleLeadingTabProducesCodeBlock(): void + { + $html = $this->render("\tfoo\tbaz\t\tbim"); + + $this->assertSame("
foo\tbaz\t\tbim\n
\n", $html); + } + + /** + * CommonMark spec §2.1 example 2: two spaces + tab = 4 columns, still an indented code block. + */ + public function testSpecExample2_TwoSpacesPlusTabProducesCodeBlock(): void + { + $html = $this->render(" \tfoo\tbaz\t\tbim"); + + $this->assertSame("
foo\tbaz\t\tbim\n
\n", $html); + } + + /** + * CommonMark spec §2.1 example 4: tab continuation in a loose list item. + * Deferred — requires Parser-level list continuation changes. + */ + public function testSpecExample4_TabContinuationInLooseListItem(): void + { + // TODO: story 54+ — list continuation tab handling requires Parser changes + $this->markTestSkipped('TODO: story 54+ — list continuation tab handling requires Parser changes'); + } + + /** + * CommonMark spec §2.1 example 5: double tab in list item produces indented code block. + * Deferred — requires Parser-level list continuation changes. + */ + public function testSpecExample5_DoubleTabInListItemProducesCodeBlock(): void + { + // TODO: story 54+ — list continuation tab handling requires Parser changes + $this->markTestSkipped('TODO: story 54+ — list continuation tab handling requires Parser changes'); + } + + /** + * CommonMark spec §2.1 example 6: a tab after a blockquote marker `>` can contribute + * surplus spaces that push the inner content into an indented code block. + * `>\t\tfoo` → the first tab after `>` overshoots the mandatory separator by 2 spaces; + * those 2 surplus spaces plus the second tab (2 more cols to reach col 4) form 4-col indent. + */ + public function testSpecExample6_TabAfterBlockquoteMarkerProducesCodeBlock(): void + { + $html = $this->render(">\t\tfoo"); + + $this->assertSame("
\n
  foo\n
\n
\n", $html); + } + + /** + * CommonMark spec §2.1 example 10: a tab after a heading marker `#` is a valid separator. + * This already worked before the story; this is a regression guard. + */ + public function testSpecExample10_TabAfterHeadingMarkerIsValidSeparator(): void + { + $html = $this->render("#\tFoo"); + + $this->assertSame("

Foo

\n", $html); + } + + /** + * CommonMark spec §2.1 example 11: asterisks separated by tabs form a thematic break. + */ + public function testSpecExample11_TabSeparatedAsterisksFormThematicBreak(): void + { + $html = $this->render("*\t*\t*\t"); + + $this->assertSame("
\n", $html); + } + + /** + * Edge case A: three spaces + tab = exactly 4 columns → indented code block. + * Content after stripping 4 cols is "foo" (no leading spaces). + */ + public function testEdgeCaseA_ThreeSpacesPlusTabEqualsExactlyFourColumns(): void + { + $html = $this->render(" \tfoo"); + + $this->assertSame("
foo\n
\n", $html); + } + + /** + * Edge case B: four spaces + tab = 8 columns → indented code block with extra indent. + * Stripping 4 cols leaves a literal tab as the first content character. + */ + public function testEdgeCaseB_FourSpacesPlusTabPreservesExtraIndent(): void + { + $html = $this->render(" \tfoo"); + + $this->assertSame("
\tfoo\n
\n", $html); + } + + /** + * Edge case C: a single tab after `>` gives only 3 surplus spaces (cols 1–3), + * which is NOT enough for an indented code block (needs 4 cols). + * The blockquote content must therefore be a plain paragraph. + */ + public function testEdgeCaseC_SingleTabAfterBlockquoteIsNotCodeBlock(): void + { + $html = $this->render(">\tbar"); + + $this->assertStringContainsString('
', $html); + $this->assertStringContainsString('

', $html); + $this->assertStringContainsString('bar', $html); + $this->assertStringNotContainsString('

', $html);
+    }
+
+    /**
+     * Regression D: the classic 4-space indented code block must still work.
+     */
+    public function testRegressionD_PlainFourSpaceIndentStillProducesCodeBlock(): void
+    {
+        $html = $this->render("    foo");
+
+        $this->assertSame("
foo\n
\n", $html);
+    }
+
+    /**
+     * Regression E: a plain paragraph must be completely unaffected by tab expansion logic.
+     */
+    public function testRegressionE_PlainParagraphUnaffectedByTabExpansionLogic(): void
+    {
+        $html = $this->render("hello world");
+
+        $this->assertSame("
hello world
\n", $html);
+    }
+
+    /**
+     * Regression G: trailing double spaces inside a blockquote must produce a hard line break.
+     * rtrim on blockquote inner content would strip the spaces and lose the hard-break signal.
+     */
+    public function testRegressionG_HardLineBreakInsideBlockquoteIsPreserved(): void
+    {
+        $html = $this->render("> foo  \n> bar");
+
+        $this->assertStringContainsString('assertStringContainsString('foo', $html);
+        $this->assertStringContainsString('bar', $html);
+    }
+
+    /**
+     * Edge case F: a tab inside a fenced code block must be preserved verbatim.
+     */
+    public function testEdgeCaseF_TabInsideFencedCodeBlockPreservedVerbatim(): void
+    {
+        $markdown = "```\nfoo\tbar\n```";
+        $html     = $this->render($markdown);
+
+        $this->assertStringContainsString("foo\tbar", $html);
+        $this->assertStringNotContainsString('foo    bar', $html);
+        $this->assertStringNotContainsString('foo   bar', $html);
+    }
+}

From f0cfb64affb634d00ec47a9cc0920f4a6fae22bd Mon Sep 17 00:00:00 2001
From: jmcollin 
Date: Mon, 1 Jun 2026 08:30:58 +0200
Subject: [PATCH 2/2] fix(parser): remove dead assignment and redundant guard
 in buildBlockquote

---
 src/Parser/Parser.php | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/src/Parser/Parser.php b/src/Parser/Parser.php
index f9e7736..26d0a5f 100644
--- a/src/Parser/Parser.php
+++ b/src/Parser/Parser.php
@@ -493,7 +493,6 @@ private function buildBlockquote(array $tokens, int &$i, int $minLevel = 1): Blo
                 // Level decrease: flush buffer and yield cursor to caller.
                 if ($buffer !== []) {
                     array_push($children, ...$this->parseBlockquoteBuffer($buffer));
-                    $buffer = [];
                 }
                 return new BlockquoteNode(children: $children);
             }
@@ -507,7 +506,7 @@ private function buildBlockquote(array $tokens, int &$i, int $minLevel = 1): Blo
                     && $tokens[$i]->type === TokenType::BLOCKQUOTE
                     && $tokens[$i]->meta['level'] === $minLevel;
 
-                if (!$nextIsCurrentLevel && $buffer !== []) {
+                if (!$nextIsCurrentLevel) {
                     array_push($children, ...$this->parseBlockquoteBuffer($buffer));
                     $buffer = [];
                 }