Skip to content

feat(lexer): implement CommonMark tab expansion (§2.1) #39

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
jmcollin merged 2 commits into from
Jun 1, 2026
Merged

feat(lexer): implement CommonMark tab expansion (§2.1) #39

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
f4fc30f
feat(lexer): implement CommonMark tab expansion (§2.1)
jmcollin May 31, 2026
f0cfb64
fix(parser): remove dead assignment and redundant guard in buildBlock…
jmcollin Jun 1, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
98 changes: 89 additions & 9 deletions src/Lexer/Lexer.php
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@ final class Lexer
private const PATTERN_BLOCKQUOTE = '/^((?:>[ \t]*)++)(.*)/';
private const PATTERN_UNORDERED_LIST = '/^( *)[-*+]\s+(.+)/';
private const PATTERN_ORDERED_LIST = '/^( *)\d+\.\s+(.+)/';
private const PATTERN_HORIZONTAL_RULE = '/^(-{3,}|\*{3,}|_{3,})\s*$/';
private const PATTERN_HORIZONTAL_RULE = '/^[ \t]{0,3}([-*_])([ \t]*\1){2,}[ \t]*$/';
private const PATTERN_LINK_DEFINITION = '/^\[([^\]\[]+)\]:\s+(?:<((?:[^<>\\\\\n]|\\\\.)*)>|(\S+))(?:\s+(?:"((?:[^"\\\\]|\\\\.)*)"|\'((?:[^\'\\\\]|\\\\.)*)\'|\(((?:[^()\\\\]|\\\\.)*)\)))?$/';
/** Matches a standalone title line (CommonMark §4.7 multiline link ref definition). */
private const PATTERN_STANDALONE_TITLE = '/^(?:"((?:[^"\\\\]|\\\\.)*)"|\'((?:[^\'\\\\]|\\\\.)*)\'|\(((?:[^()\\\\]|\\\\.)*)\))\s*$/';
Expand All @@ -28,7 +28,6 @@ final class Lexer
private const PATTERN_COLUMNS_OPEN = '/^:::\s*columns\s*$/i';
private const PATTERN_COLUMNS_CLOSE = '/^:::$/';
private const PATTERN_COLUMNS_SEP = '/^\|\|\|$/';
private const PATTERN_INDENTED_CODE = '/^( |\t)(.*)/s';
private const PATTERN_FOOTNOTE_DEF = '/^\[\^([A-Za-z0-9_-]{1,50})\]:\s+(.+)$/';

/**
Expand Down Expand Up @@ -105,7 +104,8 @@ public function tokenize(string $markdown): array
$footnoteBodyLines = [];

foreach ($lines as $raw) {
$line = rtrim($raw, "\r");
$line = rtrim($raw, "\r");
$expanded = $this->expandTabs($line);

// Collect lines for an in-progress HTML block.
// The block ends on the first blank line (CommonMark §4.6 type 6/7).
Expand Down Expand Up @@ -316,11 +316,11 @@ public function tokenize(string $markdown): array

// Indented code block drain (continuation).
if ($inIndentedBlock) {
if (preg_match(self::PATTERN_INDENTED_CODE, $line, $m)
if (preg_match('/^ (.*)$/s', $expanded, $m)
&& !preg_match(self::PATTERN_UNORDERED_LIST, $line)
&& !preg_match(self::PATTERN_ORDERED_LIST, $line)
) {
$indentedLines = [...$indentedLines, ...$pendingBlanks, $m[2]];
$indentedLines = [...$indentedLines, ...$pendingBlanks, $this->stripLeadingColumns($line, 4)];
$pendingBlanks = [];
continue;
}
Expand All @@ -343,12 +343,12 @@ public function tokenize(string $markdown): array
// and only when the line is not a list item — list items with leading spaces
// are handled by matchLine() via PATTERN_UNORDERED_LIST / PATTERN_ORDERED_LIST).
if (!$hadPendingLines
&& preg_match(self::PATTERN_INDENTED_CODE, $line, $m)
&& preg_match('/^ (.*)$/s', $expanded, $m)
&& !preg_match(self::PATTERN_UNORDERED_LIST, $line)
&& !preg_match(self::PATTERN_ORDERED_LIST, $line)
) {
$inIndentedBlock = true;
$indentedLines = [$m[2]];
$indentedLines = [$this->stripLeadingColumns($line, 4)];
$pendingBlanks = [];
continue;
}
Expand Down Expand Up @@ -457,6 +457,80 @@ private function extractTaskChecked(string $content): array
return [$content, null];
}

/**
* Expand tab characters to spaces using 4-column tab stops (CommonMark §2.1).
*
* @param int $startCol Column position of the first character of $line (default 0).
*/
private function expandTabs(string $line, int $startCol = 0): string
{
$out = '';
$col = $startCol;
$len = strlen($line);
for ($i = 0; $i < $len; $i++) {
$ch = $line[$i];
if ($ch === "\t") {
$spaces = 4 - ($col % 4);
$out .= str_repeat(' ', $spaces);
$col += $spaces;
} else {
$out .= $ch;
$col++;
}
}
return $out;
}

/**
* Return the suffix of $line after consuming exactly $cols columns,
* prepending any overshoot spaces when a tab spans the boundary.
*/
private function stripLeadingColumns(string $line, int $cols): string
{
$col = 0;
$len = strlen($line);
for ($i = 0; $i < $len; $i++) {
if ($col >= $cols) {
return substr($line, $i);
}
$ch = $line[$i];
if ($ch === "\t") {
$tabStop = 4 - ($col % 4);
$newCol = $col + $tabStop;
if ($newCol > $cols) {
$surplus = $newCol - $cols;
return str_repeat(' ', $surplus) . substr($line, $i + 1);
}
$col = $newCol;
} else {
$col++;
}
}
return '';
}

/**
* Strip blockquote markers from an already-expanded line.
*
* Implements CommonMark §5.1: each `>` consumes one mandatory character plus one
* optional space. Operates on the expanded form so tab overshoot is already resolved.
*/
private function stripBlockquoteMarkers(string $expandedLine, int $level): string
{
$pos = 0;
$len = strlen($expandedLine);
for ($l = 0; $l < $level; $l++) {
if ($pos < $len && $expandedLine[$pos] === '>') {
$pos++;
}
// Consume at most one optional space following the marker.
if ($pos < $len && $expandedLine[$pos] === ' ') {
$pos++;
}
}
return substr($expandedLine, $pos);
}

private function matchLine(string $line): Token
{
if ($line === '' || ctype_space($line)) {
Expand All @@ -477,10 +551,16 @@ private function matchLine(string $line): Token
}

if (preg_match(self::PATTERN_BLOCKQUOTE, $line, $m)) {
// Compute inner content from the expanded line using the CommonMark §5.1 rule:
// each '>' marker consumes the '>' character and optionally one space.
// Operating on the expanded form ensures tabs in the prefix are correctly handled.
$level = substr_count($m[1], '>');
$expandedLine = $this->expandTabs($line);
$innerContent = $this->stripBlockquoteMarkers($expandedLine, $level);
return new Token(
TokenType::BLOCKQUOTE,
trim($m[2]),
['level' => substr_count($m[1], '>')],
$innerContent,
['level' => $level],
);
}

Expand Down
45 changes: 26 additions & 19 deletions src/Parser/Parser.php
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -492,38 +492,30 @@ private function buildBlockquote(array $tokens, int &$i, int $minLevel = 1): Blo
if ($level < $minLevel) {
// Level decrease: flush buffer and yield cursor to caller.
if ($buffer !== []) {
$children[] = new ParagraphNode(
children: $this->inlineParser->parse(implode(' ', $buffer), $this->linkRefs, $this->footnoteDefs),
);
array_push($children, ...$this->parseBlockquoteBuffer($buffer));
}
return new BlockquoteNode(children: $children);
}

if ($level === $minLevel) {
if ($tokens[$i]->content !== '') {
$buffer[] = $tokens[$i]->content;
}
$buffer[] = $tokens[$i]->content;
$i++;

// Flush buffer when the next token changes level or ends the blockquote run.
$nextIsCurrentLevel = $i < $count
&& $tokens[$i]->type === TokenType::BLOCKQUOTE
&& $tokens[$i]->meta['level'] === $minLevel;

if (!$nextIsCurrentLevel && $buffer !== []) {
$children[] = new ParagraphNode(
children: $this->inlineParser->parse(implode(' ', $buffer), $this->linkRefs, $this->footnoteDefs),
);
if (!$nextIsCurrentLevel) {
array_push($children, ...$this->parseBlockquoteBuffer($buffer));
$buffer = [];
}
continue;
}

// $level > $minLevel: flush buffer then recurse.
if ($buffer !== []) {
$children[] = new ParagraphNode(
children: $this->inlineParser->parse(implode(' ', $buffer), $this->linkRefs, $this->footnoteDefs),
);
array_push($children, ...$this->parseBlockquoteBuffer($buffer));
$buffer = [];
}

Expand All @@ -534,20 +526,35 @@ private function buildBlockquote(array $tokens, int &$i, int $minLevel = 1): Blo
// are intentionally rendered as content inside the level-32 blockquote rather
// than triggering unbounded recursion. The extra ">" markers are consumed and
// discarded; only the text content is preserved.
if ($tokens[$i]->content !== '') {
$buffer[] = $tokens[$i]->content;
}
$buffer[] = $tokens[$i]->content;
$i++;
}
}

// Flush any remaining buffer at end of token stream.
if ($buffer !== []) {
$children[] = new ParagraphNode(
children: $this->inlineParser->parse(implode(' ', $buffer), $this->linkRefs, $this->footnoteDefs),
);
array_push($children, ...$this->parseBlockquoteBuffer($buffer));
}

return new BlockquoteNode(children: $children);
}

/**
* Re-tokenize a collected blockquote content buffer and parse it into block nodes.
*
* Each entry in $buffer is the raw content string of one BLOCKQUOTE token (one line).
* Empty strings (blank lines) become blank-line separators between blocks.
* The content is joined with newlines and passed through the Lexer so that
* indented code blocks, headings, etc. within a blockquote are correctly detected.
*
* @param string[] $buffer
* @return array<int, \PhpMarkdown\Node\BlockNodeInterface>
*/
private function parseBlockquoteBuffer(array $buffer): array
{
$lexer = new Lexer();
$raw = implode("\n", $buffer);
$innerTokens = $lexer->tokenize($raw);
return $this->parseBlocks($innerTokens);
}
}
7 changes: 4 additions & 3 deletions tests/Integration/MarkdownParserTest.php
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -373,13 +373,14 @@ public function testDepthGuardAbove32ContentPreserved(): void
$this->assertStringContainsString('text', $html);
}

public function testMultiLineSameLevelJoinedWithSpace(): void
public function testMultiLineSameLevelJoinedWithSoftBreak(): void
{
// Two consecutive level-1 tokens accumulate into one ParagraphNode joined by space.
// Two consecutive level-1 tokens re-tokenize as a single paragraph with a soft line break.
// CommonMark §2.3: a soft line break is a newline that is not a hard line break.
$md = "> line one\n> line two";
$html = $this->parser->parse($md);

$this->assertSame("<blockquote>\n<p>line one line two</p>\n</blockquote>\n", $html);
$this->assertSame("<blockquote>\n<p>line one\nline two</p>\n</blockquote>\n", $html);
}

public function testLevelSkipOneToThree(): void
Expand Down
7 changes: 5 additions & 2 deletions tests/Unit/LexerTest.php
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -163,12 +163,15 @@ public function testBlockquoteNoSpaceAfterInnerMarker(): void
$this->assertSame('text', $tokens[0]->content);
}

public function testBlockquoteTrailingSpacesInContentAreTrimmed(): void
public function testBlockquoteLeadingSurplusSpacesKeptTrailingSpacesPreserved(): void
{
// CommonMark §5.1: '>' strips one optional space; extra leading spaces are content.
// The two extra spaces in '> ' remain as leading content spaces.
// Trailing spaces are preserved — the inner re-tokenisation detects hard line breaks.
$tokens = $this->lexer->tokenize('> lots of spaces ');

$this->assertSame(TokenType::BLOCKQUOTE, $tokens[0]->type);
$this->assertSame('lots of spaces', $tokens[0]->content);
$this->assertSame(' lots of spaces ', $tokens[0]->content);
}

public function testHorizontalRuleVariants(): void
Expand Down
Loading
Loading