diff --git a/src/Lexer/Lexer.php b/src/Lexer/Lexer.php index db14404..e19a918 100644 --- a/src/Lexer/Lexer.php +++ b/src/Lexer/Lexer.php @@ -84,7 +84,7 @@ public function tokenize(string $markdown): array $fencedLines = []; $fenceChar = ''; $fenceLength = 0; - $pendingToken = null; + $pendingLines = []; $pendingLinkDef = null; // LINK_DEFINITION token awaiting a possible next-line title $inHtmlBlock = false; @@ -149,10 +149,10 @@ public function tokenize(string $markdown): array if ($inColumnsBlock) { if (preg_match(self::PATTERN_COLUMNS_CLOSE, $line)) { - if ($pendingToken !== null) { - $tokens[] = $pendingToken; - $pendingToken = null; + foreach ($pendingLines as $pt) { + $tokens[] = $pt; } + $pendingLines = []; $tokens[] = new Token( TokenType::COLUMNS_CONTAINER, '', @@ -176,10 +176,10 @@ public function tokenize(string $markdown): array } if (preg_match(self::PATTERN_COLUMNS_OPEN, $line)) { - if ($pendingToken !== null) { - $tokens[] = $pendingToken; - $pendingToken = null; + foreach ($pendingLines as $pt) { + $tokens[] = $pt; } + $pendingLines = []; $inColumnsBlock = true; $columnsSepFound = false; $columnsLeftLines = []; @@ -189,10 +189,10 @@ public function tokenize(string $markdown): array if ($inFencedBlock) { if (preg_match('/^ {0,3}' . preg_quote($fenceChar, '/') . '{' . $fenceLength . ',}\s*$/', $line)) { - if ($pendingToken !== null) { - $tokens[] = $pendingToken; - $pendingToken = null; + foreach ($pendingLines as $pt) { + $tokens[] = $pt; } + $pendingLines = []; $tokens[] = new Token( TokenType::FENCED_CODE, implode("\n", $fencedLines), @@ -210,10 +210,10 @@ public function tokenize(string $markdown): array } if (preg_match(self::PATTERN_FENCED_OPEN, $line, $m)) { - if ($pendingToken !== null) { - $tokens[] = $pendingToken; - $pendingToken = null; + foreach ($pendingLines as $pt) { + $tokens[] = $pt; } + $pendingLines = []; $inFencedBlock = true; $fencedLanguage = $m[2]; $fencedLines = []; @@ -226,10 +226,10 @@ public function tokenize(string $markdown): array // Must run before setext/paragraph logic so that block-level HTML tags // are not consumed as paragraphs. if (preg_match($this->patternHtmlBlockStart, $line)) { - if ($pendingToken !== null) { - $tokens[] = $pendingToken; - $pendingToken = null; + foreach ($pendingLines as $pt) { + $tokens[] = $pt; } + $pendingLines = []; // Detect whether this is an HTML comment opener. $isCommentStart = str_starts_with(ltrim($line), ''); @@ -269,22 +269,22 @@ public function tokenize(string $markdown): array } // Capture paragraph-active state before setext promotion may clear it. - $hadPendingToken = $pendingToken !== null; + $hadPendingLines = $pendingLines !== []; - // Setext heading detection: a pending text line followed by === or --- - if ($pendingToken !== null) { + // Setext heading detection: pending text lines followed by === or --- + if ($pendingLines !== []) { if (preg_match(self::PATTERN_SETEXT_H1, $line)) { - $tokens[] = new Token(TokenType::HEADING, $pendingToken->content, ['level' => 1]); - $pendingToken = null; + $content = implode(' ', array_map(fn(Token $t) => $t->content, $pendingLines)); + $tokens[] = new Token(TokenType::HEADING, $content, ['level' => 1]); + $pendingLines = []; continue; } if (preg_match(self::PATTERN_SETEXT_H2, $line)) { - $tokens[] = new Token(TokenType::HEADING, $pendingToken->content, ['level' => 2]); - $pendingToken = null; + $content = implode(' ', array_map(fn(Token $t) => $t->content, $pendingLines)); + $tokens[] = new Token(TokenType::HEADING, $content, ['level' => 2]); + $pendingLines = []; continue; } - $tokens[] = $pendingToken; - $pendingToken = null; } // Multiline link ref title (CommonMark §4.7): a LINK_DEFINITION with no title @@ -342,7 +342,7 @@ public function tokenize(string $markdown): array // Start new indented code block (only when no paragraph was active, // and only when the line is not a list item — list items with leading spaces // are handled by matchLine() via PATTERN_UNORDERED_LIST / PATTERN_ORDERED_LIST). - if (!$hadPendingToken + if (!$hadPendingLines && preg_match(self::PATTERN_INDENTED_CODE, $line, $m) && !preg_match(self::PATTERN_UNORDERED_LIST, $line) && !preg_match(self::PATTERN_ORDERED_LIST, $line) @@ -364,11 +364,10 @@ public function tokenize(string $markdown): array // A FOOTNOTE_DEFINITION token opens multi-line body accumulation. // Flush any pending setext heading candidate and pending link def first. if ($token->type === TokenType::FOOTNOTE_DEFINITION) { - /** @psalm-suppress TypeDoesNotContainType @phpstan-ignore notIdentical.alwaysFalse */ - if ($pendingToken !== null) { - $tokens[] = $pendingToken; - $pendingToken = null; + foreach ($pendingLines as $pt) { + $tokens[] = $pt; } + $pendingLines = []; /** @psalm-suppress TypeDoesNotContainType @phpstan-ignore notIdentical.alwaysFalse */ if ($pendingLinkDef !== null) { $tokens[] = $pendingLinkDef; @@ -382,10 +381,14 @@ public function tokenize(string $markdown): array // A plain paragraph line is held as pending to allow setext promotion on the next line. if ($token->type === TokenType::PARAGRAPH && ($line !== '' && !ctype_space($line))) { - $pendingToken = $token; + $pendingLines[] = $token; continue; } + foreach ($pendingLines as $pt) { + $tokens[] = $pt; + } + $pendingLines = []; $tokens[] = $token; } @@ -394,9 +397,9 @@ public function tokenize(string $markdown): array $tokens[] = $pendingLinkDef; } - // Flush any remaining pending token - if ($pendingToken !== null) { - $tokens[] = $pendingToken; + // Flush any remaining pending lines + foreach ($pendingLines as $pt) { + $tokens[] = $pt; } // Flush any in-progress footnote definition body diff --git a/tests/Integration/fixtures/setext-headings.html b/tests/Integration/fixtures/setext-headings.html index 4712458..818e929 100644 --- a/tests/Integration/fixtures/setext-headings.html +++ b/tests/Integration/fixtures/setext-headings.html @@ -1 +1 @@ -

My Title

Sub Title

bold title

text

Title

\ No newline at end of file +

My Title

Sub Title

bold title

text

Title

foo bar

line one line two line three

foo bar

\ No newline at end of file diff --git a/tests/Integration/fixtures/setext-headings.md b/tests/Integration/fixtures/setext-headings.md index 23e8815..1232c88 100644 --- a/tests/Integration/fixtures/setext-headings.md +++ b/tests/Integration/fixtures/setext-headings.md @@ -8,3 +8,13 @@ text --- Title = +foo +bar +=== +line one +line two +line three +--- +*foo +bar* +=== diff --git a/tests/Unit/SetextHeadingMultilineTest.php b/tests/Unit/SetextHeadingMultilineTest.php new file mode 100644 index 0000000..ad3f953 --- /dev/null +++ b/tests/Unit/SetextHeadingMultilineTest.php @@ -0,0 +1,157 @@ +tokenize("foo\nbar\n==="); + + $headingTokens = array_values(array_filter( + $tokens, + static fn($t) => $t->type === TokenType::HEADING, + )); + + self::assertCount(1, $headingTokens); + self::assertSame(1, $headingTokens[0]->meta['level']); + self::assertSame('foo bar', $headingTokens[0]->content); + } + + public function testThreeLineH2Token(): void + { + $lexer = new Lexer(); + $tokens = $lexer->tokenize("line one\nline two\nline three\n---"); + + $headingTokens = array_values(array_filter( + $tokens, + static fn($t) => $t->type === TokenType::HEADING, + )); + + self::assertCount(1, $headingTokens); + self::assertSame(2, $headingTokens[0]->meta['level']); + self::assertSame('line one line two line three', $headingTokens[0]->content); + } + + public function testSingleLineRegressionToken(): void + { + $lexer = new Lexer(); + $tokens = $lexer->tokenize("foo\n==="); + + $headingTokens = array_values(array_filter( + $tokens, + static fn($t) => $t->type === TokenType::HEADING, + )); + + self::assertCount(1, $headingTokens); + self::assertSame(1, $headingTokens[0]->meta['level']); + self::assertSame('foo', $headingTokens[0]->content); + } + + public function testBlankBreaksAccumulationTokens(): void + { + $lexer = new Lexer(); + $tokens = $lexer->tokenize("foo\n\nbar\n==="); + + $types = array_map(static fn($t) => $t->type, $tokens); + + self::assertContains(TokenType::PARAGRAPH, $types); + self::assertContains(TokenType::BLANK, $types); + self::assertContains(TokenType::HEADING, $types); + + $paragraphs = array_values(array_filter($tokens, static fn($t) => $t->type === TokenType::PARAGRAPH)); + $headings = array_values(array_filter($tokens, static fn($t) => $t->type === TokenType::HEADING)); + + self::assertCount(1, $paragraphs); + self::assertSame('foo', $paragraphs[0]->content); + + self::assertCount(1, $headings); + self::assertSame(1, $headings[0]->meta['level']); + self::assertSame('bar', $headings[0]->content); + } + + public function testInlineMarkupSpansLinesToken(): void + { + $lexer = new Lexer(); + $tokens = $lexer->tokenize("*foo\nbar*\n==="); + + $headingTokens = array_values(array_filter( + $tokens, + static fn($t) => $t->type === TokenType::HEADING, + )); + + self::assertCount(1, $headingTokens); + self::assertSame(1, $headingTokens[0]->meta['level']); + self::assertSame('*foo bar*', $headingTokens[0]->content); + } + + public function testAtxHeadingInterruptsPendingLines(): void + { + $lexer = new Lexer(); + $tokens = $lexer->tokenize("foo\n## bar"); + + $paragraphs = array_values(array_filter($tokens, static fn($t) => $t->type === TokenType::PARAGRAPH)); + $headings = array_values(array_filter($tokens, static fn($t) => $t->type === TokenType::HEADING)); + + self::assertCount(1, $paragraphs); + self::assertSame('foo', $paragraphs[0]->content); + + self::assertCount(1, $headings); + self::assertSame(2, $headings[0]->meta['level']); + self::assertSame('bar', $headings[0]->content); + } + + // ------------------------------------------------------------------------- + // Render-level tests + // ------------------------------------------------------------------------- + + private function render(string $markdown): string + { + $lexer = new Lexer(); + $parser = new Parser(); + $renderer = new HtmlRenderer(); + + return trim($renderer->render($parser->parse($lexer->tokenize($markdown)))); + } + + public function testTwoLineH1Renders(): void + { + self::assertSame('

foo bar

', $this->render("foo\nbar\n===")); + } + + public function testThreeLineH2Renders(): void + { + self::assertSame('

line one line two line three

', $this->render("line one\nline two\nline three\n---")); + } + + public function testSingleLineRegressionRenders(): void + { + self::assertSame('

foo

', $this->render("foo\n===")); + } + + public function testBlankBreaksAccumulationRenders(): void + { + self::assertSame('

foo

bar

', $this->render("foo\n\nbar\n===")); + } + + public function testInlineMarkupSpansLinesRenders(): void + { + self::assertSame('

foo bar

', $this->render("*foo\nbar*\n===")); + } +}