feat(lexer): support multiline setext heading content

src/Lexer/Lexer.php

@@ -84,7 +84,7 @@ public function tokenize(string $markdown): array
         $fencedLines = [];
         $fenceChar = '';
         $fenceLength = 0;
-        $pendingToken = null;
+        $pendingLines = [];
         $pendingLinkDef = null; // LINK_DEFINITION token awaiting a possible next-line title
 
         $inHtmlBlock = false;
@@ -149,10 +149,10 @@ public function tokenize(string $markdown): array
 
             if ($inColumnsBlock) {
                 if (preg_match(self::PATTERN_COLUMNS_CLOSE, $line)) {
-                    if ($pendingToken !== null) {
-                        $tokens[] = $pendingToken;
-                        $pendingToken = null;
+                    foreach ($pendingLines as $pt) {
+                        $tokens[] = $pt;
                     }
+                    $pendingLines = [];
                     $tokens[] = new Token(
                         TokenType::COLUMNS_CONTAINER,
                         '',
@@ -176,10 +176,10 @@ public function tokenize(string $markdown): array
             }
 
             if (preg_match(self::PATTERN_COLUMNS_OPEN, $line)) {
-                if ($pendingToken !== null) {
-                    $tokens[] = $pendingToken;
-                    $pendingToken = null;
+                foreach ($pendingLines as $pt) {
+                    $tokens[] = $pt;
                 }
+                $pendingLines = [];
                 $inColumnsBlock    = true;
                 $columnsSepFound   = false;
                 $columnsLeftLines  = [];
@@ -189,10 +189,10 @@ public function tokenize(string $markdown): array
 
             if ($inFencedBlock) {
                 if (preg_match('/^ {0,3}' . preg_quote($fenceChar, '/') . '{' . $fenceLength . ',}\s*$/', $line)) {
-                    if ($pendingToken !== null) {
-                        $tokens[] = $pendingToken;
-                        $pendingToken = null;
+                    foreach ($pendingLines as $pt) {
+                        $tokens[] = $pt;
                     }
+                    $pendingLines = [];
                     $tokens[] = new Token(
                         TokenType::FENCED_CODE,
                         implode("\n", $fencedLines),
@@ -210,10 +210,10 @@ public function tokenize(string $markdown): array
             }
 
             if (preg_match(self::PATTERN_FENCED_OPEN, $line, $m)) {
-                if ($pendingToken !== null) {
-                    $tokens[] = $pendingToken;
-                    $pendingToken = null;
+                foreach ($pendingLines as $pt) {
+                    $tokens[] = $pt;
                 }
+                $pendingLines = [];
                 $inFencedBlock  = true;
                 $fencedLanguage = $m[2];
                 $fencedLines    = [];
@@ -226,10 +226,10 @@ public function tokenize(string $markdown): array
             // Must run before setext/paragraph logic so that block-level HTML tags
             // are not consumed as paragraphs.
             if (preg_match($this->patternHtmlBlockStart, $line)) {
-                if ($pendingToken !== null) {
-                    $tokens[] = $pendingToken;
-                    $pendingToken = null;
+                foreach ($pendingLines as $pt) {
+                    $tokens[] = $pt;
                 }
+                $pendingLines = [];
                 // Detect whether this is an HTML comment opener.
                 $isCommentStart = str_starts_with(ltrim($line), '<!--');
                 $isCommentClosed = $isCommentStart && str_contains($line, '-->');
@@ -269,22 +269,22 @@ public function tokenize(string $markdown): array
             }
 
             // Capture paragraph-active state before setext promotion may clear it.
-            $hadPendingToken = $pendingToken !== null;
+            $hadPendingLines = $pendingLines !== [];
 
-            // Setext heading detection: a pending text line followed by === or ---
-            if ($pendingToken !== null) {
+            // Setext heading detection: pending text lines followed by === or ---
+            if ($pendingLines !== []) {
                 if (preg_match(self::PATTERN_SETEXT_H1, $line)) {
-                    $tokens[] = new Token(TokenType::HEADING, $pendingToken->content, ['level' => 1]);
-                    $pendingToken = null;
+                    $content = implode(' ', array_map(fn(Token $t) => $t->content, $pendingLines));
+                    $tokens[] = new Token(TokenType::HEADING, $content, ['level' => 1]);
+                    $pendingLines = [];
                     continue;
                 }
                 if (preg_match(self::PATTERN_SETEXT_H2, $line)) {
-                    $tokens[] = new Token(TokenType::HEADING, $pendingToken->content, ['level' => 2]);
-                    $pendingToken = null;
+                    $content = implode(' ', array_map(fn(Token $t) => $t->content, $pendingLines));
+                    $tokens[] = new Token(TokenType::HEADING, $content, ['level' => 2]);
+                    $pendingLines = [];
                     continue;
                 }
-                $tokens[] = $pendingToken;
-                $pendingToken = null;
             }
 
             // Multiline link ref title (CommonMark §4.7): a LINK_DEFINITION with no title
@@ -342,7 +342,7 @@ public function tokenize(string $markdown): array
             // Start new indented code block (only when no paragraph was active,
             // and only when the line is not a list item — list items with leading spaces
             // are handled by matchLine() via PATTERN_UNORDERED_LIST / PATTERN_ORDERED_LIST).
-            if (!$hadPendingToken
+            if (!$hadPendingLines
                 && preg_match(self::PATTERN_INDENTED_CODE, $line, $m)
                 && !preg_match(self::PATTERN_UNORDERED_LIST, $line)
                 && !preg_match(self::PATTERN_ORDERED_LIST, $line)
@@ -364,11 +364,10 @@ public function tokenize(string $markdown): array
             // A FOOTNOTE_DEFINITION token opens multi-line body accumulation.
             // Flush any pending setext heading candidate and pending link def first.
             if ($token->type === TokenType::FOOTNOTE_DEFINITION) {
-                /** @psalm-suppress TypeDoesNotContainType @phpstan-ignore notIdentical.alwaysFalse */
-                if ($pendingToken !== null) {
-                    $tokens[] = $pendingToken;
-                    $pendingToken = null;
+                foreach ($pendingLines as $pt) {
+                    $tokens[] = $pt;
                 }
+                $pendingLines = [];
                 /** @psalm-suppress TypeDoesNotContainType @phpstan-ignore notIdentical.alwaysFalse */
                 if ($pendingLinkDef !== null) {
                     $tokens[] = $pendingLinkDef;
@@ -382,10 +381,14 @@ public function tokenize(string $markdown): array
 
             // A plain paragraph line is held as pending to allow setext promotion on the next line.
             if ($token->type === TokenType::PARAGRAPH && ($line !== '' && !ctype_space($line))) {
-                $pendingToken = $token;
+                $pendingLines[] = $token;
                 continue;
             }
 
+            foreach ($pendingLines as $pt) {
+                $tokens[] = $pt;
+            }
+            $pendingLines = [];
             $tokens[] = $token;
         }
 
@@ -394,9 +397,9 @@ public function tokenize(string $markdown): array
             $tokens[] = $pendingLinkDef;
         }
 
-        // Flush any remaining pending token
-        if ($pendingToken !== null) {
-            $tokens[] = $pendingToken;
+        // Flush any remaining pending lines
+        foreach ($pendingLines as $pt) {
+            $tokens[] = $pt;
         }
 
         // Flush any in-progress footnote definition body

tests/Integration/fixtures/setext-headings.html

@@ -1 +1 @@
-<h1>My Title</h1><h2>Sub Title</h2><h1><strong>bold</strong> title</h1><h2>text</h2><h1>Title</h1>
+<h1>My Title</h1><h2>Sub Title</h2><h1><strong>bold</strong> title</h1><h2>text</h2><h1>Title</h1><h1>foo bar</h1><h2>line one line two line three</h2><h1><em>foo bar</em></h1>

tests/Integration/fixtures/setext-headings.md

@@ -8,3 +8,13 @@ text
 ---
 Title
 =
+foo
+bar
+===
+line one
+line two
+line three
+---
+*foo
+bar*
+===

tests/Unit/SetextHeadingMultilineTest.php

@@ -0,0 +1,157 @@
+<?php
+
+declare(strict_types=1);
+
+namespace PhpMarkdown\Tests\Unit;
+
+use PhpMarkdown\Lexer\Lexer;
+use PhpMarkdown\Lexer\TokenType;
+use PhpMarkdown\Parser\Parser;
+use PhpMarkdown\Renderer\HtmlRenderer;
+use PHPUnit\Framework\TestCase;
+
+/**
+ * Tests for setext headings with multiple preceding paragraph lines.
+ */
+final class SetextHeadingMultilineTest extends TestCase
+{
+    // -------------------------------------------------------------------------
+    // Token-level tests
+    // -------------------------------------------------------------------------
+
+    public function testTwoLineH1Token(): void
+    {
+        $lexer  = new Lexer();
+        $tokens = $lexer->tokenize("foo\nbar\n===");
+
+        $headingTokens = array_values(array_filter(
+            $tokens,
+            static fn($t) => $t->type === TokenType::HEADING,
+        ));
+
+        self::assertCount(1, $headingTokens);
+        self::assertSame(1, $headingTokens[0]->meta['level']);
+        self::assertSame('foo bar', $headingTokens[0]->content);
+    }
+
+    public function testThreeLineH2Token(): void
+    {
+        $lexer  = new Lexer();
+        $tokens = $lexer->tokenize("line one\nline two\nline three\n---");
+
+        $headingTokens = array_values(array_filter(
+            $tokens,
+            static fn($t) => $t->type === TokenType::HEADING,
+        ));
+
+        self::assertCount(1, $headingTokens);
+        self::assertSame(2, $headingTokens[0]->meta['level']);
+        self::assertSame('line one line two line three', $headingTokens[0]->content);
+    }
+
+    public function testSingleLineRegressionToken(): void
+    {
+        $lexer  = new Lexer();
+        $tokens = $lexer->tokenize("foo\n===");
+
+        $headingTokens = array_values(array_filter(
+            $tokens,
+            static fn($t) => $t->type === TokenType::HEADING,
+        ));
+
+        self::assertCount(1, $headingTokens);
+        self::assertSame(1, $headingTokens[0]->meta['level']);
+        self::assertSame('foo', $headingTokens[0]->content);
+    }
+
+    public function testBlankBreaksAccumulationTokens(): void
+    {
+        $lexer  = new Lexer();
+        $tokens = $lexer->tokenize("foo\n\nbar\n===");
+
+        $types = array_map(static fn($t) => $t->type, $tokens);
+
+        self::assertContains(TokenType::PARAGRAPH, $types);
+        self::assertContains(TokenType::BLANK, $types);
+        self::assertContains(TokenType::HEADING, $types);
+
+        $paragraphs = array_values(array_filter($tokens, static fn($t) => $t->type === TokenType::PARAGRAPH));
+        $headings   = array_values(array_filter($tokens, static fn($t) => $t->type === TokenType::HEADING));
+
+        self::assertCount(1, $paragraphs);
+        self::assertSame('foo', $paragraphs[0]->content);
+
+        self::assertCount(1, $headings);
+        self::assertSame(1, $headings[0]->meta['level']);
+        self::assertSame('bar', $headings[0]->content);
+    }
+
+    public function testInlineMarkupSpansLinesToken(): void
+    {
+        $lexer  = new Lexer();
+        $tokens = $lexer->tokenize("*foo\nbar*\n===");
+
+        $headingTokens = array_values(array_filter(
+            $tokens,
+            static fn($t) => $t->type === TokenType::HEADING,
+        ));
+
+        self::assertCount(1, $headingTokens);
+        self::assertSame(1, $headingTokens[0]->meta['level']);
+        self::assertSame('*foo bar*', $headingTokens[0]->content);
+    }
+
+    public function testAtxHeadingInterruptsPendingLines(): void
+    {
+        $lexer  = new Lexer();
+        $tokens = $lexer->tokenize("foo\n## bar");
+
+        $paragraphs = array_values(array_filter($tokens, static fn($t) => $t->type === TokenType::PARAGRAPH));
+        $headings   = array_values(array_filter($tokens, static fn($t) => $t->type === TokenType::HEADING));
+
+        self::assertCount(1, $paragraphs);
+        self::assertSame('foo', $paragraphs[0]->content);
+
+        self::assertCount(1, $headings);
+        self::assertSame(2, $headings[0]->meta['level']);
+        self::assertSame('bar', $headings[0]->content);
+    }
+
+    // -------------------------------------------------------------------------
+    // Render-level tests
+    // -------------------------------------------------------------------------
+
+    private function render(string $markdown): string
+    {
+        $lexer    = new Lexer();
+        $parser   = new Parser();
+        $renderer = new HtmlRenderer();
+
+        return trim($renderer->render($parser->parse($lexer->tokenize($markdown))));
+    }
+
+    public function testTwoLineH1Renders(): void
+    {
+        self::assertSame('<h1>foo bar</h1>', $this->render("foo\nbar\n==="));
+    }
+
+    public function testThreeLineH2Renders(): void
+    {
+        self::assertSame('<h2>line one line two line three</h2>', $this->render("line one\nline two\nline three\n---"));
+    }
+
+    public function testSingleLineRegressionRenders(): void
+    {
+        self::assertSame('<h1>foo</h1>', $this->render("foo\n==="));
+    }
+
+    public function testBlankBreaksAccumulationRenders(): void
+    {
+        self::assertSame('<p>foo</p><h1>bar</h1>', $this->render("foo\n\nbar\n==="));
+    }
+
+    public function testInlineMarkupSpansLinesRenders(): void
+    {
+        self::assertSame('<h1><em>foo bar</em></h1>', $this->render("*foo\nbar*\n==="));
+    }
+}