From 80c141bad8d6f4591ad4a7f68bbbdd6e1ca7bcaa Mon Sep 17 00:00:00 2001 From: Dylan Conway Date: Thu, 16 Jul 2026 02:29:53 -0700 Subject: [PATCH 01/13] Yarr JIT: compile lookbehind assertions Patterns containing a lookbehind previously fell back entirely to the bytecode interpreter (JITFailureReason::Lookbehind at compile time, and a pre-check in RegExp::compile that never even attempted the JIT). A single lookbehind therefore sent an otherwise cheap pattern to the interpreter; the isbot user-agent regex (~110 alternatives, four of them lookbehinds) ran ~40x slower than with the assertions JIT'd. A lookbehind body matches right-to-left ending at the assertion point. Matching backward over the input is equivalent to matching a term-reversed copy of the body forward over the reversed prefix, so each body is compiled as a mirrored copy (terms reversed, input positions reassigned with the same width rules as setupAlternativeOffsets) driven through the existing forward machinery. The original YarrPattern is untouched, keeping the bytecode fallback valid. Each op carries a MatchDirection; in a backward body the index register is the leftward-moving left frontier, and only the primitives flip: input claims subtract (branchSub32 on the borrow's sign bit), characters are read at input[index + (k - 1)], end-of-input is index == 0, greedy consumption and give-back reverse sign, captures write the real position with start/end slots swapped (backward captures record end-first, matching the interpreter), and BOL/EOL/word-boundary anchors get mirrored edge tests. Nested lookbehinds are mirrored on demand. Still handled by the interpreter, exactly as before: surrogate-pair decoding (unicode 16-bit), backreferences inside a body, lookaheads nested inside a body, and quantified groups with maxCount > 1 inside a body. Character fusion (multi-char wide loads) is disabled inside backward bodies since ascending memory holds descending pattern positions. collectBoyerMooreInfoFromTerm now treats a parenthetical assertion as zero-width (like BOL/EOL/word-boundary) instead of aborting collection: an assertion consumes no input, so the leading-character sets remain a valid over-approximation. --- JSTests/stress/regexp-lookbehind-jit.js | 136 ++++++ Source/JavaScriptCore/runtime/RegExp.cpp | 2 - Source/JavaScriptCore/yarr/YarrJIT.cpp | 594 +++++++++++++++++++++-- 3 files changed, 686 insertions(+), 46 deletions(-) create mode 100644 JSTests/stress/regexp-lookbehind-jit.js diff --git a/JSTests/stress/regexp-lookbehind-jit.js b/JSTests/stress/regexp-lookbehind-jit.js new file mode 100644 index 000000000000..011ea9253c44 --- /dev/null +++ b/JSTests/stress/regexp-lookbehind-jit.js @@ -0,0 +1,136 @@ +//@ runDefault("--useRegExpJIT=true") +//@ runNoJIT("--useRegExpJIT=false") + +// Lookbehind assertions compiled by the Yarr JIT (mirrored bodies matched +// right-to-left) must produce results identical to the interpreter. + +function shouldBe(actual, expected, message) { + if (JSON.stringify(actual) !== JSON.stringify(expected)) + throw new Error( + (message ? message + ": " : "") + "expected " + JSON.stringify(expected) + " but got " + JSON.stringify(actual), + ); +} + +function matchOf(re, s) { + let m = re.exec(s); + return m ? [m.index, [...m]] : null; +} + +// Fixed-width bodies. +shouldBe(matchOf(/(?<=x)a/, "xa"), [1, ["a"]]); +shouldBe(matchOf(/(?<=x)a/, "ya"), null); +shouldBe(matchOf(/(? re.test(l)), + [true, false, false, true, true, false, true, true], + ); +} diff --git a/Source/JavaScriptCore/runtime/RegExp.cpp b/Source/JavaScriptCore/runtime/RegExp.cpp index f01fc40ffa73..81dbb090a605 100644 --- a/Source/JavaScriptCore/runtime/RegExp.cpp +++ b/Source/JavaScriptCore/runtime/RegExp.cpp @@ -318,7 +318,6 @@ void RegExp::compile(VM* vm, Yarr::CharSize charSize, std::optional #if !ENABLE(YARR_JIT_BACKREFERENCES) && !pattern.m_containsBackreferences #endif - && !pattern.m_containsLookbehinds ) { auto& jitCode = ensureRegExpJITCode(); Yarr::jitCompile(pattern, m_patternString, charSize, sampleString, vm, jitCode, Yarr::ExecutionMode::IncludeSubpatterns); @@ -386,7 +385,6 @@ void RegExp::compileMatchOnly(VM* vm, Yarr::CharSize charSize, std::optional characterOffset = static_cast(negativeCharacterOffset.value()) - 1; + unsigned positiveCharacterOffset = negativeCharacterOffset.value() ? negativeCharacterOffset.value() - 1 : 0; + if (positiveCharacterOffset > maximumNegativeOffsetForCharacterSize) { + base = tempReg; + m_jit.move(m_regs.input, base); + while (positiveCharacterOffset > maximumNegativeOffsetForCharacterSize) { + m_jit.addPtr(MacroAssembler::TrustedImm32(offsetAdjustAmount), base); + if (m_charSize != CharSize::Char8) + m_jit.addPtr(MacroAssembler::TrustedImm32(offsetAdjustAmount), base); + positiveCharacterOffset -= offsetAdjustAmount; + } + characterOffset = static_cast(positiveCharacterOffset); + } + if (m_charSize == CharSize::Char8) + return MacroAssembler::BaseIndex(base, indexReg, MacroAssembler::TimesOne, characterOffset.value() * static_cast(sizeof(char))); + return MacroAssembler::BaseIndex(base, indexReg, MacroAssembler::TimesTwo, characterOffset.value() * static_cast(sizeof(char16_t))); + } + if (negativeCharacterOffset > maximumNegativeOffsetForCharacterSize) { base = tempReg; m_jit.move(m_regs.input, base); @@ -1678,16 +1727,64 @@ class YarrGenerator final : public YarrJITInfo { m_jit.store32(MacroAssembler::TrustedImm32(0), duplicateNamedGroupAddress(i)); } + // Compute the real input position of a point that is `inputOffset` frame + // slots behind the current frame position, into `reg`. Forward: behind the + // frontier means a lower address (subtract); in a backward (mirrored) frame + // it is ahead of the leftward cursor (add). + void emitFramePosition(unsigned inputOffset, MacroAssembler::RegisterID reg) + { + if (m_direction == Backward) + m_jit.add32(MacroAssembler::Imm32(inputOffset), m_regs.index, reg); + else + m_jit.sub32(m_regs.index, MacroAssembler::Imm32(inputOffset), reg); + } + + // Consume `count` characters at the frame position: advance the forward + // frontier, or move the backward (mirrored) leftward cursor down. + void consumeIndex(unsigned count) + { + if (m_direction == Backward) + m_jit.sub32(MacroAssembler::Imm32(count), m_regs.index); + else + m_jit.add32(MacroAssembler::Imm32(count), m_regs.index); + } + + // Give `count` characters back (undo a consume). + void rewindIndex(unsigned count) + { + if (m_direction == Backward) + m_jit.add32(MacroAssembler::Imm32(count), m_regs.index); + else + m_jit.sub32(MacroAssembler::Imm32(count), m_regs.index); + } + + // Give back the number of characters held in `countRegister`. + void rewindIndex(MacroAssembler::RegisterID countRegister) + { + if (m_direction == Backward) + m_jit.add32(countRegister, m_regs.index); + else + m_jit.sub32(countRegister, m_regs.index); + } + + // In a backward (mirrored) frame a capture group's virtual start is its + // real end and vice versa, so the start/end slots are written swapped - + // matching the interpreter, which records backward captures end-first. + void emitCaptureStart(PatternTerm* term, Checked checkedOffset, MacroAssembler::RegisterID indexTemporary, unsigned extraOffset = 0) { // FIXME: could avoid offsetting this value in JIT code, apply offsets only afterwards, at the point the results array is being accessed. ASSERT(term->capture() && shouldRecordSubpatterns()); unsigned inputOffset = checkedOffset - term->inputPosition + extraOffset; + MacroAssembler::RegisterID positionReg = m_regs.index; if (inputOffset) { - m_jit.sub32(m_regs.index, MacroAssembler::Imm32(inputOffset), indexTemporary); - setSubpatternStart(indexTemporary, term->parentheses.subpatternId); - } else - setSubpatternStart(m_regs.index, term->parentheses.subpatternId); + emitFramePosition(inputOffset, indexTemporary); + positionReg = indexTemporary; + } + if (m_direction == Backward) + setSubpatternEnd(positionReg, term->parentheses.subpatternId); + else + setSubpatternStart(positionReg, term->parentheses.subpatternId); } void emitCaptureEnd(PatternTerm* term, Checked checkedOffset, MacroAssembler::RegisterID indexTemporary) @@ -1696,11 +1793,15 @@ class YarrGenerator final : public YarrJITInfo { ASSERT(term->capture() && shouldRecordSubpatterns()); auto subpatternId = term->parentheses.subpatternId; unsigned inputOffset = checkedOffset - term->inputPosition; + MacroAssembler::RegisterID positionReg = m_regs.index; if (inputOffset) { - m_jit.sub32(m_regs.index, MacroAssembler::Imm32(inputOffset), indexTemporary); - setSubpatternEnd(indexTemporary, subpatternId); - } else - setSubpatternEnd(m_regs.index, subpatternId); + emitFramePosition(inputOffset, indexTemporary); + positionReg = indexTemporary; + } + if (m_direction == Backward) + setSubpatternStart(positionReg, subpatternId); + else + setSubpatternEnd(positionReg, subpatternId); if (m_pattern.m_numDuplicateNamedCaptureGroups) { if (auto duplicateNamedGroupId = m_pattern.m_duplicateNamedGroupForSubpatternId[subpatternId]) storeDuplicateNamedGroupSubpatternId(duplicateNamedGroupId, subpatternId); @@ -1836,6 +1937,12 @@ class YarrGenerator final : public YarrJITInfo { PatternTerm* m_term; YarrOpCode m_op; + // The matching direction in effect for the code this op generates. Ops + // belonging to a lookbehind body (compiled from a mirrored copy of the + // body, see mirrorDisjunctionForLookbehind) are Backward: the index register + // is a leftward-moving cursor rather than the usual rightward frontier. + MatchDirection m_direction { Forward }; + // Used to record a set of Jumps out of the generated code, typically // used for jumps out to backtracking code, and a single reentry back // into the code for a node (likely where a backtrack will trigger @@ -2049,6 +2156,11 @@ class YarrGenerator final : public YarrJITInfo { YarrOp& op = m_ops[opIndex]; PatternTerm* term = op.m_term; + if (m_direction == Backward) { + generateAssertionBOLBackward(opIndex); + return; + } + if (term->multiline()) { const MacroAssembler::RegisterID character = m_regs.regT0; const MacroAssembler::RegisterID scratch = m_regs.regT1; @@ -2070,6 +2182,39 @@ class YarrGenerator final : public YarrJITInfo { op.m_jumps.append(m_jit.branch32(MacroAssembler::NotEqual, m_regs.index, MacroAssembler::Imm32(op.m_checkedOffset))); } } + + // BOL inside a lookbehind body (mirrored frame). The anchor's real position + // is index + (checkedOffset - inputPosition). It can be the real start of + // input only when it sits at the frame's right edge (inputPosition == + // checkedOffset) and the cursor has reached position 0; the real character + // preceding the anchor is at frame offset (checkedOffset - inputPosition), + // one less than in the forward case. + void generateAssertionBOLBackward(size_t opIndex) + { + YarrOp& op = m_ops[opIndex]; + PatternTerm* term = op.m_term; + bool atFrameEdge = term->inputPosition == op.m_checkedOffset; + + if (term->multiline()) { + const MacroAssembler::RegisterID character = m_regs.regT0; + const MacroAssembler::RegisterID scratch = m_regs.regT1; + + MacroAssembler::JumpList matchDest; + if (atFrameEdge) + matchDest.append(m_jit.branchTest32(MacroAssembler::Zero, m_regs.index)); + + readCharacter(op.m_checkedOffset - term->inputPosition, character); + matchCharacterClass(character, scratch, matchDest, m_pattern.newlineCharacterClass()); + op.m_jumps.append(m_jit.jump()); + + matchDest.link(&m_jit); + } else { + if (atFrameEdge) + op.m_jumps.append(m_jit.branchTest32(MacroAssembler::NonZero, m_regs.index)); + else + op.m_jumps.append(m_jit.jump()); + } + } void backtrackAssertionBOL(size_t opIndex) { backtrackTermDefault(opIndex); @@ -2080,6 +2225,11 @@ class YarrGenerator final : public YarrJITInfo { YarrOp& op = m_ops[opIndex]; PatternTerm* term = op.m_term; + if (m_direction == Backward) { + generateAssertionEOLBackward(opIndex); + return; + } + if (term->multiline()) { const MacroAssembler::RegisterID character = m_regs.regT0; const MacroAssembler::RegisterID scratch = m_regs.regT1; @@ -2101,6 +2251,44 @@ class YarrGenerator final : public YarrJITInfo { op.m_jumps.append(m_jit.jump()); } } + + // EOL inside a lookbehind body (mirrored frame). The anchor's real position + // is index + (checkedOffset - inputPosition). It can be the real end of input + // only when it sits at the frame's left edge (inputPosition == 0, i.e. the + // assertion point itself) and index + checkedOffset == length; the real + // character following the anchor is at frame offset (checkedOffset - + // inputPosition + 1), one more than in the forward case. + void generateAssertionEOLBackward(size_t opIndex) + { + YarrOp& op = m_ops[opIndex]; + PatternTerm* term = op.m_term; + bool atFrameEdge = !term->inputPosition; + const MacroAssembler::RegisterID scratch = m_regs.regT1; + + auto atRealEnd = [&](MacroAssembler::RelationalCondition condition) { + m_jit.add32(MacroAssembler::Imm32(op.m_checkedOffset), m_regs.index, scratch); + return m_jit.branch32(condition, scratch, m_regs.length); + }; + + if (term->multiline()) { + const MacroAssembler::RegisterID character = m_regs.regT0; + + MacroAssembler::JumpList matchDest; + if (atFrameEdge) + matchDest.append(atRealEnd(MacroAssembler::Equal)); + + readCharacter(op.m_checkedOffset - term->inputPosition + 1, character); + matchCharacterClass(character, scratch, matchDest, m_pattern.newlineCharacterClass()); + op.m_jumps.append(m_jit.jump()); + + matchDest.link(&m_jit); + } else { + if (atFrameEdge) + op.m_jumps.append(atRealEnd(MacroAssembler::NotEqual)); + else + op.m_jumps.append(m_jit.jump()); + } + } void backtrackAssertionEOL(size_t opIndex) { backtrackTermDefault(opIndex); @@ -2115,10 +2303,22 @@ class YarrGenerator final : public YarrJITInfo { const MacroAssembler::RegisterID character = m_regs.regT0; const MacroAssembler::RegisterID scratch = m_regs.regT1; - if (term->inputPosition == op.m_checkedOffset) - nextIsNotWordChar.append(atEndOfInput()); + if (m_direction == Backward) { + // Mirrored frame: the boundary's real position is index + (checked - + // pos). It is the real end of input only at the frame's left edge + // (pos == 0) with index + checked == length; the following real + // character is at frame offset (checked - pos + 1). + if (!term->inputPosition) { + m_jit.add32(MacroAssembler::Imm32(op.m_checkedOffset), m_regs.index, scratch); + nextIsNotWordChar.append(m_jit.branch32(MacroAssembler::Equal, scratch, m_regs.length)); + } + readCharacter(op.m_checkedOffset - term->inputPosition + 1, character); + } else { + if (term->inputPosition == op.m_checkedOffset) + nextIsNotWordChar.append(atEndOfInput()); - readCharacter(op.m_checkedOffset - term->inputPosition, character); + readCharacter(op.m_checkedOffset - term->inputPosition, character); + } CharacterClass* wordcharCharacterClass; @@ -2145,9 +2345,22 @@ class YarrGenerator final : public YarrJITInfo { MacroAssembler::Jump atBegin; MacroAssembler::JumpList matchDest; - if (!term->inputPosition) - atBegin = m_jit.branch32(MacroAssembler::Equal, m_regs.index, MacroAssembler::Imm32(op.m_checkedOffset)); - readCharacter(op.m_checkedOffset - term->inputPosition + 1, character); + // Forward: the boundary is at the real start of input only if it is at + // the frame start (pos == 0) with index == checked; the preceding real + // character is at frame offset (checked - pos + 1). Backward (mirrored): + // the real start test is at the frame's right edge (pos == checked) with + // index == 0, and the preceding real character is at (checked - pos). + bool atStartEdge = m_direction == Backward ? term->inputPosition == op.m_checkedOffset : !term->inputPosition; + if (atStartEdge) { + if (m_direction == Backward) + atBegin = m_jit.branchTest32(MacroAssembler::Zero, m_regs.index); + else + atBegin = m_jit.branch32(MacroAssembler::Equal, m_regs.index, MacroAssembler::Imm32(op.m_checkedOffset)); + } + if (m_direction == Backward) + readCharacter(op.m_checkedOffset - term->inputPosition, character); + else + readCharacter(op.m_checkedOffset - term->inputPosition + 1, character); CharacterClass* wordcharCharacterClass; @@ -2157,7 +2370,7 @@ class YarrGenerator final : public YarrJITInfo { wordcharCharacterClass = m_pattern.wordcharCharacterClass(); matchCharacterClass(character, scratch, matchDest, wordcharCharacterClass); - if (!term->inputPosition) + if (atStartEdge) atBegin.link(&m_jit); // We fall through to here if the last character was not a wordchar. @@ -2596,8 +2809,14 @@ class YarrGenerator final : public YarrJITInfo { // upper & lower case representations are converted to a character class. ASSERT(!op.m_term->ignoreCase() || isASCIIAlpha(firstChar) || isCanonicallyUnique(firstChar, m_canonicalMode)); - if (m_decodeSurrogatePairs && (!U_IS_BMP(firstChar) || U16_IS_SURROGATE(firstChar))) { - // The first term we are considering is a non-BMP or dangling surrogate char in unicode pattern. Just try matching it and be done. + if (m_direction == Backward || (m_decodeSurrogatePairs && (!U_IS_BMP(firstChar) || U16_IS_SURROGATE(firstChar)))) { + // Match a single character on its own, without fusing neighbours into a + // wide load. In a backward (lookbehind) frame ascending memory holds + // *descending* pattern positions, so the forward multi-character packing + // does not apply (a run could be fused by comparing against the reversed + // packed constant loaded from the run's lowest address; not done yet); + // and a non-BMP / dangling surrogate char in a unicode pattern is likewise + // matched individually. uint64_t charToMatch = firstChar; auto offset = op.m_checkedOffset - op.m_term->inputPosition; @@ -2971,7 +3190,10 @@ class YarrGenerator final : public YarrJITInfo { Checked scaledMaxCount = term->quantityMaxCount; scaledMaxCount *= U_IS_BMP(ch) ? 1 : 2; - m_jit.sub32(m_regs.index, MacroAssembler::Imm32(scaledMaxCount), countRegister); + // The count register walks toward index over the run of characters: + // upward from index - count in a forward frame, downward from index + count + // in a backward (mirrored) frame where index is the left frontier. + emitFramePosition(scaledMaxCount, countRegister); MacroAssembler::Label loop(&m_jit); readCharacter(op.m_checkedOffset - term->inputPosition - scaledMaxCount, character, countRegister); @@ -2984,11 +3206,13 @@ class YarrGenerator final : public YarrJITInfo { } op.m_jumps.append(m_jit.branch32(MacroAssembler::NotEqual, character, MacroAssembler::Imm32(ch))); + if (m_direction == Backward) + m_jit.sub32(MacroAssembler::TrustedImm32(1), countRegister); #if ENABLE(YARR_JIT_UNICODE_EXPRESSIONS) - if (m_decodeSurrogatePairs && !U_IS_BMP(ch)) + else if (m_decodeSurrogatePairs && !U_IS_BMP(ch)) m_jit.add32(MacroAssembler::TrustedImm32(2), countRegister); - else #endif + else m_jit.add32(MacroAssembler::TrustedImm32(1), countRegister); m_jit.branch32(MacroAssembler::NotEqual, countRegister, m_regs.index).linkTo(loop, &m_jit); } @@ -3015,7 +3239,12 @@ class YarrGenerator final : public YarrJITInfo { failures.append(atEndOfInput()); failures.append(jumpIfCharNotEquals(ch, op.m_checkedOffset - term->inputPosition, character, term->ignoreCase())); - m_jit.add32(MacroAssembler::TrustedImm32(1), m_regs.index); + // Consume the character: advance the frontier forward, or move the + // leftward cursor down in a backward (mirrored) frame. + if (m_direction == Backward) + m_jit.sub32(MacroAssembler::TrustedImm32(1), m_regs.index); + else + m_jit.add32(MacroAssembler::TrustedImm32(1), m_regs.index); #if ENABLE(YARR_JIT_UNICODE_EXPRESSIONS) if (m_decodeSurrogatePairs && !U_IS_BMP(ch)) { MacroAssembler::Jump surrogatePairOk = notAtEndOfInput(); @@ -3053,7 +3282,7 @@ class YarrGenerator final : public YarrJITInfo { loadFromFrame(term->frameLocation + BackTrackInfoPatternCharacter::matchAmountIndex(), countRegister); if (m_decodeSurrogatePairs && !U_IS_BMP(term->patternCharacter)) m_jit.lshift32(MacroAssembler::TrustedImm32(1), countRegister); - m_jit.sub32(countRegister, m_regs.index); + rewindIndex(countRegister); m_backtrackingState.fallthrough(); return; } @@ -3061,7 +3290,11 @@ class YarrGenerator final : public YarrJITInfo { loadFromFrame(term->frameLocation + BackTrackInfoPatternCharacter::matchAmountIndex(), countRegister); m_backtrackingState.append(m_jit.branchTest32(MacroAssembler::Zero, countRegister)); m_jit.sub32(MacroAssembler::TrustedImm32(1), countRegister); - if (!m_decodeSurrogatePairs || U_IS_BMP(term->patternCharacter)) + // Give one character back: retreat the forward frontier, or move the + // leftward cursor back up in a backward (mirrored) frame. + if (m_direction == Backward) + m_jit.add32(MacroAssembler::TrustedImm32(1), m_regs.index); + else if (!m_decodeSurrogatePairs || U_IS_BMP(term->patternCharacter)) m_jit.sub32(MacroAssembler::TrustedImm32(1), m_regs.index); else m_jit.sub32(MacroAssembler::TrustedImm32(2), m_regs.index); @@ -3100,7 +3333,10 @@ class YarrGenerator final : public YarrJITInfo { nonGreedyFailures.append(m_jit.branch32(MacroAssembler::Equal, countRegister, MacroAssembler::Imm32(term->quantityMaxCount))); nonGreedyFailures.append(jumpIfCharNotEquals(ch, op.m_checkedOffset - term->inputPosition, character, term->ignoreCase())); - m_jit.add32(MacroAssembler::TrustedImm32(1), m_regs.index); + if (m_direction == Backward) + m_jit.sub32(MacroAssembler::TrustedImm32(1), m_regs.index); + else + m_jit.add32(MacroAssembler::TrustedImm32(1), m_regs.index); #if ENABLE(YARR_JIT_UNICODE_EXPRESSIONS) if (m_decodeSurrogatePairs && !U_IS_BMP(ch)) { MacroAssembler::Jump surrogatePairOk = notAtEndOfInput(); @@ -3121,7 +3357,8 @@ class YarrGenerator final : public YarrJITInfo { m_jit.lshift32(MacroAssembler::TrustedImm32(1), countRegister); } - m_jit.sub32(countRegister, m_regs.index); + // Rewind everything this term consumed. + rewindIndex(countRegister); m_backtrackingState.fallthrough(); } @@ -3225,7 +3462,8 @@ class YarrGenerator final : public YarrJITInfo { } #endif - m_jit.sub32(m_regs.index, MacroAssembler::Imm32(scaledMaxCount), countRegister); + // See generatePatternCharacterFixed for the forward/backward count register walk. + emitFramePosition(scaledMaxCount, countRegister); MacroAssembler::Label loop(&m_jit); readCharacter(op.m_checkedOffset - term->inputPosition - scaledMaxCount, character, countRegister); @@ -3234,8 +3472,10 @@ class YarrGenerator final : public YarrJITInfo { matchCharacterClassTermInner(term, op.m_jumps, character, scratch); + if (m_direction == Backward) + m_jit.sub32(MacroAssembler::TrustedImm32(1), countRegister); #if ENABLE(YARR_JIT_UNICODE_EXPRESSIONS) - if (m_decodeSurrogatePairs) { + else if (m_decodeSurrogatePairs) { if (term->isFixedWidthCharacterClass()) m_jit.add32(MacroAssembler::TrustedImm32(term->characterClass->hasNonBMPCharacters() ? 2 : 1), countRegister); else { @@ -3246,8 +3486,9 @@ class YarrGenerator final : public YarrJITInfo { m_jit.add32(MacroAssembler::TrustedImm32(1), m_regs.index); isBMPChar.link(&m_jit); } - } else + } #endif + else m_jit.add32(MacroAssembler::TrustedImm32(1), countRegister); #if ENABLE(YARR_JIT_UNICODE_EXPRESSIONS) @@ -3335,11 +3576,13 @@ class YarrGenerator final : public YarrJITInfo { matchCharacterClassTermInner(term, failures, character, scratch); + if (m_direction == Backward) + m_jit.sub32(MacroAssembler::TrustedImm32(1), m_regs.index); #if ENABLE(YARR_JIT_UNICODE_EXPRESSIONS) - if (m_decodeSurrogatePairs) + else if (m_decodeSurrogatePairs) advanceIndexAfterCharacterClassTermMatch(term, failuresDecrementIndex, character); - else #endif + else m_jit.add32(MacroAssembler::TrustedImm32(1), m_regs.index); m_jit.add32(MacroAssembler::TrustedImm32(1), countRegister); @@ -3379,7 +3622,7 @@ class YarrGenerator final : public YarrJITInfo { loadFromFrame(term->frameLocation + BackTrackInfoCharacterClass::matchAmountIndex(), countRegister); if (m_decodeSurrogatePairs && term->characterClass->hasNonBMPCharacters()) m_jit.lshift32(MacroAssembler::TrustedImm32(1), countRegister); // 2 code units per match. - m_jit.sub32(countRegister, m_regs.index); + rewindIndex(countRegister); m_backtrackingState.fallthrough(); return; } @@ -3389,7 +3632,9 @@ class YarrGenerator final : public YarrJITInfo { m_jit.sub32(MacroAssembler::TrustedImm32(1), countRegister); storeToFrame(countRegister, term->frameLocation + BackTrackInfoCharacterClass::matchAmountIndex()); - if (!m_decodeSurrogatePairs) + if (m_direction == Backward) + m_jit.add32(MacroAssembler::TrustedImm32(1), m_regs.index); + else if (!m_decodeSurrogatePairs) m_jit.sub32(MacroAssembler::TrustedImm32(1), m_regs.index); else if (term->isFixedWidthCharacterClass()) m_jit.sub32(MacroAssembler::TrustedImm32(term->characterClass->hasNonBMPCharacters() ? 2 : 1), m_regs.index); @@ -3464,11 +3709,13 @@ class YarrGenerator final : public YarrJITInfo { matchCharacterClassTermInner(term, nonGreedyFailures, character, scratch); + if (m_direction == Backward) + m_jit.sub32(MacroAssembler::TrustedImm32(1), m_regs.index); #if ENABLE(YARR_JIT_UNICODE_EXPRESSIONS) - if (m_decodeSurrogatePairs) + else if (m_decodeSurrogatePairs) advanceIndexAfterCharacterClassTermMatch(term, nonGreedyFailuresDecrementIndex, character); - else #endif + else m_jit.add32(MacroAssembler::TrustedImm32(1), m_regs.index); m_jit.add32(MacroAssembler::TrustedImm32(1), countRegister); @@ -3480,11 +3727,13 @@ class YarrGenerator final : public YarrJITInfo { } nonGreedyFailures.link(&m_jit); + if (m_direction == Backward) + m_jit.add32(countRegister, m_regs.index); #if ENABLE(YARR_JIT_UNICODE_EXPRESSIONS) - if (m_decodeSurrogatePairs) + else if (m_decodeSurrogatePairs) loadFromFrame(term->frameLocation + BackTrackInfoCharacterClass::beginIndex(), m_regs.index); - else #endif + else m_jit.sub32(countRegister, m_regs.index); m_backtrackingState.fallthrough(); } @@ -3734,6 +3983,7 @@ class YarrGenerator final : public YarrJITInfo { m_disassembler->setForGenerate(opIndex, m_jit.label()); YarrOp& op = m_ops[opIndex]; + m_direction = op.m_direction; switch (op.m_op) { case YarrOpCode::Term: @@ -4496,8 +4746,16 @@ class YarrGenerator final : public YarrJITInfo { unsigned parenthesesFrameLocation = term->frameLocation; storeToFrame(m_regs.index, parenthesesFrameLocation + BackTrackInfoParentheticalAssertion::beginIndex()); - if (op.m_checkAdjust) - m_jit.sub32(MacroAssembler::Imm32(op.m_checkAdjust), m_regs.index); + // Realign the index register to the assertion point. In a forward + // frame that point is checkAdjust characters behind the frontier; + // in a backward (mirrored) frame it is checkAdjust characters ahead + // of the leftward cursor. + if (op.m_checkAdjust) { + if (op.m_direction == Backward) + m_jit.add32(MacroAssembler::Imm32(op.m_checkAdjust), m_regs.index); + else + m_jit.sub32(MacroAssembler::Imm32(op.m_checkAdjust), m_regs.index); + } break; } case YarrOpCode::ParentheticalAssertionEnd: { @@ -4543,6 +4801,7 @@ class YarrGenerator final : public YarrJITInfo { m_disassembler->setForBacktrack(opIndex, m_jit.label()); YarrOp& op = m_ops[opIndex]; + m_direction = op.m_direction; switch (op.m_op) { case YarrOpCode::Term: @@ -4856,7 +5115,11 @@ class YarrGenerator final : public YarrJITInfo { if (!m_backtrackingState.isEmpty()) { // Handle the cases where we need to link the backtracks here. m_backtrackingState.link(*this, op); - m_jit.sub32(MacroAssembler::Imm32(op.m_checkAdjust), m_regs.index); + // Give back the input this alternative claimed on entry. + if (op.m_direction == Backward) + m_jit.add32(MacroAssembler::Imm32(op.m_checkAdjust), m_regs.index); + else + m_jit.sub32(MacroAssembler::Imm32(op.m_checkAdjust), m_regs.index); if (!isLastAlternative) { // An alternative that is not the last should jump to its successor. m_jit.jump(nextOp.m_reentry); @@ -5289,8 +5552,13 @@ class YarrGenerator final : public YarrJITInfo { if (op.m_checkAdjust || term->invert()) { m_backtrackingState.link(*this, op); - if (op.m_checkAdjust) - m_jit.add32(MacroAssembler::Imm32(op.m_checkAdjust), m_regs.index); + // Undo the realignment done on entry (see the forward path). + if (op.m_checkAdjust) { + if (op.m_direction == Backward) + m_jit.sub32(MacroAssembler::Imm32(op.m_checkAdjust), m_regs.index); + else + m_jit.add32(MacroAssembler::Imm32(op.m_checkAdjust), m_regs.index); + } // In an inverted assertion failure to match the subpattern // is treated as a successful match - jump to the end of the @@ -5515,6 +5783,199 @@ class YarrGenerator final : public YarrJITInfo { m_ops[parenEnd].m_checkedOffset = checkedOffset; } + // Lookbehind body mirroring. + // + // A lookbehind body matches right-to-left ending at the assertion point. + // Matching backward over the input is equivalent to matching a term-reversed + // copy of the body forward over the virtually reversed input, so we build + // such a reversed copy and compile it with the standard forward machinery, + // marking its ops Backward so the input-check/read/end-of-input primitives + // use the leftward cursor variants. Positions are reassigned in virtual + // (reversed) coordinates using exactly the width rules of + // YarrPatternConstructor::setupAlternativeOffsets. Frame locations are kept + // from the original terms; the original body is never itself compiled, so its + // slots are free for the mirrored copy to use. + // + // Returns false (with m_failureReason set) if the body uses a construct the + // backward JIT does not support yet; the whole pattern then falls back to the + // interpreter as it did before lookbehind JIT support existed. + + bool assignMirroredAlternativeOffsets(PatternAlternative* alternative, unsigned initialInputPosition) + { + CheckedUint32 currentInputPosition = initialInputPosition; + + for (auto& term : alternative->m_terms) { + switch (term.type) { + case PatternTerm::Type::AssertionBOL: + case PatternTerm::Type::AssertionEOL: + case PatternTerm::Type::AssertionWordBoundary: + term.inputPosition = currentInputPosition; + break; + + case PatternTerm::Type::NumberedForwardReference: + case PatternTerm::Type::NamedForwardReference: + break; + + case PatternTerm::Type::PatternCharacter: + term.inputPosition = currentInputPosition; + if (term.quantityType == QuantifierType::FixedCount) { + if (m_pattern.eitherUnicode()) { + CheckedUint32 tempCount = term.quantityMaxCount; + tempCount *= U16_LENGTH(term.patternCharacter); + currentInputPosition += tempCount; + } else + currentInputPosition += term.quantityMaxCount; + } + break; + + case PatternTerm::Type::CharacterClass: + term.inputPosition = currentInputPosition; + if (term.quantityType == QuantifierType::FixedCount) { + if (m_pattern.eitherUnicode()) { + if (term.characterClass->hasOneCharacterSize() && !term.invert()) { + CheckedUint32 tempCount = term.quantityMaxCount; + tempCount *= term.characterClass->hasNonBMPCharacters() ? 2 : 1; + currentInputPosition += tempCount; + } else + currentInputPosition += term.quantityMaxCount; + } else + currentInputPosition += term.quantityMaxCount; + } + break; + + case PatternTerm::Type::ParenthesesSubpattern: + // Only the quantityMaxCount == 1, !isCopy shape reaches here (see + // mirrorAlternative). Nested group positions continue the enclosing + // numbering, as in setupAlternativeOffsets. + if (!assignMirroredDisjunctionOffsets(term.parentheses.disjunction, currentInputPosition)) + return false; + if (term.quantityType == QuantifierType::FixedCount) + currentInputPosition += term.parentheses.disjunction->m_minimumSize; + term.inputPosition = currentInputPosition; + break; + + case PatternTerm::Type::ParentheticalAssertion: + // A nested lookbehind keeps its original (0-based) body and is mirrored + // on demand when compiled. Its own position is the virtual assertion point. + term.inputPosition = currentInputPosition; + break; + + default: + RELEASE_ASSERT_NOT_REACHED(); + } + if (currentInputPosition.hasOverflowed()) { + m_failureReason = JITFailureReason::OffsetTooLarge; + return false; + } + } + + // Mirroring reverses term order but must preserve the alternative's width. + RELEASE_ASSERT(alternative->m_minimumSize == (currentInputPosition - initialInputPosition)); + return true; + } + + bool assignMirroredDisjunctionOffsets(PatternDisjunction* disjunction, unsigned initialInputPosition) + { + for (auto& alternative : disjunction->m_alternatives) { + if (!assignMirroredAlternativeOffsets(alternative.get(), initialInputPosition)) + return false; + } + return true; + } + + // Structurally reverse `source` into `destination`. Nested plain groups are + // deep-copied and reversed as part of the same backward match. + bool mirrorAlternativeInto(PatternAlternative& source, PatternDisjunction& destination) + { + PatternAlternative* mirrored = destination.addNewAlternative(source.m_firstSubpatternId, Backward); + mirrored->m_minimumSize = source.m_minimumSize; + mirrored->m_lastSubpatternId = source.m_lastSubpatternId; + mirrored->m_hasFixedSize = source.m_hasFixedSize; + mirrored->m_containsBOL = source.m_containsBOL; + mirrored->m_isLastAlternative = source.m_isLastAlternative; + + for (size_t i = source.m_terms.size(); i--;) { + PatternTerm term = source.m_terms[i]; + + switch (term.type) { + case PatternTerm::Type::AssertionBOL: + case PatternTerm::Type::AssertionEOL: + case PatternTerm::Type::AssertionWordBoundary: + case PatternTerm::Type::PatternCharacter: + case PatternTerm::Type::CharacterClass: + case PatternTerm::Type::NumberedForwardReference: + case PatternTerm::Type::NamedForwardReference: + break; + + case PatternTerm::Type::NumberedBackReference: + case PatternTerm::Type::NamedBackReference: + // Backreferences inside a lookbehind body: not yet supported by the + // backward JIT (they must compare right-to-left). + m_failureReason = JITFailureReason::Lookbehind; + return false; + + case PatternTerm::Type::ParenthesesSubpattern: + if (term.quantityMaxCount != 1 || term.parentheses.isCopy || term.parentheses.isTerminal) { + // Quantified groups with maxCount > 1 inside a lookbehind body: not + // yet supported by the backward JIT. + m_failureReason = JITFailureReason::Lookbehind; + return false; + } + // The string-list / terminal group optimizations assume forward, + // trailing-context conditions; take the general path for the copy. + term.parentheses.isStringList = false; + term.parentheses.isEOLStringList = false; + term.parentheses.disjunction = mirrorDisjunctionForLookbehind(term.parentheses.disjunction); + if (!term.parentheses.disjunction) + return false; + break; + + case PatternTerm::Type::ParentheticalAssertion: + if (term.matchDirection() != Backward) { + // A lookahead nested inside a lookbehind body: its body would need + // re-basing into the mirrored coordinate space. Not yet supported. + m_failureReason = JITFailureReason::Lookbehind; + return false; + } + // Nested lookbehind: keep pointing at its original body; it is mirrored + // when opCompileParentheticalAssertion reaches it. + break; + + case PatternTerm::Type::DotStarEnclosure: + m_failureReason = JITFailureReason::Lookbehind; + return false; + } + + mirrored->m_terms.append(term); + } + return true; + } + + // Build the reversed copy of a lookbehind body disjunction (recursively + // reversing nested groups), then assign virtual input positions from 0. + // Returns null with m_failureReason set on unsupported content. + PatternDisjunction* mirrorDisjunctionForLookbehind(PatternDisjunction* disjunction) + { + if (!isSafeToRecurse()) [[unlikely]] { + m_failureReason = JITFailureReason::ParenthesisNestedTooDeep; + return nullptr; + } + + auto mirroredDisjunction = makeUnique(); + mirroredDisjunction->m_minimumSize = disjunction->m_minimumSize; + mirroredDisjunction->m_callFrameSize = disjunction->m_callFrameSize; + mirroredDisjunction->m_hasFixedSize = disjunction->m_hasFixedSize; + + for (auto& alternative : disjunction->m_alternatives) { + if (!mirrorAlternativeInto(*alternative, *mirroredDisjunction)) + return nullptr; + } + + PatternDisjunction* result = mirroredDisjunction.get(); + m_mirroredDisjunctions.append(WTF::move(mirroredDisjunction)); + return result; + } + // opCompileParentheticalAssertion // Emits ops for a parenthetical assertion. These consist of an // YarrOpCode::SimpleNestedAlternativeBegin/Next/End set of nodes wrapping @@ -5530,17 +5991,37 @@ class YarrGenerator final : public YarrJITInfo { return; } + PatternDisjunction* disjunction = term->parentheses.disjunction; + MatchDirection bodyDirection = term->matchDirection(); + if (bodyDirection == Backward) { + // Lookbehind: compile a term-reversed mirrored copy of the body whose + // input positions are in virtual (reversed) coordinates starting at 0. + disjunction = mirrorDisjunctionForLookbehind(disjunction); + if (!disjunction) + return; + if (!assignMirroredDisjunctionOffsets(disjunction, 0)) + return; + } + auto originalCheckedOffset = checkedOffset; size_t parenBegin = m_ops.size(); + // The Begin/End ops realign the index register to the assertion point; + // they belong to the enclosing direction (m_direction is unchanged here). appendOp(YarrOpCode::ParentheticalAssertionBegin); m_ops.last().m_checkAdjust = checkedOffset - term->inputPosition; checkedOffset -= m_ops.last().m_checkAdjust; m_ops.last().m_checkedOffset = checkedOffset; + // The body's ops are emitted in the body's own direction. A lookbehind + // body's mirrored frame starts fresh at virtual position 0. + MatchDirection enclosingDirection = m_direction; + m_direction = bodyDirection; + if (bodyDirection == Backward) + checkedOffset = 0; + appendOp(YarrOpCode::SimpleNestedAlternativeBegin); m_ops.last().m_previousOp = notFound; m_ops.last().m_term = term; - PatternDisjunction* disjunction = term->parentheses.disjunction; auto& alternatives = disjunction->m_alternatives; for (unsigned i = 0; i < alternatives.size(); ++i) { size_t lastOpIndex = m_ops.size() - 1; @@ -5574,6 +6055,9 @@ class YarrGenerator final : public YarrJITInfo { lastOp.m_nextOp = notFound; lastOp.m_checkedOffset = checkedOffset; + // The End op is back in the enclosing direction. + m_direction = enclosingDirection; + size_t parenEnd = m_ops.size(); appendOp(YarrOpCode::ParentheticalAssertionEnd); @@ -5889,7 +6373,12 @@ class YarrGenerator final : public YarrJITInfo { } case PatternTerm::Type::ParentheticalAssertion: - return std::nullopt; + // An assertion consumes no input, so it contributes no characters at any + // offset of the leading-character sets. A lookbehind constrains input + // before the match start (not tracked here); a lookahead only narrows what + // the following terms may match, so ignoring it keeps the sets a valid + // over-approximation. Conservatively say it just matches, like BOL/EOL/\b. + return cursor; case PatternTerm::Type::DotStarEnclosure: return std::nullopt; @@ -6867,7 +7356,12 @@ class YarrGenerator final : public YarrJITInfo { } #endif - if (m_pattern.m_containsLookbehinds) { + // Lookbehind bodies are JIT-compiled via mirroring (see + // mirrorDisjunctionForLookbehind), except when the pattern also decodes + // surrogate pairs: reading backward across a UTF-16 pair (trail first, + // then lead) is not implemented in the backward JIT yet. Unsupported body + // content is detected during op-compilation and falls back the same way. + if (m_pattern.m_containsLookbehinds && m_decodeSurrogatePairs) { codeBlock.setFallBackWithFailureReason(JITFailureReason::Lookbehind); return; } @@ -7401,6 +7895,7 @@ class YarrGenerator final : public YarrJITInfo { unsigned index = m_ops.size(); m_ops.constructAndAppend(std::forward(args)...); m_ops.last().m_index = index; + m_ops.last().m_direction = m_direction; } private: @@ -7446,6 +7941,17 @@ class YarrGenerator final : public YarrJITInfo { MacroAssembler::JumpList m_inlinedMatched; MacroAssembler::JumpList m_inlinedFailedMatch; + // The matching direction currently in effect. During op-compilation this is + // the direction ops are being emitted in (stamped onto each YarrOp); during + // code generation and backtracking it is set from the op being processed so + // that the direction-aware primitives (input checks, character addressing, + // end-of-input) emit the leftward variants inside lookbehind bodies. + MatchDirection m_direction { Forward }; + + // Owns the mirrored (term-reversed) copies of lookbehind bodies. The original + // YarrPattern is left untouched so the bytecode fallback remains valid. + Vector> m_mirroredDisjunctions; + // The regular expression expressed as a linear sequence of operations. Vector m_ops; Vector, 4> m_bmInfos; From 416378670ab7e9222ccf60d350496b0d51bb5eac Mon Sep 17 00:00:00 2001 From: Dylan Conway Date: Thu, 16 Jul 2026 03:08:38 -0700 Subject: [PATCH 02/13] Yarr: factor alternation prefixes and fold large alternations into a group A large top-level alternation is matched by attempting each alternative in turn at every candidate position, so cost grows linearly with the alternative count regardless of how quickly each alternative fails. Two pattern-level rewrites cut the number of alternatives entered. Alternation prefix factoring: alternatives are tried leftmost-first, but their order is only observable between alternatives that can match at the same starting position. Alternatives that must begin with different literal characters have disjoint starting points, so a maximal run of consecutive alternatives that each begin with a fixed, case-sensitive literal character is stably sorted by that character (stability keeps same-first-character alternatives in source order) and each group sharing its leading literal is merged into one alternative whose common prefix is hoisted over a non-capturing group of the suffixes: /aq|bx|ar|by/ becomes /a(?:q|r)|b(?:x|y)/. Any alternative that does not begin with such a character (class, group, anchor, optional atom, the empty alternative) is a barrier no reordering crosses, and the rewrite recurses into the factored suffixes. Group folding: the body's repeated (non-once-through) alternatives, when numerous, are folded into a single alternative holding one non-capturing group -- /X|Y|Z/ is exactly /(?:X|Y|Z)/ -- whose capture range brackets the captures it absorbs. Nested alternatives claim and release their own input, giving each alternative an index-neutral entry that a following change dispatches on the first character. Both rewrites are engine-neutral pattern equivalences (the interpreter and the JIT see the same factored form) and run before setupOffsets(), which lays the synthesized groups out like hand-written ones. Controlled by useRegExpAlternationFactoring / regExpAlternationGroupThreshold. --- Source/JavaScriptCore/runtime/OptionsList.h | 2 + Source/JavaScriptCore/yarr/YarrPattern.cpp | 265 +++++++++++++++++++- 2 files changed, 266 insertions(+), 1 deletion(-) diff --git a/Source/JavaScriptCore/runtime/OptionsList.h b/Source/JavaScriptCore/runtime/OptionsList.h index dba9d6e9411b..8316dc1c9988 100644 --- a/Source/JavaScriptCore/runtime/OptionsList.h +++ b/Source/JavaScriptCore/runtime/OptionsList.h @@ -87,6 +87,8 @@ bool hasCapacityToUseLargeGigacage(); v(Bool, useBaselineJIT, true, Normal, "allows the baseline JIT to be used if true"_s) \ v(Bool, useDFGJIT, is64Bit(), Normal, "allows the DFG JIT to be used if true"_s) \ v(Bool, useRegExpJIT, jitEnabledByDefault() && is64Bit(), Normal, "allows the RegExp JIT to be used if true"_s) \ + v(Bool, useRegExpAlternationFactoring, true, Normal, "factors common prefixes out of an alternation's literal-leading alternatives and folds large top-level alternations into a group"_s) \ + v(Unsigned, regExpAlternationGroupThreshold, 8, Normal, "minimum number of repeated top-level alternatives before the alternation is folded into a group"_s) \ v(Bool, useDOMJIT, is64Bit(), Normal, "allows the DOMJIT to be used if true"_s) \ \ v(Bool, reportMustSucceedExecutableAllocations, false, Normal, nullptr) \ diff --git a/Source/JavaScriptCore/yarr/YarrPattern.cpp b/Source/JavaScriptCore/yarr/YarrPattern.cpp index 09bf2a30e9cb..9d589c98ceae 100644 --- a/Source/JavaScriptCore/yarr/YarrPattern.cpp +++ b/Source/JavaScriptCore/yarr/YarrPattern.cpp @@ -2334,11 +2334,273 @@ class YarrPatternConstructor { // Move alternatives from loopDisjunction to disjunction for (unsigned alt = 0; alt < loopDisjunction->m_alternatives.size(); ++alt) disjunction->m_alternatives.append(loopDisjunction->m_alternatives[alt].release()); - + loopDisjunction->m_alternatives.clear(); } } + // Alternation prefix factoring. + // + // Alternatives are tried leftmost-first, so their order is observable -- but + // only between alternatives that can match at the same starting position. + // Two alternatives that must begin with different literal characters have + // disjoint starting points, so a maximal run of consecutive alternatives + // that each begin with a (non-optional, case-sensitive) literal character + // may be stably sorted by that character and then merged on common + // prefixes: /aq|bx|ar|by/ becomes /a(?:q|r)|b(?:x|y)/. Stability keeps + // same-first-character alternatives in source order, and any alternative + // that does not start with such a character (a class, group, anchor, + // optional atom, or the empty alternative) is a barrier that no reordering + // crosses. The rewrite is applied recursively to the factored suffixes. It + // is a pure pattern-level equivalence, so both the JIT and the interpreter + // see the factored form. + + // The leading literal character of an alternative, if its first term is a + // fixed, case-sensitive pattern character that must consume input. + std::optional firstLiteralCharacter(const PatternAlternative& alternative) + { + if (alternative.m_terms.isEmpty()) + return std::nullopt; + const PatternTerm& term = alternative.m_terms[0]; + if (term.type != PatternTerm::Type::PatternCharacter) + return std::nullopt; + if (term.quantityType != QuantifierType::FixedCount || term.quantityMinCount != 1 || term.quantityMaxCount != 1) + return std::nullopt; + if (term.ignoreCase() || term.matchDirection() != Forward) + return std::nullopt; + return term.patternCharacter; + } + + // Two leading terms match for prefix purposes only if they are the same + // fixed, case-sensitive pattern character. + static bool isSameLiteralTerm(const PatternTerm& a, const PatternTerm& b) + { + return a.type == PatternTerm::Type::PatternCharacter + && b.type == PatternTerm::Type::PatternCharacter + && a.quantityType == QuantifierType::FixedCount && b.quantityType == QuantifierType::FixedCount + && a.quantityMinCount == 1 && a.quantityMaxCount == 1 + && b.quantityMinCount == 1 && b.quantityMaxCount == 1 + && !a.ignoreCase() && !b.ignoreCase() + && a.matchDirection() == Forward && b.matchDirection() == Forward + && a.patternCharacter == b.patternCharacter; + } + + // Merge a group of alternatives (already known to share their leading + // literal term) into a single alternative: a X | a Y | a Z -> a (?: X | Y | Z), + // with the longest common literal prefix hoisted and the suffixes factored + // recursively. `members` are removed from their owner and re-parented. + std::unique_ptr mergeSharedPrefix(Vector>&& members) + { + ASSERT(members.size() >= 2); + + // Longest common prefix of literal terms across all members. Never + // consume a member's entire term list, so no member's suffix depends on + // an empty tail being ordered against a longer sibling's tail through the + // prefix (the empty suffix is handled explicitly below). + size_t prefixLength = 1; + for (;; ++prefixLength) { + const auto& first = members[0]->m_terms; + if (prefixLength >= first.size()) + break; + bool allShare = true; + for (auto& member : members) { + if (prefixLength >= member->m_terms.size() || !isSameLiteralTerm(first[prefixLength], member->m_terms[prefixLength])) { + allShare = false; + break; + } + } + if (!allShare) + break; + } + + auto merged = makeUnique(members[0]->m_parent, members[0]->m_firstSubpatternId); + for (size_t i = 0; i < prefixLength; ++i) + merged->m_terms.append(members[0]->m_terms[i]); + + // The suffix disjunction holds each member's remaining terms, in the + // members' (stable, first-character-sorted) order. + auto suffixDisjunction = makeUnique(); + unsigned firstSubpatternId = members[0]->m_firstSubpatternId; + unsigned lastSubpatternId = 0; + bool containsBOL = false; + for (auto& member : members) { + PatternAlternative* suffix = suffixDisjunction->addNewAlternative(member->m_firstSubpatternId); + suffix->m_lastSubpatternId = member->m_lastSubpatternId; + suffix->m_containsBOL = member->m_containsBOL; + for (size_t i = prefixLength; i < member->m_terms.size(); ++i) + suffix->m_terms.append(member->m_terms[i]); + reparentNestedDisjunctions(*suffix); + lastSubpatternId = std::max(lastSubpatternId, member->m_lastSubpatternId); + containsBOL |= member->m_containsBOL; + } + suffixDisjunction->m_alternatives.last()->m_isLastAlternative = true; + + // Factor the suffixes themselves (a shared second character, and so on). + factorAlternatives(*suffixDisjunction); + + merged->m_lastSubpatternId = lastSubpatternId; + merged->m_containsBOL = containsBOL; + PatternTerm group(PatternTerm::Type::ParenthesesSubpattern, firstSubpatternId + 1, suffixDisjunction.get(), m_flags, /* capture */ false); + group.parentheses.lastSubpatternId = lastSubpatternId; + merged->m_terms.append(group); + suffixDisjunction->m_parent = merged.get(); + m_pattern.m_disjunctions.append(WTF::move(suffixDisjunction)); + return merged; + } + + // A nested group's disjunction points back at its owning alternative; keep + // that consistent when terms move to a new alternative. + void reparentNestedDisjunctions(PatternAlternative& alternative) + { + for (auto& term : alternative.m_terms) { + if ((term.type == PatternTerm::Type::ParenthesesSubpattern || term.type == PatternTerm::Type::ParentheticalAssertion) && term.parentheses.disjunction) + term.parentheses.disjunction->m_parent = &alternative; + } + } + + // Rewrite `disjunction`'s alternatives in place: sort each barrier-free run + // by leading literal and merge shared prefixes into nested groups. + void factorAlternatives(PatternDisjunction& disjunction) + { + if (!isSafeToRecurse()) [[unlikely]] + return; + + auto& alternatives = disjunction.m_alternatives; + Vector> result; + result.reserveInitialCapacity(alternatives.size()); + + size_t i = 0; + while (i < alternatives.size()) { + // A barrier (no fixed leading literal, or the empty alternative) is + // copied through untouched and never reordered across. + if (!firstLiteralCharacter(*alternatives[i]) || alternatives[i]->onceThrough()) { + result.append(WTF::move(alternatives[i])); + ++i; + continue; + } + + // Gather the maximal run of literal-leading alternatives. + size_t runStart = i; + while (i < alternatives.size() && firstLiteralCharacter(*alternatives[i]) && !alternatives[i]->onceThrough()) + ++i; + size_t runEnd = i; + + if (runEnd - runStart == 1) { + result.append(WTF::move(alternatives[runStart])); + continue; + } + + Vector> run; + for (size_t j = runStart; j < runEnd; ++j) + run.append(WTF::move(alternatives[j])); + std::stable_sort(run.begin(), run.end(), [&](auto& a, auto& b) { + return *firstLiteralCharacter(*a) < *firstLiteralCharacter(*b); + }); + + // Walk the sorted run, merging each maximal group that shares the + // leading literal term. + size_t j = 0; + while (j < run.size()) { + size_t groupEnd = j + 1; + while (groupEnd < run.size() && isSameLiteralTerm(run[j]->m_terms[0], run[groupEnd]->m_terms[0])) + ++groupEnd; + if (groupEnd - j == 1) + result.append(WTF::move(run[j])); + else { + Vector> members; + for (size_t k = j; k < groupEnd; ++k) + members.append(WTF::move(run[k])); + auto merged = mergeSharedPrefix(WTF::move(members)); + merged->m_parent = &disjunction; + result.append(WTF::move(merged)); + } + j = groupEnd; + } + } + + // Every alternative was moved into `result`; always reinstall the list + // (a run that sorted or merged nothing is simply the original order). + for (auto& alternative : result) { + alternative->m_parent = &disjunction; + alternative->m_isLastAlternative = false; + } + result.last()->m_isLastAlternative = true; + alternatives = WTF::move(result); + } + + // A large top-level alternation costs one entry attempt per alternative at + // every candidate position when the alternatives are tried in sequence. Fold + // the repeated (non-once-through) alternatives into a single alternative + // holding one non-capturing group -- /X|Y|Z/ is exactly /(?:X|Y|Z)/ -- so + // that the group's alternatives can be dispatched on their first character. + // The rewrite itself is engine-neutral; setupOffsets() (which runs after + // this) lays out the group like any hand-written one. + void factorAndWrapAlternatives() + { + if (!Options::useRegExpAlternationFactoring()) + return; + + factorAlternatives(*m_pattern.m_body); + wrapAlternativesForDispatch(); + } + + void wrapAlternativesForDispatch() + { + PatternDisjunction* body = m_pattern.m_body; + auto& alternatives = body->m_alternatives; + + // The body is laid out as [onceThrough..., repeated...] (see optimizeBOL). + size_t firstRepeated = 0; + while (firstRepeated < alternatives.size() && alternatives[firstRepeated]->onceThrough()) + ++firstRepeated; + size_t repeatedCount = alternatives.size() - firstRepeated; + if (repeatedCount < Options::regExpAlternationGroupThreshold()) + return; + + // A DotStarEnclosure records match bounds through the enclosing body + // alternative; keep such bodies in their existing shape. + for (size_t i = firstRepeated; i < alternatives.size(); ++i) { + for (auto& term : alternatives[i]->m_terms) { + if (term.type == PatternTerm::Type::DotStarEnclosure) + return; + } + } + + // The group's captures are exactly those of the alternatives it absorbs: + // subpatternId is the id a capture opened at the group start would take, + // lastSubpatternId the last id closed inside it (matching atomParenthesesEnd). + unsigned firstSubpatternId = alternatives[firstRepeated]->m_firstSubpatternId; + unsigned lastSubpatternId = 0; + bool containsBOL = false; + unsigned startsWithBOLCount = 0; + + auto groupDisjunction = makeUnique(); + for (size_t i = firstRepeated; i < alternatives.size(); ++i) { + PatternAlternative* alternative = alternatives[i].get(); + alternative->m_parent = groupDisjunction.get(); + alternative->m_isLastAlternative = false; + lastSubpatternId = std::max(lastSubpatternId, alternative->m_lastSubpatternId); + containsBOL |= alternative->m_containsBOL; + if (alternative->m_startsWithBOL) + ++startsWithBOLCount; + groupDisjunction->m_alternatives.append(WTF::move(alternatives[i])); + } + groupDisjunction->m_alternatives.last()->m_isLastAlternative = true; + alternatives.shrink(firstRepeated); + + PatternAlternative* wrapped = body->addNewAlternative(firstSubpatternId); + wrapped->m_lastSubpatternId = lastSubpatternId; + wrapped->m_containsBOL = containsBOL; + wrapped->m_startsWithBOL = startsWithBOLCount == groupDisjunction->m_alternatives.size(); + groupDisjunction->m_parent = wrapped; + + PatternTerm group(PatternTerm::Type::ParenthesesSubpattern, firstSubpatternId + 1, groupDisjunction.get(), m_flags, /* capture */ false); + group.parentheses.lastSubpatternId = lastSubpatternId; + wrapped->m_terms.append(group); + + m_pattern.m_disjunctions.append(WTF::move(groupDisjunction)); + } + bool containsCapturingTerms(PatternAlternative* alternative, size_t firstTermIndex, size_t endIndex) { if (!isSafeToRecurse()) [[unlikely]] { @@ -2930,6 +3192,7 @@ ErrorCode YarrPattern::compile(StringView patternString) constructor.checkForTerminalParentheses(); constructor.optimizeDotStarWrappedExpressions(); constructor.optimizeBOL(); + constructor.factorAndWrapAlternatives(); constructor.optimizePossessiveQuantifiers(); if (hasError(constructor.error())) From 3037b8d51f3b525412503aa988e96ca1552f5439 Mon Sep 17 00:00:00 2001 From: Dylan Conway Date: Thu, 16 Jul 2026 03:28:55 -0700 Subject: [PATCH 03/13] Yarr JIT: dispatch a group's alternatives on the first character Trying each alternative of a nested disjunction in sequence costs an entry attempt (claim, character load, compare, release, jump) per alternative at every candidate position, so a large alternation scales linearly with its alternative count even when each alternative fails on its first character. For a once-quantified fixed group whose alternatives all consume at least one character, compute per alternative a first-character set (an over-approximation of the characters a match can start with; Latin-1 tracked precisely, wider characters and the empty prefix widen to "any"). Every alternative's first character sits at the group's start frame position, which is inside input the enclosing alternative already claimed, so it is read once with no bounds check. A binary decision tree over that character's value then jumps to the ordered chain of exactly the alternatives whose set contains it; an "any" alternative appears in every chain, so leftmost-alternative-wins ordering is preserved. When no alternative can start with the character the group fails immediately. Each chain is a sequence of stubs that store the address of the next stub into a new frame slot (BackTrackInfoParenthesesOnce::chainResume, patched at link time) before jumping into the alternative; a failing alternative releases its claim and continues its own chain through that slot rather than falling to a fixed successor, and an exhausted chain joins the group's failure flow. Entry into an alternative is now a labelled chain target rather than a fallthrough, and backtracking back into a matched alternative reuses the still-valid resume address. Combined with the alternation prefix factoring, the isbot user-agent regex over the 361,829-string corpus goes from ~620 ms (factoring alone) to ~120 ms; the sequential JIT was ~750 ms and the interpreter ~32,000 ms. Controlled by useRegExpAlternationDispatch / regExpAlternationDispatchThreshold; interpreter behaviour is unchanged and JIT results match it byte-for-byte across differential corpora. --- Source/JavaScriptCore/runtime/OptionsList.h | 2 + Source/JavaScriptCore/yarr/Yarr.h | 2 +- Source/JavaScriptCore/yarr/YarrJIT.cpp | 491 ++++++++++++++++++++ Source/JavaScriptCore/yarr/YarrPattern.h | 5 + 4 files changed, 499 insertions(+), 1 deletion(-) diff --git a/Source/JavaScriptCore/runtime/OptionsList.h b/Source/JavaScriptCore/runtime/OptionsList.h index 8316dc1c9988..b0287562b3be 100644 --- a/Source/JavaScriptCore/runtime/OptionsList.h +++ b/Source/JavaScriptCore/runtime/OptionsList.h @@ -89,6 +89,8 @@ bool hasCapacityToUseLargeGigacage(); v(Bool, useRegExpJIT, jitEnabledByDefault() && is64Bit(), Normal, "allows the RegExp JIT to be used if true"_s) \ v(Bool, useRegExpAlternationFactoring, true, Normal, "factors common prefixes out of an alternation's literal-leading alternatives and folds large top-level alternations into a group"_s) \ v(Unsigned, regExpAlternationGroupThreshold, 8, Normal, "minimum number of repeated top-level alternatives before the alternation is folded into a group"_s) \ + v(Bool, useRegExpAlternationDispatch, true, Normal, "reads a group's first character once and jumps to the chain of alternatives that can start with it"_s) \ + v(Unsigned, regExpAlternationDispatchThreshold, 4, Normal, "minimum number of alternatives in a group before its alternatives are dispatched on the first character"_s) \ v(Bool, useDOMJIT, is64Bit(), Normal, "allows the DOMJIT to be used if true"_s) \ \ v(Bool, reportMustSucceedExecutableAllocations, false, Normal, nullptr) \ diff --git a/Source/JavaScriptCore/yarr/Yarr.h b/Source/JavaScriptCore/yarr/Yarr.h index a7bf9954cdd6..e68b380e5941 100644 --- a/Source/JavaScriptCore/yarr/Yarr.h +++ b/Source/JavaScriptCore/yarr/Yarr.h @@ -38,7 +38,7 @@ namespace JSC { namespace Yarr { #define YarrStackSpaceForBackTrackInfoBackReference 3 #define YarrStackSpaceForBackTrackInfoAlternative 1 // One per alternative. #define YarrStackSpaceForBackTrackInfoParentheticalAssertion 1 -#define YarrStackSpaceForBackTrackInfoParenthesesOnce 2 +#define YarrStackSpaceForBackTrackInfoParenthesesOnce 3 #define YarrStackSpaceForBackTrackInfoParenthesesTerminal 1 #define YarrStackSpaceForBackTrackInfoParentheses 4 #define YarrStackSpaceForDotStarEnclosure 1 diff --git a/Source/JavaScriptCore/yarr/YarrJIT.cpp b/Source/JavaScriptCore/yarr/YarrJIT.cpp index 6f2781fb8ced..7d388d8ced2a 100644 --- a/Source/JavaScriptCore/yarr/YarrJIT.cpp +++ b/Source/JavaScriptCore/yarr/YarrJIT.cpp @@ -314,6 +314,83 @@ void BoyerMooreBitmap::dump(PrintStream& out) const // // FIXME: Currently we are only supporting 2 alternative cases at first, aligned to V8's optimization. // We will extend it to 1 and 3 later. +// First-character set of an alternative: an over-approximation of the set of +// characters a match can start with. Latin-1 characters are tracked precisely; +// `wide` means a first character >= 0x100 is possible; `any` is the top +// element (every character possible). An over-approximation is always sound +// for dispatch: membership merely means the alternative is tried there. +struct FirstCharacterSet { + WTF::BitSet<256> latin1; + bool wide { false }; + bool any { false }; + // Whether analysis reached a term that must consume a character. If false + // the alternative can match the empty string, which for dispatch is `any`. + bool consumed { false }; + + void add(char32_t ch) + { + if (ch < 256) + latin1.set(ch); + else + wide = true; + } + void addRange(char32_t begin, char32_t end) + { + for (char32_t ch = begin; ch <= end && ch < 256; ++ch) + latin1.set(ch); + if (end >= 256) + wide = true; + } + void merge(const FirstCharacterSet& other) + { + latin1.merge(other.latin1); + wide |= other.wide; + any |= other.any; + } + bool matches(unsigned latin1Char) const { return any || latin1.get(latin1Char); } + bool matchesWide() const { return any || wide; } +}; + +// First-character dispatch for a nested disjunction with many alternatives. +// +// Trying N alternatives in sequence costs an entry attempt each. When every +// alternative must consume a first character, that character is already inside +// input the enclosing alternative claimed, so it can be read once (no bounds +// check) and a decision tree over its value can jump straight to the ordered +// chain of alternatives whose first-character set contains it. Alternatives +// are still tried in source order within a chain, so leftmost-first semantics +// hold; an alternative whose set is `any` appears in every chain. A failing +// alternative continues its chain through a code address the chain stored in +// the group's frame (BackTrackInfoParenthesesOnce::chainResume) before entry. +struct DispatchInfo { +private: + WTF_MAKE_TZONE_ALLOCATED(DispatchInfo); +public: + struct Chain { + Vector alternativeIndices; // ascending; tried in this order + MacroAssembler::Label head; // start of the chain's stub sequence + }; + + Vector sets; // per alternative + Vector chains; // distinct Latin-1 chains + std::array chainForCharacter { }; // Latin-1 char -> chain index, or noChain + Chain wideChain; // characters >= 0x100 (16-bit strings only) + unsigned frameLocation { 0 }; // the group's frame; chainResume lives here + Checked firstCharacterOffset; // read offset of the group's first character + + // Per alternative: the entry label (bound during generation) and jumps + // from chain stubs awaiting that label. + Vector entries; + Vector entryBound; + Vector pendingEntryJumps; + + // Failures that mean "no alternative can match here" (empty chains, chain + // exhaustion). Routed as the group's failure with the index unmoved. + MacroAssembler::JumpList groupFailJumps; + + static constexpr unsigned noChain = std::numeric_limits::max(); +}; + struct MaskedAlternativeInfo { private: WTF_MAKE_TZONE_ALLOCATED(MaskedAlternativeInfo); @@ -1979,6 +2056,13 @@ class YarrGenerator final : public YarrJITInfo { BoyerMooreInfo* m_bmInfo { nullptr }; MaskedAlternativeInfo* m_maskedAltInfo { nullptr }; + // Set on the alternative Begin/Next/End ops of a nested disjunction whose + // alternatives are dispatched on their first character (see DispatchInfo). + struct DispatchInfo* m_dispatch { nullptr }; + // Position of this Begin/Next op's alternative within the dispatched + // disjunction. + unsigned m_dispatchAlternativeIndex { 0 }; + // Shared UTF-16 lead surrogate across all repeated alternatives' first character. std::optional m_bodyAltSharedLead; @@ -2039,6 +2123,13 @@ class YarrGenerator final : public YarrJITInfo { { m_pendingReturns.append(returnAddress); } + // Bind a patchable frame store directly to a known code location (used by + // first-character dispatch chains, whose targets are forward-generated + // stubs rather than backtracking sites). + void addReturnAddressRecord(const MacroAssembler::DataLabelPtr& dataLabel, MacroAssembler::Label target) + { + m_backtrackRecords.append(ReturnAddressRecord(dataLabel, target)); + } void NODELETE fallthrough() { ASSERT(!m_pendingFallthrough); @@ -4305,6 +4396,14 @@ class YarrGenerator final : public YarrJITInfo { termMatchTargets.last() = alternative->m_isLastAlternative ? MatchTargets(MatchTargets::PreferredTarget::MatchSuccessFallThrough) : MatchTargets(endOp->m_jumps, op.m_jumps, MatchTargets::PreferredTarget::MatchFailFallThrough); } + // First-character dispatch: read the character once and route to + // the applicable chain, then fall into alternative 0's entry (which + // is now a labelled chain target rather than a fallthrough). + if (op.m_dispatch) { + generateDispatch(op); + bindDispatchEntry(*op.m_dispatch, 0); + } + // Calculate how much input we need to check for, and if non-zero check. op.m_checkAdjust = Checked(alternative->m_minimumSize); if ((term->quantityType == QuantifierType::FixedCount) && (term->quantityMaxCount == 1) && (term->type != PatternTerm::Type::ParentheticalAssertion)) @@ -4358,6 +4457,8 @@ class YarrGenerator final : public YarrJITInfo { // This is the entry point for the next alternative. defineReentryLabel(op); + if (op.m_dispatch) + bindDispatchEntry(*op.m_dispatch, op.m_dispatchAlternativeIndex); // Calculate how much input we need to check for, and if non-zero check. op.m_checkAdjust = alternative->m_minimumSize; @@ -5091,6 +5192,38 @@ class YarrGenerator final : public YarrJITInfo { // Treat an input check failure the same as a failed match. m_backtrackingState.append(op.m_jumps); + // Dispatched alternatives (first-character dispatch): a failing + // alternative releases its claim and continues its own chain through + // the resume address the chain stored in the frame, rather than + // falling to a fixed successor. Failures that mean "no alternative can + // match here" (empty routes, exhausted chains) join the group's + // failure flow at the Begin op, where the index is at the group frame. + if (op.m_dispatch) { + if (!m_backtrackingState.isEmpty()) { + m_backtrackingState.link(*this, op); + // Give back the input this alternative claimed on entry. + if (op.m_checkAdjust) + m_jit.sub32(MacroAssembler::Imm32(op.m_checkAdjust), m_regs.index); + loadFromFrameAndJump(op.m_dispatch->frameLocation + BackTrackInfoParenthesesOnce::chainResumeIndex()); + } + if (isBegin) { + // Group-level failures gathered by the dispatch, plus any jumps + // planted on the End (none are expected under dispatch, kept + // for parity with the sequential path). + m_backtrackingState.append(op.m_dispatch->groupFailJumps); + YarrOp* endOp = &m_ops[op.m_nextOp]; + while (endOp->m_nextOp != notFound) + endOp = &m_ops[endOp->m_nextOp]; + m_backtrackingState.append(endOp->m_jumps); + } + // For non-simple alternatives, link the alternative's 'return + // address' so we backtrack back out into it correctly. + if (op.m_op == YarrOpCode::NestedAlternativeNext) + m_backtrackingState.append(op.m_returnAddress); + op.m_contentBacktrackEntryLabel = m_jit.label(); + break; + } + // Set the backtracks to jump to the appropriate place. We may need // to link the backtracks in one of three different way depending on // the type of alternative we are dealing with: @@ -5721,12 +5854,25 @@ class YarrGenerator final : public YarrJITInfo { #endif } + // Try first-character dispatch for a plain once-group with several + // alternatives (its ops are the ParenthesesSubpatternOnce + Nested* set). + // The sets must be computed before opCompileAlternative reorders terms. + DispatchInfo* dispatch = nullptr; + if (parenthesesBeginOpCode == YarrOpCode::ParenthesesSubpatternOnceBegin && alternativeBeginOpCode == YarrOpCode::NestedAlternativeBegin) + dispatch = tryPrepareDispatch(term, checkedOffset); + if (Options::verboseRegExpCompilation() && term->parentheses.disjunction->m_alternatives.size() >= Options::regExpAlternationDispatchThreshold()) { + dumpFirstCharacterSets(*term->parentheses.disjunction); + dataLogLn(dispatch ? "First-character dispatch enabled" : "First-character dispatch not used"); + } + size_t parenBegin = m_ops.size(); appendOp(parenthesesBeginOpCode); appendOp(alternativeBeginOpCode); m_ops.last().m_previousOp = notFound; m_ops.last().m_term = term; + m_ops.last().m_dispatch = dispatch; + m_ops.last().m_dispatchAlternativeIndex = 0; PatternDisjunction* disjunction = term->parentheses.disjunction; auto& alternatives = disjunction->m_alternatives; for (unsigned i = 0; i < alternatives.size(); ++i) { @@ -5762,6 +5908,8 @@ class YarrGenerator final : public YarrJITInfo { lastOp.m_nextOp = thisOpIndex; thisOp.m_previousOp = lastOpIndex; thisOp.m_term = term; + thisOp.m_dispatch = dispatch; + thisOp.m_dispatchAlternativeIndex = i + 1; } YarrOp& lastOp = m_ops.last(); ASSERT(lastOp.m_op == alternativeNextOpCode); @@ -6297,6 +6445,346 @@ class YarrGenerator final : public YarrJITInfo { } } + // First-character set analysis (see FirstCharacterSet). + // + // Adds the characters that `term`, matched at the start of what remains, + // can begin with. Returns true if the term must consume at least one + // character (so the alternative's first character is now fully accounted + // for), false if the term can match the empty string and the analysis must + // continue into the following term. + bool addFirstCharactersOfTerm(const PatternTerm& term, FirstCharacterSet& set) + { + switch (term.type) { + case PatternTerm::Type::AssertionBOL: + case PatternTerm::Type::AssertionEOL: + case PatternTerm::Type::AssertionWordBoundary: + case PatternTerm::Type::NumberedForwardReference: + case PatternTerm::Type::NamedForwardReference: + // Zero-width: contributes no first character. + return false; + + case PatternTerm::Type::NumberedBackReference: + case PatternTerm::Type::NamedBackReference: + case PatternTerm::Type::DotStarEnclosure: + set.any = true; + return true; + + case PatternTerm::Type::ParentheticalAssertion: + // An assertion consumes no input; a positive lookahead could narrow the + // set but ignoring it keeps the set a valid over-approximation. + return false; + + case PatternTerm::Type::PatternCharacter: { + char32_t ch = term.patternCharacter; + if (term.ignoreCase() && isASCIIAlpha(ch)) { + set.add(toASCIIUpper(ch)); + set.add(toASCIILower(ch)); + } else if (term.ignoreCase() && !isASCII(ch)) + set.any = true; // Non-ASCII case folding: don't try to be clever. + else + set.add(ch); + return !!term.quantityMinCount; + } + + case PatternTerm::Type::CharacterClass: { + const CharacterClass* characterClass = term.characterClass; + if (term.invert() || characterClass->m_anyCharacter || characterClass->m_table || term.ignoreCase()) + set.any = true; + else { + for (char32_t ch : characterClass->m_matches8) + set.add(ch); + for (auto& range : characterClass->m_ranges8) + set.addRange(range.begin, range.end); + if (!characterClass->m_matches32.isEmpty() || !characterClass->m_ranges32.isEmpty() || characterClass->hasStrings()) + set.wide = true; + } + return !!term.quantityMinCount; + } + + case PatternTerm::Type::ParenthesesSubpattern: { + PatternDisjunction* disjunction = term.parentheses.disjunction; + bool allConsumed = true; + for (auto& alternative : disjunction->m_alternatives) { + FirstCharacterSet nested = computeFirstCharacterSet(*alternative); + set.merge(nested); + allConsumed &= nested.consumed; + } + // The group only guarantees a consumed first character if it is + // required (min count >= 1) and every alternative consumes. + return term.quantityMinCount && allConsumed; + } + } + RELEASE_ASSERT_NOT_REACHED(); + } + + // First-character dispatch code generation. + // + // Emitted at the group's alternative Begin op, before any alternative body: + // + // char = input[index - firstCharacterOffset] ; inside claimed input + // [16-bit: char >= 0x100 -> wide chain] + // binary decision tree over char -> jump to chain head, or group failure + // chain stubs: head_C: frame.chainResume = &stub_C[1]; jump entry(alt i0) + // stub_C[1]: frame.chainResume = &stub_C[2]; jump entry(alt i1) + // ... stub_C[n]: jump groupFailure + // + // A failing alternative releases its claim and jumps to frame.chainResume, so + // it continues its own chain rather than a fixed successor. Entry labels of + // later alternatives are bound as they are generated; stubs that target them + // are patched then via DispatchInfo::pendingEntryJumps. + + void emitJumpToDispatchEntry(DispatchInfo& info, unsigned alternativeIndex) + { + if (info.entryBound[alternativeIndex]) + m_jit.jump(info.entries[alternativeIndex]); + else + info.pendingEntryJumps[alternativeIndex].append(m_jit.jump()); + } + + void bindDispatchEntry(DispatchInfo& info, unsigned alternativeIndex) + { + info.entries[alternativeIndex] = m_jit.label(); + info.entryBound[alternativeIndex] = true; + info.pendingEntryJumps[alternativeIndex].link(&m_jit); + info.pendingEntryJumps[alternativeIndex].clear(); + } + + // Emit one chain: for each member, store the address of the following stub + // as the chain-resume point, then jump into the member; the final stub is the + // group failure. Returns the label of the chain head. + MacroAssembler::Label emitDispatchChainStubs(DispatchInfo& info, const Vector& alternativeIndices) + { + JIT_COMMENT(m_jit, "dispatch chain (", alternativeIndices.size(), " alternatives)"); + MacroAssembler::Label head = m_jit.label(); + Vector pendingStores; + for (unsigned index : alternativeIndices) { + // Bind the previous stub's stored resume address to here. + if (!pendingStores.isEmpty()) + m_backtrackingState.addReturnAddressRecord(pendingStores.takeLast(), m_jit.label()); + pendingStores.append(storeToFrameWithPatch(info.frameLocation + BackTrackInfoParenthesesOnce::chainResumeIndex())); + emitJumpToDispatchEntry(info, index); + } + // Chain exhausted: no more applicable alternatives at this position. + m_backtrackingState.addReturnAddressRecord(pendingStores.takeLast(), m_jit.label()); + info.groupFailJumps.append(m_jit.jump()); + return head; + } + + // Binary decision tree over the character's Latin-1 value. `ranges` are + // maximal [begin,end] intervals with a constant chain id, sorted and + // contiguous over 0..255. Leaves jump to the chain head (or group failure). + struct DispatchRange { + unsigned begin; + unsigned end; + unsigned chainId; + }; + void emitDispatchTree(DispatchInfo& info, MacroAssembler::RegisterID character, const Vector& ranges, unsigned low, unsigned high) + { + if (low == high) { + unsigned chainId = ranges[low].chainId; + if (chainId == DispatchInfo::noChain) + info.groupFailJumps.append(m_jit.jump()); + else + m_jit.jump().linkTo(info.chains[chainId].head, &m_jit); + return; + } + unsigned middle = (low + high + 1) / 2; + MacroAssembler::Jump upper = m_jit.branch32(MacroAssembler::AboveOrEqual, character, MacroAssembler::TrustedImm32(ranges[middle].begin)); + emitDispatchTree(info, character, ranges, low, middle - 1); + upper.link(&m_jit); + emitDispatchTree(info, character, ranges, middle, high); + } + + void generateDispatch(YarrOp& op) + { + DispatchInfo& info = *op.m_dispatch; + const MacroAssembler::RegisterID character = m_regs.regT0; + + JIT_COMMENT(m_jit, "first-character dispatch"); + readCharacter(info.firstCharacterOffset, character); + + MacroAssembler::JumpList wideCharacter; + if (m_charSize == CharSize::Char16) + wideCharacter.append(m_jit.branch32(MacroAssembler::AboveOrEqual, character, MacroAssembler::TrustedImm32(256))); + + // Emit the chain stubs first so tree leaves can jump backward to them; + // the tree itself is entered by jumping over the stubs. + MacroAssembler::Jump toTree = m_jit.jump(); + for (auto& chain : info.chains) + chain.head = emitDispatchChainStubs(info, chain.alternativeIndices); + if (m_charSize == CharSize::Char16) { + if (info.wideChain.alternativeIndices.isEmpty()) { + MacroAssembler::Label failHead(m_jit); + info.wideChain.head = failHead; + info.groupFailJumps.append(m_jit.jump()); + } else + info.wideChain.head = emitDispatchChainStubs(info, info.wideChain.alternativeIndices); + } + toTree.link(&m_jit); + if (m_charSize == CharSize::Char16) + wideCharacter.linkTo(info.wideChain.head, &m_jit); + + // Group the routing table into maximal ranges of equal chain id. + Vector ranges; + for (unsigned ch = 0; ch < 256;) { + unsigned chainId = info.chainForCharacter[ch]; + unsigned begin = ch; + while (ch < 256 && info.chainForCharacter[ch] == chainId) + ++ch; + ranges.append({ begin, ch - 1, chainId }); + } + emitDispatchTree(info, character, ranges, 0, ranges.size() - 1); + } + + // Decide whether a nested group's alternatives are worth dispatching on the + // first character, and if so build the DispatchInfo (chains and character + // routing). Called at op-compile time, before the alternatives' ops are + // emitted (so the first term of each alternative is still in position order). + DispatchInfo* tryPrepareDispatch(PatternTerm* term, Checked checkedOffset) + { + if (!Options::useRegExpAlternationDispatch()) + return nullptr; + // Only once-quantified fixed groups take the ParenthesesSubpatternOnce + + // NestedAlternative path whose frame holds BackTrackInfoParenthesesOnce. + if (term->type != PatternTerm::Type::ParenthesesSubpattern) + return nullptr; + if (term->quantityType != QuantifierType::FixedCount || term->quantityMinCount != 1 || term->quantityMaxCount != 1) + return nullptr; + if (term->parentheses.isCopy || term->parentheses.isTerminal || term->parentheses.isStringList) + return nullptr; + if (term->matchDirection() != Forward || m_direction != Forward) + return nullptr; + if (m_decodeSurrogatePairs) + return nullptr; + + PatternDisjunction* disjunction = term->parentheses.disjunction; + auto& alternatives = disjunction->m_alternatives; + if (alternatives.size() < Options::regExpAlternationDispatchThreshold()) + return nullptr; + // The first character is only guaranteed inside claimed input when every + // alternative must consume at least one character. + if (!disjunction->m_minimumSize) + return nullptr; + + auto info = makeUniqueRef(); + unsigned anyCount = 0; + for (auto& alternative : alternatives) { + FirstCharacterSet set = computeFirstCharacterSet(*alternative); + if (set.any) + ++anyCount; + info->sets.append(WTF::move(set)); + } + // Dispatch is pointless when most alternatives can start with anything. + if (anyCount * 4 > alternatives.size()) + return nullptr; + + // Route each Latin-1 character to the ordered chain of alternatives whose + // set contains it; identical chains are shared (linear dedupe: chains are + // few in practice). + for (unsigned ch = 0; ch < 256; ++ch) { + Vector members; + for (unsigned i = 0; i < alternatives.size(); ++i) { + if (info->sets[i].matches(ch)) + members.append(i); + } + if (members.isEmpty()) { + info->chainForCharacter[ch] = DispatchInfo::noChain; + continue; + } + unsigned chainId = DispatchInfo::noChain; + for (unsigned c = 0; c < info->chains.size(); ++c) { + if (info->chains[c].alternativeIndices == members) { + chainId = c; + break; + } + } + if (chainId == DispatchInfo::noChain) { + chainId = info->chains.size(); + DispatchInfo::Chain chain; + chain.alternativeIndices = WTF::move(members); + info->chains.append(WTF::move(chain)); + } + info->chainForCharacter[ch] = chainId; + } + // Bound the emitted stub code. + unsigned stubCount = 0; + for (auto& chain : info->chains) + stubCount += chain.alternativeIndices.size(); + if (info->chains.size() > 96 || stubCount > 768) + return nullptr; + if (m_charSize == CharSize::Char16) { + for (unsigned i = 0; i < alternatives.size(); ++i) { + if (info->sets[i].matchesWide()) + info->wideChain.alternativeIndices.append(i); + } + } + + // The group's first character is at the frame position where the group + // starts, inside the input the enclosing alternative already claimed. + Checked groupStart = term->inputPosition - disjunction->m_minimumSize; + if (checkedOffset < term->inputPosition) + return nullptr; + info->firstCharacterOffset = checkedOffset - groupStart; + if (info->firstCharacterOffset.hasOverflowed()) + return nullptr; + info->frameLocation = term->frameLocation; + info->entries.resize(alternatives.size()); + info->entryBound.fill(false, alternatives.size()); + info->pendingEntryJumps.resize(alternatives.size()); + + DispatchInfo* result = info.ptr(); + m_dispatchInfos.append(WTF::move(info)); + return result; + } + + void dumpFirstCharacterSets(PatternDisjunction& disjunction) + { + dataLogLn("First-character sets for ", disjunction.m_alternatives.size(), " alternatives:"); + for (size_t i = 0; i < disjunction.m_alternatives.size(); ++i) { + FirstCharacterSet set = computeFirstCharacterSet(*disjunction.m_alternatives[i]); + dataLog(" alt ", i, ": "); + if (set.any) + dataLog("ANY"); + else { + for (unsigned ch = 0; ch < 256; ++ch) { + if (set.latin1.get(ch)) { + if (isASCIIPrintable(ch)) + dataLog("'", static_cast(ch), "'"); + else + dataLog("\\x", ch); + } + } + if (set.wide) + dataLog(" +wide"); + } + dataLogLn(); + } + } + + FirstCharacterSet computeFirstCharacterSet(const PatternAlternative& alternative) + { + FirstCharacterSet set; + if (!isSafeToRecurse()) [[unlikely]] { + set.any = true; + return set; + } + for (auto& term : alternative.m_terms) { + if (addFirstCharactersOfTerm(term, set)) { + set.consumed = true; + break; + } + if (set.any) { + set.consumed = true; + break; + } + } + // An alternative that can match without consuming a character can start + // at any character (its "first character" is whatever follows). + if (!set.consumed) + set.any = true; + return set; + } + std::optional collectBoyerMooreInfoFromTerm(PatternTerm& term, unsigned cursor, BoyerMooreInfo& bmInfo) { switch (term.type) { @@ -7952,6 +8440,9 @@ class YarrGenerator final : public YarrJITInfo { // YarrPattern is left untouched so the bytecode fallback remains valid. Vector> m_mirroredDisjunctions; + // First-character dispatch state for nested disjunctions (see DispatchInfo). + Vector, 2> m_dispatchInfos; + // The regular expression expressed as a linear sequence of operations. Vector m_ops; Vector, 4> m_bmInfos; diff --git a/Source/JavaScriptCore/yarr/YarrPattern.h b/Source/JavaScriptCore/yarr/YarrPattern.h index ade52491a9fa..21eec64779cb 100644 --- a/Source/JavaScriptCore/yarr/YarrPattern.h +++ b/Source/JavaScriptCore/yarr/YarrPattern.h @@ -863,9 +863,14 @@ struct YarrPattern { struct BackTrackInfoParenthesesOnce { uintptr_t begin; uintptr_t returnAddress; + // Address of the code that continues a first-character dispatch chain + // when the alternative just entered fails (see the dispatched-alternatives + // path in YarrJIT). + uintptr_t chainResume; static unsigned beginIndex() { return offsetof(BackTrackInfoParenthesesOnce, begin) / sizeof(uintptr_t); } static unsigned returnAddressIndex() { return offsetof(BackTrackInfoParenthesesOnce, returnAddress) / sizeof(uintptr_t); } + static unsigned chainResumeIndex() { return offsetof(BackTrackInfoParenthesesOnce, chainResume) / sizeof(uintptr_t); } }; struct BackTrackInfoParenthesesTerminal { From de02df79b5993fe59ea3204c5dd7789f14b6a7aa Mon Sep 17 00:00:00 2001 From: Dylan Conway Date: Thu, 16 Jul 2026 03:38:59 -0700 Subject: [PATCH 04/13] Yarr: also factor alternation prefixes inside pre-existing groups The prefix-factoring pass previously ran only over the body's top-level alternatives, so a hand-written group such as /\b(?:about|above|after)\b/ kept re-comparing its shared "a" prefix in every alternative. Recurse into the disjunctions of source-written (non-copy, forward) groups after factoring each level, so nested alternations are factored the same way. Groups synthesized by the pass were already factored when built; revisiting them is a no-op. --- Source/JavaScriptCore/yarr/YarrPattern.cpp | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/Source/JavaScriptCore/yarr/YarrPattern.cpp b/Source/JavaScriptCore/yarr/YarrPattern.cpp index 9d589c98ceae..99788dc4922e 100644 --- a/Source/JavaScriptCore/yarr/YarrPattern.cpp +++ b/Source/JavaScriptCore/yarr/YarrPattern.cpp @@ -2526,6 +2526,18 @@ class YarrPatternConstructor { } result.last()->m_isLastAlternative = true; alternatives = WTF::move(result); + + // Factor inside pre-existing (source-written) groups too, so that + // /\b(?:about|above|after)\b/ shares its "a" prefix like a top-level + // alternation would. Groups synthesized by mergeSharedPrefix were + // already factored when built; re-running is a harmless no-op. + for (auto& alternative : alternatives) { + for (auto& term : alternative->m_terms) { + if (term.type == PatternTerm::Type::ParenthesesSubpattern && term.parentheses.disjunction && !term.parentheses.isCopy + && term.parentheses.disjunction->m_alternatives.size() > 1 && term.matchDirection() == Forward) + factorAlternatives(*term.parentheses.disjunction); + } + } } // A large top-level alternation costs one entry attempt per alternative at From 3557cf1403a4ccd3b3e9800f79c8834fcd378f4d Mon Sep 17 00:00:00 2001 From: Dylan Conway Date: Thu, 16 Jul 2026 04:03:41 -0700 Subject: [PATCH 05/13] Yarr: harden alternation factoring against specialized group shapes Fixes found in review of the factoring/folding passes: - Do not restructure a nested disjunction whose group was already committed to a specialized code shape by checkForTerminalParentheses. Factoring an isStringList group turned /^(?:xa|xb|y)/ into [x(?:a|b), y]; the JIT's string-list path treats a non-last alternative's final character as a full match, and the merged alternative's "final character" is only the shared prefix, so /^(?:GET|POST|PUT|SEND)/.test("Pzz") wrongly returned true. Terminal groups are skipped for the same reason. - Clear parentheses.isTerminal on every alternative absorbed into a synthesized group (prefix suffixes and folded body alternatives). A terminal parenthesis is only valid as the last term of a body alternative; once nested, TerminalEnd's empty-backtrack-state assertion no longer holds. The general once/greedy path is used. - Compute a synthesized group's capture span from the terms it contains rather than per-alternative bookkeeping: sorting reorders which alternative is first, and the parser leaves the last body alternative's m_lastSubpatternId unset, so the old span could exclude captures the group holds (or invert subpatternId/lastSubpatternId). - Leave small alternations unfactored (runs below the group threshold). The sequential JIT path there already has fused compares, the two-alternative SIMD scan, and frame-free inlinable code, which factoring would forfeit for no gain; measured /agggtaaa|acccttta/ and /on|off/ regressions are gone. --- Source/JavaScriptCore/yarr/YarrPattern.cpp | 99 +++++++++++++++++----- 1 file changed, 76 insertions(+), 23 deletions(-) diff --git a/Source/JavaScriptCore/yarr/YarrPattern.cpp b/Source/JavaScriptCore/yarr/YarrPattern.cpp index 99788dc4922e..cc1360590b48 100644 --- a/Source/JavaScriptCore/yarr/YarrPattern.cpp +++ b/Source/JavaScriptCore/yarr/YarrPattern.cpp @@ -2393,10 +2393,9 @@ class YarrPatternConstructor { { ASSERT(members.size() >= 2); - // Longest common prefix of literal terms across all members. Never - // consume a member's entire term list, so no member's suffix depends on - // an empty tail being ordered against a longer sibling's tail through the - // prefix (the empty suffix is handled explicitly below). + // Longest common prefix of literal terms across all members. A member + // that is a prefix of a longer sibling contributes an empty suffix + // alternative, which keeps its own position in the (stable) order. size_t prefixLength = 1; for (;; ++prefixLength) { const auto& first = members[0]->m_terms; @@ -2418,10 +2417,11 @@ class YarrPatternConstructor { merged->m_terms.append(members[0]->m_terms[i]); // The suffix disjunction holds each member's remaining terms, in the - // members' (stable, first-character-sorted) order. + // members' (stable, first-character-sorted) order. The group's capture + // span is computed from the terms (see accumulateCaptureRange). auto suffixDisjunction = makeUnique(); - unsigned firstSubpatternId = members[0]->m_firstSubpatternId; - unsigned lastSubpatternId = 0; + unsigned firstCaptureId = std::numeric_limits::max(); + unsigned lastCaptureId = 0; bool containsBOL = false; for (auto& member : members) { PatternAlternative* suffix = suffixDisjunction->addNewAlternative(member->m_firstSubpatternId); @@ -2430,7 +2430,8 @@ class YarrPatternConstructor { for (size_t i = prefixLength; i < member->m_terms.size(); ++i) suffix->m_terms.append(member->m_terms[i]); reparentNestedDisjunctions(*suffix); - lastSubpatternId = std::max(lastSubpatternId, member->m_lastSubpatternId); + clearTerminalMarks(*suffix); + accumulateCaptureRange(*member, firstCaptureId, lastCaptureId); containsBOL |= member->m_containsBOL; } suffixDisjunction->m_alternatives.last()->m_isLastAlternative = true; @@ -2438,10 +2439,11 @@ class YarrPatternConstructor { // Factor the suffixes themselves (a shared second character, and so on). factorAlternatives(*suffixDisjunction); - merged->m_lastSubpatternId = lastSubpatternId; + bool hasCaptures = firstCaptureId <= lastCaptureId; + merged->m_lastSubpatternId = hasCaptures ? lastCaptureId : 0; merged->m_containsBOL = containsBOL; - PatternTerm group(PatternTerm::Type::ParenthesesSubpattern, firstSubpatternId + 1, suffixDisjunction.get(), m_flags, /* capture */ false); - group.parentheses.lastSubpatternId = lastSubpatternId; + PatternTerm group(PatternTerm::Type::ParenthesesSubpattern, hasCaptures ? firstCaptureId : m_pattern.m_numSubpatterns + 1, suffixDisjunction.get(), m_flags, /* capture */ false); + group.parentheses.lastSubpatternId = hasCaptures ? lastCaptureId : 0; merged->m_terms.append(group); suffixDisjunction->m_parent = merged.get(); m_pattern.m_disjunctions.append(WTF::move(suffixDisjunction)); @@ -2458,6 +2460,39 @@ class YarrPatternConstructor { } } + // The [first, last] capture-subpattern ids actually contained in an + // alternative's terms (recursively). Parser bookkeeping on the alternative + // (m_firstSubpatternId / m_lastSubpatternId) is not reliable enough here: + // m_lastSubpatternId is only set once a following sibling is parsed, and + // sorting reorders which alternative comes first. + static void accumulateCaptureRange(const PatternAlternative& alternative, unsigned& first, unsigned& last) + { + for (auto& term : alternative.m_terms) { + if (term.type != PatternTerm::Type::ParenthesesSubpattern && term.type != PatternTerm::Type::ParentheticalAssertion) + continue; + if (term.capture()) { + first = std::min(first, term.parentheses.subpatternId); + last = std::max(last, term.parentheses.subpatternId); + } + if (auto* nested = term.parentheses.disjunction) { + for (auto& nestedAlternative : nested->m_alternatives) + accumulateCaptureRange(*nestedAlternative, first, last); + } + } + } + + // A "terminal" parenthesis is valid only as the last term of a body + // alternative (nothing after the body can force a backtrack into it). Once + // an alternative moves inside a nested group that guarantee is gone, so + // drop the marks (the general once/greedy path is used instead). + static void clearTerminalMarks(PatternAlternative& alternative) + { + for (auto& term : alternative.m_terms) { + if (term.type == PatternTerm::Type::ParenthesesSubpattern) + term.parentheses.isTerminal = false; + } + } + // Rewrite `disjunction`'s alternatives in place: sort each barrier-free run // by leading literal and merge shared prefixes into nested groups. void factorAlternatives(PatternDisjunction& disjunction) @@ -2485,8 +2520,13 @@ class YarrPatternConstructor { ++i; size_t runEnd = i; - if (runEnd - runStart == 1) { - result.append(WTF::move(alternatives[runStart])); + // Small alternations are left alone: the sequential JIT path (fused + // compares, the two-alternative SIMD scan, frame-free inlinable code) + // is already optimal there, and factoring would forfeit those. The + // rewrite pays for itself only on large alternations. + if (runEnd - runStart < Options::regExpAlternationGroupThreshold()) { + for (size_t j = runStart; j < runEnd; ++j) + result.append(WTF::move(alternatives[j])); continue; } @@ -2530,10 +2570,14 @@ class YarrPatternConstructor { // Factor inside pre-existing (source-written) groups too, so that // /\b(?:about|above|after)\b/ shares its "a" prefix like a top-level // alternation would. Groups synthesized by mergeSharedPrefix were - // already factored when built; re-running is a harmless no-op. + // already factored when built; re-running is a harmless no-op. Groups + // that checkForTerminalParentheses already committed to a specialized + // code shape (string lists, terminal parentheses) are left alone: that + // shape is fixed by their alternatives, which restructuring would break. for (auto& alternative : alternatives) { for (auto& term : alternative->m_terms) { if (term.type == PatternTerm::Type::ParenthesesSubpattern && term.parentheses.disjunction && !term.parentheses.isCopy + && !term.parentheses.isStringList && !term.parentheses.isTerminal && term.parentheses.disjunction->m_alternatives.size() > 1 && term.matchDirection() == Forward) factorAlternatives(*term.parentheses.disjunction); } @@ -2579,10 +2623,12 @@ class YarrPatternConstructor { } // The group's captures are exactly those of the alternatives it absorbs: - // subpatternId is the id a capture opened at the group start would take, - // lastSubpatternId the last id closed inside it (matching atomParenthesesEnd). - unsigned firstSubpatternId = alternatives[firstRepeated]->m_firstSubpatternId; - unsigned lastSubpatternId = 0; + // The group's capture span brackets exactly the captures its alternatives + // contain, computed from the terms themselves (sorting reorders which + // alternative is first, and the parser leaves the last body alternative's + // m_lastSubpatternId unset, so per-alternative bookkeeping is not reliable). + unsigned firstCaptureId = std::numeric_limits::max(); + unsigned lastCaptureId = 0; bool containsBOL = false; unsigned startsWithBOLCount = 0; @@ -2591,7 +2637,8 @@ class YarrPatternConstructor { PatternAlternative* alternative = alternatives[i].get(); alternative->m_parent = groupDisjunction.get(); alternative->m_isLastAlternative = false; - lastSubpatternId = std::max(lastSubpatternId, alternative->m_lastSubpatternId); + clearTerminalMarks(*alternative); + accumulateCaptureRange(*alternative, firstCaptureId, lastCaptureId); containsBOL |= alternative->m_containsBOL; if (alternative->m_startsWithBOL) ++startsWithBOLCount; @@ -2600,14 +2647,20 @@ class YarrPatternConstructor { groupDisjunction->m_alternatives.last()->m_isLastAlternative = true; alternatives.shrink(firstRepeated); - PatternAlternative* wrapped = body->addNewAlternative(firstSubpatternId); - wrapped->m_lastSubpatternId = lastSubpatternId; + // With no captures inside, subpatternId > lastSubpatternId makes + // containsAnyCaptures() false (the convention for capture-free groups). + bool hasCaptures = firstCaptureId <= lastCaptureId; + unsigned groupSubpatternId = hasCaptures ? firstCaptureId : m_pattern.m_numSubpatterns + 1; + unsigned groupLastSubpatternId = hasCaptures ? lastCaptureId : 0; + + PatternAlternative* wrapped = body->addNewAlternative(hasCaptures ? firstCaptureId - 1 : m_pattern.m_numSubpatterns); + wrapped->m_lastSubpatternId = groupLastSubpatternId; wrapped->m_containsBOL = containsBOL; wrapped->m_startsWithBOL = startsWithBOLCount == groupDisjunction->m_alternatives.size(); groupDisjunction->m_parent = wrapped; - PatternTerm group(PatternTerm::Type::ParenthesesSubpattern, firstSubpatternId + 1, groupDisjunction.get(), m_flags, /* capture */ false); - group.parentheses.lastSubpatternId = lastSubpatternId; + PatternTerm group(PatternTerm::Type::ParenthesesSubpattern, groupSubpatternId, groupDisjunction.get(), m_flags, /* capture */ false); + group.parentheses.lastSubpatternId = groupLastSubpatternId; wrapped->m_terms.append(group); m_pattern.m_disjunctions.append(WTF::move(groupDisjunction)); From 4261b012f51d88269c2ad1d210c92a67ea1a4e69 Mon Sep 17 00:00:00 2001 From: Dylan Conway Date: Thu, 16 Jul 2026 04:11:53 -0700 Subject: [PATCH 06/13] Yarr JIT: compare dispatched literal alternatives inline A dispatched alternative that is a fixed-length literal string (only fixed-count pattern characters) can never be backtracked into: it matches its exact characters or it does not. Emit such alternatives as inline compare blocks inside the chain instead of entering them through the frame-slot resume mechanism: claim the alternative's extra input, compare its characters at their frame offsets, and on mismatch release the claim and fall statically to the next chain element. On a full match the block joins the group's success path after the End op's own return-address store, having stored a return address pointing at an unwind thunk (release the claim, continue the chain) so that a later failure can still backtrack into the matched literal and try the following alternatives. This removes the per-alternative frame store, frame load, and indirect jump for literal alternatives; complex alternatives keep the existing resume mechanism at chain boundaries. --- Source/JavaScriptCore/yarr/YarrJIT.cpp | 93 ++++++++++++++++++++++++-- 1 file changed, 86 insertions(+), 7 deletions(-) diff --git a/Source/JavaScriptCore/yarr/YarrJIT.cpp b/Source/JavaScriptCore/yarr/YarrJIT.cpp index 7d388d8ced2a..ba5e9517039f 100644 --- a/Source/JavaScriptCore/yarr/YarrJIT.cpp +++ b/Source/JavaScriptCore/yarr/YarrJIT.cpp @@ -377,6 +377,9 @@ struct DispatchInfo { Chain wideChain; // characters >= 0x100 (16-bit strings only) unsigned frameLocation { 0 }; // the group's frame; chainResume lives here Checked firstCharacterOffset; // read offset of the group's first character + PatternDisjunction* disjunction { nullptr }; // the dispatched alternatives + Checked enclosingCheckedOffset; // checkedOffset at the group term + size_t endOpIndex { 0 }; // the group's NestedAlternativeEnd op (success join point) // Per alternative: the entry label (bound during generation) and jumps // from chain stubs awaiting that label. @@ -5917,6 +5920,8 @@ class YarrGenerator final : public YarrJITInfo { lastOp.m_alternative = nullptr; lastOp.m_nextOp = notFound; lastOp.m_checkedOffset = checkedOffset; + if (dispatch) + dispatch->endOpIndex = m_ops.size() - 1; size_t parenEnd = m_ops.size(); appendOp(parenthesesEndOpCode); @@ -6549,23 +6554,95 @@ class YarrGenerator final : public YarrJITInfo { info.pendingEntryJumps[alternativeIndex].clear(); } - // Emit one chain: for each member, store the address of the following stub - // as the chain-resume point, then jump into the member; the final stub is the - // group failure. Returns the label of the chain head. + // An alternative that is a fixed-length literal string (only fixed-count + // pattern characters) can never be backtracked into: it either matches its + // exact characters or not. Such alternatives are emitted as inline compare + // blocks inside the chain (no frame store/load, no indirect jump), with a + // static fall-through to the next chain element on mismatch. + bool isInlineLiteralAlternative(const PatternAlternative& alternative) + { + if (alternative.m_terms.isEmpty() || !alternative.m_hasFixedSize) + return false; + for (auto& term : alternative.m_terms) { + if (term.type != PatternTerm::Type::PatternCharacter) + return false; + if (term.quantityType != QuantifierType::FixedCount || !term.quantityMaxCount || term.quantityMinCount != term.quantityMaxCount) + return false; + if (term.matchDirection() != Forward) + return false; + } + return true; + } + + // Emit an inline literal alternative as part of a chain: claim its extra + // input, compare its characters, and on a full match join the group's + // success path; on mismatch release the claim and fall to the next chain + // element. The return-address slot points at an unwind thunk that releases + // the claim and continues the chain, so backtracking into this matched + // alternative from after the group tries the following alternatives. + void emitInlineLiteralAlternative(DispatchInfo& info, const PatternAlternative& alternative, MacroAssembler::JumpList& toNextElement) + { + const MacroAssembler::RegisterID character = m_regs.regT0; + unsigned extraClaim = alternative.m_minimumSize - info.disjunction->m_minimumSize; + Checked frameChecked = info.enclosingCheckedOffset + extraClaim; + + MacroAssembler::JumpList mismatch; + if (extraClaim) { + m_jit.add32(MacroAssembler::Imm32(extraClaim), m_regs.index); + mismatch.append(m_jit.branch32(MacroAssembler::Above, m_regs.index, m_regs.length)); + } + for (auto& term : alternative.m_terms) { + char32_t ch = term.patternCharacter; + for (unsigned repeat = 0; repeat < term.quantityMaxCount; ++repeat) { + Checked offset = frameChecked - term.inputPosition - repeat; + mismatch.append(jumpIfCharNotEquals(ch, offset, character, term.ignoreCase())); + } + } + + // Full match: record where to unwind to, then skip to the group's End. + MacroAssembler::DataLabelPtr returnAddress = storeToFrameWithPatch(info.frameLocation + BackTrackInfoParenthesesOnce::returnAddressIndex()); + m_ops[info.endOpIndex].m_jumps.append(m_jit.jump()); + + // Unwind thunk (reached only through the return-address slot when + // something after the group fails): release the claim and continue the + // chain by falling through to the next element. + MacroAssembler::Label unwind(m_jit); + m_backtrackingState.addReturnAddressRecord(returnAddress, unwind); + mismatch.link(&m_jit); + if (extraClaim) + m_jit.sub32(MacroAssembler::Imm32(extraClaim), m_regs.index); + toNextElement.append(m_jit.jump()); + } + + // Emit one chain: literal alternatives are compared inline; other + // alternatives are entered through a stub that stores the address of the + // following element as the chain-resume point before jumping in. The final + // element is the group failure. Returns the label of the chain head. MacroAssembler::Label emitDispatchChainStubs(DispatchInfo& info, const Vector& alternativeIndices) { JIT_COMMENT(m_jit, "dispatch chain (", alternativeIndices.size(), " alternatives)"); MacroAssembler::Label head = m_jit.label(); Vector pendingStores; + MacroAssembler::JumpList toNextElement; for (unsigned index : alternativeIndices) { - // Bind the previous stub's stored resume address to here. + // Bind the previous element's continuations to here. if (!pendingStores.isEmpty()) m_backtrackingState.addReturnAddressRecord(pendingStores.takeLast(), m_jit.label()); - pendingStores.append(storeToFrameWithPatch(info.frameLocation + BackTrackInfoParenthesesOnce::chainResumeIndex())); - emitJumpToDispatchEntry(info, index); + toNextElement.link(&m_jit); + toNextElement.clear(); + + const PatternAlternative& alternative = *info.disjunction->m_alternatives[index]; + if (isInlineLiteralAlternative(alternative)) + emitInlineLiteralAlternative(info, alternative, toNextElement); + else { + pendingStores.append(storeToFrameWithPatch(info.frameLocation + BackTrackInfoParenthesesOnce::chainResumeIndex())); + emitJumpToDispatchEntry(info, index); + } } // Chain exhausted: no more applicable alternatives at this position. - m_backtrackingState.addReturnAddressRecord(pendingStores.takeLast(), m_jit.label()); + if (!pendingStores.isEmpty()) + m_backtrackingState.addReturnAddressRecord(pendingStores.takeLast(), m_jit.label()); + toNextElement.link(&m_jit); info.groupFailJumps.append(m_jit.jump()); return head; } @@ -6728,6 +6805,8 @@ class YarrGenerator final : public YarrJITInfo { if (info->firstCharacterOffset.hasOverflowed()) return nullptr; info->frameLocation = term->frameLocation; + info->disjunction = disjunction; + info->enclosingCheckedOffset = checkedOffset; info->entries.resize(alternatives.size()); info->entryBound.fill(false, alternatives.size()); info->pendingEntryJumps.resize(alternatives.size()); From b9615a1760db0de233f69215b2dce307a4b9d0ea Mon Sep 17 00:00:00 2001 From: Dylan Conway Date: Thu, 16 Jul 2026 06:49:52 -0700 Subject: [PATCH 07/13] Yarr: fix review findings in the lookbehind/alternation series Correctness: - YarrPattern::atomParenthesesEnd bubbled m_startsWithBOL from a group whose alternatives all start with ^ onto the enclosing alternative even when the group was not the alternative's first term or was a negative assertion; and quantifyAtom kept the flag after a zero minimum count. optimizeBOL then marked the body once-through and the JIT tried only index 0. Enabling lookbehind in the JIT exposed this: /a(?m_jumps append under dispatch. - Correct the regExpAlternationGroupThreshold description (it gates prefix factoring at every level as well as the top-level fold). Adds the reviewer's reproducers to regexp-lookbehind-jit.js. --- JSTests/stress/regexp-lookbehind-jit.js | 23 ++++++++ Source/JavaScriptCore/runtime/OptionsList.h | 2 +- Source/JavaScriptCore/yarr/YarrJIT.cpp | 60 +++++++++++---------- Source/JavaScriptCore/yarr/YarrPattern.cpp | 19 +++++-- 4 files changed, 72 insertions(+), 32 deletions(-) diff --git a/JSTests/stress/regexp-lookbehind-jit.js b/JSTests/stress/regexp-lookbehind-jit.js index 011ea9253c44..cd475823dd0b 100644 --- a/JSTests/stress/regexp-lookbehind-jit.js +++ b/JSTests/stress/regexp-lookbehind-jit.js @@ -134,3 +134,26 @@ shouldBe(matchOf(/(?<=(a))bz|(?<=(a))b/, "ab"), [1, ["b", undefined, "a"]]); [true, false, false, true, true, false, true, true], ); } + +// Lookbehind patterns must not force BOL anchoring onto the enclosing +// alternative through a group that starts with ^ but is optional, quantified, +// negated, or not the alternative's first term (these previously matched only +// because lookbehind kept the whole pattern on the interpreter). +shouldBe(JSON.stringify(/a(? sets; // per alternative Vector chains; // distinct Latin-1 chains std::array chainForCharacter { }; // Latin-1 char -> chain index, or noChain Chain wideChain; // characters >= 0x100 (16-bit strings only) @@ -381,10 +380,8 @@ struct DispatchInfo { Checked enclosingCheckedOffset; // checkedOffset at the group term size_t endOpIndex { 0 }; // the group's NestedAlternativeEnd op (success join point) - // Per alternative: the entry label (bound during generation) and jumps - // from chain stubs awaiting that label. - Vector entries; - Vector entryBound; + // Per alternative: jumps from chain stubs awaiting that alternative's + // entry point (always a forward reference; see emitJumpToDispatchEntry). Vector pendingEntryJumps; // Failures that mean "no alternative can match here" (empty chains, chain @@ -394,6 +391,8 @@ struct DispatchInfo { static constexpr unsigned noChain = std::numeric_limits::max(); }; +WTF_MAKE_TZONE_ALLOCATED_IMPL(DispatchInfo); + struct MaskedAlternativeInfo { private: WTF_MAKE_TZONE_ALLOCATED(MaskedAlternativeInfo); @@ -5210,14 +5209,12 @@ class YarrGenerator final : public YarrJITInfo { loadFromFrameAndJump(op.m_dispatch->frameLocation + BackTrackInfoParenthesesOnce::chainResumeIndex()); } if (isBegin) { - // Group-level failures gathered by the dispatch, plus any jumps - // planted on the End (none are expected under dispatch, kept - // for parity with the sequential path). + // Group-level failures gathered by the dispatch (empty routes, + // exhausted chains) resume at the Begin, where the index is at + // the group frame. (Under dispatch nothing plants jumps on the + // End: Next backtracks jump through chainResume, and dispatch + // requires a FixedCount group, so no zero-length-match check.) m_backtrackingState.append(op.m_dispatch->groupFailJumps); - YarrOp* endOp = &m_ops[op.m_nextOp]; - while (endOp->m_nextOp != notFound) - endOp = &m_ops[endOp->m_nextOp]; - m_backtrackingState.append(endOp->m_jumps); } // For non-simple alternatives, link the alternative's 'return // address' so we backtrack back out into it correctly. @@ -6538,18 +6535,16 @@ class YarrGenerator final : public YarrJITInfo { // later alternatives are bound as they are generated; stubs that target them // are patched then via DispatchInfo::pendingEntryJumps. + // Chain stubs are all emitted at the Begin op, before any alternative's + // entry point exists, so every stub target is a forward reference: collected + // per alternative here and linked when the alternative's entry is generated. void emitJumpToDispatchEntry(DispatchInfo& info, unsigned alternativeIndex) { - if (info.entryBound[alternativeIndex]) - m_jit.jump(info.entries[alternativeIndex]); - else - info.pendingEntryJumps[alternativeIndex].append(m_jit.jump()); + info.pendingEntryJumps[alternativeIndex].append(m_jit.jump()); } void bindDispatchEntry(DispatchInfo& info, unsigned alternativeIndex) { - info.entries[alternativeIndex] = m_jit.label(); - info.entryBound[alternativeIndex] = true; info.pendingEntryJumps[alternativeIndex].link(&m_jit); info.pendingEntryJumps[alternativeIndex].clear(); } @@ -6559,10 +6554,17 @@ class YarrGenerator final : public YarrJITInfo { // exact characters or not. Such alternatives are emitted as inline compare // blocks inside the chain (no frame store/load, no indirect jump), with a // static fall-through to the next chain element on mismatch. + // Inlining unrolls one compare per character, so it is only for genuinely + // short literals; a counted literal like a{200000000} keeps the entered + // path, whose fixed-count loop runs at match time instead of being unrolled. + static constexpr unsigned maxInlineLiteralLength = 32; + bool isInlineLiteralAlternative(const PatternAlternative& alternative) { if (alternative.m_terms.isEmpty() || !alternative.m_hasFixedSize) return false; + if (alternative.m_minimumSize > maxInlineLiteralLength) + return false; for (auto& term : alternative.m_terms) { if (term.type != PatternTerm::Type::PatternCharacter) return false; @@ -6744,12 +6746,13 @@ class YarrGenerator final : public YarrJITInfo { return nullptr; auto info = makeUniqueRef(); + Vector sets; // per alternative; only needed to build the routing unsigned anyCount = 0; for (auto& alternative : alternatives) { FirstCharacterSet set = computeFirstCharacterSet(*alternative); if (set.any) ++anyCount; - info->sets.append(WTF::move(set)); + sets.append(WTF::move(set)); } // Dispatch is pointless when most alternatives can start with anything. if (anyCount * 4 > alternatives.size()) @@ -6761,7 +6764,7 @@ class YarrGenerator final : public YarrJITInfo { for (unsigned ch = 0; ch < 256; ++ch) { Vector members; for (unsigned i = 0; i < alternatives.size(); ++i) { - if (info->sets[i].matches(ch)) + if (sets[i].matches(ch)) members.append(i); } if (members.isEmpty()) { @@ -6791,7 +6794,7 @@ class YarrGenerator final : public YarrJITInfo { return nullptr; if (m_charSize == CharSize::Char16) { for (unsigned i = 0; i < alternatives.size(); ++i) { - if (info->sets[i].matchesWide()) + if (sets[i].matchesWide()) info->wideChain.alternativeIndices.append(i); } } @@ -6807,8 +6810,6 @@ class YarrGenerator final : public YarrJITInfo { info->frameLocation = term->frameLocation; info->disjunction = disjunction; info->enclosingCheckedOffset = checkedOffset; - info->entries.resize(alternatives.size()); - info->entryBound.fill(false, alternatives.size()); info->pendingEntryJumps.resize(alternatives.size()); DispatchInfo* result = info.ptr(); @@ -7924,11 +7925,14 @@ class YarrGenerator final : public YarrJITInfo { #endif // Lookbehind bodies are JIT-compiled via mirroring (see - // mirrorDisjunctionForLookbehind), except when the pattern also decodes - // surrogate pairs: reading backward across a UTF-16 pair (trail first, - // then lead) is not implemented in the backward JIT yet. Unsupported body - // content is detected during op-compilation and falls back the same way. - if (m_pattern.m_containsLookbehinds && m_decodeSurrogatePairs) { + // mirrorDisjunctionForLookbehind), except for unicode patterns: reading + // backward across a UTF-16 surrogate pair (trail first, then lead) is + // not implemented in the backward JIT yet. The decision is made per + // pattern (eitherUnicode), not per subject encoding (m_decodeSurrogatePairs), + // so the 8-bit and 16-bit specializations of one RegExp always run the + // same engine. Unsupported body content is detected during op-compilation + // and falls back the same way. + if (m_pattern.m_containsLookbehinds && m_pattern.eitherUnicode()) { codeBlock.setFallBackWithFailureReason(JITFailureReason::Lookbehind); return; } diff --git a/Source/JavaScriptCore/yarr/YarrPattern.cpp b/Source/JavaScriptCore/yarr/YarrPattern.cpp index cc1360590b48..4389a2b6a956 100644 --- a/Source/JavaScriptCore/yarr/YarrPattern.cpp +++ b/Source/JavaScriptCore/yarr/YarrPattern.cpp @@ -1572,8 +1572,12 @@ class YarrPatternConstructor { if (numBOLAnchoredAlts) { m_alternative->m_containsBOL = true; - // If all the alternatives in parens start with BOL, then so does this one - if (numBOLAnchoredAlts == numParenAlternatives) + // If all the alternatives in parens start with BOL, then this alternative + // starts with BOL too -- but only when the group is its first term (mirroring + // assertionBOL) and is a group whose match anchors the position, i.e. not a + // negative assertion (which anchors nothing on success). A later quantifier + // with a zero minimum removes the anchoring again; see quantifyAtom. + if (numBOLAnchoredAlts == numParenAlternatives && m_alternative->m_terms.size() == 1 && !lastTerm.invert()) m_alternative->m_startsWithBOL = true; } @@ -1765,6 +1769,12 @@ class YarrPatternConstructor { ASSERT(term.type > PatternTerm::Type::AssertionWordBoundary); ASSERT(term.quantityMinCount == 1 && term.quantityMaxCount == 1 && term.quantityType == QuantifierType::FixedCount); + // A term with a zero minimum count may not match at all, so an alternative + // whose BOL anchoring came from this (first) term no longer starts with BOL: + // e.g. /(^)?a/ or /(?:^b)?a/ can match away from a line start. + if (!min && m_alternative->m_terms.size() == 1 && m_alternative->m_startsWithBOL) + m_alternative->m_startsWithBOL = false; + if (term.type == PatternTerm::Type::ParentheticalAssertion) { // If an assertion is quantified with a minimum count of zero, it can simply be removed. // This arises from the RepeatMatcher behaviour in the spec. Matching an assertion never @@ -2610,7 +2620,10 @@ class YarrPatternConstructor { while (firstRepeated < alternatives.size() && alternatives[firstRepeated]->onceThrough()) ++firstRepeated; size_t repeatedCount = alternatives.size() - firstRepeated; - if (repeatedCount < Options::regExpAlternationGroupThreshold()) + // Folding one (or no) alternative into a group is meaningless whatever + // the configured threshold is (e.g. regExpAlternationGroupThreshold=0 + // with an all-once-through body). + if (repeatedCount < 2 || repeatedCount < Options::regExpAlternationGroupThreshold()) return; // A DotStarEnclosure records match bounds through the enclosing body From 1e1228712871f5e66bfc05cb466a12fb32a0b105 Mon Sep 17 00:00:00 2001 From: Dylan Conway Date: Thu, 16 Jul 2026 11:04:39 -0700 Subject: [PATCH 08/13] Yarr: keep BOL-group lookbehind patterns on the interpreter instead of narrowing BOL bubbling Revert the m_startsWithBOL edits from the previous commit and gate instead. An alternative's m_startsWithBOL is not just an anchoring hint: optimizeBOL uses it to exclude the alternative from the loop copy, and copyTerm's filterStartsWithBOL silently drops BOL-anchored sub-alternatives when copying. That dropping is only semantics-preserving because the eager bubbling in atomParenthesesEnd guarantees such alternatives never reach the loop copy. Narrowing the bubbling (as the previous commit did) let /.(^)X/ reach the loop copy, where the (^) group was deleted and the pattern over-matched ("aX" against /.(^)X/) -- worse than the bug it was addressing. So restore trunk's bubbling exactly. The lookbehind JIT still must not newly route these patterns into the JIT's stricter once-through handling (a pre-existing JIT-only defect the interpreter tolerates). Record when a group's alternatives all start with BOL (m_containsBOLGroupBubble) and keep lookbehind patterns with such a bubble on the interpreter, i.e. their exact previous behaviour. Bare ^ alternatives (as in the isbot pattern) do not set the flag, so the target case still JITs. The (?:^)?a family without lookbehind is a pre-existing upstream JIT defect (interpreter and V8 agree); the stress test now pins its per-tier output rather than deleting the cases. --- JSTests/stress/regexp-lookbehind-jit.js | 26 +++++++++++++--------- Source/JavaScriptCore/yarr/YarrJIT.cpp | 21 ++++++++++------- Source/JavaScriptCore/yarr/YarrPattern.cpp | 21 ++++++++--------- Source/JavaScriptCore/yarr/YarrPattern.h | 2 ++ 4 files changed, 40 insertions(+), 30 deletions(-) diff --git a/JSTests/stress/regexp-lookbehind-jit.js b/JSTests/stress/regexp-lookbehind-jit.js index cd475823dd0b..36ca74059659 100644 --- a/JSTests/stress/regexp-lookbehind-jit.js +++ b/JSTests/stress/regexp-lookbehind-jit.js @@ -135,10 +135,9 @@ shouldBe(matchOf(/(?<=(a))bz|(?<=(a))b/, "ab"), [1, ["b", undefined, "a"]]); ); } -// Lookbehind patterns must not force BOL anchoring onto the enclosing -// alternative through a group that starts with ^ but is optional, quantified, -// negated, or not the alternative's first term (these previously matched only -// because lookbehind kept the whole pattern on the interpreter). +// Lookbehind patterns whose alternatives contain a group that is BOL-anchored +// (an optional/quantified/negated group starting with ^) keep the results the +// interpreter gives: the JIT must not (mis)handle their once-through split. shouldBe(JSON.stringify(/a(?m_containsBOL = true; - // If all the alternatives in parens start with BOL, then this alternative - // starts with BOL too -- but only when the group is its first term (mirroring - // assertionBOL) and is a group whose match anchors the position, i.e. not a - // negative assertion (which anchors nothing on success). A later quantifier - // with a zero minimum removes the anchoring again; see quantifyAtom. - if (numBOLAnchoredAlts == numParenAlternatives && m_alternative->m_terms.size() == 1 && !lastTerm.invert()) + // If all the alternatives in parens start with BOL, then so does this one + if (numBOLAnchoredAlts == numParenAlternatives) { m_alternative->m_startsWithBOL = true; + // A whole group being BOL-anchored feeds optimizeBOL's once-through / + // loop-copy split, which the JIT and interpreter treat differently for + // some shapes (e.g. an optional (^)? group); noted so a JIT compile can + // keep such patterns on the interpreter (see YarrJIT::compile). + m_pattern.m_containsBOLGroupBubble = true; + } } lastTerm.parentheses.lastSubpatternId = m_pattern.m_numSubpatterns; @@ -1769,12 +1771,6 @@ class YarrPatternConstructor { ASSERT(term.type > PatternTerm::Type::AssertionWordBoundary); ASSERT(term.quantityMinCount == 1 && term.quantityMaxCount == 1 && term.quantityType == QuantifierType::FixedCount); - // A term with a zero minimum count may not match at all, so an alternative - // whose BOL anchoring came from this (first) term no longer starts with BOL: - // e.g. /(^)?a/ or /(?:^b)?a/ can match away from a line start. - if (!min && m_alternative->m_terms.size() == 1 && m_alternative->m_startsWithBOL) - m_alternative->m_startsWithBOL = false; - if (term.type == PatternTerm::Type::ParentheticalAssertion) { // If an assertion is quantified with a minimum count of zero, it can simply be removed. // This arises from the RepeatMatcher behaviour in the spec. Matching an assertion never @@ -3295,6 +3291,7 @@ ErrorCode YarrPattern::compile(StringView patternString) YarrPattern::YarrPattern(StringView pattern, OptionSet flags, ErrorCode& error, ExecutionMode executionMode) : m_containsBackreferences(false) , m_containsBOL(false) + , m_containsBOLGroupBubble(false) , m_containsLookbehinds(false) , m_containsUnsignedLengthPattern(false) , m_containsModifiers(false) diff --git a/Source/JavaScriptCore/yarr/YarrPattern.h b/Source/JavaScriptCore/yarr/YarrPattern.h index 21eec64779cb..ce3bc9561d8f 100644 --- a/Source/JavaScriptCore/yarr/YarrPattern.h +++ b/Source/JavaScriptCore/yarr/YarrPattern.h @@ -605,6 +605,7 @@ struct YarrPattern { m_containsBackreferences = false; m_containsBOL = false; + m_containsBOLGroupBubble = false; m_containsLookbehinds = false; m_containsUnsignedLengthPattern = false; m_hasCopiedParenSubexpressions = false; @@ -776,6 +777,7 @@ struct YarrPattern { bool m_containsBackreferences : 1; bool m_containsBOL : 1; + bool m_containsBOLGroupBubble : 1; // a group's alternatives all start with BOL (optimizeBOL bubbling) bool m_containsLookbehinds : 1; bool m_containsUnsignedLengthPattern : 1; bool m_containsModifiers : 1; From 5cd184beebfa29fb010554c03feaf2f98271fef6 Mon Sep 17 00:00:00 2001 From: Dylan Conway Date: Thu, 16 Jul 2026 19:48:17 -0700 Subject: [PATCH 09/13] Yarr: JIT-compile unicode lookbehinds through the mirrored-body machinery Lookbehind bodies are compiled as term-reversed mirrored copies through the forward generator with Backward ops, so unicode (surrogate-pair) lookbehinds no longer fall back to the interpreter. This adds leftward decoding (readUnicodeCharBackward, matching the interpreter's tryReadBackward), backward astral literals and character classes for every quantifier form, backreferences and quantified/nested groups inside mirrored bodies, and nested forward assertions renumbered from their mirrored parent position. The lookbehind compile gate and its options are removed. Correctness work required to make that sound: - General real-position tests for ^ and \b (index == checked - pos) replace the "not at position 0" shortcuts, which are wrong for an anchor whose alternative is numbered from a nonzero origin; ^ inside a lookbehind never marks startsWithBOL, and quantifying a BOL group {0} withdraws the anchor before the term is elided. - A nested forward assertion's body is deep-copied for the mirror so renumbering never mutates the pattern the interpreter fallback reuses. - The NonGreedy paren "try one more iteration" guard is direction-aware. - The dotAll any-character class rejects a dangling surrogate in unicode compiles like every other class path. - The first-non-BMP-character register is written only by the match-start read (ambient-off, enabled while emitting that one term), so loop peeks and backtrack re-reads can't leave a stale astral flag that over-advances the next start; the whole optimization is additionally disabled for patterns that can match zero-width, since a zero-width alternative can legitimately succeed at a mid-surrogate-pair position that the skip drops. - Quantified splits (X{min,max} -> mandatory copy + optional {0,max-min} copy sharing X's capture ids): the optional copy's abandoned iteration clears the shared capture, and control re-emerges through the mandatory group's End, so each paren End emitter re-establishes a copy owner's entry-side capture endpoint from that iteration's saved start index (parentheses.hasCopySibling). Backward parens now take this same path. - A leftward pair-lead borrow undoes its failed borrow before joining the failure jumps so a negative frontier is never left live for a sibling op. --- JSTests/stress/regexp-lookbehind-jit.js | 40 +- Source/JavaScriptCore/runtime/OptionsList.h | 4 - Source/JavaScriptCore/yarr/Yarr.h | 2 +- Source/JavaScriptCore/yarr/YarrJIT.cpp | 946 +++++++++++++++----- Source/JavaScriptCore/yarr/YarrJIT.h | 1 - Source/JavaScriptCore/yarr/YarrPattern.cpp | 186 +++- Source/JavaScriptCore/yarr/YarrPattern.h | 7 +- 7 files changed, 908 insertions(+), 278 deletions(-) diff --git a/JSTests/stress/regexp-lookbehind-jit.js b/JSTests/stress/regexp-lookbehind-jit.js index 36ca74059659..76beede8fbf7 100644 --- a/JSTests/stress/regexp-lookbehind-jit.js +++ b/JSTests/stress/regexp-lookbehind-jit.js @@ -101,9 +101,8 @@ shouldBe(matchOf(/(?<=^x)y/, "xy"), [1, ["y"]]); shouldBe(matchOf(/(?<=^x)y/, "axy"), null); shouldBe(matchOf(/(?<=^)a/m, "\na"), [1, ["a"]]); -// Bodies the JIT does not compile fall back to the interpreter and must still -// be correct: backreference, quantified group, nested lookahead, and unicode -// surrogate-pair bodies. +// Backreference, quantified group, nested lookahead and unicode surrogate-pair +// bodies (all compiled natively by the JIT). shouldBe(matchOf(/(?<=(a)\1)b/, "aab"), [2, ["b", "a"]]); shouldBe(matchOf(/(?<=(?:ab)+)c/, "ababc"), [4, ["c"]]); shouldBe(matchOf(/(?<=(?:ab)+)c/, "xc"), null); @@ -145,19 +144,28 @@ shouldBe(JSON.stringify(/(?^)/.exec("qx"), null); +shouldBe(/c(?:^(?:x))/.exec("acx"), null); +shouldBe(JSON.stringify(/(\w(^)\s|$)/i.exec("a x")), '["","",null]'); // Genuine anchoring is unchanged. shouldBe(/^a/.exec("ba"), null); shouldBe(/(?:^a)/.exec("ba"), null); diff --git a/Source/JavaScriptCore/runtime/OptionsList.h b/Source/JavaScriptCore/runtime/OptionsList.h index 8ccf1f4e20d9..dba9d6e9411b 100644 --- a/Source/JavaScriptCore/runtime/OptionsList.h +++ b/Source/JavaScriptCore/runtime/OptionsList.h @@ -87,10 +87,6 @@ bool hasCapacityToUseLargeGigacage(); v(Bool, useBaselineJIT, true, Normal, "allows the baseline JIT to be used if true"_s) \ v(Bool, useDFGJIT, is64Bit(), Normal, "allows the DFG JIT to be used if true"_s) \ v(Bool, useRegExpJIT, jitEnabledByDefault() && is64Bit(), Normal, "allows the RegExp JIT to be used if true"_s) \ - v(Bool, useRegExpAlternationFactoring, true, Normal, "factors common prefixes out of an alternation's literal-leading alternatives and folds large top-level alternations into a group"_s) \ - v(Unsigned, regExpAlternationGroupThreshold, 8, Normal, "minimum number of alternatives in a run before shared prefixes are factored out (at any nesting level) and before repeated top-level alternatives are folded into a group"_s) \ - v(Bool, useRegExpAlternationDispatch, true, Normal, "reads a group's first character once and jumps to the chain of alternatives that can start with it"_s) \ - v(Unsigned, regExpAlternationDispatchThreshold, 4, Normal, "minimum number of alternatives in a group before its alternatives are dispatched on the first character"_s) \ v(Bool, useDOMJIT, is64Bit(), Normal, "allows the DOMJIT to be used if true"_s) \ \ v(Bool, reportMustSucceedExecutableAllocations, false, Normal, nullptr) \ diff --git a/Source/JavaScriptCore/yarr/Yarr.h b/Source/JavaScriptCore/yarr/Yarr.h index e68b380e5941..e85d38a8050e 100644 --- a/Source/JavaScriptCore/yarr/Yarr.h +++ b/Source/JavaScriptCore/yarr/Yarr.h @@ -35,7 +35,7 @@ namespace JSC { namespace Yarr { #define YarrStackSpaceForBackTrackInfoPatternCharacter 2 // Only for !fixed quantifiers. #define YarrStackSpaceForBackTrackInfoCharacterClass 2 // Greedy/NonGreedy, or FixedCount with unicode/unicodeSets flag. -#define YarrStackSpaceForBackTrackInfoBackReference 3 +#define YarrStackSpaceForBackTrackInfoBackReference 4 #define YarrStackSpaceForBackTrackInfoAlternative 1 // One per alternative. #define YarrStackSpaceForBackTrackInfoParentheticalAssertion 1 #define YarrStackSpaceForBackTrackInfoParenthesesOnce 3 diff --git a/Source/JavaScriptCore/yarr/YarrJIT.cpp b/Source/JavaScriptCore/yarr/YarrJIT.cpp index 230163fe3823..970162bdc783 100644 --- a/Source/JavaScriptCore/yarr/YarrJIT.cpp +++ b/Source/JavaScriptCore/yarr/YarrJIT.cpp @@ -1303,13 +1303,19 @@ class YarrGenerator final : public YarrJITInfo { matchCharacterClassOnlyOneRange(character, scratch, failures, characterClassToProcess->m_ranges8.size() ? characterClassToProcess->m_ranges8 : characterClassToProcess->m_ranges32); } else { MacroAssembler::JumpList matchDest; - // If we are matching the "any character" builtin class for non-unicode patterns, - // we only need to read the character and don't need to match as it will always succeed. + // If we are matching the "any character" builtin class we only need to read the + // character and don't need to match, as it will always succeed -- except that in a + // unicode pattern a dangling surrogate reads as errorCodePoint, which is not a + // character and must fail (the inverted-class path above rejects it likewise). if (!characterClassToProcess->m_anyCharacter) { matchCharacterClass(character, scratch, MatchTargets(matchDest, failures, MatchTargets::PreferredTarget::MatchSuccessFallThrough), characterClassToProcess); if (!matchDest.empty()) failures.append(m_jit.jump()); } +#if ENABLE(YARR_JIT_UNICODE_EXPRESSIONS) + else if (m_decodeSurrogatePairs) + failures.append(m_jit.branch32(MacroAssembler::Equal, character, MacroAssembler::TrustedImm32(errorCodePoint))); +#endif matchDest.link(&m_jit); } }; @@ -1355,13 +1361,33 @@ class YarrGenerator final : public YarrJITInfo { // Jumps if input not available; will have (incorrectly) incremented already! MacroAssembler::Jump jumpIfNoAvailableInput(unsigned countToCheck = 0) { - if (m_direction == Backward && countToCheck) - return m_jit.branchSub32(MacroAssembler::Signed, MacroAssembler::TrustedImm32(countToCheck), m_regs.index); + if (m_direction == Backward) { + // Backward: the frontier is exhausted once index would go below zero. + // With a count, claim it and branch on the borrow; with none, just test + // the current frontier's sign (the leftward mirror of index > length). + if (countToCheck) + return m_jit.branchSub32(MacroAssembler::Signed, MacroAssembler::TrustedImm32(countToCheck), m_regs.index); + return m_jit.branch32(MacroAssembler::LessThan, m_regs.index, MacroAssembler::TrustedImm32(0)); + } if (countToCheck) m_jit.add32(MacroAssembler::Imm32(countToCheck), m_regs.index); return m_jit.branch32(MacroAssembler::Above, m_regs.index, m_regs.length); } + // Extend the leftward claim by one unit (a decoded pair's lead). On the + // failure path the borrowed unit is restored before joining `failures`, so + // a negative frontier is never left live where another term (e.g. a sibling + // op's non-greedy re-entry) could address input through it before this term's + // own backtrack restores its beginIndex. + void claimBackwardPairLead(MacroAssembler::JumpList& failures) + { + ASSERT(m_direction == Backward); + MacroAssembler::Jump claimed = m_jit.branchSub32(MacroAssembler::PositiveOrZero, MacroAssembler::TrustedImm32(1), m_regs.index); + m_jit.add32(MacroAssembler::TrustedImm32(1), m_regs.index); // undo the failed borrow + failures.append(m_jit.jump()); + claimed.link(&m_jit); + } + MacroAssembler::Jump jumpIfAvailableInput(unsigned countToCheck) { if (m_direction == Backward) @@ -1377,6 +1403,43 @@ class YarrGenerator final : public YarrJITInfo { return m_jit.branch32(MacroAssembler::Above, additionalAmount, m_regs.length); } + // Backward (lookbehind) form: a dynamic span of `amount` units must lie + // wholly left of the cursor. Computes the span's left edge (index - amount) + // into `edge` and jumps if it would be negative; the caller then walks the + // span rightward from `edge` and finally moves the frontier down to it. + MacroAssembler::Jump checkNotEnoughInputBackward(MacroAssembler::RegisterID amount, MacroAssembler::RegisterID edge) + { + ASSERT(m_direction == Backward); + m_jit.move(m_regs.index, edge); + m_jit.sub32(amount, edge); + return m_jit.branch32(MacroAssembler::LessThan, edge, MacroAssembler::TrustedImm32(0)); + } + + // A backreference's span of `size` units: forward, it lies right of the + // frontier (availability check only; the walk advances the frontier through + // it); in a mirrored frame it lies left of the frontier, so the frontier is + // first lowered to the span's left edge -- recorded in the frame so the walk, + // which raises index back up as it compares, can be undone by + // emitBackReferenceSpanFrontier -- and the walk proceeds rightward from there. + void emitBackReferenceSpanClaim(MacroAssembler::JumpList& notAvailable, MacroAssembler::RegisterID size, MacroAssembler::RegisterID edge, unsigned frameLocation) + { + if (m_direction != Backward) { + notAvailable.append(checkNotEnoughInput(size)); + return; + } + notAvailable.append(checkNotEnoughInputBackward(size, edge)); + storeToFrame(edge, frameLocation + BackTrackInfoBackReference::backwardSpanEdgeIndex()); + m_jit.move(edge, m_regs.index); + } + + // After a mirrored backreference walk, the span has been consumed leftward: + // the frontier rests at the span's left edge, not where the walk left it. + void emitBackReferenceSpanFrontier(unsigned frameLocation) + { + if (m_direction == Backward) + loadFromFrame(frameLocation + BackTrackInfoBackReference::backwardSpanEdgeIndex(), m_regs.index); + } + MacroAssembler::Jump checkInput() { if (m_direction == Backward) @@ -1485,6 +1548,51 @@ class YarrGenerator final : public YarrJITInfo { tryReadUnicodeCharImpl(*m_vm, m_jit, resultReg); } + // Backward unicode read: decode the code point whose LAST code unit lives at + // `address` (the mirror of tryReadUnicodeChar, which decodes from a first + // unit). Matches the bytecode interpreter's tryReadBackward exactly: + // - load the unit at `address`; + // - if it is a trail surrogate and the unit before it (within the string) + // is a lead surrogate, the result is the supplementary code point; + // - otherwise the result is the unit itself. A lone lead or lone trail is + // returned as-is, never as an error (deliberately unlike the forward + // reader), because a backward match position can legitimately sit inside + // a surrogate pair the pattern splits. + // Callers recover the consumed width from the result: >= 0x10000 means a + // decoded pair (2 units), anything lower is a single unit -- exactly as the + // forward paths branch on `character < 0x10000`. `temp2` is a scratch. + void readUnicodeCharBackward(MacroAssembler::BaseIndex address, MacroAssembler::RegisterID resultReg, MacroAssembler::RegisterID temp2) + { + ASSERT(m_charSize == CharSize::Char16); + ASSERT(resultReg != temp2); + const MacroAssembler::RegisterID unitAddress = m_regs.regUnicodeInputAndTrail; + const MacroAssembler::RegisterID temp = m_regs.unicodeAndSubpatternIdTemp; + + m_jit.getEffectiveAddress(address, unitAddress); + m_jit.load16(MacroAssembler::Address(unitAddress), resultReg); + + MacroAssembler::JumpList notAPair; + // resultReg is a trail surrogate iff (resultReg & 0xfc00) == 0xdc00. + m_jit.move(resultReg, temp); + m_jit.and32(MacroAssembler::TrustedImm32(0xfc00), temp); + notAPair.append(m_jit.branch32(MacroAssembler::NotEqual, temp, MacroAssembler::TrustedImm32(0xdc00))); + // The unit before must exist (unitAddress > input) ... + notAPair.append(m_jit.branchPtr(MacroAssembler::BelowOrEqual, unitAddress, m_regs.input)); + // ... and be a lead surrogate: (unit & 0xfc00) == 0xd800. + m_jit.load16(MacroAssembler::Address(unitAddress, -static_cast(sizeof(char16_t))), temp2); + m_jit.move(temp2, temp); + m_jit.and32(MacroAssembler::TrustedImm32(0xfc00), temp); + notAPair.append(m_jit.branch32(MacroAssembler::NotEqual, temp, MacroAssembler::TrustedImm32(0xd800))); + + // A proper pair: U16_GET_SUPPLEMENTARY(lead, trail) + // = (lead << 10) + trail - U16_SURROGATE_OFFSET + m_jit.lshift32(MacroAssembler::TrustedImm32(10), temp2); + m_jit.add32(temp2, resultReg); + m_jit.sub32(MacroAssembler::TrustedImm32(U16_SURROGATE_OFFSET), resultReg); + + notAPair.link(&m_jit); + } + // Specialized read+match for character classes whose codepoints all share a single UTF-16 // lead surrogate. Loads the (lead, trail) pair as a single 32-bit value, branches to // |failures| if the lead doesn't match |sharedLead|, otherwise leaves the trail surrogate @@ -1625,10 +1733,47 @@ class YarrGenerator final : public YarrJITInfo { m_jit.load16Unaligned(address, resultReg); } + // Read the code point relative to the term's direction. A forward term is + // addressed by its first unit (decode rightward, tryReadUnicodeChar); a + // backward (lookbehind) term is addressed by its last unit (decode leftward, + // the interpreter's tryReadBackward). Reads that are inherently forward -- a + // capture's own units, the Boyer-Moore scans -- keep using readCharacter(). + void readCharacterForDirection(Checked negativeCharacterOffset, MacroAssembler::RegisterID resultReg) + { +#if ENABLE(YARR_JIT_UNICODE_EXPRESSIONS) + if (m_direction == Backward && m_decodeSurrogatePairs) { + readUnicodeCharBackward(negativeOffsetIndexedAddress(negativeCharacterOffset, resultReg), resultReg, m_regs.regT2); + m_usesT2 = true; + return; + } +#endif + readCharacter(negativeCharacterOffset, resultReg, m_regs.index); + } + + // Load the single code unit at the offset, with no surrogate-pair decoding. + // Used where a term compares exact UTF-16 units (a fixed astral literal is + // matched as its two known units), so pair decoding would be wrong. + void readCodeUnit(Checked negativeCharacterOffset, MacroAssembler::RegisterID resultReg) + { + readCodeUnit(negativeCharacterOffset, resultReg, m_regs.index); + } + + void readCodeUnit(Checked negativeCharacterOffset, MacroAssembler::RegisterID resultReg, MacroAssembler::RegisterID indexReg) + { + MacroAssembler::BaseIndex address = negativeOffsetIndexedAddress(negativeCharacterOffset, resultReg, indexReg); + if (m_charSize == CharSize::Char8) + m_jit.load8(address, resultReg); + else + m_jit.load16Unaligned(address, resultReg); + } + template MacroAssembler::Jump jumpIfCharCond(char32_t ch, Checked negativeCharacterOffset, MacroAssembler::RegisterID character, bool ignoreCase) { - readCharacter(negativeCharacterOffset, character); + // A term compares the code point that begins (forward) or ends (backward) + // at its position; the direction picks the decode. Astral literals never + // reach here in a backward frame (they compare units directly). + readCharacterForDirection(negativeCharacterOffset, character); // For case-insesitive compares, non-ascii characters that have different // upper & lower case representations are converted to a character class. @@ -2037,6 +2182,14 @@ class YarrGenerator final : public YarrJITInfo { // multiple are fused to match as a group. bool m_isDeadCode { false }; + // True for the one term whose first read is guaranteed to be at the match + // start: the alternative's first term, provided no earlier term can move + // the frontier at runtime. Only such a read may establish + // firstCharacterAdditionalReadSize ("the start character is a non-BMP + // pair"); a term after a zero-or-more (min-0) sibling reads past the + // start whenever that sibling consumed input. + bool m_readsStartCharacter { false }; + // Currently used in the case of some of the more complex management of // 'm_checkedOffset', to cache the offset used in this alternative, to avoid // recalculating it. @@ -2254,26 +2407,28 @@ class YarrGenerator final : public YarrJITInfo { return; } + // The anchor's real string position is index - (checkedOffset - position), so + // it is the start of input iff index == checkedOffset - position. A top-level + // ^ at position 0 gives the classic index == checkedOffset; the general form + // also covers a ^ in the body of an assertion nested inside a lookbehind, whose + // numbering continues from a nonzero origin (and makes a ^ that can never be + // at position 0 -- e.g. /a^b/ -- simply never satisfied, as before). + MacroAssembler::Imm32 startFrontier(static_cast((op.m_checkedOffset - term->inputPosition).value())); + if (term->multiline()) { const MacroAssembler::RegisterID character = m_regs.regT0; const MacroAssembler::RegisterID scratch = m_regs.regT1; MacroAssembler::JumpList matchDest; - if (!term->inputPosition) - matchDest.append(m_jit.branch32(MacroAssembler::Equal, m_regs.index, MacroAssembler::Imm32(op.m_checkedOffset))); + matchDest.append(m_jit.branch32(MacroAssembler::Equal, m_regs.index, startFrontier)); readCharacter(op.m_checkedOffset - term->inputPosition + 1, character); matchCharacterClass(character, scratch, matchDest, m_pattern.newlineCharacterClass()); op.m_jumps.append(m_jit.jump()); matchDest.link(&m_jit); - } else { - // Erk, really should poison out these alternatives early. :-/ - if (term->inputPosition) - op.m_jumps.append(m_jit.jump()); - else - op.m_jumps.append(m_jit.branch32(MacroAssembler::NotEqual, m_regs.index, MacroAssembler::Imm32(op.m_checkedOffset))); - } + } else + op.m_jumps.append(m_jit.branch32(MacroAssembler::NotEqual, m_regs.index, startFrontier)); } // BOL inside a lookbehind body (mirrored frame). The anchor's real position @@ -2296,7 +2451,9 @@ class YarrGenerator final : public YarrJITInfo { if (atFrameEdge) matchDest.append(m_jit.branchTest32(MacroAssembler::Zero, m_regs.index)); - readCharacter(op.m_checkedOffset - term->inputPosition, character); + // The character just left of the anchor ends at the anchor: a leftward + // decode in a unicode pattern. + readCharacterForDirection(op.m_checkedOffset - term->inputPosition, character); matchCharacterClass(character, scratch, matchDest, m_pattern.newlineCharacterClass()); op.m_jumps.append(m_jit.jump()); @@ -2400,7 +2557,9 @@ class YarrGenerator final : public YarrJITInfo { // Mirrored frame: the boundary's real position is index + (checked - // pos). It is the real end of input only at the frame's left edge // (pos == 0) with index + checked == length; the following real - // character is at frame offset (checked - pos + 1). + // character is at frame offset (checked - pos + 1). The character AFTER + // a boundary starts at it, so it is a forward decode from that unit -- + // readCharacter, not the direction-relative reader. if (!term->inputPosition) { m_jit.add32(MacroAssembler::Imm32(op.m_checkedOffset), m_regs.index, scratch); nextIsNotWordChar.append(m_jit.branch32(MacroAssembler::Equal, scratch, m_regs.length)); @@ -2432,26 +2591,31 @@ class YarrGenerator final : public YarrJITInfo { const MacroAssembler::RegisterID scratch = m_regs.regT1; #if ENABLE(YARR_JIT_UNICODE_EXPRESSIONS) && ENABLE(YARR_JIT_UNICODE_CAN_INCREMENT_INDEX_FOR_NON_BMP) - // Prevent word boundary assertion reads from corrupting firstCharacterAdditionalReadSize. - SetForScope useOptimizationScope(m_useFirstNonBMPCharacterOptimization, false); #endif MacroAssembler::Jump atBegin; MacroAssembler::JumpList matchDest; - // Forward: the boundary is at the real start of input only if it is at - // the frame start (pos == 0) with index == checked; the preceding real - // character is at frame offset (checked - pos + 1). Backward (mirrored): - // the real start test is at the frame's right edge (pos == checked) with - // index == 0, and the preceding real character is at (checked - pos). - bool atStartEdge = m_direction == Backward ? term->inputPosition == op.m_checkedOffset : !term->inputPosition; + // Forward: the boundary's real position is index - (checked - pos), so it + // is the real start of input iff index == checked - pos (the classic + // index == checked when the ^-like term is at pos 0; the general form also + // covers a \b in the body of an assertion nested inside a lookbehind, whose + // numbering continues from a nonzero origin). The preceding real character + // is at frame offset (checked - pos + 1). Backward (mirrored): the real + // start test is at the frame's right edge (pos == checked) with index == 0, + // and the preceding real character is at (checked - pos). + bool atStartEdge = m_direction == Backward ? term->inputPosition == op.m_checkedOffset : true; if (atStartEdge) { if (m_direction == Backward) atBegin = m_jit.branchTest32(MacroAssembler::Zero, m_regs.index); else - atBegin = m_jit.branch32(MacroAssembler::Equal, m_regs.index, MacroAssembler::Imm32(op.m_checkedOffset)); + atBegin = m_jit.branch32(MacroAssembler::Equal, m_regs.index, MacroAssembler::Imm32(static_cast((op.m_checkedOffset - term->inputPosition).value()))); } + // The character BEFORE the boundary ends at the unit just left of it. In + // a mirrored frame that unit is at frame offset (checked - pos), and in a + // unicode pattern it is a leftward decode (an astral pair ending at the + // boundary is classified as its code point). Forward: (checked - pos + 1). if (m_direction == Backward) - readCharacter(op.m_checkedOffset - term->inputPosition, character); + readCharacterForDirection(op.m_checkedOffset - term->inputPosition, character); else readCharacter(op.m_checkedOffset - term->inputPosition + 1, character); @@ -2510,24 +2674,35 @@ class YarrGenerator final : public YarrJITInfo { unsigned duplicateNamedGroupId = m_pattern.hasDuplicateNamedCaptureGroups() ? m_pattern.m_duplicateNamedGroupForSubpatternId[subpatternId] : 0; #if ENABLE(YARR_JIT_UNICODE_EXPRESSIONS) && ENABLE(YARR_JIT_UNICODE_CAN_INCREMENT_INDEX_FOR_NON_BMP) - SetForScope useOptimizationScope(m_useFirstNonBMPCharacterOptimization, false); #endif MacroAssembler::Label loop(&m_jit); + // The referenced capture is an ordinary substring: its units are read + // forward from patternIndex, and the input span is compared ascending in + // lockstep. Both operand offsets are expressed in the frame's addressing: + // forward reads input[reg - k], a mirrored (lookbehind) frame reads + // input[reg + k - 1] -- so the capture unit itself is k = 1 there, and + // the input span (walked upward from its left edge, which the caller put + // in index) begins one unit past the term's own offset. + unsigned captureUnitOffset = m_direction == Backward ? 1 : 0; + Checked inputSpanOffset = op.m_checkedOffset - term->inputPosition; + if (m_direction == Backward) + inputSpanOffset += 1; + #if ENABLE(YARR_JIT_BACKREFERENCES_FOR_16BIT_EXPRS) if (!m_decodeSurrogatePairs) - readCharacter(0, patternCharacter, patternIndex); + readCharacter(captureUnitOffset, patternCharacter, patternIndex); else { // For reading Unicode characters, use the standard resultReg so we can call the standard tryReadUnicodeChar() // helper instead of emitting an inlined version. - readCharacter(0, character, patternIndex); + readCharacter(captureUnitOffset, character, patternIndex); m_jit.move(character, patternCharacter); } #else - readCharacter(0, patternCharacter, patternIndex); + readCharacter(captureUnitOffset, patternCharacter, patternIndex); #endif - readCharacter(op.m_checkedOffset - term->inputPosition, character); + readCharacter(inputSpanOffset, character); if (!term->ignoreCase()) { // When !m_decodeSurrogatePairs, readCharacter() emits load8/load16 @@ -2688,9 +2863,10 @@ class YarrGenerator final : public YarrJITInfo { // PatternTemp should contain pattern end index at this point. Compute pattern size. m_jit.sub32(patternIndex, patternTemp); - op.m_jumps.append(checkNotEnoughInput(patternTemp)); + emitBackReferenceSpanClaim(op.m_jumps, patternTemp, characterOrTemp, parenthesesFrameLocation); matchBackreference(opIndex, op.m_jumps, characterOrTemp, patternIndex, patternTemp, subpatternIdReg == m_regs.unicodeAndSubpatternIdTemp ? subpatternIdReg : InvalidGPRReg); + emitBackReferenceSpanFrontier(parenthesesFrameLocation); if (term->quantityMaxCount != 1) { loadFromFrame(parenthesesFrameLocation + BackTrackInfoBackReference::matchAmountIndex(), characterOrTemp); @@ -2721,9 +2897,10 @@ class YarrGenerator final : public YarrJITInfo { m_jit.sub32(patternIndex, patternTemp); storeToFrame(patternTemp, parenthesesFrameLocation + BackTrackInfoBackReference::backReferenceSizeIndex()); - matches.append(checkNotEnoughInput(patternTemp)); + emitBackReferenceSpanClaim(matches, patternTemp, characterOrTemp, parenthesesFrameLocation); matchBackreference(opIndex, incompleteMatches, characterOrTemp, patternIndex, patternTemp, subpatternIdReg == m_regs.unicodeAndSubpatternIdTemp ? subpatternIdReg : InvalidGPRReg); + emitBackReferenceSpanFrontier(parenthesesFrameLocation); loadFromFrame(parenthesesFrameLocation + BackTrackInfoBackReference::matchAmountIndex(), characterOrTemp); m_jit.add32(MacroAssembler::TrustedImm32(1), characterOrTemp); @@ -2791,9 +2968,10 @@ class YarrGenerator final : public YarrJITInfo { // when checkNotEnoughInput bails out early. storeToFrame(m_regs.index, parenthesesFrameLocation + BackTrackInfoBackReference::beginIndex()); m_jit.sub32(patternIndex, patternTemp); - matches.append(checkNotEnoughInput(patternTemp)); + emitBackReferenceSpanClaim(matches, patternTemp, characterOrTemp, parenthesesFrameLocation); matchBackreference(opIndex, incompleteMatches, characterOrTemp, patternIndex, patternTemp, subpatternIdReg == m_regs.unicodeAndSubpatternIdTemp ? subpatternIdReg : InvalidGPRReg); + emitBackReferenceSpanFrontier(parenthesesFrameLocation); matches.append(m_jit.jump()); @@ -2829,7 +3007,12 @@ class YarrGenerator final : public YarrJITInfo { failures.append(m_jit.branchTest32(MacroAssembler::Zero, matchAmount)); loadFromFrame(parenthesesFrameLocation + BackTrackInfoBackReference::backReferenceSizeIndex(), matchSize); - m_jit.sub32(matchSize, m_regs.index); + // Give one span back: retreat the forward frontier, or raise the + // leftward-consuming frontier of a mirrored frame, by the span size. + if (m_direction == Backward) + m_jit.add32(matchSize, m_regs.index); + else + m_jit.sub32(matchSize, m_regs.index); m_jit.sub32(MacroAssembler::TrustedImm32(1), matchAmount); storeToFrame(matchAmount, parenthesesFrameLocation + BackTrackInfoBackReference::matchAmountIndex()); @@ -2914,6 +3097,22 @@ class YarrGenerator final : public YarrJITInfo { auto offset = op.m_checkedOffset - op.m_term->inputPosition; + if (m_direction == Backward && m_decodeSurrogatePairs && !U_IS_BMP(firstChar)) { + // Backward astral literal: the term is a fixed two-unit code point. + // Its higher frame offset addresses the later (trail) unit and the + // next-lower offset the lead unit, so comparing the two units at + // their fixed positions equals comparing the code point (a lead+trail + // pair encodes exactly one code point). No decode, no dynamic width. + // Such a literal is canonically unique or ASCII by construction + // (atomPatternCharacter), so no case folding applies. + ASSERT(!op.m_term->ignoreCase() || isCanonicallyUnique(firstChar, m_canonicalMode)); + readCodeUnit(offset, character); + defaultMatchTargets.appendFailed(m_jit.branch32(MacroAssembler::NotEqual, character, MacroAssembler::Imm32(U16_TRAIL(firstChar)))); + readCodeUnit(offset - 1, character); + defaultMatchTargets.appendFailed(m_jit.branch32(MacroAssembler::NotEqual, character, MacroAssembler::Imm32(U16_LEAD(firstChar)))); + return; + } + if (!matchTargets.hasSucceedTarget() || m_ops[opIndex + 1].m_op == YarrOpCode::Term) defaultMatchTargets.appendFailed(jumpIfCharNotEquals(charToMatch, offset, character, op.m_term->ignoreCase())); else @@ -3288,25 +3487,43 @@ class YarrGenerator final : public YarrJITInfo { // in a backward (mirrored) frame where index is the left frontier. emitFramePosition(scaledMaxCount, countRegister); - MacroAssembler::Label loop(&m_jit); - readCharacter(op.m_checkedOffset - term->inputPosition - scaledMaxCount, character, countRegister); + Checked baseOffset = op.m_checkedOffset - term->inputPosition - scaledMaxCount; + // For case-insesitive compares, non-ascii characters that have different // upper & lower case representations are converted to a character class. ASSERT(!term->ignoreCase() || isASCIIAlpha(ch) || isCanonicallyUnique(ch, m_canonicalMode)); - if (term->ignoreCase() && isASCIIAlpha(ch)) { - m_jit.or32(MacroAssembler::TrustedImm32(0x20), character); - ch |= 0x20; - } - op.m_jumps.append(m_jit.branch32(MacroAssembler::NotEqual, character, MacroAssembler::Imm32(ch))); - if (m_direction == Backward) + MacroAssembler::Label loop(&m_jit); + if (m_direction == Backward && m_decodeSurrogatePairs && !U_IS_BMP(ch)) { + // Backward run of a fixed astral literal: each occurrence is a known + // (lead, trail) unit pair. At loop entry countRegister addresses the last + // occurrence's trail unit; compare it, step the count register down one + // unit to the lead, compare that, and step down once more. The lead is + // reached by moving the index register rather than the (unsigned) frame + // offset, which is 0 at the lowest occurrence. Unit-exact, no decoding. + readCodeUnit(baseOffset, character, countRegister); + op.m_jumps.append(m_jit.branch32(MacroAssembler::NotEqual, character, MacroAssembler::Imm32(U16_TRAIL(ch)))); m_jit.sub32(MacroAssembler::TrustedImm32(1), countRegister); + readCodeUnit(baseOffset, character, countRegister); + op.m_jumps.append(m_jit.branch32(MacroAssembler::NotEqual, character, MacroAssembler::Imm32(U16_LEAD(ch)))); + m_jit.sub32(MacroAssembler::TrustedImm32(1), countRegister); + } else { + readCharacter(baseOffset, character, countRegister); + if (term->ignoreCase() && isASCIIAlpha(ch)) { + m_jit.or32(MacroAssembler::TrustedImm32(0x20), character); + ch |= 0x20; + } + + op.m_jumps.append(m_jit.branch32(MacroAssembler::NotEqual, character, MacroAssembler::Imm32(ch))); + if (m_direction == Backward) + m_jit.sub32(MacroAssembler::TrustedImm32(1), countRegister); #if ENABLE(YARR_JIT_UNICODE_EXPRESSIONS) - else if (m_decodeSurrogatePairs && !U_IS_BMP(ch)) - m_jit.add32(MacroAssembler::TrustedImm32(2), countRegister); + else if (m_decodeSurrogatePairs && !U_IS_BMP(ch)) + m_jit.add32(MacroAssembler::TrustedImm32(2), countRegister); #endif - else - m_jit.add32(MacroAssembler::TrustedImm32(1), countRegister); + else + m_jit.add32(MacroAssembler::TrustedImm32(1), countRegister); + } m_jit.branch32(MacroAssembler::NotEqual, countRegister, m_regs.index).linkTo(loop, &m_jit); } void backtrackPatternCharacterFixed(size_t opIndex) @@ -3330,23 +3547,44 @@ class YarrGenerator final : public YarrJITInfo { MacroAssembler::JumpList failures; MacroAssembler::Label loop(&m_jit); failures.append(atEndOfInput()); - failures.append(jumpIfCharNotEquals(ch, op.m_checkedOffset - term->inputPosition, character, term->ignoreCase())); - - // Consume the character: advance the frontier forward, or move the - // leftward cursor down in a backward (mirrored) frame. - if (m_direction == Backward) - m_jit.sub32(MacroAssembler::TrustedImm32(1), m_regs.index); - else + if (m_direction == Backward && m_decodeSurrogatePairs && !U_IS_BMP(ch)) { + // Backward astral literal: the frontier-adjacent unit must be the + // trail; take it, then require and compare the lead below it, taking + // that too. Unit-exact (a lead+trail pair is one code point). If the + // lead is absent or wrong the trail unit is given back before failing. + Checked offset = op.m_checkedOffset - term->inputPosition; + readCodeUnit(offset, character); + failures.append(m_jit.branch32(MacroAssembler::NotEqual, character, MacroAssembler::Imm32(U16_TRAIL(ch)))); + m_jit.sub32(MacroAssembler::TrustedImm32(1), m_regs.index); // the trail unit + MacroAssembler::JumpList giveBackTrail; + giveBackTrail.append(m_jit.branchTest32(MacroAssembler::Zero, m_regs.index)); // no lead below + readCodeUnit(offset, character); // now addresses the unit below the trail + giveBackTrail.append(m_jit.branch32(MacroAssembler::NotEqual, character, MacroAssembler::Imm32(U16_LEAD(ch)))); + m_jit.sub32(MacroAssembler::TrustedImm32(1), m_regs.index); // the lead unit + MacroAssembler::Jump matched = m_jit.jump(); + giveBackTrail.link(&m_jit); m_jit.add32(MacroAssembler::TrustedImm32(1), m_regs.index); -#if ENABLE(YARR_JIT_UNICODE_EXPRESSIONS) - if (m_decodeSurrogatePairs && !U_IS_BMP(ch)) { - MacroAssembler::Jump surrogatePairOk = notAtEndOfInput(); - m_jit.sub32(MacroAssembler::TrustedImm32(1), m_regs.index); failures.append(m_jit.jump()); - surrogatePairOk.link(&m_jit); - m_jit.add32(MacroAssembler::TrustedImm32(1), m_regs.index); - } + matched.link(&m_jit); + } else { + failures.append(jumpIfCharNotEquals(ch, op.m_checkedOffset - term->inputPosition, character, term->ignoreCase())); + + // Consume the character: advance the frontier forward, or move the + // leftward cursor down in a backward (mirrored) frame. + if (m_direction == Backward) + m_jit.sub32(MacroAssembler::TrustedImm32(1), m_regs.index); + else + m_jit.add32(MacroAssembler::TrustedImm32(1), m_regs.index); +#if ENABLE(YARR_JIT_UNICODE_EXPRESSIONS) + if (m_decodeSurrogatePairs && !U_IS_BMP(ch)) { + MacroAssembler::Jump surrogatePairOk = notAtEndOfInput(); + m_jit.sub32(MacroAssembler::TrustedImm32(1), m_regs.index); + failures.append(m_jit.jump()); + surrogatePairOk.link(&m_jit); + m_jit.add32(MacroAssembler::TrustedImm32(1), m_regs.index); + } #endif + } m_jit.add32(MacroAssembler::TrustedImm32(1), countRegister); if (term->quantityMaxCount == quantifyInfinite) @@ -3384,13 +3622,13 @@ class YarrGenerator final : public YarrJITInfo { m_backtrackingState.append(m_jit.branchTest32(MacroAssembler::Zero, countRegister)); m_jit.sub32(MacroAssembler::TrustedImm32(1), countRegister); // Give one character back: retreat the forward frontier, or move the - // leftward cursor back up in a backward (mirrored) frame. + // leftward cursor back up in a backward (mirrored) frame. The character + // is a single unit unless it is an astral literal in a unicode pattern. + unsigned characterWidth = (m_decodeSurrogatePairs && !U_IS_BMP(term->patternCharacter)) ? 2 : 1; if (m_direction == Backward) - m_jit.add32(MacroAssembler::TrustedImm32(1), m_regs.index); - else if (!m_decodeSurrogatePairs || U_IS_BMP(term->patternCharacter)) - m_jit.sub32(MacroAssembler::TrustedImm32(1), m_regs.index); + m_jit.add32(MacroAssembler::TrustedImm32(characterWidth), m_regs.index); else - m_jit.sub32(MacroAssembler::TrustedImm32(2), m_regs.index); + m_jit.sub32(MacroAssembler::TrustedImm32(characterWidth), m_regs.index); m_jit.jump(op.m_reentry); } @@ -3424,21 +3662,40 @@ class YarrGenerator final : public YarrJITInfo { nonGreedyFailures.append(atEndOfInput()); if (term->quantityMaxCount != quantifyInfinite) nonGreedyFailures.append(m_jit.branch32(MacroAssembler::Equal, countRegister, MacroAssembler::Imm32(term->quantityMaxCount))); - nonGreedyFailures.append(jumpIfCharNotEquals(ch, op.m_checkedOffset - term->inputPosition, character, term->ignoreCase())); - - if (m_direction == Backward) - m_jit.sub32(MacroAssembler::TrustedImm32(1), m_regs.index); - else + if (m_direction == Backward && m_decodeSurrogatePairs && !U_IS_BMP(ch)) { + // Backward astral literal: trail at the frontier, then the lead below + // it (see generatePatternCharacterGreedy); a partial take is undone. + Checked offset = op.m_checkedOffset - term->inputPosition; + readCodeUnit(offset, character); + nonGreedyFailures.append(m_jit.branch32(MacroAssembler::NotEqual, character, MacroAssembler::Imm32(U16_TRAIL(ch)))); + m_jit.sub32(MacroAssembler::TrustedImm32(1), m_regs.index); // the trail unit + MacroAssembler::JumpList giveBackTrail; + giveBackTrail.append(m_jit.branchTest32(MacroAssembler::Zero, m_regs.index)); // no lead below + readCodeUnit(offset, character); + giveBackTrail.append(m_jit.branch32(MacroAssembler::NotEqual, character, MacroAssembler::Imm32(U16_LEAD(ch)))); + m_jit.sub32(MacroAssembler::TrustedImm32(1), m_regs.index); // the lead unit + MacroAssembler::Jump matched = m_jit.jump(); + giveBackTrail.link(&m_jit); m_jit.add32(MacroAssembler::TrustedImm32(1), m_regs.index); -#if ENABLE(YARR_JIT_UNICODE_EXPRESSIONS) - if (m_decodeSurrogatePairs && !U_IS_BMP(ch)) { - MacroAssembler::Jump surrogatePairOk = notAtEndOfInput(); - m_jit.sub32(MacroAssembler::TrustedImm32(1), m_regs.index); nonGreedyFailures.append(m_jit.jump()); - surrogatePairOk.link(&m_jit); - m_jit.add32(MacroAssembler::TrustedImm32(1), m_regs.index); - } + matched.link(&m_jit); + } else { + nonGreedyFailures.append(jumpIfCharNotEquals(ch, op.m_checkedOffset - term->inputPosition, character, term->ignoreCase())); + + if (m_direction == Backward) + m_jit.sub32(MacroAssembler::TrustedImm32(1), m_regs.index); + else + m_jit.add32(MacroAssembler::TrustedImm32(1), m_regs.index); +#if ENABLE(YARR_JIT_UNICODE_EXPRESSIONS) + if (m_decodeSurrogatePairs && !U_IS_BMP(ch)) { + MacroAssembler::Jump surrogatePairOk = notAtEndOfInput(); + m_jit.sub32(MacroAssembler::TrustedImm32(1), m_regs.index); + nonGreedyFailures.append(m_jit.jump()); + surrogatePairOk.link(&m_jit); + m_jit.add32(MacroAssembler::TrustedImm32(1), m_regs.index); + } #endif + } m_jit.add32(MacroAssembler::TrustedImm32(1), countRegister); m_jit.jump(op.m_reentry); @@ -3469,7 +3726,10 @@ class YarrGenerator final : public YarrJITInfo { } #if ENABLE(YARR_JIT_UNICODE_EXPRESSIONS) - if (m_decodeSurrogatePairs && !term->invert()) { + // The shared-lead fast path loads the (lead, trail) pair forward from the + // term's first unit; a backward frame addresses the term by its last unit + // and uses the general path below. + if (m_decodeSurrogatePairs && !term->invert() && m_direction == Forward) { if (auto sharedLead = term->characterClass->hasSharedLeadSurrogate()) { CharacterClass trailsOnlyClass; buildTrailsOnlyClass(*term->characterClass, trailsOnlyClass); @@ -3479,6 +3739,27 @@ class YarrGenerator final : public YarrJITInfo { } #endif + if (m_direction == Backward && m_decodeSurrogatePairs) { + // Backward + unicode: read the code point that ends at the term's + // position (its last unit is at frame offset checkedOffset - position). + // The class match runs first and leaves the frontier alone, preserving + // `character` for the width test just as the forward path does; + // beginIndex (stored above) restores the frontier on backtrack. + readUnicodeCharBackward(negativeOffsetIndexedAddress(op.m_checkedOffset - term->inputPosition, character), character, scratch); + matchCharacterClassTermInner(term, op.m_jumps, character, scratch); + // A static-width class was laid out at its true width (2 units when its + // members are all astral) and its claim already covers the whole pair. + // Only a variable-width class was budgeted a single unit, so only it + // grows the leftward claim by one when the code point was a pair + // (failing when the frontier is exhausted). + if (!term->isFixedWidthCharacterClass()) { + MacroAssembler::Jump isBMPChar = m_jit.branch32(MacroAssembler::LessThan, character, MacroAssembler::TrustedImm32(0x10000)); + claimBackwardPairLead(op.m_jumps); // claim the pair's lead unit + isBMPChar.link(&m_jit); + } + return; + } + readCharacter(op.m_checkedOffset - term->inputPosition, character); matchCharacterClassTermInner(term, op.m_jumps, character, scratch); @@ -3537,7 +3818,9 @@ class YarrGenerator final : public YarrJITInfo { scaledMaxCount *= 2; nonBMPOnly = true; - if (auto sharedLead = term->characterClass->hasSharedLeadSurrogate()) { + // The shared-lead pair fast path reads (lead, trail) forward from the + // count position; a backward frame uses the general loop below. + if (auto sharedLead = m_direction == Forward ? term->characterClass->hasSharedLeadSurrogate() : std::nullopt) { // Replace tryReadUnicodeCharImpl + full-codepoint class match // with a single 32-bit pair load + lead check + trail-only class match. ASSERT(term->isFixedWidthCharacterClass()); @@ -3558,8 +3841,42 @@ class YarrGenerator final : public YarrJITInfo { // See generatePatternCharacterFixed for the forward/backward count register walk. emitFramePosition(scaledMaxCount, countRegister); + Checked baseOffset = op.m_checkedOffset - term->inputPosition - scaledMaxCount; + MacroAssembler::Label loop(&m_jit); - readCharacter(op.m_checkedOffset - term->inputPosition - scaledMaxCount, character, countRegister); + if (m_direction == Backward && m_decodeSurrogatePairs) { + // Backward + unicode. At loop entry countRegister addresses the last + // unit of the next-lower occurrence; decode the code point ending + // there. Width is recovered from the code point itself (>= 0x10000 is + // a pair), after the class match, which preserves `character`. + readUnicodeCharBackward(negativeOffsetIndexedAddress(baseOffset, character, countRegister), character, scratch); + matchCharacterClassTermInner(term, op.m_jumps, character, scratch); + if (nonBMPOnly) { + // Static width 2: budgeted at two units per match. + m_jit.sub32(MacroAssembler::TrustedImm32(2), countRegister); + } else if (term->isFixedWidthCharacterClass()) { + // Static width 1 (all-BMP class). The read still decodes so that a + // unit trailing a lead is presented as its astral code point, exactly + // as the interpreter's tryReadBackward presents it; such a code point + // is never in this class, so no width consequence arises. + m_jit.sub32(MacroAssembler::TrustedImm32(1), countRegister); + } else { + // Variable width: a pair consumed one more unit than budgeted, so + // grow the leftward claim by one (fail if exhausted) and step over + // both units. beginIndex, stored above, restores the frontier when + // this term backtracks. + m_jit.sub32(MacroAssembler::TrustedImm32(1), countRegister); + MacroAssembler::Jump isBMPChar = m_jit.branch32(MacroAssembler::LessThan, character, MacroAssembler::TrustedImm32(0x10000)); + claimBackwardPairLead(op.m_jumps); // claim the pair's lead unit + m_jit.sub32(MacroAssembler::TrustedImm32(1), countRegister); // extra unit of the pair + isBMPChar.link(&m_jit); + } + m_jit.branch32(MacroAssembler::NotEqual, countRegister, m_regs.index).linkTo(loop, &m_jit); + done.link(&m_jit); + return; + } + + readCharacter(baseOffset, character, countRegister); MacroAssembler::Label nonBMPLoop(&m_jit); @@ -3587,7 +3904,7 @@ class YarrGenerator final : public YarrJITInfo { #if ENABLE(YARR_JIT_UNICODE_EXPRESSIONS) if (nonBMPOnly) { done.append(m_jit.branch32(MacroAssembler::Equal, countRegister, m_regs.index)); - tryReadNonBMPUnicodeChar(op.m_checkedOffset - term->inputPosition - scaledMaxCount, character, countRegister); + tryReadNonBMPUnicodeChar(baseOffset, character, countRegister); m_jit.jump().linkTo(nonBMPLoop, &m_jit); } else #endif @@ -3629,7 +3946,8 @@ class YarrGenerator final : public YarrJITInfo { m_jit.move(MacroAssembler::TrustedImm32(0), countRegister); #if ENABLE(YARR_JIT_UNICODE_EXPRESSIONS) - if (m_decodeSurrogatePairs && !term->invert()) { + // Forward-only: the shared-lead fast path reads the pair from its first unit. + if (m_decodeSurrogatePairs && !term->invert() && m_direction == Forward) { if (auto sharedLead = term->characterClass->hasSharedLeadSurrogate()) { ASSERT(term->isFixedWidthCharacterClass()); CharacterClass trailsOnlyClass; @@ -3665,18 +3983,38 @@ class YarrGenerator final : public YarrJITInfo { MacroAssembler::Label loop(&m_jit); failures.append(atEndOfInput()); - readCharacter(op.m_checkedOffset - term->inputPosition, character); + if (m_direction == Backward && m_decodeSurrogatePairs) { + // Backward + unicode: decode the code point ending just left of the + // frontier (frame offset 0), class-match it (frontier untouched, so a + // mismatch simply ends the greedy run), then consume it by its width, + // read off the code point exactly as the forward path does. A pair's + // lead is the unit below its trail: once the trail unit is taken and the + // frontier is 0, there is no lead to take, so that path rejoins through + // failuresDecrementIndex, which gives back the trail unit. + readUnicodeCharBackward(negativeOffsetIndexedAddress(op.m_checkedOffset - term->inputPosition, character), character, scratch); + matchCharacterClassTermInner(term, failures, character, scratch); + m_jit.sub32(MacroAssembler::TrustedImm32(1), m_regs.index); // the code point's last unit + MacroAssembler::Jump isBMPChar = m_jit.branch32(MacroAssembler::LessThan, character, MacroAssembler::TrustedImm32(0x10000)); + // The pair's lead is one further unit; mirroring the forward path, test + // for it BEFORE taking it, so a failure here has consumed exactly one unit + // (the trail) -- which failuresDecrementIndex gives back. + failuresDecrementIndex.append(atEndOfInput()); + m_jit.sub32(MacroAssembler::TrustedImm32(1), m_regs.index); // the pair's lead unit + isBMPChar.link(&m_jit); + } else { + readCharacter(op.m_checkedOffset - term->inputPosition, character); - matchCharacterClassTermInner(term, failures, character, scratch); + matchCharacterClassTermInner(term, failures, character, scratch); - if (m_direction == Backward) - m_jit.sub32(MacroAssembler::TrustedImm32(1), m_regs.index); + if (m_direction == Backward) + m_jit.sub32(MacroAssembler::TrustedImm32(1), m_regs.index); #if ENABLE(YARR_JIT_UNICODE_EXPRESSIONS) - else if (m_decodeSurrogatePairs) - advanceIndexAfterCharacterClassTermMatch(term, failuresDecrementIndex, character); + else if (m_decodeSurrogatePairs) + advanceIndexAfterCharacterClassTermMatch(term, failuresDecrementIndex, character); #endif - else - m_jit.add32(MacroAssembler::TrustedImm32(1), m_regs.index); + else + m_jit.add32(MacroAssembler::TrustedImm32(1), m_regs.index); + } m_jit.add32(MacroAssembler::TrustedImm32(1), countRegister); if (term->quantityMaxCount == quantifyInfinite) @@ -3691,7 +4029,12 @@ class YarrGenerator final : public YarrJITInfo { if (!failuresDecrementIndex.empty()) { failuresDecrementIndex.link(&m_jit); - m_jit.sub32(MacroAssembler::TrustedImm32(1), m_regs.index); + // Give back the unit that was already consumed before the failure: the + // frontier moved up in a forward frame, down in a backward frame. + if (m_direction == Backward) + m_jit.add32(MacroAssembler::TrustedImm32(1), m_regs.index); + else + m_jit.sub32(MacroAssembler::TrustedImm32(1), m_regs.index); } failures.link(&m_jit); @@ -3725,14 +4068,20 @@ class YarrGenerator final : public YarrJITInfo { m_jit.sub32(MacroAssembler::TrustedImm32(1), countRegister); storeToFrame(countRegister, term->frameLocation + BackTrackInfoCharacterClass::matchAmountIndex()); - if (m_direction == Backward) + if (m_direction == Backward && !m_decodeSurrogatePairs) m_jit.add32(MacroAssembler::TrustedImm32(1), m_regs.index); - else if (!m_decodeSurrogatePairs) + else if (m_direction == Backward && term->isFixedWidthCharacterClass()) + m_jit.add32(MacroAssembler::TrustedImm32(term->characterClass->hasNonBMPCharacters() ? 2 : 1), m_regs.index); + else if (m_direction != Backward && !m_decodeSurrogatePairs) m_jit.sub32(MacroAssembler::TrustedImm32(1), m_regs.index); - else if (term->isFixedWidthCharacterClass()) + else if (m_direction != Backward && term->isFixedWidthCharacterClass()) m_jit.sub32(MacroAssembler::TrustedImm32(term->characterClass->hasNonBMPCharacters() ? 2 : 1), m_regs.index); else { - // Rematch one less + // Rematch one less. The consumed characters have data-dependent widths, + // so the frontier after count-1 matches is recovered by re-consuming + // that many characters from the recorded begin index: rightward in a + // forward frame, leftward (decoding each code point from its last + // unit) in a backward frame. const MacroAssembler::RegisterID character = m_regs.regT0; loadFromFrame(term->frameLocation + BackTrackInfoCharacterClass::beginIndex(), m_regs.index); @@ -3740,16 +4089,26 @@ class YarrGenerator final : public YarrJITInfo { MacroAssembler::Label rematchLoop(&m_jit); MacroAssembler::Jump doneRematching = m_jit.branchTest32(MacroAssembler::Zero, countRegister); - readCharacter(op.m_checkedOffset - term->inputPosition, character); + if (m_direction == Backward) { + readUnicodeCharBackward(negativeOffsetIndexedAddress(op.m_checkedOffset - term->inputPosition, character), character, m_regs.regT2); + m_usesT2 = true; + m_jit.sub32(MacroAssembler::TrustedImm32(1), countRegister); + m_jit.sub32(MacroAssembler::TrustedImm32(1), m_regs.index); + MacroAssembler::Jump isBMPChar = m_jit.branch32(MacroAssembler::LessThan, character, MacroAssembler::TrustedImm32(0x10000)); + m_jit.sub32(MacroAssembler::TrustedImm32(1), m_regs.index); + isBMPChar.link(&m_jit); + } else { + readCharacter(op.m_checkedOffset - term->inputPosition, character); - m_jit.sub32(MacroAssembler::TrustedImm32(1), countRegister); - m_jit.add32(MacroAssembler::TrustedImm32(1), m_regs.index); + m_jit.sub32(MacroAssembler::TrustedImm32(1), countRegister); + m_jit.add32(MacroAssembler::TrustedImm32(1), m_regs.index); #if ENABLE(YARR_JIT_UNICODE_EXPRESSIONS) - MacroAssembler::Jump isBMPChar = m_jit.branch32(MacroAssembler::LessThan, character, MacroAssembler::TrustedImm32(0x10000)); - m_jit.add32(MacroAssembler::TrustedImm32(1), m_regs.index); - isBMPChar.link(&m_jit); + MacroAssembler::Jump isBMPChar = m_jit.branch32(MacroAssembler::LessThan, character, MacroAssembler::TrustedImm32(0x10000)); + m_jit.add32(MacroAssembler::TrustedImm32(1), m_regs.index); + isBMPChar.link(&m_jit); #endif + } m_jit.jump(rematchLoop); doneRematching.link(&m_jit); @@ -3798,34 +4157,60 @@ class YarrGenerator final : public YarrJITInfo { nonGreedyFailures.append(atEndOfInput()); nonGreedyFailures.append(m_jit.branch32(MacroAssembler::Equal, countRegister, MacroAssembler::Imm32(term->quantityMaxCount))); - readCharacter(op.m_checkedOffset - term->inputPosition, character); + if (m_direction == Backward && m_decodeSurrogatePairs) { + // Backward + unicode: as in the greedy loop, decode the code point + // ending just left of the frontier, class-match it, then consume it by + // the width read off the code point; the pair's lead needs a further + // unit below the frontier or the take is undone via + // nonGreedyFailuresDecrementIndex. + readUnicodeCharBackward(negativeOffsetIndexedAddress(op.m_checkedOffset - term->inputPosition, character), character, scratch); + matchCharacterClassTermInner(term, nonGreedyFailures, character, scratch); + m_jit.sub32(MacroAssembler::TrustedImm32(1), m_regs.index); // the code point's last unit + MacroAssembler::Jump isBMPChar = m_jit.branch32(MacroAssembler::LessThan, character, MacroAssembler::TrustedImm32(0x10000)); + // As in the greedy loop: test for the lead before taking it, so the failure + // path (nonGreedyFailuresDecrementIndex) gives back exactly the trail. + nonGreedyFailuresDecrementIndex.append(atEndOfInput()); + m_jit.sub32(MacroAssembler::TrustedImm32(1), m_regs.index); // the pair's lead unit + isBMPChar.link(&m_jit); + } else { + readCharacter(op.m_checkedOffset - term->inputPosition, character); - matchCharacterClassTermInner(term, nonGreedyFailures, character, scratch); + matchCharacterClassTermInner(term, nonGreedyFailures, character, scratch); - if (m_direction == Backward) - m_jit.sub32(MacroAssembler::TrustedImm32(1), m_regs.index); + if (m_direction == Backward) + m_jit.sub32(MacroAssembler::TrustedImm32(1), m_regs.index); #if ENABLE(YARR_JIT_UNICODE_EXPRESSIONS) - else if (m_decodeSurrogatePairs) - advanceIndexAfterCharacterClassTermMatch(term, nonGreedyFailuresDecrementIndex, character); + else if (m_decodeSurrogatePairs) + advanceIndexAfterCharacterClassTermMatch(term, nonGreedyFailuresDecrementIndex, character); #endif - else - m_jit.add32(MacroAssembler::TrustedImm32(1), m_regs.index); + else + m_jit.add32(MacroAssembler::TrustedImm32(1), m_regs.index); + } m_jit.add32(MacroAssembler::TrustedImm32(1), countRegister); m_jit.jump(op.m_reentry); if (!nonGreedyFailuresDecrementIndex.empty()) { nonGreedyFailuresDecrementIndex.link(&m_jit); - m_jit.sub32(MacroAssembler::TrustedImm32(1), m_regs.index); + // Give back the unit consumed before the failure (frontier moved up in + // a forward frame, down in a backward frame). + if (m_direction == Backward) + m_jit.add32(MacroAssembler::TrustedImm32(1), m_regs.index); + else + m_jit.sub32(MacroAssembler::TrustedImm32(1), m_regs.index); } nonGreedyFailures.link(&m_jit); - if (m_direction == Backward) - m_jit.add32(countRegister, m_regs.index); + // Restore the frontier to where this term began. With per-match widths + // (unicode) the distance is not derivable from the match count, so it is + // reloaded from beginIndex; otherwise each match was exactly one unit. #if ENABLE(YARR_JIT_UNICODE_EXPRESSIONS) - else if (m_decodeSurrogatePairs) + if (m_decodeSurrogatePairs) loadFromFrame(term->frameLocation + BackTrackInfoCharacterClass::beginIndex(), m_regs.index); + else #endif + if (m_direction == Backward) + m_jit.add32(countRegister, m_regs.index); else m_jit.sub32(countRegister, m_regs.index); m_backtrackingState.fallthrough(); @@ -3910,6 +4295,15 @@ class YarrGenerator final : public YarrJITInfo { YarrOp& op = m_ops[opIndex]; PatternTerm* term = op.m_term; +#if ENABLE(YARR_JIT_UNICODE_EXPRESSIONS) && ENABLE(YARR_JIT_UNICODE_CAN_INCREMENT_INDEX_FOR_NON_BMP) + // firstCharacterAdditionalReadSize means "the character at the match START + // was a non-BMP pair, so advancing the start by one code point takes an + // extra unit". Only the term whose first read is at the match start may + // establish that (see YarrOp::m_readsStartCharacter); a non-BMP read + // anywhere else says nothing about the start character. + SetForScope firstCharacterOnly(m_useFirstNonBMPCharacterOptimization, m_canUseFirstNonBMPCharacterOptimization && op.m_readsStartCharacter); +#endif + switch (term->type) { case PatternTerm::Type::PatternCharacter: switch (term->quantityType) { @@ -4112,7 +4506,7 @@ class YarrGenerator final : public YarrJITInfo { #if ENABLE(YARR_JIT_UNICODE_EXPRESSIONS) && ENABLE(YARR_JIT_UNICODE_CAN_INCREMENT_INDEX_FOR_NON_BMP) // Initialize before input check to prevent uninitialized data in arithmetic. - if (m_useFirstNonBMPCharacterOptimization) + if (m_canUseFirstNonBMPCharacterOptimization) m_jit.move(MacroAssembler::TrustedImm32(0), m_regs.firstCharacterAdditionalReadSize); #endif @@ -4127,7 +4521,7 @@ class YarrGenerator final : public YarrJITInfo { #if ENABLE(YARR_JIT_UNICODE_EXPRESSIONS) && ENABLE(YARR_JIT_UNICODE_CAN_INCREMENT_INDEX_FOR_NON_BMP) // Initialize after reentry to ensure fresh register state on backtrack retries, // since character reading operations modify this register during execution. - if (m_useFirstNonBMPCharacterOptimization) + if (m_canUseFirstNonBMPCharacterOptimization) m_jit.move(MacroAssembler::TrustedImm32(0), m_regs.firstCharacterAdditionalReadSize); #endif @@ -4337,7 +4731,7 @@ class YarrGenerator final : public YarrJITInfo { // Reset on reentry: a prior alternative may have read a non-BMP codepoint // and left this register set to 1. The next alternative starts a fresh // first-character read. - if (m_useFirstNonBMPCharacterOptimization) + if (m_canUseFirstNonBMPCharacterOptimization) m_jit.move(MacroAssembler::TrustedImm32(0), m_regs.firstCharacterAdditionalReadSize); #endif if (shouldRecordSubpatterns() && priorAlternative->needToCleanupCaptures()) { @@ -4355,7 +4749,7 @@ class YarrGenerator final : public YarrJITInfo { defineReentryLabel(op); #if ENABLE(YARR_JIT_UNICODE_EXPRESSIONS) && ENABLE(YARR_JIT_UNICODE_CAN_INCREMENT_INDEX_FOR_NON_BMP) // Reset on reentry: see BodyAlternativeNext above. - if (m_useFirstNonBMPCharacterOptimization) + if (m_canUseFirstNonBMPCharacterOptimization) m_jit.move(MacroAssembler::TrustedImm32(0), m_regs.firstCharacterAdditionalReadSize); #endif m_jit.sub32(MacroAssembler::Imm32(priorAlternative->m_minimumSize), m_regs.index); @@ -4986,7 +5380,7 @@ class YarrGenerator final : public YarrJITInfo { if (!m_pattern.m_body->m_hasFixedSize) { if (alternative->m_minimumSize == 1) #if ENABLE(YARR_JIT_UNICODE_EXPRESSIONS) && ENABLE(YARR_JIT_UNICODE_CAN_INCREMENT_INDEX_FOR_NON_BMP) - if (m_useFirstNonBMPCharacterOptimization) { + if (m_canUseFirstNonBMPCharacterOptimization) { m_jit.add32(m_regs.firstCharacterAdditionalReadSize, m_regs.index, m_regs.regT0); setMatchStart(m_regs.regT0); } else @@ -4998,7 +5392,7 @@ class YarrGenerator final : public YarrJITInfo { else m_jit.add32(MacroAssembler::TrustedImm32(1), m_regs.index, m_regs.regT0); #if ENABLE(YARR_JIT_UNICODE_EXPRESSIONS) && ENABLE(YARR_JIT_UNICODE_CAN_INCREMENT_INDEX_FOR_NON_BMP) - if (m_useFirstNonBMPCharacterOptimization) + if (m_canUseFirstNonBMPCharacterOptimization) m_jit.add32(m_regs.firstCharacterAdditionalReadSize, m_regs.regT0); #endif setMatchStart(m_regs.regT0); @@ -5013,7 +5407,7 @@ class YarrGenerator final : public YarrJITInfo { unsigned delta = alternative->m_minimumSize - beginOp->m_alternative->m_minimumSize; ASSERT(delta); #if ENABLE(YARR_JIT_UNICODE_EXPRESSIONS) && ENABLE(YARR_JIT_UNICODE_CAN_INCREMENT_INDEX_FOR_NON_BMP) - if (m_useFirstNonBMPCharacterOptimization) { + if (m_canUseFirstNonBMPCharacterOptimization) { m_jit.add32(m_regs.firstCharacterAdditionalReadSize, m_regs.index); if (delta != 1) m_jit.sub32(MacroAssembler::Imm32(delta - 1), m_regs.index); @@ -5033,7 +5427,7 @@ class YarrGenerator final : public YarrJITInfo { if (delta != 0xFFFFFFFFu) { // We need to check input because we are incrementing the input. #if ENABLE(YARR_JIT_UNICODE_EXPRESSIONS) && ENABLE(YARR_JIT_UNICODE_CAN_INCREMENT_INDEX_FOR_NON_BMP) - if (m_useFirstNonBMPCharacterOptimization) + if (m_canUseFirstNonBMPCharacterOptimization) m_jit.add32(m_regs.firstCharacterAdditionalReadSize, m_regs.index); #endif m_jit.add32(MacroAssembler::Imm32(delta + 1), m_regs.index); @@ -5116,7 +5510,7 @@ class YarrGenerator final : public YarrJITInfo { // If the last alternative had the same minimum size as the disjunction, // just simply increment input pos by 1, no adjustment based on minimum size. #if ENABLE(YARR_JIT_UNICODE_EXPRESSIONS) && ENABLE(YARR_JIT_UNICODE_CAN_INCREMENT_INDEX_FOR_NON_BMP) - if (m_useFirstNonBMPCharacterOptimization) + if (m_canUseFirstNonBMPCharacterOptimization) m_jit.add32(m_regs.firstCharacterAdditionalReadSize, m_regs.index); #endif m_jit.add32(MacroAssembler::TrustedImm32(1), m_regs.index); @@ -5553,7 +5947,13 @@ class YarrGenerator final : public YarrJITInfo { // This is because NonGreedy already tried with count = 0, failing and reaching here. // So there is no way to proceed further, just propagate a failure. noPreviousIteration.link(&m_jit); - emitClearCapturesForTerm(term); + // Zero iterations of THIS quantified group means its captures are + // undefined -- except for a quantified split's optional copy, whose + // capture ids belong to the mandatory sibling: restoreParenContext has + // already restored the sibling's committed values, and clearing them + // here would erase captures the copy never owned. + if (!term->parentheses.isCopy) + emitClearCapturesForTerm(term); storeToFrame(MacroAssembler::TrustedImm32(-1), parenthesesFrameLocation + BackTrackInfoParentheses::beginIndex()); m_backtrackingState.fallthrough(); break; @@ -5585,7 +5985,9 @@ class YarrGenerator final : public YarrJITInfo { // We need to clear the state, and go back to the further former backtracking. noPreviousIteration.link(&m_jit); op.m_jumps.link(&m_jit); - emitClearCapturesForTerm(term); + // As above: an optional copy owns none of its capture ids. + if (!term->parentheses.isCopy) + emitClearCapturesForTerm(term); } else { // Try fewer iterations. m_jit.jump(endOp.m_reentry); @@ -5650,7 +6052,11 @@ class YarrGenerator final : public YarrJITInfo { exceededMatchLimit.append(m_jit.branch32(MacroAssembler::AboveOrEqual, countTemporary, MacroAssembler::Imm32(term->quantityMaxCount))); } - m_jit.branch32(MacroAssembler::Above, m_regs.index, beginTemporary).linkTo(beginOp.m_reentry, &m_jit); + // Try one more (lazy) iteration only if the last one made progress: + // the frontier moved right in a forward frame, left (down) in a + // mirrored one. Without the direction split a leftward-consuming + // lazy group stops re-entering as soon as its frontier has moved. + m_jit.branch32(op.m_direction == Backward ? MacroAssembler::Below : MacroAssembler::Above, m_regs.index, beginTemporary).linkTo(beginOp.m_reentry, &m_jit); exceededMatchLimit.link(&m_jit); break; @@ -5681,12 +6087,19 @@ class YarrGenerator final : public YarrJITInfo { // We need to handle the backtracks upon backtracking back out // of a parenthetical assertion if either we need to correct - // the input index, or the assertion was inverted. - if (op.m_checkAdjust || term->invert()) { + // the input index, or the assertion was inverted, or the body ran + // in a different direction from this (enclosing) frame -- then the + // body left index on its own coordinate line, and only the position + // saved at entry (beginIndex) is meaningful to the code we back + // out to. An arithmetic undo cannot cross that boundary. + bool bodyDirectionDiffers = term->matchDirection() != op.m_direction; + if (op.m_checkAdjust || term->invert() || bodyDirectionDiffers) { m_backtrackingState.link(*this, op); - // Undo the realignment done on entry (see the forward path). - if (op.m_checkAdjust) { + if (bodyDirectionDiffers) + loadFromFrame(term->frameLocation + BackTrackInfoParentheticalAssertion::beginIndex(), m_regs.index); + else if (op.m_checkAdjust) { + // Undo the realignment done on entry (see the forward path). if (op.m_direction == Backward) m_jit.sub32(MacroAssembler::Imm32(op.m_checkAdjust), m_regs.index); else @@ -5860,7 +6273,7 @@ class YarrGenerator final : public YarrJITInfo { DispatchInfo* dispatch = nullptr; if (parenthesesBeginOpCode == YarrOpCode::ParenthesesSubpatternOnceBegin && alternativeBeginOpCode == YarrOpCode::NestedAlternativeBegin) dispatch = tryPrepareDispatch(term, checkedOffset); - if (Options::verboseRegExpCompilation() && term->parentheses.disjunction->m_alternatives.size() >= Options::regExpAlternationDispatchThreshold()) { + if (Options::verboseRegExpCompilation() && term->parentheses.disjunction->m_alternatives.size() >= alternationDispatchMinAlternatives) { dumpFirstCharacterSets(*term->parentheses.disjunction); dataLogLn(dispatch ? "First-character dispatch enabled" : "First-character dispatch not used"); } @@ -5940,17 +6353,20 @@ class YarrGenerator final : public YarrJITInfo { // copy of the body forward over the virtually reversed input, so we build // such a reversed copy and compile it with the standard forward machinery, // marking its ops Backward so the input-check/read/end-of-input primitives - // use the leftward cursor variants. Positions are reassigned in virtual - // (reversed) coordinates using exactly the width rules of - // YarrPatternConstructor::setupAlternativeOffsets. Frame locations are kept - // from the original terms; the original body is never itself compiled, so its - // slots are free for the mirrored copy to use. + // use the leftward cursor variants. // - // Returns false (with m_failureReason set) if the body uses a construct the - // backward JIT does not support yet; the whole pattern then falls back to the - // interpreter as it did before lookbehind JIT support existed. + // assignAlternativeOffsets numbers a stored term list from a given origin + // using exactly the width rules of + // YarrPatternConstructor::setupAlternativeOffsets. Applied to a mirrored + // copy (stored reversed) it yields the virtual reversed coordinates; applied + // to an ordinary forward body it renumbers that body from the origin. Frame + // locations always stay those the pattern constructor assigned: the + // original lookbehind body is never itself compiled, so a mirrored copy may + // reuse its slots, and a renumbered forward body keeps its own. + // + // Returns false (with m_failureReason set) only on offset overflow. - bool assignMirroredAlternativeOffsets(PatternAlternative* alternative, unsigned initialInputPosition) + bool assignAlternativeOffsets(PatternAlternative* alternative, unsigned initialInputPosition) { CheckedUint32 currentInputPosition = initialInputPosition; @@ -5966,6 +6382,13 @@ class YarrGenerator final : public YarrJITInfo { case PatternTerm::Type::NamedForwardReference: break; + case PatternTerm::Type::NumberedBackReference: + case PatternTerm::Type::NamedBackReference: + // Dynamic length: positioned but contributing no width, as in + // setupAlternativeOffsets. Its frame slot (allocated there) is kept. + term.inputPosition = currentInputPosition; + break; + case PatternTerm::Type::PatternCharacter: term.inputPosition = currentInputPosition; if (term.quantityType == QuantifierType::FixedCount) { @@ -5994,20 +6417,39 @@ class YarrGenerator final : public YarrJITInfo { break; case PatternTerm::Type::ParenthesesSubpattern: - // Only the quantityMaxCount == 1, !isCopy shape reaches here (see - // mirrorAlternative). Nested group positions continue the enclosing - // numbering, as in setupAlternativeOffsets. - if (!assignMirroredDisjunctionOffsets(term.parentheses.disjunction, currentInputPosition)) - return false; - if (term.quantityType == QuantifierType::FixedCount) - currentInputPosition += term.parentheses.disjunction->m_minimumSize; - term.inputPosition = currentInputPosition; + // Nested group positions continue the enclosing numbering, as in + // setupAlternativeOffsets: the once shape (max count 1, not a copy) + // pre-claims its body minimum and is positioned after it; every + // counted/copied shape claims dynamically and is positioned before it. + if (term.quantityMaxCount == 1 && !term.parentheses.isCopy) { + if (!assignDisjunctionOffsets(term.parentheses.disjunction, currentInputPosition)) + return false; + if (term.quantityType == QuantifierType::FixedCount) + currentInputPosition += term.parentheses.disjunction->m_minimumSize; + term.inputPosition = currentInputPosition; + } else { + term.inputPosition = currentInputPosition; + if (!assignDisjunctionOffsets(term.parentheses.disjunction, currentInputPosition)) + return false; + } break; case PatternTerm::Type::ParentheticalAssertion: - // A nested lookbehind keeps its original (0-based) body and is mirrored - // on demand when compiled. Its own position is the virtual assertion point. + // The assertion's own position is the virtual assertion point. A + // nested lookbehind keeps its original 0-based body (mirrored and + // renumbered on demand when compiled); a nested forward assertion's + // body is numbered from this position, as in + // setupAlternativeOffsets: the enclosing checkedOffset continues into + // the body, so read offsets (checkedOffset - inputPosition) are the + // same as any forward body's, and the body is renumbered from this + // position to stay on that line -- recursively, through further + // nesting. BOL/EOL inside such a body test the real assertion point + // (see generateAssertionBOL/EOL), never the absolute inputPosition. term.inputPosition = currentInputPosition; + if (term.matchDirection() != Backward) { + if (!assignDisjunctionOffsets(term.parentheses.disjunction, currentInputPosition)) + return false; + } break; default: @@ -6024,10 +6466,10 @@ class YarrGenerator final : public YarrJITInfo { return true; } - bool assignMirroredDisjunctionOffsets(PatternDisjunction* disjunction, unsigned initialInputPosition) + bool assignDisjunctionOffsets(PatternDisjunction* disjunction, unsigned initialInputPosition) { for (auto& alternative : disjunction->m_alternatives) { - if (!assignMirroredAlternativeOffsets(alternative.get(), initialInputPosition)) + if (!assignAlternativeOffsets(alternative.get(), initialInputPosition)) return false; } return true; @@ -6059,41 +6501,41 @@ class YarrGenerator final : public YarrJITInfo { case PatternTerm::Type::NumberedBackReference: case PatternTerm::Type::NamedBackReference: - // Backreferences inside a lookbehind body: not yet supported by the - // backward JIT (they must compare right-to-left). - m_failureReason = JITFailureReason::Lookbehind; - return false; + break; case PatternTerm::Type::ParenthesesSubpattern: - if (term.quantityMaxCount != 1 || term.parentheses.isCopy || term.parentheses.isTerminal) { - // Quantified groups with maxCount > 1 inside a lookbehind body: not - // yet supported by the backward JIT. - m_failureReason = JITFailureReason::Lookbehind; - return false; - } - // The string-list / terminal group optimizations assume forward, - // trailing-context conditions; take the general path for the copy. - term.parentheses.isStringList = false; - term.parentheses.isEOLStringList = false; + // Groups of any quantifier are mirrored recursively: the paren + // iteration/backtracking machinery keeps positions in saved frame + // slots and per-iteration ParenContexts rather than deriving them + // from counts, so it is direction-neutral. The string-list and + // terminal group forms are only ever produced for the top-level + // pattern's own alternatives, never inside a lookbehind body. + ASSERT(!term.parentheses.isStringList); + ASSERT(!term.parentheses.isEOLStringList); + ASSERT(!term.parentheses.isTerminal); term.parentheses.disjunction = mirrorDisjunctionForLookbehind(term.parentheses.disjunction); if (!term.parentheses.disjunction) return false; break; case PatternTerm::Type::ParentheticalAssertion: + // A nested lookbehind keeps its 0-based body: it is mirrored into its + // own copy on demand when compiled and never mutated in place. A nested + // FORWARD assertion's body is about to be renumbered onto this frame's + // line by assignAlternativeOffsets, so the mirror must own it -- a + // shallow copy would rewrite the pattern's own body, which the + // bytecode interpreter reuses if this JIT compile later bails. if (term.matchDirection() != Backward) { - // A lookahead nested inside a lookbehind body: its body would need - // re-basing into the mirrored coordinate space. Not yet supported. - m_failureReason = JITFailureReason::Lookbehind; - return false; + term.parentheses.disjunction = copyForwardDisjunctionForMirror(term.parentheses.disjunction); + if (!term.parentheses.disjunction) + return false; } - // Nested lookbehind: keep pointing at its original body; it is mirrored - // when opCompileParentheticalAssertion reaches it. break; case PatternTerm::Type::DotStarEnclosure: - m_failureReason = JITFailureReason::Lookbehind; - return false; + // Only synthesized for the top-level pattern body (see + // optimizeDotStarWrappedExpressions), never inside a lookbehind. + RELEASE_ASSERT_NOT_REACHED(); } mirrored->m_terms.append(term); @@ -6126,6 +6568,69 @@ class YarrGenerator final : public YarrJITInfo { return result; } + // Structural (in-order) deep copy of a forward body reached from inside a + // mirrored lookbehind body, so that renumbering it (assignAlternativeOffsets) + // touches only mirror-owned terms and never the pattern's own disjunctions. + // Nested groups and nested forward assertions are copied recursively for the + // same reason; a nested lookbehind's body is left shared (it is mirrored into + // its own copy on demand and never renumbered in place). + bool copyForwardAlternativeInto(PatternAlternative& source, PatternDisjunction& destination) + { + PatternAlternative* copy = destination.addNewAlternative(source.m_firstSubpatternId, source.matchDirection()); + copy->m_minimumSize = source.m_minimumSize; + copy->m_lastSubpatternId = source.m_lastSubpatternId; + copy->m_hasFixedSize = source.m_hasFixedSize; + copy->m_containsBOL = source.m_containsBOL; + copy->m_startsWithBOL = source.m_startsWithBOL; + copy->m_onceThrough = source.m_onceThrough; + copy->m_isLastAlternative = source.m_isLastAlternative; + + copy->m_terms.reserveInitialCapacity(source.m_terms.size()); + for (auto& sourceTerm : source.m_terms) { + PatternTerm term = sourceTerm; + switch (term.type) { + case PatternTerm::Type::ParenthesesSubpattern: + term.parentheses.disjunction = copyForwardDisjunctionForMirror(term.parentheses.disjunction); + if (!term.parentheses.disjunction) + return false; + break; + case PatternTerm::Type::ParentheticalAssertion: + if (term.matchDirection() != Backward) { + term.parentheses.disjunction = copyForwardDisjunctionForMirror(term.parentheses.disjunction); + if (!term.parentheses.disjunction) + return false; + } + break; + default: + break; + } + copy->m_terms.append(term); + } + return true; + } + + PatternDisjunction* copyForwardDisjunctionForMirror(PatternDisjunction* disjunction) + { + if (!isSafeToRecurse()) [[unlikely]] { + m_failureReason = JITFailureReason::ParenthesisNestedTooDeep; + return nullptr; + } + + auto copiedDisjunction = makeUnique(); + copiedDisjunction->m_minimumSize = disjunction->m_minimumSize; + copiedDisjunction->m_callFrameSize = disjunction->m_callFrameSize; + copiedDisjunction->m_hasFixedSize = disjunction->m_hasFixedSize; + + for (auto& alternative : disjunction->m_alternatives) { + if (!copyForwardAlternativeInto(*alternative, *copiedDisjunction)) + return nullptr; + } + + PatternDisjunction* result = copiedDisjunction.get(); + m_mirroredDisjunctions.append(WTF::move(copiedDisjunction)); + return result; + } + // opCompileParentheticalAssertion // Emits ops for a parenthetical assertion. These consist of an // YarrOpCode::SimpleNestedAlternativeBegin/Next/End set of nodes wrapping @@ -6149,7 +6654,7 @@ class YarrGenerator final : public YarrJITInfo { disjunction = mirrorDisjunctionForLookbehind(disjunction); if (!disjunction) return; - if (!assignMirroredDisjunctionOffsets(disjunction, 0)) + if (!assignDisjunctionOffsets(disjunction, 0)) return; } @@ -6162,8 +6667,14 @@ class YarrGenerator final : public YarrJITInfo { checkedOffset -= m_ops.last().m_checkAdjust; m_ops.last().m_checkedOffset = checkedOffset; - // The body's ops are emitted in the body's own direction. A lookbehind - // body's mirrored frame starts fresh at virtual position 0. + // The body's ops are emitted in the body's own direction. A body starts a + // fresh frame at position 0 whenever its coordinates are not the enclosing + // forward line: a lookbehind body is mirrored and numbered from 0, and a + // forward body nested inside a mirrored (Backward) body is renumbered from + // 0 by assignAlternativeOffsets, since it cannot inherit mirrored offsets. + // A forward body under a forward frame keeps the pattern layer's numbering + // and continues the enclosing checkedOffset. The Begin op has already + // placed the index register at the assertion point in every case. MatchDirection enclosingDirection = m_direction; m_direction = bodyDirection; if (bodyDirection == Backward) @@ -6222,25 +6733,49 @@ class YarrGenerator final : public YarrJITInfo { // opCompileAlternative // Called to emit nodes for all terms in an alternative. - void opCompileAlternative(Checked checkedOffset, PatternAlternative* alternative) + // `beginsAtMatchStart` is true only for the pattern's own body alternatives: + // a nested group's or assertion's alternative begins wherever its enclosing + // term sits, so its head term's read says nothing about the character at the + // match start (see YarrOp::m_readsStartCharacter). + void opCompileAlternative(Checked checkedOffset, PatternAlternative* alternative, bool beginsAtMatchStart = false) { optimizeAlternative(alternative); + // The first term reads the match start's character until some term can + // move the frontier at runtime; from then on a read may land anywhere. + bool frontierMayHaveMoved = !beginsAtMatchStart; for (unsigned i = 0; i < alternative->m_terms.size(); ++i) { PatternTerm* term = &alternative->m_terms[i]; switch (term->type) { case PatternTerm::Type::ParenthesesSubpattern: opCompileParenthesesSubpattern(checkedOffset, term); + frontierMayHaveMoved = true; break; case PatternTerm::Type::ParentheticalAssertion: opCompileParentheticalAssertion(checkedOffset, term); break; - default: + default: { appendOp(term); m_ops.last().m_checkedOffset = checkedOffset; + // Only a literal or a class term makes a single fixed-position read of + // the start character; assertions, forward references, and backreferences + // (a variable-length span match, never the start's one code point) do not. + bool readsInput = term->type == PatternTerm::Type::PatternCharacter || term->type == PatternTerm::Type::CharacterClass; + if (readsInput) { + // Only a term that reads exactly once has its single read pinned to + // the match start; a quantified term's later iterations peek beyond + // it. The read variant only records that the start character is + // non-BMP (it never moves the frontier), so a literal or a class head + // that reads once at the start both qualify. + bool readsOnce = term->quantityType == QuantifierType::FixedCount && term->quantityMaxCount == 1; + m_ops.last().m_readsStartCharacter = !frontierMayHaveMoved && readsOnce; + frontierMayHaveMoved = true; + } + break; + } } } } @@ -6278,7 +6813,7 @@ class YarrGenerator final : public YarrJITInfo { PatternAlternative* alternative = alternatives[currentAlternativeIndex].get(); m_ops[lastOpIndex].m_checkedOffset = alternative->m_minimumSize; - opCompileAlternative(alternative->m_minimumSize, alternative); + opCompileAlternative(alternative->m_minimumSize, alternative, /* beginsAtMatchStart */ true); size_t thisOpIndex = m_ops.size(); appendOp(YarrOp(YarrOpCode::BodyAlternativeNext)); @@ -6379,7 +6914,7 @@ class YarrGenerator final : public YarrJITInfo { PatternAlternative* alternative = alternatives[currentAlternativeIndex].get(); ASSERT(!alternative->onceThrough()); m_ops[lastOpIndex].m_checkedOffset = alternative->m_minimumSize; - opCompileAlternative(alternative->m_minimumSize, alternative); + opCompileAlternative(alternative->m_minimumSize, alternative, /* beginsAtMatchStart */ true); size_t thisOpIndex = m_ops.size(); appendOp(YarrOp(YarrOpCode::BodyAlternativeNext)); @@ -6719,10 +7254,12 @@ class YarrGenerator final : public YarrJITInfo { // first character, and if so build the DispatchInfo (chains and character // routing). Called at op-compile time, before the alternatives' ops are // emitted (so the first term of each alternative is still in position order). + // First-character dispatch pays for its extra read only from a handful of + // alternatives up; below that the sequential chain is faster. + static constexpr size_t alternationDispatchMinAlternatives = 4; + DispatchInfo* tryPrepareDispatch(PatternTerm* term, Checked checkedOffset) { - if (!Options::useRegExpAlternationDispatch()) - return nullptr; // Only once-quantified fixed groups take the ParenthesesSubpatternOnce + // NestedAlternative path whose frame holds BackTrackInfoParenthesesOnce. if (term->type != PatternTerm::Type::ParenthesesSubpattern) @@ -6738,7 +7275,7 @@ class YarrGenerator final : public YarrJITInfo { PatternDisjunction* disjunction = term->parentheses.disjunction; auto& alternatives = disjunction->m_alternatives; - if (alternatives.size() < Options::regExpAlternationDispatchThreshold()) + if (alternatives.size() < alternationDispatchMinAlternatives) return nullptr; // The first character is only guaranteed inside claimed input when every // alternative must consume at least one character. @@ -7925,27 +8462,21 @@ class YarrGenerator final : public YarrJITInfo { #endif // Lookbehind bodies are JIT-compiled via mirroring (see - // mirrorDisjunctionForLookbehind), except when the pattern would change - // engines in ways that are not yet safe: - // - unicode patterns: reading backward across a UTF-16 surrogate pair - // (trail first, then lead) is not implemented in the backward JIT; and - // - a BOL-group bubble (m_containsBOLGroupBubble): optimizeBOL's - // once-through/loop-copy split of such patterns is only handled - // correctly by the interpreter today (a pre-existing JIT-only defect), - // so those patterns must not newly move to the JIT with this change. - // The decision is per pattern, not per subject encoding, so the 8-bit and - // 16-bit specializations of one RegExp always run the same engine. - // Unsupported body content is detected during op-compilation and falls - // back the same way. - if (m_pattern.m_containsLookbehinds && (m_pattern.eitherUnicode() || m_pattern.m_containsBOLGroupBubble)) { - codeBlock.setFallBackWithFailureReason(JITFailureReason::Lookbehind); - return; - } - + // mirrorDisjunctionForLookbehind): the reversed body copy runs through the + // forward machinery with Backward ops, decoding unicode text leftward + // (readUnicodeCharBackward, matching the interpreter's tryReadBackward). + + // The first-non-BMP-character skip advances the next match start past the + // trailing surrogate of an astral start character. That is only sound when + // no match can begin at a mid-pair position -- i.e. when the pattern cannot + // match zero-width (a body minimum size of 0 means some alternative can, and + // a zero-width alternative such as a negative lookahead over . or an + // assertion can then succeed at a dangling-trail position, whose match the + // skip would drop). #if ENABLE(YARR_JIT_UNICODE_EXPRESSIONS) && ENABLE(YARR_JIT_UNICODE_CAN_INCREMENT_INDEX_FOR_NON_BMP) - if (m_decodeSurrogatePairs && m_executionMode != ExecutionMode::InlineTest && !m_pattern.multiline() && !m_pattern.m_containsBOL && !m_pattern.m_containsLookbehinds && !m_pattern.m_containsModifiers) { + if (m_decodeSurrogatePairs && m_executionMode != ExecutionMode::InlineTest && !m_pattern.multiline() && !m_pattern.m_containsBOL && !m_pattern.m_containsLookbehinds && !m_pattern.m_containsModifiers && m_pattern.m_body->m_minimumSize) { ASSERT(m_regs.firstCharacterAdditionalReadSize != InvalidGPRReg); - m_useFirstNonBMPCharacterOptimization = true; + m_canUseFirstNonBMPCharacterOptimization = true; } #endif @@ -8509,6 +9040,12 @@ class YarrGenerator final : public YarrJITInfo { ParenContextSizes m_parenContextSizes; #endif #if ENABLE(YARR_JIT_UNICODE_EXPRESSIONS) && ENABLE(YARR_JIT_UNICODE_CAN_INCREMENT_INDEX_FOR_NON_BMP) + // Whether this pattern may use the index-incrementing first-character read + // (decided once at compile). m_useFirstNonBMPCharacterOptimization is the + // AMBIENT state consulted at emission time: it is off everywhere except while + // generating the one term whose read is the match start (m_readsStartCharacter), + // so loop peeks and backtrack-pass re-reads can never write the register. + bool m_canUseFirstNonBMPCharacterOptimization { false }; bool m_useFirstNonBMPCharacterOptimization { false }; #endif RegisterAtOffsetList m_calleeSaves; @@ -8674,9 +9211,6 @@ static void dumpCompileFailure(JITFailureReason failure) case JITFailureReason::BackReference: dataLog("Can't JIT some patterns containing back references\n"); break; - case JITFailureReason::Lookbehind: - dataLog("Can't JIT a pattern containing lookbehinds\n"); - break; case JITFailureReason::ParenthesizedSubpattern: dataLog("Can't JIT a pattern containing parenthesized subpatterns\n"); break; diff --git a/Source/JavaScriptCore/yarr/YarrJIT.h b/Source/JavaScriptCore/yarr/YarrJIT.h index 4932afa28a40..124a6de19736 100644 --- a/Source/JavaScriptCore/yarr/YarrJIT.h +++ b/Source/JavaScriptCore/yarr/YarrJIT.h @@ -58,7 +58,6 @@ class YarrCodeBlock; enum class JITFailureReason : uint8_t { DecodeSurrogatePair, BackReference, - Lookbehind, ParenthesizedSubpattern, ParenthesisNestedTooDeep, ExecutableMemoryAllocationFailure, diff --git a/Source/JavaScriptCore/yarr/YarrPattern.cpp b/Source/JavaScriptCore/yarr/YarrPattern.cpp index a4add50c5bb0..fb82b8f0f766 100644 --- a/Source/JavaScriptCore/yarr/YarrPattern.cpp +++ b/Source/JavaScriptCore/yarr/YarrPattern.cpp @@ -1268,7 +1268,10 @@ class YarrPatternConstructor { void assertionBOL() { - if (!m_alternative->m_terms.size() && !parenthesisInvert() && parenthesisMatchDirection() == Forward) { + // A ^ anywhere inside a lookbehind (even within a lookahead nested in one) + // constrains a position BEHIND the match: it never anchors an alternative to + // the match start, so it must not feed the once-through/loop-copy split. + if (!m_alternative->m_terms.size() && !parenthesisInvert() && parenthesisMatchDirection() == Forward && !insideLookbehind()) { m_alternative->m_startsWithBOL = true; m_alternative->m_containsBOL = true; m_pattern.m_containsBOL = true; @@ -1572,15 +1575,18 @@ class YarrPatternConstructor { if (numBOLAnchoredAlts) { m_alternative->m_containsBOL = true; - // If all the alternatives in parens start with BOL, then so does this one - if (numBOLAnchoredAlts == numParenAlternatives) { + // The enclosing alternative can only match at the start of input (what + // m_startsWithBOL means) if this group requires a BOL and nothing can + // match before it: every alternative in the parens starts with BOL, the + // group is the alternative's first term, and it is not an inverted + // assertion. A later {0,n} / ? / * on this group makes the anchor + // optional again; quantifyAtom withdraws the flag in that case. + // A BOL matched behind the match (in a lookbehind body) constrains a + // position to the left, not where the enclosing alternative starts, so + // it never anchors that alternative (copyDisjunction likewise exempts + // backward alternatives from the once-through filter). + if (numBOLAnchoredAlts == numParenAlternatives && m_alternative->m_terms.size() == 1 && !lastTerm.invert() && lastTerm.matchDirection() != Backward) m_alternative->m_startsWithBOL = true; - // A whole group being BOL-anchored feeds optimizeBOL's once-through / - // loop-copy split, which the JIT and interpreter treat differently for - // some shapes (e.g. an optional (^)? group); noted so a JIT compile can - // keep such patterns on the interpreter (see YarrJIT::compile). - m_pattern.m_containsBOLGroupBubble = true; - } } lastTerm.parentheses.lastSubpatternId = m_pattern.m_numSubpatterns; @@ -1703,20 +1709,37 @@ class YarrPatternConstructor { for (unsigned alt = 0; alt < disjunction->m_alternatives.size(); ++alt) { PatternAlternative* alternative = disjunction->m_alternatives[alt].get(); if (!filterStartsWithBOL || !alternative->m_startsWithBOL || alternative->m_direction == Backward) { + // Copy the terms first: under filterStartsWithBOL a term can turn out to + // be dead (a required group with no satisfiable alternative), which + // kills this whole alternative -- it is then omitted, never emitted + // with the term silently gone. + Vector copiedTerms; + copiedTerms.reserveInitialCapacity(alternative->m_terms.size()); + bool alternativeIsDead = false; + for (auto& term : alternative->m_terms) { + TermCopy copy = copyTerm(term, filterStartsWithBOL); + if (hasError(error())) + return nullptr; + if (copy.isDead) { + alternativeIsDead = true; + break; + } + if (copy.term) + copiedTerms.append(WTF::move(*copy.term)); + } + if (alternativeIsDead) + continue; + if (!newDisjunction) { newDisjunction = makeUnique(); newDisjunction->m_parent = disjunction->m_parent; } PatternAlternative* newAlternative = newDisjunction->addNewAlternative(alternative->m_firstSubpatternId, alternative->matchDirection()); newAlternative->m_lastSubpatternId = alternative->m_lastSubpatternId; - newAlternative->m_terms.reserveCapacity(alternative->m_terms.size()); - for (auto& term : alternative->m_terms) { - if (auto copied = copyTerm(term, filterStartsWithBOL)) - newAlternative->m_terms.append(WTF::move(*copied)); - } + newAlternative->m_terms = WTF::move(copiedTerms); } } - + if (hasError(error())) { newDisjunction = nullptr; return nullptr; @@ -1729,24 +1752,46 @@ class YarrPatternConstructor { m_pattern.m_disjunctions.append(WTF::move(newDisjunction)); return copiedDisjunction; } - - std::optional copyTerm(PatternTerm& term, bool filterStartsWithBOL) + + // A copied term is one of: the term (recursively copied); nothing, when an + // optional group's every alternative was filtered away (matching its empty + // option is exactly the remaining semantics); or dead, when a REQUIRED group + // has no satisfiable alternative left, which makes any alternative + // containing it unmatchable (copyDisjunction drops such an alternative). + struct TermCopy { + std::optional term; + bool isDead { false }; + }; + + TermCopy copyTerm(PatternTerm& term, bool filterStartsWithBOL) { if (!isSafeToRecurse()) [[unlikely]] { m_error = ErrorCode::PatternTooLarge; - return PatternTerm(term); + return { PatternTerm(term), false }; } if ((term.type != PatternTerm::Type::ParenthesesSubpattern) && (term.type != PatternTerm::Type::ParentheticalAssertion)) - return PatternTerm(term); - + return { PatternTerm(term), false }; + if (auto* newDisjunction = copyDisjunction(term.parentheses.disjunction, filterStartsWithBOL)) { PatternTerm termCopy = term; termCopy.parentheses.disjunction = newDisjunction; m_pattern.m_hasCopiedParenSubexpressions = true; - return termCopy; - } - return std::nullopt; + return { WTF::move(termCopy), false }; + } + // copyDisjunction produced nothing. On an error (recursion limit) report + // no term and no death; the caller checks hasError(). Otherwise every + // alternative was filtered away, i.e. the parenthetical construct's body + // can never match here: + // - a group that may match zero times (or a NEGATIVE assertion, which is + // satisfied precisely when its body cannot match) reduces to matching + // empty, so the term is simply gone; + // - a group that must match, or a POSITIVE assertion, can never be + // satisfied, so the term -- and any alternative containing it -- is dead. + if (hasError(error())) + return { std::nullopt, false }; + bool satisfiedByEmpty = term.type == PatternTerm::Type::ParentheticalAssertion ? term.invert() : !term.quantityMinCount; + return { std::nullopt, !satisfiedByEmpty }; } void quantifyAtom(unsigned min, unsigned max, bool greedy) @@ -1754,6 +1799,13 @@ class YarrPatternConstructor { ASSERT(min <= max); ASSERT(m_alternative->m_terms.size()); + // A group or assertion that anchored its alternative to the start of input + // (see atomParenthesesEnd) no longer does so once it may match zero times: + // the anchor is optional, so the alternative can match anywhere. This must + // precede the {0} handling below, which removes the term entirely. + if (!min && m_alternative->m_terms.size() == 1) + m_alternative->m_startsWithBOL = false; + if (!max) { // In a case of backwards parentheses matching, we may have a forward reference that has // been quantified with {0}, meaning that we can elide it. We should check if we added an @@ -1796,23 +1848,35 @@ class YarrPatternConstructor { else if (!min || (term.type == PatternTerm::Type::ParenthesesSubpattern && (m_pattern.m_hasCopiedParenSubexpressions || term.matchDirection() == Forward))) { - // Forward-direction parenthesized subpatterns with non-zero minimum are kept - // as a single PatternTerm with quantityMinCount > 0; YarrJIT compiles these - // natively (see opCompileParenthesesSubpattern) without splitting into - // FixedCount{min} + Greedy/NonGreedy{0,max-min}. The split would deep-copy - // the disjunction subtree, which can hit OffsetTooLarge / pattern-size limits - // for very large bounds (e.g. (?:x){2147483648,...}). Backward parens - // (lookbehinds) still use the expansion path; the JIT's right-to-left - // backtracking machinery hasn't been generalized for single-term VariableMin. + // A single quantified term, compiled natively by YarrJIT + // (opCompileParenthesesSubpattern for groups). Forward parenthesized + // subpatterns keep this form even with a non-zero minimum; so does any + // group once the pattern already contains a split copy + // (m_hasCopiedParenSubexpressions): splitting a body that itself contains + // a copy re-copies it, so nested min>0 groups (e.g. (?:(?:(?:a)+)+)+, in a + // lookbehind or forward) would grow the pattern -- and both the JIT's op + // vector and the interpreter's work -- exponentially in the nesting + // depth. Only the innermost quantification of such a nest splits. This + // also avoids the split's other copy costs (OffsetTooLarge / size limits + // for huge bounds, e.g. (?:x){2147483648,...}). term.quantify(min, max, greedy ? QuantifierType::Greedy : QuantifierType::NonGreedy); } else { + // Split X{min,max} into a mandatory FixedCount piece and an optional + // {0,max-min} copy sharing X's capture ids: reached by non-parenthesis + // atoms and by the innermost quantified group of a backward (lookbehind) + // parenthesized subpattern, whose mirrored body runs through the copy + // machinery. Term order is match + // order: source order forward, reversed by the mirror for backward + // bodies. isCopy marks the optional piece; a copy that ran zero + // iterations must not clear the capture ids it shares with the + // mandatory piece (see the paren Begin backtrack in YarrJIT). if (term.matchDirection() == Forward) { term.quantify(min, min, QuantifierType::FixedCount); - auto copied = copyTerm(term, /* filterStartsWithBOL */ false); + // Unfiltered copies never come back dead; a missing term is the error path. + auto copied = copyTerm(term, /* filterStartsWithBOL */ false).term; if (!copied) [[unlikely]] return; m_alternative->m_terms.append(WTF::move(*copied)); - // NOTE: this term is interesting from an analysis perspective, in that it can be ignored..... m_alternative->lastTerm().quantify((max == quantifyInfinite) ? max : max - min, greedy ? QuantifierType::Greedy : QuantifierType::NonGreedy); if (m_alternative->lastTerm().type == PatternTerm::Type::ParenthesesSubpattern) m_alternative->lastTerm().parentheses.isCopy = true; @@ -1820,7 +1884,8 @@ class YarrPatternConstructor { term.quantify((max == quantifyInfinite) ? max : max - min, greedy ? QuantifierType::Greedy : QuantifierType::NonGreedy); if (term.type == PatternTerm::Type::ParenthesesSubpattern) term.parentheses.isCopy = true; - auto copied = copyTerm(term, /* filterStartsWithBOL */ false); + // Unfiltered copies never come back dead; a missing term is the error path. + auto copied = copyTerm(term, /* filterStartsWithBOL */ false).term; if (!copied) [[unlikely]] return; m_alternative->m_terms.append(WTF::move(*copied)); @@ -2345,6 +2410,12 @@ class YarrPatternConstructor { } } + // Below these alternative counts the plainer code shapes the JIT emits + // for a short alternation (fused literal compares, the two-alternative + // SIMD scan, frame-free inlinable groups) beat the rewrites; the transforms + // pay for themselves only on wide alternations. + static constexpr size_t alternationFactoringMinRun = 8; // prefix factoring / top-level fold + // Alternation prefix factoring. // // Alternatives are tried leftmost-first, so their order is observable -- but @@ -2530,7 +2601,7 @@ class YarrPatternConstructor { // compares, the two-alternative SIMD scan, frame-free inlinable code) // is already optimal there, and factoring would forfeit those. The // rewrite pays for itself only on large alternations. - if (runEnd - runStart < Options::regExpAlternationGroupThreshold()) { + if (runEnd - runStart < alternationFactoringMinRun) { for (size_t j = runStart; j < runEnd; ++j) result.append(WTF::move(alternatives[j])); continue; @@ -2599,9 +2670,6 @@ class YarrPatternConstructor { // this) lays out the group like any hand-written one. void factorAndWrapAlternatives() { - if (!Options::useRegExpAlternationFactoring()) - return; - factorAlternatives(*m_pattern.m_body); wrapAlternativesForDispatch(); } @@ -2616,10 +2684,7 @@ class YarrPatternConstructor { while (firstRepeated < alternatives.size() && alternatives[firstRepeated]->onceThrough()) ++firstRepeated; size_t repeatedCount = alternatives.size() - firstRepeated; - // Folding one (or no) alternative into a group is meaningless whatever - // the configured threshold is (e.g. regExpAlternationGroupThreshold=0 - // with an all-once-through body). - if (repeatedCount < 2 || repeatedCount < Options::regExpAlternationGroupThreshold()) + if (repeatedCount < alternationFactoringMinRun) return; // A DotStarEnclosure records match bounds through the enclosing body @@ -3073,19 +3138,21 @@ class YarrPatternConstructor { private: class SavedContext { public: - SavedContext(bool isModifier, bool invert, MatchDirection matchDirection, OptionSet flags) + SavedContext(bool isModifier, bool invert, MatchDirection matchDirection, bool insideLookbehind, OptionSet flags) : m_isModifier(isModifier) , m_invert(invert) , m_matchDirection(matchDirection) + , m_insideLookbehind(insideLookbehind) , m_flags(flags) { } - void NODELETE restore(bool& isModifier, bool& invert, MatchDirection& matchDirection, OptionSet& flags) + void NODELETE restore(bool& isModifier, bool& invert, MatchDirection& matchDirection, bool& insideLookbehind, OptionSet& flags) { isModifier = m_isModifier; invert = m_invert; matchDirection = m_matchDirection; + insideLookbehind = m_insideLookbehind; flags = m_flags; } @@ -3093,6 +3160,7 @@ class YarrPatternConstructor { bool m_isModifier { false }; bool m_invert { false }; MatchDirection m_matchDirection { Forward }; + bool m_insideLookbehind { false }; OptionSet m_flags; }; @@ -3106,9 +3174,10 @@ class YarrPatternConstructor { ASSERT(m_stackDepth < std::numeric_limits::max()); if (m_stackDepth++ > 0) - m_backingStack.append(SavedContext(m_isModifier, m_invert, m_matchDirection, m_flags)); + m_backingStack.append(SavedContext(m_isModifier, m_invert, m_matchDirection, m_insideLookbehind, m_flags)); - // isModifier should only apply to one frame at a time + // isModifier should only apply to one frame at a time. m_insideLookbehind is + // inherited: a nested parenthesis stays inside any enclosing lookbehind. m_isModifier = false; } @@ -3118,11 +3187,12 @@ class YarrPatternConstructor { if (--m_stackDepth > 0) { SavedContext context = m_backingStack.takeLast(); - context.restore(m_isModifier, m_invert, m_matchDirection, m_flags); + context.restore(m_isModifier, m_invert, m_matchDirection, m_insideLookbehind, m_flags); } else { m_isModifier = false; m_invert = false; m_matchDirection = Forward; + m_insideLookbehind = false; m_flags = { }; } } @@ -3150,6 +3220,10 @@ class YarrPatternConstructor { void setMatchDirection(MatchDirection matchDirection) { m_matchDirection = matchDirection; + // Entering a lookbehind puts every deeper context inside one; the bit is + // inherited through push() and cleared only when this frame pops. + if (matchDirection == Backward) + m_insideLookbehind = true; } MatchDirection matchDirection() const @@ -3157,6 +3231,14 @@ class YarrPatternConstructor { return m_matchDirection; } + // True inside a lookbehind at ANY nesting depth (including within a lookahead + // nested in a lookbehind), unlike matchDirection(), which is only the innermost + // assertion's own direction. + bool NODELETE insideLookbehind() const + { + return m_insideLookbehind; + } + void NODELETE setFlags(OptionSet flags) { m_flags = flags; @@ -3175,6 +3257,7 @@ class YarrPatternConstructor { m_isModifier = false; m_invert = false; m_matchDirection = Forward; + m_insideLookbehind = false; m_flags = { }; } @@ -3184,6 +3267,7 @@ class YarrPatternConstructor { bool m_isModifier { false }; bool m_invert { false }; MatchDirection m_matchDirection { Forward }; + bool m_insideLookbehind { false }; OptionSet m_flags; }; @@ -3217,6 +3301,13 @@ class YarrPatternConstructor { return m_parenthesisContext.matchDirection(); } + // Inside a lookbehind at any nesting depth (a lookahead nested within a + // lookbehind still counts). + bool NODELETE insideLookbehind() const + { + return m_parenthesisContext.insideLookbehind(); + } + bool ignoreCase() const { return m_flags.contains(Flags::IgnoreCase); @@ -3291,7 +3382,6 @@ ErrorCode YarrPattern::compile(StringView patternString) YarrPattern::YarrPattern(StringView pattern, OptionSet flags, ErrorCode& error, ExecutionMode executionMode) : m_containsBackreferences(false) , m_containsBOL(false) - , m_containsBOLGroupBubble(false) , m_containsLookbehinds(false) , m_containsUnsignedLengthPattern(false) , m_containsModifiers(false) diff --git a/Source/JavaScriptCore/yarr/YarrPattern.h b/Source/JavaScriptCore/yarr/YarrPattern.h index ce3bc9561d8f..712c4c4ddc30 100644 --- a/Source/JavaScriptCore/yarr/YarrPattern.h +++ b/Source/JavaScriptCore/yarr/YarrPattern.h @@ -605,7 +605,6 @@ struct YarrPattern { m_containsBackreferences = false; m_containsBOL = false; - m_containsBOLGroupBubble = false; m_containsLookbehinds = false; m_containsUnsignedLengthPattern = false; m_hasCopiedParenSubexpressions = false; @@ -777,7 +776,6 @@ struct YarrPattern { bool m_containsBackreferences : 1; bool m_containsBOL : 1; - bool m_containsBOLGroupBubble : 1; // a group's alternatives all start with BOL (optimizeBOL bubbling) bool m_containsLookbehinds : 1; bool m_containsUnsignedLengthPattern : 1; bool m_containsModifiers : 1; @@ -844,10 +842,15 @@ struct YarrPattern { uintptr_t begin; // Not really needed for greedy quantifiers. uintptr_t matchAmount; // Not really needed for fixed quantifiers. uintptr_t backReferenceSize; // Used by greedy quantifiers to backtrack. + // A backward (lookbehind) backreference's span lies left of the cursor; + // this holds that span's left edge across the compare walk (the JIT's + // forward form gets the equivalent from the moving frontier for free). + uintptr_t backwardSpanEdge; static unsigned beginIndex() { return offsetof(BackTrackInfoBackReference, begin) / sizeof(uintptr_t); } static unsigned matchAmountIndex() { return offsetof(BackTrackInfoBackReference, matchAmount) / sizeof(uintptr_t); } static unsigned backReferenceSizeIndex() { return offsetof(BackTrackInfoBackReference, backReferenceSize) / sizeof(uintptr_t); } + static unsigned backwardSpanEdgeIndex() { return offsetof(BackTrackInfoBackReference, backwardSpanEdge) / sizeof(uintptr_t); } }; struct BackTrackInfoAlternative { From 4d91f2c9ab2e5523cf239b76547d5346f18f7694 Mon Sep 17 00:00:00 2001 From: Dylan Conway Date: Thu, 16 Jul 2026 21:01:59 -0700 Subject: [PATCH 10/13] Yarr: address review nits in the lookbehind/alternation series - Print non-printable first-set characters in hex in the verboseRegExpCompilation dump; the \x prefix was followed by a decimal value. - Remove the two #if/#endif pairs left empty when the ambient first-non-BMP-character scopes were deleted, and a duplicated draft comment line in wrapAlternativesForDispatch. - In regexp-lookbehind-jit.js, compare results with a strict deep equality that distinguishes an undefined array element (a non-participating capture) from null. JSON.stringify maps both to null, so a wrongly-nulled capture could have passed the capture-clearing assertions. No-Verification-Needed: comment, verbose-dump, and test-only edits; no product code path changes --- JSTests/stress/regexp-lookbehind-jit.js | 22 ++++++++++++++++++---- Source/JavaScriptCore/yarr/YarrJIT.cpp | 6 +----- Source/JavaScriptCore/yarr/YarrPattern.cpp | 1 - 3 files changed, 19 insertions(+), 10 deletions(-) diff --git a/JSTests/stress/regexp-lookbehind-jit.js b/JSTests/stress/regexp-lookbehind-jit.js index 76beede8fbf7..1e24fd0c7955 100644 --- a/JSTests/stress/regexp-lookbehind-jit.js +++ b/JSTests/stress/regexp-lookbehind-jit.js @@ -4,11 +4,25 @@ // Lookbehind assertions compiled by the Yarr JIT (mirrored bodies matched // right-to-left) must produce results identical to the interpreter. +// Strict structural equality that distinguishes an undefined array element +// (a non-participating capture group) from null; JSON.stringify would map +// both to null and let a wrongly-nulled capture pass. +function isSameValue(actual, expected) { + if (Array.isArray(actual) || Array.isArray(expected)) { + return Array.isArray(actual) && Array.isArray(expected) + && actual.length === expected.length + && actual.every((value, i) => isSameValue(value, expected[i])); + } + return Object.is(actual, expected); +} + +function describe(value) { + return JSON.stringify(value, (key, v) => (v === undefined ? "" : v)); +} + function shouldBe(actual, expected, message) { - if (JSON.stringify(actual) !== JSON.stringify(expected)) - throw new Error( - (message ? message + ": " : "") + "expected " + JSON.stringify(expected) + " but got " + JSON.stringify(actual), - ); + if (!isSameValue(actual, expected)) + throw new Error((message ? message + ": " : "") + "expected " + describe(expected) + " but got " + describe(actual)); } function matchOf(re, s) { diff --git a/Source/JavaScriptCore/yarr/YarrJIT.cpp b/Source/JavaScriptCore/yarr/YarrJIT.cpp index 970162bdc783..3a1c6a0a2e80 100644 --- a/Source/JavaScriptCore/yarr/YarrJIT.cpp +++ b/Source/JavaScriptCore/yarr/YarrJIT.cpp @@ -2590,8 +2590,6 @@ class YarrGenerator final : public YarrJITInfo { const MacroAssembler::RegisterID character = m_regs.regT0; const MacroAssembler::RegisterID scratch = m_regs.regT1; -#if ENABLE(YARR_JIT_UNICODE_EXPRESSIONS) && ENABLE(YARR_JIT_UNICODE_CAN_INCREMENT_INDEX_FOR_NON_BMP) -#endif MacroAssembler::Jump atBegin; MacroAssembler::JumpList matchDest; @@ -2673,8 +2671,6 @@ class YarrGenerator final : public YarrJITInfo { unsigned subpatternId = term->backReferenceSubpatternId; unsigned duplicateNamedGroupId = m_pattern.hasDuplicateNamedCaptureGroups() ? m_pattern.m_duplicateNamedGroupForSubpatternId[subpatternId] : 0; -#if ENABLE(YARR_JIT_UNICODE_EXPRESSIONS) && ENABLE(YARR_JIT_UNICODE_CAN_INCREMENT_INDEX_FOR_NON_BMP) -#endif MacroAssembler::Label loop(&m_jit); @@ -7368,7 +7364,7 @@ class YarrGenerator final : public YarrJITInfo { if (isASCIIPrintable(ch)) dataLog("'", static_cast(ch), "'"); else - dataLog("\\x", ch); + dataLog("\\x", hex(ch, 2)); } } if (set.wide) diff --git a/Source/JavaScriptCore/yarr/YarrPattern.cpp b/Source/JavaScriptCore/yarr/YarrPattern.cpp index fb82b8f0f766..6d7c04553d0a 100644 --- a/Source/JavaScriptCore/yarr/YarrPattern.cpp +++ b/Source/JavaScriptCore/yarr/YarrPattern.cpp @@ -2696,7 +2696,6 @@ class YarrPatternConstructor { } } - // The group's captures are exactly those of the alternatives it absorbs: // The group's capture span brackets exactly the captures its alternatives // contain, computed from the terms themselves (sorting reorders which // alternative is first, and the parser leaves the last body alternative's From e2ed5f8eef4b9a77e75af39aab5132be1cf98857 Mon Sep 17 00:00:00 2001 From: Dylan Conway Date: Fri, 17 Jul 2026 02:23:03 -0700 Subject: [PATCH 11/13] Yarr: score every Boyer-Moore sub-range, and allow the search on 8-bit unicode subjects The JIT's Boyer-Moore search reads one character and skips a stride of positions when that character cannot start a match. Choosing the range to test only ever scored maximal runs of positions under a candidate-count limit, so a short and highly selective range was thrown away whenever an adjacent position widened its union. /zalpha|qbravo|xcharlie/ is led by three rare characters, but merging the positions behind them pulls in most of the alphabet; the merged set then matches almost every character, scores below the 50% cutoff, and the search was dropped altogether rather than falling back to testing the leading characters alone. Score every sub-range instead. The candidate-count eligibility is kept, since a position matching many characters hits far more often than its sampled frequency suggests once the pattern itself matches often. Extending a range only adds characters, so a range is abandoned as soon as its match probability turns negative: a longer stride can only scale an already-negative score further down. A range whose union is empty is skipped, because the search needs at least one character to test and a frequency of zero would otherwise outscore every real range; an 8-bit compile of a class holding only characters above Latin1 produces exactly that, and no alternative can match there at all. The search was also refused for every unicode pattern, because a stride counts code units and could land inside a surrogate pair. A Latin1 subject holds no surrogates whatever the pattern's flags, and the collection code already keys off m_decodeSurrogatePairs, so gate the search on that rather than on the flag. This enables it for /u and /v patterns matched against 8-bit subjects and leaves 16-bit unicode subjects, where the hazard is real, exactly as they were. Alternation scans over 1.5MB of prose speed up by 2.4-5.9x, and the unicode penalty on 8-bit subjects disappears: /zalpha|qbravo|xcharlie/gu goes from 4.8ms to 1.0ms, matching its non-unicode compile. Compile time grows 4-6% on the longest patterns. Matches are unchanged -- the JIT and the interpreter stay byte-identical across 52,000 randomized differential cases and three 6MB corpora. --- Source/JavaScriptCore/yarr/YarrJIT.cpp | 90 ++++++++++++++------------ 1 file changed, 48 insertions(+), 42 deletions(-) diff --git a/Source/JavaScriptCore/yarr/YarrJIT.cpp b/Source/JavaScriptCore/yarr/YarrJIT.cpp index 3a1c6a0a2e80..fdcec2b32f47 100644 --- a/Source/JavaScriptCore/yarr/YarrJIT.cpp +++ b/Source/JavaScriptCore/yarr/YarrJIT.cpp @@ -216,7 +216,7 @@ class BoyerMooreInfo { void dump(PrintStream&) const; private: - std::tuple findBestCharacterSequence(const SubjectSampler&, unsigned numberOfCandidatesLimit) const; + std::tuple findBestCharacterSequence(const SubjectSampler&) const; Vector m_characters; CharSize m_charSize; @@ -224,33 +224,53 @@ class BoyerMooreInfo { WTF_MAKE_TZONE_ALLOCATED_IMPL(BoyerMooreInfo); -std::tuple BoyerMooreInfo::findBestCharacterSequence(const SubjectSampler& sampler, unsigned numberOfCandidatesLimit) const +std::tuple BoyerMooreInfo::findBestCharacterSequence(const SubjectSampler& sampler) const { + // The search loop tests one character against the union of the candidate sets in [begin, end) + // and shifts by the length of that range, so a longer range shifts further while its union + // matches more often. Score every sub-range: restricting the search to maximal runs would miss + // a short selective range whose neighbours pollute the union, e.g. the distinct leading + // characters of /zalpha|qbravo|xcharlie/, whose union with the following positions matches + // almost everything. + // A position matching many characters is not worth testing: the sampled frequency underestimates + // how often such a set hits when the pattern itself matches often, and the search loop then costs + // more than the match attempts it avoids. + constexpr unsigned maxCandidatesPerCharacter = 16; + static_assert(maxCandidatesPerCharacter < BoyerMooreBitmap::mapSize); int32_t biggestPoint = INT32_MIN; unsigned beginResult = 0; unsigned endResult = 0; - for (unsigned index = 0; index < length();) { - while (index < length() && m_characters[index].count() > numberOfCandidatesLimit) - ++index; - if (index == length()) - break; - unsigned begin = index; + for (unsigned begin = 0; begin < length(); ++begin) { BoyerMooreBitmap::Map map { }; - for (; index < length() && m_characters[index].count() <= numberOfCandidatesLimit; ++index) - map.merge(m_characters[index].map()); - int32_t frequency = 0; - map.forEachSetBit([&](unsigned index) { - frequency += sampler.frequency(index); - }); + for (unsigned end = begin + 1; end <= length(); ++end) { + auto& candidates = m_characters[end - 1]; + if (candidates.count() > maxCandidatesPerCharacter) + break; + candidates.map().forEachSetBit([&](unsigned character) { + if (!map.testAndSet(character)) + frequency += sampler.frequency(character); + }); + + // An empty union is not searchable -- the caller needs at least one candidate character to + // test against. Its frequency of zero would otherwise score higher than any real range and + // suppress the search entirely. This happens in 8-bit compiles of a class holding only + // characters above Latin1, where no alternative can match at all. + if (map.isEmpty()) + continue; - // Cutoff at 50%. If we could encounter the character more than 50%, then BM search would be useless probably. - int32_t matchingProbability = (BoyerMooreBitmap::mapSize / 2) - frequency; - int32_t point = (index - begin) * matchingProbability; - if (point > biggestPoint) { - biggestPoint = point; - beginResult = begin; - endResult = index; + // Cutoff at 50%. If we could encounter the character more than 50%, then BM search would be useless probably. + int32_t matchingProbability = (BoyerMooreBitmap::mapSize / 2) - frequency; + int32_t point = static_cast(end - begin) * matchingProbability; + if (point > biggestPoint) { + biggestPoint = point; + beginResult = begin; + endResult = end; + } + // Extending the range only adds characters, so the probability never recovers and a + // longer stride only scales an already-negative score further down. + if (matchingProbability < 0) + break; } } return std::tuple { biggestPoint, beginResult, endResult }; @@ -258,23 +278,7 @@ std::tuple BoyerMooreInfo::findBestCharacterSequenc std::optional> BoyerMooreInfo::findWorthwhileCharacterSequenceForLookahead(const SubjectSampler& sampler) const { - // If candiates-per-character becomes larger, then sequence is not profitable since this sequence will match against - // too many characters. But if we limit candiates-per-character smaller, it is possible that we only find very short - // character sequence. We start with low limit, then enlarging the limit to find more and more profitable - // character sequence. - int32_t biggestPoint = INT32_MIN; - unsigned begin = 0; - unsigned end = 0; - constexpr unsigned maxCandidatesPerCharacter = 32; - static_assert(maxCandidatesPerCharacter < BoyerMooreBitmap::mapSize); - for (unsigned limit = 4; limit < maxCandidatesPerCharacter; limit *= 2) { - auto [newPoint, newBegin, newEnd] = findBestCharacterSequence(sampler, limit); - if (newPoint > biggestPoint) { - biggestPoint = newPoint; - begin = newBegin; - end = newEnd; - } - } + auto [biggestPoint, begin, end] = findBestCharacterSequence(sampler); if (biggestPoint < 0) return std::nullopt; return std::tuple { begin, end }; @@ -6845,12 +6849,14 @@ class YarrGenerator final : public YarrJITInfo { m_ops.last().m_previousOp = notFound; if (disjunction->m_minimumSize && !m_pattern.sticky()) { - if (!m_pattern.eitherUnicode()) { + if (!m_decodeSurrogatePairs) { // Collect BoyerMooreInfo if it is possible and profitable. BoyerMooreInfo will be used to emit fast skip path with large stride // at the beginning of the body alternatives. - // We do not emit these fast path when RegExp has sticky or unicode flag. Sticky case does not need this since - // it fails when the body alternatives fail to match with the current offset. - // FIXME: Support unicode flag. + // We do not emit these fast path when RegExp has a sticky flag, since it fails when the body alternatives + // fail to match with the current offset. The skip advances by a count of code units, which could land inside + // a surrogate pair, so this is also limited to compiles that decode no pairs: every 8-bit subject (a Latin1 + // string holds no surrogates, whatever the flags), and 16-bit subjects of non-unicode patterns. + // FIXME: Support unicode flag for 16-bit subjects. // https://bugs.webkit.org/show_bug.cgi?id=228611 auto bmInfo = BoyerMooreInfo::create(m_charSize, std::min(disjunction->m_minimumSize, BoyerMooreInfo::maxLength)); if (collectBoyerMooreInfo(disjunction, currentAlternativeIndex, bmInfo.get())) { From 8810af372af7e3004a1d56554188116ea2fb60e8 Mon Sep 17 00:00:00 2001 From: Dylan Conway Date: Fri, 17 Jul 2026 02:58:33 -0700 Subject: [PATCH 12/13] Yarr: pick the Boyer-Moore search loop by how often its candidates appear The JIT has two Boyer-Moore searches: a vector one reading sixteen characters per iteration, and a scalar one reading a character per iteration. The vector search was preferred wherever it applies, but leaving its loop costs a vector to general purpose register transfer, so it pays that price once per candidate that fails to match. It wins by 8x on a subject holding no candidate at all and loses by 6.5x once half the characters are candidates, and nothing measured which side of that it was on. How often a candidate fails is not known when compiling, so use the frequency the subject sampler already collects for choosing the search range. The sampler scales its count to a rate, which a subject shorter than the sample cannot resolve -- one candidate in four characters reads as a rate of 32 -- and such a subject says little about the later ones a compile is reused for, since a RegExp keeps the first one's code. Count the candidates instead in that case, as an absent sample already does, so an incidental first subject cannot decide the search for the whole process. That choice then replaces the candidate-count limit on which positions the range search may test. The limit stood in for "this set matches too often", but the sampled frequency measures that directly, and the cost it was really standing in for was the vector loop's. It also rejected the best filters: /([a-z0-9])([A-Z])/ over source text was left with no search at all because both of its positions hold more characters than the limit allows, though upper case runs at nearly zero frequency there and rejects every position at once. Positions matching every character still have to be excluded, for a different and load-bearing reason: setAll() records them in the count alone and leaves the map empty, so merging one drops every character it matches out of the union and the search skips positions that can start a match. The count limit had been hiding that, since such positions count as the whole map. Test it directly instead. /([a-z0-9])([A-Z])/ over source text goes from 2.2ms to 0.1ms, keyword alternation scans get 1.8-2.4x faster, and no case measured across a 26 benchmark suite, 20 library patterns and a scan suite gets slower. Matches are unchanged: the JIT and the interpreter stay byte-identical over 34,000 randomized differential cases and three 6MB corpora, and test262 and the regexp stress tests are unmoved on both tiers. The crossover was measured on arm64. --- Source/JavaScriptCore/yarr/YarrJIT.cpp | 40 ++++++++++++++++++++++---- 1 file changed, 34 insertions(+), 6 deletions(-) diff --git a/Source/JavaScriptCore/yarr/YarrJIT.cpp b/Source/JavaScriptCore/yarr/YarrJIT.cpp index fdcec2b32f47..f2fd9aab2fd1 100644 --- a/Source/JavaScriptCore/yarr/YarrJIT.cpp +++ b/Source/JavaScriptCore/yarr/YarrJIT.cpp @@ -108,11 +108,27 @@ class SubjectSampler { public: static constexpr unsigned sampleSize = 128; + // Frequency, in units of the sample above, past which the vector search stops paying for + // itself. Leaving the vector loop costs a vector to general purpose register transfer, so its + // price is paid per candidate that fails to match. How often a candidate fails is not known + // when compiling, leaving this frequency as the proxy. Measured on arm64: from here the scalar + // loop is 2.2x faster on a keyword alternation whose candidates almost all fail, while below it + // the vector loop reaches 8x on subjects holding no candidate at all. The vector loop leaves + // more cheaply on x86_64, where the crossover sits higher and this gives up some of its reach. + static constexpr int32_t maxCandidateFrequencyForSIMDSearch = 8; + + // Below this, one occurrence alone already scales past maxCandidateFrequencyForSIMDSearch, so + // the sample cannot tell a rare candidate from a common one. Such a subject also says little + // about the later subjects a compile is reused for, since RegExp caches keep the first one's. + static constexpr unsigned minimumSampleSizeForFrequency = sampleSize / maxCandidateFrequencyForSIMDSearch; + explicit SubjectSampler(CharSize charSize) : m_is8Bit(charSize == CharSize::Char8) { } + bool canResolveCandidateFrequency() const { return m_size >= minimumSampleSizeForFrequency; } + int32_t NODELETE frequency(char16_t character) const { if (!m_size) @@ -232,11 +248,6 @@ std::tuple BoyerMooreInfo::findBestCharacterSequenc // a short selective range whose neighbours pollute the union, e.g. the distinct leading // characters of /zalpha|qbravo|xcharlie/, whose union with the following positions matches // almost everything. - // A position matching many characters is not worth testing: the sampled frequency underestimates - // how often such a set hits when the pattern itself matches often, and the search loop then costs - // more than the match attempts it avoids. - constexpr unsigned maxCandidatesPerCharacter = 16; - static_assert(maxCandidatesPerCharacter < BoyerMooreBitmap::mapSize); int32_t biggestPoint = INT32_MIN; unsigned beginResult = 0; unsigned endResult = 0; @@ -245,7 +256,11 @@ std::tuple BoyerMooreInfo::findBestCharacterSequenc int32_t frequency = 0; for (unsigned end = begin + 1; end <= length(); ++end) { auto& candidates = m_characters[end - 1]; - if (candidates.count() > maxCandidatesPerCharacter) + // A position matching every character records that in its count alone, leaving its map + // empty, so merging one would drop every character it matches out of the union and the + // search would skip positions that can start a match. It could only widen the union to + // everything in any case, which no search can profit from. + if (candidates.isAllSet()) break; candidates.map().forEachSetBit([&](unsigned character) { if (!map.testAndSet(character)) @@ -7972,6 +7987,19 @@ class YarrGenerator final : public YarrJITInfo { if (strideLength != 1) return std::nullopt; + // Leave the search scalar once candidates are too common for the vector loop's exit to pay + // for its reads. A sample too short to rate them falls back to counting them, as an absent + // sample already does. + int32_t frequency = 0; + if (m_sampler.canResolveCandidateFrequency()) { + bitmap.forEachSetBit([&](unsigned character) { + frequency += m_sampler.frequency(character); + }); + } else + frequency = bitmap.count(); + if (frequency > SubjectSampler::maxCandidateFrequencyForSIMDSearch) + return std::nullopt; + // Call can clobber SIMD registers if (mayCall()) return std::nullopt; From ab39369a3698b8c0c898a176f6b87c80e23980e0 Mon Sep 17 00:00:00 2001 From: Dylan Conway Date: Fri, 17 Jul 2026 03:24:25 -0700 Subject: [PATCH 13/13] Yarr: cover the Boyer-Moore search's any-character positions The search may not span a position that matches every character: setAll() records one in its count alone and leaves its map empty, so merging it drops every character it matches out of the range's union and the search then skips positions that can start a match. Nothing states that invariant, and nothing tested it -- all eight existing bm-search tests pass against a build that violates it, while "bab".match(/.(a)./) returns null. Every case expects a match, because each failure in this class is a false negative and an expectation of null would be exactly what a broken search produces. Each is written with single dots rather than a quantified count, since a count aborts collection before any position is recorded and would leave nothing to get wrong: of an earlier draft's assertions, only the unquantified ones failed against the violating build. The remaining cases cover the search now being available to unicode patterns over Latin1 subjects. No-Verification-Needed: test-only; the engine change it covers landed with its own verification --- ...regexp-bm-search-any-character-position.js | 39 +++++++++++++++++++ 1 file changed, 39 insertions(+) create mode 100644 JSTests/stress/regexp-bm-search-any-character-position.js diff --git a/JSTests/stress/regexp-bm-search-any-character-position.js b/JSTests/stress/regexp-bm-search-any-character-position.js new file mode 100644 index 000000000000..ed46ac8a04f8 --- /dev/null +++ b/JSTests/stress/regexp-bm-search-any-character-position.js @@ -0,0 +1,39 @@ +function shouldBe(actual, expected) { + if (actual !== expected) + throw new Error("bad value: " + actual + " (expected " + expected + ")"); +} + +// A BM search range must never span a position that matches every character. setAll() records +// such a position in its count alone and leaves its map empty, so merging one drops every +// character it matches out of the range's union, and the search then skips past positions that +// can start a match. Every case below therefore expects a MATCH: the failure is a false negative, +// so an expectation of null could not catch it. Each one is written with single dots rather than +// `.{n}`, because a quantified count aborts BM collection before any position is recorded and +// would leave nothing for the search to get wrong. +for (var i = 0; i < 1e2; ++i) { + shouldBe(JSON.stringify("bab".match(/.(a)./)), '["bab","a"]'); + shouldBe(JSON.stringify("bab".match(/.(a)./u)), '["bab","a"]'); + shouldBe(JSON.stringify("bab".match(/.(a)./du).indices), "[[0,3],[1,2]]"); + shouldBe(JSON.stringify("xaz".match(/.a./)), '["xaz"]'); + shouldBe(JSON.stringify("qqzaaqq".match(/z.a/)), '["zaa"]'); + shouldBe(JSON.stringify("aaaXb".match(/X./)), '["Xb"]'); + shouldBe(JSON.stringify("abcQdef".match(/...Q.../)), '["abcQdef"]'); + shouldBe(JSON.stringify("zzza1b2c".match(/a.b.c/)), '["a1b2c"]'); + shouldBe(JSON.stringify("zzq12r".match(/q..r/)), '["q12r"]'); + // A dot that cannot match a newline, and inverted classes, are all-set positions too. + shouldBe(JSON.stringify("a\nbXc".match(/b.c/)), '["bXc"]'); + shouldBe(JSON.stringify("aaaaK".match(/[^0-9]K/)), '["aK"]'); + shouldBe(JSON.stringify(("z".repeat(37) + "yP").match(/[^P]P/)), '["yP"]'); + // A range wider than the map turns into an all-set position the same way. + shouldBe(JSON.stringify("aaaŐeaaa".match(/[Ā-Ȁ]e/)), '["Őe"]'); +} + +// The search is available to unicode patterns whose subject is Latin1, where no surrogate pair +// exists for a code-unit stride to land inside. These cover that newly reachable path; they too +// expect matches, so an over-skipping search fails them. +var subject = "the quick brown fox jumps over the lazy dog. ".repeat(64); +for (var i = 0; i < 1e2; ++i) { + shouldBe(JSON.stringify(("xxqbravo" + subject + "zalpha").match(/zalpha|qbravo|xcharlie/gu)), '["qbravo","zalpha"]'); + shouldBe(subject.match(/\bfox\b/gu).length, 64); + shouldBe(JSON.stringify("abcd".match(new RegExp("[\\q{abc}d]", "gv"))), '["abc","d"]'); +}