diff --git a/JSTests/stress/regexp-bm-search-any-character-position.js b/JSTests/stress/regexp-bm-search-any-character-position.js new file mode 100644 index 000000000000..ed46ac8a04f8 --- /dev/null +++ b/JSTests/stress/regexp-bm-search-any-character-position.js @@ -0,0 +1,39 @@ +function shouldBe(actual, expected) { + if (actual !== expected) + throw new Error("bad value: " + actual + " (expected " + expected + ")"); +} + +// A BM search range must never span a position that matches every character. setAll() records +// such a position in its count alone and leaves its map empty, so merging one drops every +// character it matches out of the range's union, and the search then skips past positions that +// can start a match. Every case below therefore expects a MATCH: the failure is a false negative, +// so an expectation of null could not catch it. Each one is written with single dots rather than +// `.{n}`, because a quantified count aborts BM collection before any position is recorded and +// would leave nothing for the search to get wrong. +for (var i = 0; i < 1e2; ++i) { + shouldBe(JSON.stringify("bab".match(/.(a)./)), '["bab","a"]'); + shouldBe(JSON.stringify("bab".match(/.(a)./u)), '["bab","a"]'); + shouldBe(JSON.stringify("bab".match(/.(a)./du).indices), "[[0,3],[1,2]]"); + shouldBe(JSON.stringify("xaz".match(/.a./)), '["xaz"]'); + shouldBe(JSON.stringify("qqzaaqq".match(/z.a/)), '["zaa"]'); + shouldBe(JSON.stringify("aaaXb".match(/X./)), '["Xb"]'); + shouldBe(JSON.stringify("abcQdef".match(/...Q.../)), '["abcQdef"]'); + shouldBe(JSON.stringify("zzza1b2c".match(/a.b.c/)), '["a1b2c"]'); + shouldBe(JSON.stringify("zzq12r".match(/q..r/)), '["q12r"]'); + // A dot that cannot match a newline, and inverted classes, are all-set positions too. + shouldBe(JSON.stringify("a\nbXc".match(/b.c/)), '["bXc"]'); + shouldBe(JSON.stringify("aaaaK".match(/[^0-9]K/)), '["aK"]'); + shouldBe(JSON.stringify(("z".repeat(37) + "yP").match(/[^P]P/)), '["yP"]'); + // A range wider than the map turns into an all-set position the same way. + shouldBe(JSON.stringify("aaaŐeaaa".match(/[Ā-Ȁ]e/)), '["Őe"]'); +} + +// The search is available to unicode patterns whose subject is Latin1, where no surrogate pair +// exists for a code-unit stride to land inside. These cover that newly reachable path; they too +// expect matches, so an over-skipping search fails them. +var subject = "the quick brown fox jumps over the lazy dog. ".repeat(64); +for (var i = 0; i < 1e2; ++i) { + shouldBe(JSON.stringify(("xxqbravo" + subject + "zalpha").match(/zalpha|qbravo|xcharlie/gu)), '["qbravo","zalpha"]'); + shouldBe(subject.match(/\bfox\b/gu).length, 64); + shouldBe(JSON.stringify("abcd".match(new RegExp("[\\q{abc}d]", "gv"))), '["abc","d"]'); +} diff --git a/JSTests/stress/regexp-lookbehind-jit.js b/JSTests/stress/regexp-lookbehind-jit.js new file mode 100644 index 000000000000..1e24fd0c7955 --- /dev/null +++ b/JSTests/stress/regexp-lookbehind-jit.js @@ -0,0 +1,187 @@ +//@ runDefault("--useRegExpJIT=true") +//@ runNoJIT("--useRegExpJIT=false") + +// Lookbehind assertions compiled by the Yarr JIT (mirrored bodies matched +// right-to-left) must produce results identical to the interpreter. + +// Strict structural equality that distinguishes an undefined array element +// (a non-participating capture group) from null; JSON.stringify would map +// both to null and let a wrongly-nulled capture pass. +function isSameValue(actual, expected) { + if (Array.isArray(actual) || Array.isArray(expected)) { + return Array.isArray(actual) && Array.isArray(expected) + && actual.length === expected.length + && actual.every((value, i) => isSameValue(value, expected[i])); + } + return Object.is(actual, expected); +} + +function describe(value) { + return JSON.stringify(value, (key, v) => (v === undefined ? "" : v)); +} + +function shouldBe(actual, expected, message) { + if (!isSameValue(actual, expected)) + throw new Error((message ? message + ": " : "") + "expected " + describe(expected) + " but got " + describe(actual)); +} + +function matchOf(re, s) { + let m = re.exec(s); + return m ? [m.index, [...m]] : null; +} + +// Fixed-width bodies. +shouldBe(matchOf(/(?<=x)a/, "xa"), [1, ["a"]]); +shouldBe(matchOf(/(?<=x)a/, "ya"), null); +shouldBe(matchOf(/(? re.test(l)), + [true, false, false, true, true, false, true, true], + ); +} + +// Lookbehind patterns whose alternatives contain a group that is BOL-anchored +// (an optional/quantified/negated group starting with ^) keep the results the +// interpreter gives: the JIT must not (mis)handle their once-through split. +shouldBe(JSON.stringify(/a(?^)/.exec("qx"), null); +shouldBe(/c(?:^(?:x))/.exec("acx"), null); +shouldBe(JSON.stringify(/(\w(^)\s|$)/i.exec("a x")), '["","",null]'); +// Genuine anchoring is unchanged. +shouldBe(/^a/.exec("ba"), null); +shouldBe(/(?:^a)/.exec("ba"), null); +shouldBe(/(^a)+/.exec("ba"), null); +shouldBe(/(?:^a|^b)/.exec("cb"), null); diff --git a/Source/JavaScriptCore/runtime/RegExp.cpp b/Source/JavaScriptCore/runtime/RegExp.cpp index f01fc40ffa73..81dbb090a605 100644 --- a/Source/JavaScriptCore/runtime/RegExp.cpp +++ b/Source/JavaScriptCore/runtime/RegExp.cpp @@ -318,7 +318,6 @@ void RegExp::compile(VM* vm, Yarr::CharSize charSize, std::optional #if !ENABLE(YARR_JIT_BACKREFERENCES) && !pattern.m_containsBackreferences #endif - && !pattern.m_containsLookbehinds ) { auto& jitCode = ensureRegExpJITCode(); Yarr::jitCompile(pattern, m_patternString, charSize, sampleString, vm, jitCode, Yarr::ExecutionMode::IncludeSubpatterns); @@ -386,7 +385,6 @@ void RegExp::compileMatchOnly(VM* vm, Yarr::CharSize charSize, std::optional= minimumSampleSizeForFrequency; } + int32_t NODELETE frequency(char16_t character) const { if (!m_size) @@ -216,7 +232,7 @@ class BoyerMooreInfo { void dump(PrintStream&) const; private: - std::tuple findBestCharacterSequence(const SubjectSampler&, unsigned numberOfCandidatesLimit) const; + std::tuple findBestCharacterSequence(const SubjectSampler&) const; Vector m_characters; CharSize m_charSize; @@ -224,33 +240,52 @@ class BoyerMooreInfo { WTF_MAKE_TZONE_ALLOCATED_IMPL(BoyerMooreInfo); -std::tuple BoyerMooreInfo::findBestCharacterSequence(const SubjectSampler& sampler, unsigned numberOfCandidatesLimit) const +std::tuple BoyerMooreInfo::findBestCharacterSequence(const SubjectSampler& sampler) const { + // The search loop tests one character against the union of the candidate sets in [begin, end) + // and shifts by the length of that range, so a longer range shifts further while its union + // matches more often. Score every sub-range: restricting the search to maximal runs would miss + // a short selective range whose neighbours pollute the union, e.g. the distinct leading + // characters of /zalpha|qbravo|xcharlie/, whose union with the following positions matches + // almost everything. int32_t biggestPoint = INT32_MIN; unsigned beginResult = 0; unsigned endResult = 0; - for (unsigned index = 0; index < length();) { - while (index < length() && m_characters[index].count() > numberOfCandidatesLimit) - ++index; - if (index == length()) - break; - unsigned begin = index; + for (unsigned begin = 0; begin < length(); ++begin) { BoyerMooreBitmap::Map map { }; - for (; index < length() && m_characters[index].count() <= numberOfCandidatesLimit; ++index) - map.merge(m_characters[index].map()); - int32_t frequency = 0; - map.forEachSetBit([&](unsigned index) { - frequency += sampler.frequency(index); - }); + for (unsigned end = begin + 1; end <= length(); ++end) { + auto& candidates = m_characters[end - 1]; + // A position matching every character records that in its count alone, leaving its map + // empty, so merging one would drop every character it matches out of the union and the + // search would skip positions that can start a match. It could only widen the union to + // everything in any case, which no search can profit from. + if (candidates.isAllSet()) + break; + candidates.map().forEachSetBit([&](unsigned character) { + if (!map.testAndSet(character)) + frequency += sampler.frequency(character); + }); + + // An empty union is not searchable -- the caller needs at least one candidate character to + // test against. Its frequency of zero would otherwise score higher than any real range and + // suppress the search entirely. This happens in 8-bit compiles of a class holding only + // characters above Latin1, where no alternative can match at all. + if (map.isEmpty()) + continue; - // Cutoff at 50%. If we could encounter the character more than 50%, then BM search would be useless probably. - int32_t matchingProbability = (BoyerMooreBitmap::mapSize / 2) - frequency; - int32_t point = (index - begin) * matchingProbability; - if (point > biggestPoint) { - biggestPoint = point; - beginResult = begin; - endResult = index; + // Cutoff at 50%. If we could encounter the character more than 50%, then BM search would be useless probably. + int32_t matchingProbability = (BoyerMooreBitmap::mapSize / 2) - frequency; + int32_t point = static_cast(end - begin) * matchingProbability; + if (point > biggestPoint) { + biggestPoint = point; + beginResult = begin; + endResult = end; + } + // Extending the range only adds characters, so the probability never recovers and a + // longer stride only scales an already-negative score further down. + if (matchingProbability < 0) + break; } } return std::tuple { biggestPoint, beginResult, endResult }; @@ -258,23 +293,7 @@ std::tuple BoyerMooreInfo::findBestCharacterSequenc std::optional> BoyerMooreInfo::findWorthwhileCharacterSequenceForLookahead(const SubjectSampler& sampler) const { - // If candiates-per-character becomes larger, then sequence is not profitable since this sequence will match against - // too many characters. But if we limit candiates-per-character smaller, it is possible that we only find very short - // character sequence. We start with low limit, then enlarging the limit to find more and more profitable - // character sequence. - int32_t biggestPoint = INT32_MIN; - unsigned begin = 0; - unsigned end = 0; - constexpr unsigned maxCandidatesPerCharacter = 32; - static_assert(maxCandidatesPerCharacter < BoyerMooreBitmap::mapSize); - for (unsigned limit = 4; limit < maxCandidatesPerCharacter; limit *= 2) { - auto [newPoint, newBegin, newEnd] = findBestCharacterSequence(sampler, limit); - if (newPoint > biggestPoint) { - biggestPoint = newPoint; - begin = newBegin; - end = newEnd; - } - } + auto [biggestPoint, begin, end] = findBestCharacterSequence(sampler); if (biggestPoint < 0) return std::nullopt; return std::tuple { begin, end }; @@ -314,6 +333,85 @@ void BoyerMooreBitmap::dump(PrintStream& out) const // // FIXME: Currently we are only supporting 2 alternative cases at first, aligned to V8's optimization. // We will extend it to 1 and 3 later. +// First-character set of an alternative: an over-approximation of the set of +// characters a match can start with. Latin-1 characters are tracked precisely; +// `wide` means a first character >= 0x100 is possible; `any` is the top +// element (every character possible). An over-approximation is always sound +// for dispatch: membership merely means the alternative is tried there. +struct FirstCharacterSet { + WTF::BitSet<256> latin1; + bool wide { false }; + bool any { false }; + // Whether analysis reached a term that must consume a character. If false + // the alternative can match the empty string, which for dispatch is `any`. + bool consumed { false }; + + void add(char32_t ch) + { + if (ch < 256) + latin1.set(ch); + else + wide = true; + } + void addRange(char32_t begin, char32_t end) + { + for (char32_t ch = begin; ch <= end && ch < 256; ++ch) + latin1.set(ch); + if (end >= 256) + wide = true; + } + void merge(const FirstCharacterSet& other) + { + latin1.merge(other.latin1); + wide |= other.wide; + any |= other.any; + } + bool matches(unsigned latin1Char) const { return any || latin1.get(latin1Char); } + bool matchesWide() const { return any || wide; } +}; + +// First-character dispatch for a nested disjunction with many alternatives. +// +// Trying N alternatives in sequence costs an entry attempt each. When every +// alternative must consume a first character, that character is already inside +// input the enclosing alternative claimed, so it can be read once (no bounds +// check) and a decision tree over its value can jump straight to the ordered +// chain of alternatives whose first-character set contains it. Alternatives +// are still tried in source order within a chain, so leftmost-first semantics +// hold; an alternative whose set is `any` appears in every chain. A failing +// alternative continues its chain through a code address the chain stored in +// the group's frame (BackTrackInfoParenthesesOnce::chainResume) before entry. +struct DispatchInfo { +private: + WTF_MAKE_TZONE_ALLOCATED(DispatchInfo); +public: + struct Chain { + Vector alternativeIndices; // ascending; tried in this order + MacroAssembler::Label head; // start of the chain's stub sequence + }; + + Vector chains; // distinct Latin-1 chains + std::array chainForCharacter { }; // Latin-1 char -> chain index, or noChain + Chain wideChain; // characters >= 0x100 (16-bit strings only) + unsigned frameLocation { 0 }; // the group's frame; chainResume lives here + Checked firstCharacterOffset; // read offset of the group's first character + PatternDisjunction* disjunction { nullptr }; // the dispatched alternatives + Checked enclosingCheckedOffset; // checkedOffset at the group term + size_t endOpIndex { 0 }; // the group's NestedAlternativeEnd op (success join point) + + // Per alternative: jumps from chain stubs awaiting that alternative's + // entry point (always a forward reference; see emitJumpToDispatchEntry). + Vector pendingEntryJumps; + + // Failures that mean "no alternative can match here" (empty chains, chain + // exhaustion). Routed as the group's failure with the index unmoved. + MacroAssembler::JumpList groupFailJumps; + + static constexpr unsigned noChain = std::numeric_limits::max(); +}; + +WTF_MAKE_TZONE_ALLOCATED_IMPL(DispatchInfo); + struct MaskedAlternativeInfo { private: WTF_MAKE_TZONE_ALLOCATED(MaskedAlternativeInfo); @@ -1224,13 +1322,19 @@ class YarrGenerator final : public YarrJITInfo { matchCharacterClassOnlyOneRange(character, scratch, failures, characterClassToProcess->m_ranges8.size() ? characterClassToProcess->m_ranges8 : characterClassToProcess->m_ranges32); } else { MacroAssembler::JumpList matchDest; - // If we are matching the "any character" builtin class for non-unicode patterns, - // we only need to read the character and don't need to match as it will always succeed. + // If we are matching the "any character" builtin class we only need to read the + // character and don't need to match, as it will always succeed -- except that in a + // unicode pattern a dangling surrogate reads as errorCodePoint, which is not a + // character and must fail (the inverted-class path above rejects it likewise). if (!characterClassToProcess->m_anyCharacter) { matchCharacterClass(character, scratch, MatchTargets(matchDest, failures, MatchTargets::PreferredTarget::MatchSuccessFallThrough), characterClassToProcess); if (!matchDest.empty()) failures.append(m_jit.jump()); } +#if ENABLE(YARR_JIT_UNICODE_EXPRESSIONS) + else if (m_decodeSurrogatePairs) + failures.append(m_jit.branch32(MacroAssembler::Equal, character, MacroAssembler::TrustedImm32(errorCodePoint))); +#endif matchDest.link(&m_jit); } }; @@ -1260,38 +1364,119 @@ class YarrGenerator final : public YarrJITInfo { } #endif + // Input availability primitives. + // + // Forward: `index` is the frontier of claimed input; claiming N characters is + // `index += N` (fails if that passes `length`). + // Backward (lookbehind body): `index` is the leftward-moving cursor / left + // frontier; claiming N characters is `index -= N` (fails if it goes below + // zero). Both `index` and any claimable N are < 2^31: `index <= length` and + // string lengths are bounded by INT32_MAX, and a claim count larger than that + // marks the pattern containsUnsignedLengthPattern(), which never reaches the + // JIT. So a borrowing subtraction sets the sign bit, letting us branch on the + // Signed/PositiveOrZero result of the same instruction, exactly mirroring the + // forward add-then-compare. + // Jumps if input not available; will have (incorrectly) incremented already! MacroAssembler::Jump jumpIfNoAvailableInput(unsigned countToCheck = 0) { + if (m_direction == Backward) { + // Backward: the frontier is exhausted once index would go below zero. + // With a count, claim it and branch on the borrow; with none, just test + // the current frontier's sign (the leftward mirror of index > length). + if (countToCheck) + return m_jit.branchSub32(MacroAssembler::Signed, MacroAssembler::TrustedImm32(countToCheck), m_regs.index); + return m_jit.branch32(MacroAssembler::LessThan, m_regs.index, MacroAssembler::TrustedImm32(0)); + } if (countToCheck) m_jit.add32(MacroAssembler::Imm32(countToCheck), m_regs.index); return m_jit.branch32(MacroAssembler::Above, m_regs.index, m_regs.length); } + // Extend the leftward claim by one unit (a decoded pair's lead). On the + // failure path the borrowed unit is restored before joining `failures`, so + // a negative frontier is never left live where another term (e.g. a sibling + // op's non-greedy re-entry) could address input through it before this term's + // own backtrack restores its beginIndex. + void claimBackwardPairLead(MacroAssembler::JumpList& failures) + { + ASSERT(m_direction == Backward); + MacroAssembler::Jump claimed = m_jit.branchSub32(MacroAssembler::PositiveOrZero, MacroAssembler::TrustedImm32(1), m_regs.index); + m_jit.add32(MacroAssembler::TrustedImm32(1), m_regs.index); // undo the failed borrow + failures.append(m_jit.jump()); + claimed.link(&m_jit); + } + MacroAssembler::Jump jumpIfAvailableInput(unsigned countToCheck) { + if (m_direction == Backward) + return m_jit.branchSub32(MacroAssembler::PositiveOrZero, MacroAssembler::TrustedImm32(countToCheck), m_regs.index); m_jit.add32(MacroAssembler::Imm32(countToCheck), m_regs.index); return m_jit.branch32(MacroAssembler::BelowOrEqual, m_regs.index, m_regs.length); } MacroAssembler::Jump checkNotEnoughInput(MacroAssembler::RegisterID additionalAmount) { + ASSERT(m_direction == Forward); m_jit.add32(m_regs.index, additionalAmount); return m_jit.branch32(MacroAssembler::Above, additionalAmount, m_regs.length); } + // Backward (lookbehind) form: a dynamic span of `amount` units must lie + // wholly left of the cursor. Computes the span's left edge (index - amount) + // into `edge` and jumps if it would be negative; the caller then walks the + // span rightward from `edge` and finally moves the frontier down to it. + MacroAssembler::Jump checkNotEnoughInputBackward(MacroAssembler::RegisterID amount, MacroAssembler::RegisterID edge) + { + ASSERT(m_direction == Backward); + m_jit.move(m_regs.index, edge); + m_jit.sub32(amount, edge); + return m_jit.branch32(MacroAssembler::LessThan, edge, MacroAssembler::TrustedImm32(0)); + } + + // A backreference's span of `size` units: forward, it lies right of the + // frontier (availability check only; the walk advances the frontier through + // it); in a mirrored frame it lies left of the frontier, so the frontier is + // first lowered to the span's left edge -- recorded in the frame so the walk, + // which raises index back up as it compares, can be undone by + // emitBackReferenceSpanFrontier -- and the walk proceeds rightward from there. + void emitBackReferenceSpanClaim(MacroAssembler::JumpList& notAvailable, MacroAssembler::RegisterID size, MacroAssembler::RegisterID edge, unsigned frameLocation) + { + if (m_direction != Backward) { + notAvailable.append(checkNotEnoughInput(size)); + return; + } + notAvailable.append(checkNotEnoughInputBackward(size, edge)); + storeToFrame(edge, frameLocation + BackTrackInfoBackReference::backwardSpanEdgeIndex()); + m_jit.move(edge, m_regs.index); + } + + // After a mirrored backreference walk, the span has been consumed leftward: + // the frontier rests at the span's left edge, not where the walk left it. + void emitBackReferenceSpanFrontier(unsigned frameLocation) + { + if (m_direction == Backward) + loadFromFrame(frameLocation + BackTrackInfoBackReference::backwardSpanEdgeIndex(), m_regs.index); + } + MacroAssembler::Jump checkInput() { + if (m_direction == Backward) + return m_jit.branch32(MacroAssembler::GreaterThanOrEqual, m_regs.index, MacroAssembler::TrustedImm32(0)); return m_jit.branch32(MacroAssembler::BelowOrEqual, m_regs.index, m_regs.length); } MacroAssembler::Jump atEndOfInput() { + if (m_direction == Backward) + return m_jit.branchTest32(MacroAssembler::Zero, m_regs.index); return m_jit.branch32(MacroAssembler::Equal, m_regs.index, m_regs.length); } MacroAssembler::Jump notAtEndOfInput() { + if (m_direction == Backward) + return m_jit.branchTest32(MacroAssembler::NonZero, m_regs.index); return m_jit.branch32(MacroAssembler::NotEqual, m_regs.index, m_regs.length); } @@ -1312,6 +1497,31 @@ class YarrGenerator final : public YarrJITInfo { // as a temp address register. unsigned maximumNegativeOffsetForCharacterSize = m_charSize == CharSize::Char8 ? 0x7fffffff : 0x3fffffff; unsigned offsetAdjustAmount = 0x40000000; + + if (m_direction == Backward) { + // In a lookbehind body the index register is the left frontier of the + // (leftward) claimed input. A forward frame offset k (1..checkedTotal) + // names the k'th character behind the forward frontier; mirrored, that + // character lives at input[index + (k - 1)]. k == 0 (a peek at the + // frontier itself, used by anchors) maps to input[index - 1]. + Checked characterOffset = static_cast(negativeCharacterOffset.value()) - 1; + unsigned positiveCharacterOffset = negativeCharacterOffset.value() ? negativeCharacterOffset.value() - 1 : 0; + if (positiveCharacterOffset > maximumNegativeOffsetForCharacterSize) { + base = tempReg; + m_jit.move(m_regs.input, base); + while (positiveCharacterOffset > maximumNegativeOffsetForCharacterSize) { + m_jit.addPtr(MacroAssembler::TrustedImm32(offsetAdjustAmount), base); + if (m_charSize != CharSize::Char8) + m_jit.addPtr(MacroAssembler::TrustedImm32(offsetAdjustAmount), base); + positiveCharacterOffset -= offsetAdjustAmount; + } + characterOffset = static_cast(positiveCharacterOffset); + } + if (m_charSize == CharSize::Char8) + return MacroAssembler::BaseIndex(base, indexReg, MacroAssembler::TimesOne, characterOffset.value() * static_cast(sizeof(char))); + return MacroAssembler::BaseIndex(base, indexReg, MacroAssembler::TimesTwo, characterOffset.value() * static_cast(sizeof(char16_t))); + } + if (negativeCharacterOffset > maximumNegativeOffsetForCharacterSize) { base = tempReg; m_jit.move(m_regs.input, base); @@ -1357,6 +1567,51 @@ class YarrGenerator final : public YarrJITInfo { tryReadUnicodeCharImpl(*m_vm, m_jit, resultReg); } + // Backward unicode read: decode the code point whose LAST code unit lives at + // `address` (the mirror of tryReadUnicodeChar, which decodes from a first + // unit). Matches the bytecode interpreter's tryReadBackward exactly: + // - load the unit at `address`; + // - if it is a trail surrogate and the unit before it (within the string) + // is a lead surrogate, the result is the supplementary code point; + // - otherwise the result is the unit itself. A lone lead or lone trail is + // returned as-is, never as an error (deliberately unlike the forward + // reader), because a backward match position can legitimately sit inside + // a surrogate pair the pattern splits. + // Callers recover the consumed width from the result: >= 0x10000 means a + // decoded pair (2 units), anything lower is a single unit -- exactly as the + // forward paths branch on `character < 0x10000`. `temp2` is a scratch. + void readUnicodeCharBackward(MacroAssembler::BaseIndex address, MacroAssembler::RegisterID resultReg, MacroAssembler::RegisterID temp2) + { + ASSERT(m_charSize == CharSize::Char16); + ASSERT(resultReg != temp2); + const MacroAssembler::RegisterID unitAddress = m_regs.regUnicodeInputAndTrail; + const MacroAssembler::RegisterID temp = m_regs.unicodeAndSubpatternIdTemp; + + m_jit.getEffectiveAddress(address, unitAddress); + m_jit.load16(MacroAssembler::Address(unitAddress), resultReg); + + MacroAssembler::JumpList notAPair; + // resultReg is a trail surrogate iff (resultReg & 0xfc00) == 0xdc00. + m_jit.move(resultReg, temp); + m_jit.and32(MacroAssembler::TrustedImm32(0xfc00), temp); + notAPair.append(m_jit.branch32(MacroAssembler::NotEqual, temp, MacroAssembler::TrustedImm32(0xdc00))); + // The unit before must exist (unitAddress > input) ... + notAPair.append(m_jit.branchPtr(MacroAssembler::BelowOrEqual, unitAddress, m_regs.input)); + // ... and be a lead surrogate: (unit & 0xfc00) == 0xd800. + m_jit.load16(MacroAssembler::Address(unitAddress, -static_cast(sizeof(char16_t))), temp2); + m_jit.move(temp2, temp); + m_jit.and32(MacroAssembler::TrustedImm32(0xfc00), temp); + notAPair.append(m_jit.branch32(MacroAssembler::NotEqual, temp, MacroAssembler::TrustedImm32(0xd800))); + + // A proper pair: U16_GET_SUPPLEMENTARY(lead, trail) + // = (lead << 10) + trail - U16_SURROGATE_OFFSET + m_jit.lshift32(MacroAssembler::TrustedImm32(10), temp2); + m_jit.add32(temp2, resultReg); + m_jit.sub32(MacroAssembler::TrustedImm32(U16_SURROGATE_OFFSET), resultReg); + + notAPair.link(&m_jit); + } + // Specialized read+match for character classes whose codepoints all share a single UTF-16 // lead surrogate. Loads the (lead, trail) pair as a single 32-bit value, branches to // |failures| if the lead doesn't match |sharedLead|, otherwise leaves the trail surrogate @@ -1497,10 +1752,47 @@ class YarrGenerator final : public YarrJITInfo { m_jit.load16Unaligned(address, resultReg); } + // Read the code point relative to the term's direction. A forward term is + // addressed by its first unit (decode rightward, tryReadUnicodeChar); a + // backward (lookbehind) term is addressed by its last unit (decode leftward, + // the interpreter's tryReadBackward). Reads that are inherently forward -- a + // capture's own units, the Boyer-Moore scans -- keep using readCharacter(). + void readCharacterForDirection(Checked negativeCharacterOffset, MacroAssembler::RegisterID resultReg) + { +#if ENABLE(YARR_JIT_UNICODE_EXPRESSIONS) + if (m_direction == Backward && m_decodeSurrogatePairs) { + readUnicodeCharBackward(negativeOffsetIndexedAddress(negativeCharacterOffset, resultReg), resultReg, m_regs.regT2); + m_usesT2 = true; + return; + } +#endif + readCharacter(negativeCharacterOffset, resultReg, m_regs.index); + } + + // Load the single code unit at the offset, with no surrogate-pair decoding. + // Used where a term compares exact UTF-16 units (a fixed astral literal is + // matched as its two known units), so pair decoding would be wrong. + void readCodeUnit(Checked negativeCharacterOffset, MacroAssembler::RegisterID resultReg) + { + readCodeUnit(negativeCharacterOffset, resultReg, m_regs.index); + } + + void readCodeUnit(Checked negativeCharacterOffset, MacroAssembler::RegisterID resultReg, MacroAssembler::RegisterID indexReg) + { + MacroAssembler::BaseIndex address = negativeOffsetIndexedAddress(negativeCharacterOffset, resultReg, indexReg); + if (m_charSize == CharSize::Char8) + m_jit.load8(address, resultReg); + else + m_jit.load16Unaligned(address, resultReg); + } + template MacroAssembler::Jump jumpIfCharCond(char32_t ch, Checked negativeCharacterOffset, MacroAssembler::RegisterID character, bool ignoreCase) { - readCharacter(negativeCharacterOffset, character); + // A term compares the code point that begins (forward) or ends (backward) + // at its position; the direction picks the decode. Astral literals never + // reach here in a backward frame (they compare units directly). + readCharacterForDirection(negativeCharacterOffset, character); // For case-insesitive compares, non-ascii characters that have different // upper & lower case representations are converted to a character class. @@ -1678,16 +1970,64 @@ class YarrGenerator final : public YarrJITInfo { m_jit.store32(MacroAssembler::TrustedImm32(0), duplicateNamedGroupAddress(i)); } + // Compute the real input position of a point that is `inputOffset` frame + // slots behind the current frame position, into `reg`. Forward: behind the + // frontier means a lower address (subtract); in a backward (mirrored) frame + // it is ahead of the leftward cursor (add). + void emitFramePosition(unsigned inputOffset, MacroAssembler::RegisterID reg) + { + if (m_direction == Backward) + m_jit.add32(MacroAssembler::Imm32(inputOffset), m_regs.index, reg); + else + m_jit.sub32(m_regs.index, MacroAssembler::Imm32(inputOffset), reg); + } + + // Consume `count` characters at the frame position: advance the forward + // frontier, or move the backward (mirrored) leftward cursor down. + void consumeIndex(unsigned count) + { + if (m_direction == Backward) + m_jit.sub32(MacroAssembler::Imm32(count), m_regs.index); + else + m_jit.add32(MacroAssembler::Imm32(count), m_regs.index); + } + + // Give `count` characters back (undo a consume). + void rewindIndex(unsigned count) + { + if (m_direction == Backward) + m_jit.add32(MacroAssembler::Imm32(count), m_regs.index); + else + m_jit.sub32(MacroAssembler::Imm32(count), m_regs.index); + } + + // Give back the number of characters held in `countRegister`. + void rewindIndex(MacroAssembler::RegisterID countRegister) + { + if (m_direction == Backward) + m_jit.add32(countRegister, m_regs.index); + else + m_jit.sub32(countRegister, m_regs.index); + } + + // In a backward (mirrored) frame a capture group's virtual start is its + // real end and vice versa, so the start/end slots are written swapped - + // matching the interpreter, which records backward captures end-first. + void emitCaptureStart(PatternTerm* term, Checked checkedOffset, MacroAssembler::RegisterID indexTemporary, unsigned extraOffset = 0) { // FIXME: could avoid offsetting this value in JIT code, apply offsets only afterwards, at the point the results array is being accessed. ASSERT(term->capture() && shouldRecordSubpatterns()); unsigned inputOffset = checkedOffset - term->inputPosition + extraOffset; + MacroAssembler::RegisterID positionReg = m_regs.index; if (inputOffset) { - m_jit.sub32(m_regs.index, MacroAssembler::Imm32(inputOffset), indexTemporary); - setSubpatternStart(indexTemporary, term->parentheses.subpatternId); - } else - setSubpatternStart(m_regs.index, term->parentheses.subpatternId); + emitFramePosition(inputOffset, indexTemporary); + positionReg = indexTemporary; + } + if (m_direction == Backward) + setSubpatternEnd(positionReg, term->parentheses.subpatternId); + else + setSubpatternStart(positionReg, term->parentheses.subpatternId); } void emitCaptureEnd(PatternTerm* term, Checked checkedOffset, MacroAssembler::RegisterID indexTemporary) @@ -1696,11 +2036,15 @@ class YarrGenerator final : public YarrJITInfo { ASSERT(term->capture() && shouldRecordSubpatterns()); auto subpatternId = term->parentheses.subpatternId; unsigned inputOffset = checkedOffset - term->inputPosition; + MacroAssembler::RegisterID positionReg = m_regs.index; if (inputOffset) { - m_jit.sub32(m_regs.index, MacroAssembler::Imm32(inputOffset), indexTemporary); - setSubpatternEnd(indexTemporary, subpatternId); - } else - setSubpatternEnd(m_regs.index, subpatternId); + emitFramePosition(inputOffset, indexTemporary); + positionReg = indexTemporary; + } + if (m_direction == Backward) + setSubpatternStart(positionReg, subpatternId); + else + setSubpatternEnd(positionReg, subpatternId); if (m_pattern.m_numDuplicateNamedCaptureGroups) { if (auto duplicateNamedGroupId = m_pattern.m_duplicateNamedGroupForSubpatternId[subpatternId]) storeDuplicateNamedGroupSubpatternId(duplicateNamedGroupId, subpatternId); @@ -1836,6 +2180,12 @@ class YarrGenerator final : public YarrJITInfo { PatternTerm* m_term; YarrOpCode m_op; + // The matching direction in effect for the code this op generates. Ops + // belonging to a lookbehind body (compiled from a mirrored copy of the + // body, see mirrorDisjunctionForLookbehind) are Backward: the index register + // is a leftward-moving cursor rather than the usual rightward frontier. + MatchDirection m_direction { Forward }; + // Used to record a set of Jumps out of the generated code, typically // used for jumps out to backtracking code, and a single reentry back // into the code for a node (likely where a backtrack will trigger @@ -1851,6 +2201,14 @@ class YarrGenerator final : public YarrJITInfo { // multiple are fused to match as a group. bool m_isDeadCode { false }; + // True for the one term whose first read is guaranteed to be at the match + // start: the alternative's first term, provided no earlier term can move + // the frontier at runtime. Only such a read may establish + // firstCharacterAdditionalReadSize ("the start character is a non-BMP + // pair"); a term after a zero-or-more (min-0) sibling reads past the + // start whenever that sibling consumed input. + bool m_readsStartCharacter { false }; + // Currently used in the case of some of the more complex management of // 'm_checkedOffset', to cache the offset used in this alternative, to avoid // recalculating it. @@ -1872,6 +2230,13 @@ class YarrGenerator final : public YarrJITInfo { BoyerMooreInfo* m_bmInfo { nullptr }; MaskedAlternativeInfo* m_maskedAltInfo { nullptr }; + // Set on the alternative Begin/Next/End ops of a nested disjunction whose + // alternatives are dispatched on their first character (see DispatchInfo). + struct DispatchInfo* m_dispatch { nullptr }; + // Position of this Begin/Next op's alternative within the dispatched + // disjunction. + unsigned m_dispatchAlternativeIndex { 0 }; + // Shared UTF-16 lead surrogate across all repeated alternatives' first character. std::optional m_bodyAltSharedLead; @@ -1932,6 +2297,13 @@ class YarrGenerator final : public YarrJITInfo { { m_pendingReturns.append(returnAddress); } + // Bind a patchable frame store directly to a known code location (used by + // first-character dispatch chains, whose targets are forward-generated + // stubs rather than backtracking sites). + void addReturnAddressRecord(const MacroAssembler::DataLabelPtr& dataLabel, MacroAssembler::Label target) + { + m_backtrackRecords.append(ReturnAddressRecord(dataLabel, target)); + } void NODELETE fallthrough() { ASSERT(!m_pendingFallthrough); @@ -2049,25 +2421,67 @@ class YarrGenerator final : public YarrJITInfo { YarrOp& op = m_ops[opIndex]; PatternTerm* term = op.m_term; + if (m_direction == Backward) { + generateAssertionBOLBackward(opIndex); + return; + } + + // The anchor's real string position is index - (checkedOffset - position), so + // it is the start of input iff index == checkedOffset - position. A top-level + // ^ at position 0 gives the classic index == checkedOffset; the general form + // also covers a ^ in the body of an assertion nested inside a lookbehind, whose + // numbering continues from a nonzero origin (and makes a ^ that can never be + // at position 0 -- e.g. /a^b/ -- simply never satisfied, as before). + MacroAssembler::Imm32 startFrontier(static_cast((op.m_checkedOffset - term->inputPosition).value())); + if (term->multiline()) { const MacroAssembler::RegisterID character = m_regs.regT0; const MacroAssembler::RegisterID scratch = m_regs.regT1; MacroAssembler::JumpList matchDest; - if (!term->inputPosition) - matchDest.append(m_jit.branch32(MacroAssembler::Equal, m_regs.index, MacroAssembler::Imm32(op.m_checkedOffset))); + matchDest.append(m_jit.branch32(MacroAssembler::Equal, m_regs.index, startFrontier)); readCharacter(op.m_checkedOffset - term->inputPosition + 1, character); matchCharacterClass(character, scratch, matchDest, m_pattern.newlineCharacterClass()); op.m_jumps.append(m_jit.jump()); + matchDest.link(&m_jit); + } else + op.m_jumps.append(m_jit.branch32(MacroAssembler::NotEqual, m_regs.index, startFrontier)); + } + + // BOL inside a lookbehind body (mirrored frame). The anchor's real position + // is index + (checkedOffset - inputPosition). It can be the real start of + // input only when it sits at the frame's right edge (inputPosition == + // checkedOffset) and the cursor has reached position 0; the real character + // preceding the anchor is at frame offset (checkedOffset - inputPosition), + // one less than in the forward case. + void generateAssertionBOLBackward(size_t opIndex) + { + YarrOp& op = m_ops[opIndex]; + PatternTerm* term = op.m_term; + bool atFrameEdge = term->inputPosition == op.m_checkedOffset; + + if (term->multiline()) { + const MacroAssembler::RegisterID character = m_regs.regT0; + const MacroAssembler::RegisterID scratch = m_regs.regT1; + + MacroAssembler::JumpList matchDest; + if (atFrameEdge) + matchDest.append(m_jit.branchTest32(MacroAssembler::Zero, m_regs.index)); + + // The character just left of the anchor ends at the anchor: a leftward + // decode in a unicode pattern. + readCharacterForDirection(op.m_checkedOffset - term->inputPosition, character); + matchCharacterClass(character, scratch, matchDest, m_pattern.newlineCharacterClass()); + op.m_jumps.append(m_jit.jump()); + matchDest.link(&m_jit); } else { - // Erk, really should poison out these alternatives early. :-/ - if (term->inputPosition) - op.m_jumps.append(m_jit.jump()); + if (atFrameEdge) + op.m_jumps.append(m_jit.branchTest32(MacroAssembler::NonZero, m_regs.index)); else - op.m_jumps.append(m_jit.branch32(MacroAssembler::NotEqual, m_regs.index, MacroAssembler::Imm32(op.m_checkedOffset))); + op.m_jumps.append(m_jit.jump()); } } void backtrackAssertionBOL(size_t opIndex) @@ -2080,6 +2494,11 @@ class YarrGenerator final : public YarrJITInfo { YarrOp& op = m_ops[opIndex]; PatternTerm* term = op.m_term; + if (m_direction == Backward) { + generateAssertionEOLBackward(opIndex); + return; + } + if (term->multiline()) { const MacroAssembler::RegisterID character = m_regs.regT0; const MacroAssembler::RegisterID scratch = m_regs.regT1; @@ -2101,6 +2520,44 @@ class YarrGenerator final : public YarrJITInfo { op.m_jumps.append(m_jit.jump()); } } + + // EOL inside a lookbehind body (mirrored frame). The anchor's real position + // is index + (checkedOffset - inputPosition). It can be the real end of input + // only when it sits at the frame's left edge (inputPosition == 0, i.e. the + // assertion point itself) and index + checkedOffset == length; the real + // character following the anchor is at frame offset (checkedOffset - + // inputPosition + 1), one more than in the forward case. + void generateAssertionEOLBackward(size_t opIndex) + { + YarrOp& op = m_ops[opIndex]; + PatternTerm* term = op.m_term; + bool atFrameEdge = !term->inputPosition; + const MacroAssembler::RegisterID scratch = m_regs.regT1; + + auto atRealEnd = [&](MacroAssembler::RelationalCondition condition) { + m_jit.add32(MacroAssembler::Imm32(op.m_checkedOffset), m_regs.index, scratch); + return m_jit.branch32(condition, scratch, m_regs.length); + }; + + if (term->multiline()) { + const MacroAssembler::RegisterID character = m_regs.regT0; + + MacroAssembler::JumpList matchDest; + if (atFrameEdge) + matchDest.append(atRealEnd(MacroAssembler::Equal)); + + readCharacter(op.m_checkedOffset - term->inputPosition + 1, character); + matchCharacterClass(character, scratch, matchDest, m_pattern.newlineCharacterClass()); + op.m_jumps.append(m_jit.jump()); + + matchDest.link(&m_jit); + } else { + if (atFrameEdge) + op.m_jumps.append(atRealEnd(MacroAssembler::NotEqual)); + else + op.m_jumps.append(m_jit.jump()); + } + } void backtrackAssertionEOL(size_t opIndex) { backtrackTermDefault(opIndex); @@ -2115,10 +2572,24 @@ class YarrGenerator final : public YarrJITInfo { const MacroAssembler::RegisterID character = m_regs.regT0; const MacroAssembler::RegisterID scratch = m_regs.regT1; - if (term->inputPosition == op.m_checkedOffset) - nextIsNotWordChar.append(atEndOfInput()); + if (m_direction == Backward) { + // Mirrored frame: the boundary's real position is index + (checked - + // pos). It is the real end of input only at the frame's left edge + // (pos == 0) with index + checked == length; the following real + // character is at frame offset (checked - pos + 1). The character AFTER + // a boundary starts at it, so it is a forward decode from that unit -- + // readCharacter, not the direction-relative reader. + if (!term->inputPosition) { + m_jit.add32(MacroAssembler::Imm32(op.m_checkedOffset), m_regs.index, scratch); + nextIsNotWordChar.append(m_jit.branch32(MacroAssembler::Equal, scratch, m_regs.length)); + } + readCharacter(op.m_checkedOffset - term->inputPosition + 1, character); + } else { + if (term->inputPosition == op.m_checkedOffset) + nextIsNotWordChar.append(atEndOfInput()); - readCharacter(op.m_checkedOffset - term->inputPosition, character); + readCharacter(op.m_checkedOffset - term->inputPosition, character); + } CharacterClass* wordcharCharacterClass; @@ -2138,16 +2609,32 @@ class YarrGenerator final : public YarrJITInfo { const MacroAssembler::RegisterID character = m_regs.regT0; const MacroAssembler::RegisterID scratch = m_regs.regT1; -#if ENABLE(YARR_JIT_UNICODE_EXPRESSIONS) && ENABLE(YARR_JIT_UNICODE_CAN_INCREMENT_INDEX_FOR_NON_BMP) - // Prevent word boundary assertion reads from corrupting firstCharacterAdditionalReadSize. - SetForScope useOptimizationScope(m_useFirstNonBMPCharacterOptimization, false); -#endif MacroAssembler::Jump atBegin; MacroAssembler::JumpList matchDest; - if (!term->inputPosition) - atBegin = m_jit.branch32(MacroAssembler::Equal, m_regs.index, MacroAssembler::Imm32(op.m_checkedOffset)); - readCharacter(op.m_checkedOffset - term->inputPosition + 1, character); + // Forward: the boundary's real position is index - (checked - pos), so it + // is the real start of input iff index == checked - pos (the classic + // index == checked when the ^-like term is at pos 0; the general form also + // covers a \b in the body of an assertion nested inside a lookbehind, whose + // numbering continues from a nonzero origin). The preceding real character + // is at frame offset (checked - pos + 1). Backward (mirrored): the real + // start test is at the frame's right edge (pos == checked) with index == 0, + // and the preceding real character is at (checked - pos). + bool atStartEdge = m_direction == Backward ? term->inputPosition == op.m_checkedOffset : true; + if (atStartEdge) { + if (m_direction == Backward) + atBegin = m_jit.branchTest32(MacroAssembler::Zero, m_regs.index); + else + atBegin = m_jit.branch32(MacroAssembler::Equal, m_regs.index, MacroAssembler::Imm32(static_cast((op.m_checkedOffset - term->inputPosition).value()))); + } + // The character BEFORE the boundary ends at the unit just left of it. In + // a mirrored frame that unit is at frame offset (checked - pos), and in a + // unicode pattern it is a leftward decode (an astral pair ending at the + // boundary is classified as its code point). Forward: (checked - pos + 1). + if (m_direction == Backward) + readCharacterForDirection(op.m_checkedOffset - term->inputPosition, character); + else + readCharacter(op.m_checkedOffset - term->inputPosition + 1, character); CharacterClass* wordcharCharacterClass; @@ -2157,7 +2644,7 @@ class YarrGenerator final : public YarrJITInfo { wordcharCharacterClass = m_pattern.wordcharCharacterClass(); matchCharacterClass(character, scratch, matchDest, wordcharCharacterClass); - if (!term->inputPosition) + if (atStartEdge) atBegin.link(&m_jit); // We fall through to here if the last character was not a wordchar. @@ -2203,25 +2690,34 @@ class YarrGenerator final : public YarrJITInfo { unsigned subpatternId = term->backReferenceSubpatternId; unsigned duplicateNamedGroupId = m_pattern.hasDuplicateNamedCaptureGroups() ? m_pattern.m_duplicateNamedGroupForSubpatternId[subpatternId] : 0; -#if ENABLE(YARR_JIT_UNICODE_EXPRESSIONS) && ENABLE(YARR_JIT_UNICODE_CAN_INCREMENT_INDEX_FOR_NON_BMP) - SetForScope useOptimizationScope(m_useFirstNonBMPCharacterOptimization, false); -#endif MacroAssembler::Label loop(&m_jit); + // The referenced capture is an ordinary substring: its units are read + // forward from patternIndex, and the input span is compared ascending in + // lockstep. Both operand offsets are expressed in the frame's addressing: + // forward reads input[reg - k], a mirrored (lookbehind) frame reads + // input[reg + k - 1] -- so the capture unit itself is k = 1 there, and + // the input span (walked upward from its left edge, which the caller put + // in index) begins one unit past the term's own offset. + unsigned captureUnitOffset = m_direction == Backward ? 1 : 0; + Checked inputSpanOffset = op.m_checkedOffset - term->inputPosition; + if (m_direction == Backward) + inputSpanOffset += 1; + #if ENABLE(YARR_JIT_BACKREFERENCES_FOR_16BIT_EXPRS) if (!m_decodeSurrogatePairs) - readCharacter(0, patternCharacter, patternIndex); + readCharacter(captureUnitOffset, patternCharacter, patternIndex); else { // For reading Unicode characters, use the standard resultReg so we can call the standard tryReadUnicodeChar() // helper instead of emitting an inlined version. - readCharacter(0, character, patternIndex); + readCharacter(captureUnitOffset, character, patternIndex); m_jit.move(character, patternCharacter); } #else - readCharacter(0, patternCharacter, patternIndex); + readCharacter(captureUnitOffset, patternCharacter, patternIndex); #endif - readCharacter(op.m_checkedOffset - term->inputPosition, character); + readCharacter(inputSpanOffset, character); if (!term->ignoreCase()) { // When !m_decodeSurrogatePairs, readCharacter() emits load8/load16 @@ -2382,9 +2878,10 @@ class YarrGenerator final : public YarrJITInfo { // PatternTemp should contain pattern end index at this point. Compute pattern size. m_jit.sub32(patternIndex, patternTemp); - op.m_jumps.append(checkNotEnoughInput(patternTemp)); + emitBackReferenceSpanClaim(op.m_jumps, patternTemp, characterOrTemp, parenthesesFrameLocation); matchBackreference(opIndex, op.m_jumps, characterOrTemp, patternIndex, patternTemp, subpatternIdReg == m_regs.unicodeAndSubpatternIdTemp ? subpatternIdReg : InvalidGPRReg); + emitBackReferenceSpanFrontier(parenthesesFrameLocation); if (term->quantityMaxCount != 1) { loadFromFrame(parenthesesFrameLocation + BackTrackInfoBackReference::matchAmountIndex(), characterOrTemp); @@ -2415,9 +2912,10 @@ class YarrGenerator final : public YarrJITInfo { m_jit.sub32(patternIndex, patternTemp); storeToFrame(patternTemp, parenthesesFrameLocation + BackTrackInfoBackReference::backReferenceSizeIndex()); - matches.append(checkNotEnoughInput(patternTemp)); + emitBackReferenceSpanClaim(matches, patternTemp, characterOrTemp, parenthesesFrameLocation); matchBackreference(opIndex, incompleteMatches, characterOrTemp, patternIndex, patternTemp, subpatternIdReg == m_regs.unicodeAndSubpatternIdTemp ? subpatternIdReg : InvalidGPRReg); + emitBackReferenceSpanFrontier(parenthesesFrameLocation); loadFromFrame(parenthesesFrameLocation + BackTrackInfoBackReference::matchAmountIndex(), characterOrTemp); m_jit.add32(MacroAssembler::TrustedImm32(1), characterOrTemp); @@ -2485,9 +2983,10 @@ class YarrGenerator final : public YarrJITInfo { // when checkNotEnoughInput bails out early. storeToFrame(m_regs.index, parenthesesFrameLocation + BackTrackInfoBackReference::beginIndex()); m_jit.sub32(patternIndex, patternTemp); - matches.append(checkNotEnoughInput(patternTemp)); + emitBackReferenceSpanClaim(matches, patternTemp, characterOrTemp, parenthesesFrameLocation); matchBackreference(opIndex, incompleteMatches, characterOrTemp, patternIndex, patternTemp, subpatternIdReg == m_regs.unicodeAndSubpatternIdTemp ? subpatternIdReg : InvalidGPRReg); + emitBackReferenceSpanFrontier(parenthesesFrameLocation); matches.append(m_jit.jump()); @@ -2523,7 +3022,12 @@ class YarrGenerator final : public YarrJITInfo { failures.append(m_jit.branchTest32(MacroAssembler::Zero, matchAmount)); loadFromFrame(parenthesesFrameLocation + BackTrackInfoBackReference::backReferenceSizeIndex(), matchSize); - m_jit.sub32(matchSize, m_regs.index); + // Give one span back: retreat the forward frontier, or raise the + // leftward-consuming frontier of a mirrored frame, by the span size. + if (m_direction == Backward) + m_jit.add32(matchSize, m_regs.index); + else + m_jit.sub32(matchSize, m_regs.index); m_jit.sub32(MacroAssembler::TrustedImm32(1), matchAmount); storeToFrame(matchAmount, parenthesesFrameLocation + BackTrackInfoBackReference::matchAmountIndex()); @@ -2596,12 +3100,34 @@ class YarrGenerator final : public YarrJITInfo { // upper & lower case representations are converted to a character class. ASSERT(!op.m_term->ignoreCase() || isASCIIAlpha(firstChar) || isCanonicallyUnique(firstChar, m_canonicalMode)); - if (m_decodeSurrogatePairs && (!U_IS_BMP(firstChar) || U16_IS_SURROGATE(firstChar))) { - // The first term we are considering is a non-BMP or dangling surrogate char in unicode pattern. Just try matching it and be done. + if (m_direction == Backward || (m_decodeSurrogatePairs && (!U_IS_BMP(firstChar) || U16_IS_SURROGATE(firstChar)))) { + // Match a single character on its own, without fusing neighbours into a + // wide load. In a backward (lookbehind) frame ascending memory holds + // *descending* pattern positions, so the forward multi-character packing + // does not apply (a run could be fused by comparing against the reversed + // packed constant loaded from the run's lowest address; not done yet); + // and a non-BMP / dangling surrogate char in a unicode pattern is likewise + // matched individually. uint64_t charToMatch = firstChar; auto offset = op.m_checkedOffset - op.m_term->inputPosition; + if (m_direction == Backward && m_decodeSurrogatePairs && !U_IS_BMP(firstChar)) { + // Backward astral literal: the term is a fixed two-unit code point. + // Its higher frame offset addresses the later (trail) unit and the + // next-lower offset the lead unit, so comparing the two units at + // their fixed positions equals comparing the code point (a lead+trail + // pair encodes exactly one code point). No decode, no dynamic width. + // Such a literal is canonically unique or ASCII by construction + // (atomPatternCharacter), so no case folding applies. + ASSERT(!op.m_term->ignoreCase() || isCanonicallyUnique(firstChar, m_canonicalMode)); + readCodeUnit(offset, character); + defaultMatchTargets.appendFailed(m_jit.branch32(MacroAssembler::NotEqual, character, MacroAssembler::Imm32(U16_TRAIL(firstChar)))); + readCodeUnit(offset - 1, character); + defaultMatchTargets.appendFailed(m_jit.branch32(MacroAssembler::NotEqual, character, MacroAssembler::Imm32(U16_LEAD(firstChar)))); + return; + } + if (!matchTargets.hasSucceedTarget() || m_ops[opIndex + 1].m_op == YarrOpCode::Term) defaultMatchTargets.appendFailed(jumpIfCharNotEquals(charToMatch, offset, character, op.m_term->ignoreCase())); else @@ -2971,25 +3497,48 @@ class YarrGenerator final : public YarrJITInfo { Checked scaledMaxCount = term->quantityMaxCount; scaledMaxCount *= U_IS_BMP(ch) ? 1 : 2; - m_jit.sub32(m_regs.index, MacroAssembler::Imm32(scaledMaxCount), countRegister); + // The count register walks toward index over the run of characters: + // upward from index - count in a forward frame, downward from index + count + // in a backward (mirrored) frame where index is the left frontier. + emitFramePosition(scaledMaxCount, countRegister); + + Checked baseOffset = op.m_checkedOffset - term->inputPosition - scaledMaxCount; - MacroAssembler::Label loop(&m_jit); - readCharacter(op.m_checkedOffset - term->inputPosition - scaledMaxCount, character, countRegister); // For case-insesitive compares, non-ascii characters that have different // upper & lower case representations are converted to a character class. ASSERT(!term->ignoreCase() || isASCIIAlpha(ch) || isCanonicallyUnique(ch, m_canonicalMode)); - if (term->ignoreCase() && isASCIIAlpha(ch)) { - m_jit.or32(MacroAssembler::TrustedImm32(0x20), character); - ch |= 0x20; - } - op.m_jumps.append(m_jit.branch32(MacroAssembler::NotEqual, character, MacroAssembler::Imm32(ch))); + MacroAssembler::Label loop(&m_jit); + if (m_direction == Backward && m_decodeSurrogatePairs && !U_IS_BMP(ch)) { + // Backward run of a fixed astral literal: each occurrence is a known + // (lead, trail) unit pair. At loop entry countRegister addresses the last + // occurrence's trail unit; compare it, step the count register down one + // unit to the lead, compare that, and step down once more. The lead is + // reached by moving the index register rather than the (unsigned) frame + // offset, which is 0 at the lowest occurrence. Unit-exact, no decoding. + readCodeUnit(baseOffset, character, countRegister); + op.m_jumps.append(m_jit.branch32(MacroAssembler::NotEqual, character, MacroAssembler::Imm32(U16_TRAIL(ch)))); + m_jit.sub32(MacroAssembler::TrustedImm32(1), countRegister); + readCodeUnit(baseOffset, character, countRegister); + op.m_jumps.append(m_jit.branch32(MacroAssembler::NotEqual, character, MacroAssembler::Imm32(U16_LEAD(ch)))); + m_jit.sub32(MacroAssembler::TrustedImm32(1), countRegister); + } else { + readCharacter(baseOffset, character, countRegister); + if (term->ignoreCase() && isASCIIAlpha(ch)) { + m_jit.or32(MacroAssembler::TrustedImm32(0x20), character); + ch |= 0x20; + } + + op.m_jumps.append(m_jit.branch32(MacroAssembler::NotEqual, character, MacroAssembler::Imm32(ch))); + if (m_direction == Backward) + m_jit.sub32(MacroAssembler::TrustedImm32(1), countRegister); #if ENABLE(YARR_JIT_UNICODE_EXPRESSIONS) - if (m_decodeSurrogatePairs && !U_IS_BMP(ch)) - m_jit.add32(MacroAssembler::TrustedImm32(2), countRegister); - else + else if (m_decodeSurrogatePairs && !U_IS_BMP(ch)) + m_jit.add32(MacroAssembler::TrustedImm32(2), countRegister); #endif - m_jit.add32(MacroAssembler::TrustedImm32(1), countRegister); + else + m_jit.add32(MacroAssembler::TrustedImm32(1), countRegister); + } m_jit.branch32(MacroAssembler::NotEqual, countRegister, m_regs.index).linkTo(loop, &m_jit); } void backtrackPatternCharacterFixed(size_t opIndex) @@ -3013,18 +3562,44 @@ class YarrGenerator final : public YarrJITInfo { MacroAssembler::JumpList failures; MacroAssembler::Label loop(&m_jit); failures.append(atEndOfInput()); - failures.append(jumpIfCharNotEquals(ch, op.m_checkedOffset - term->inputPosition, character, term->ignoreCase())); + if (m_direction == Backward && m_decodeSurrogatePairs && !U_IS_BMP(ch)) { + // Backward astral literal: the frontier-adjacent unit must be the + // trail; take it, then require and compare the lead below it, taking + // that too. Unit-exact (a lead+trail pair is one code point). If the + // lead is absent or wrong the trail unit is given back before failing. + Checked offset = op.m_checkedOffset - term->inputPosition; + readCodeUnit(offset, character); + failures.append(m_jit.branch32(MacroAssembler::NotEqual, character, MacroAssembler::Imm32(U16_TRAIL(ch)))); + m_jit.sub32(MacroAssembler::TrustedImm32(1), m_regs.index); // the trail unit + MacroAssembler::JumpList giveBackTrail; + giveBackTrail.append(m_jit.branchTest32(MacroAssembler::Zero, m_regs.index)); // no lead below + readCodeUnit(offset, character); // now addresses the unit below the trail + giveBackTrail.append(m_jit.branch32(MacroAssembler::NotEqual, character, MacroAssembler::Imm32(U16_LEAD(ch)))); + m_jit.sub32(MacroAssembler::TrustedImm32(1), m_regs.index); // the lead unit + MacroAssembler::Jump matched = m_jit.jump(); + giveBackTrail.link(&m_jit); + m_jit.add32(MacroAssembler::TrustedImm32(1), m_regs.index); + failures.append(m_jit.jump()); + matched.link(&m_jit); + } else { + failures.append(jumpIfCharNotEquals(ch, op.m_checkedOffset - term->inputPosition, character, term->ignoreCase())); - m_jit.add32(MacroAssembler::TrustedImm32(1), m_regs.index); + // Consume the character: advance the frontier forward, or move the + // leftward cursor down in a backward (mirrored) frame. + if (m_direction == Backward) + m_jit.sub32(MacroAssembler::TrustedImm32(1), m_regs.index); + else + m_jit.add32(MacroAssembler::TrustedImm32(1), m_regs.index); #if ENABLE(YARR_JIT_UNICODE_EXPRESSIONS) - if (m_decodeSurrogatePairs && !U_IS_BMP(ch)) { - MacroAssembler::Jump surrogatePairOk = notAtEndOfInput(); - m_jit.sub32(MacroAssembler::TrustedImm32(1), m_regs.index); - failures.append(m_jit.jump()); - surrogatePairOk.link(&m_jit); - m_jit.add32(MacroAssembler::TrustedImm32(1), m_regs.index); - } + if (m_decodeSurrogatePairs && !U_IS_BMP(ch)) { + MacroAssembler::Jump surrogatePairOk = notAtEndOfInput(); + m_jit.sub32(MacroAssembler::TrustedImm32(1), m_regs.index); + failures.append(m_jit.jump()); + surrogatePairOk.link(&m_jit); + m_jit.add32(MacroAssembler::TrustedImm32(1), m_regs.index); + } #endif + } m_jit.add32(MacroAssembler::TrustedImm32(1), countRegister); if (term->quantityMaxCount == quantifyInfinite) @@ -3053,7 +3628,7 @@ class YarrGenerator final : public YarrJITInfo { loadFromFrame(term->frameLocation + BackTrackInfoPatternCharacter::matchAmountIndex(), countRegister); if (m_decodeSurrogatePairs && !U_IS_BMP(term->patternCharacter)) m_jit.lshift32(MacroAssembler::TrustedImm32(1), countRegister); - m_jit.sub32(countRegister, m_regs.index); + rewindIndex(countRegister); m_backtrackingState.fallthrough(); return; } @@ -3061,10 +3636,14 @@ class YarrGenerator final : public YarrJITInfo { loadFromFrame(term->frameLocation + BackTrackInfoPatternCharacter::matchAmountIndex(), countRegister); m_backtrackingState.append(m_jit.branchTest32(MacroAssembler::Zero, countRegister)); m_jit.sub32(MacroAssembler::TrustedImm32(1), countRegister); - if (!m_decodeSurrogatePairs || U_IS_BMP(term->patternCharacter)) - m_jit.sub32(MacroAssembler::TrustedImm32(1), m_regs.index); + // Give one character back: retreat the forward frontier, or move the + // leftward cursor back up in a backward (mirrored) frame. The character + // is a single unit unless it is an astral literal in a unicode pattern. + unsigned characterWidth = (m_decodeSurrogatePairs && !U_IS_BMP(term->patternCharacter)) ? 2 : 1; + if (m_direction == Backward) + m_jit.add32(MacroAssembler::TrustedImm32(characterWidth), m_regs.index); else - m_jit.sub32(MacroAssembler::TrustedImm32(2), m_regs.index); + m_jit.sub32(MacroAssembler::TrustedImm32(characterWidth), m_regs.index); m_jit.jump(op.m_reentry); } @@ -3098,18 +3677,40 @@ class YarrGenerator final : public YarrJITInfo { nonGreedyFailures.append(atEndOfInput()); if (term->quantityMaxCount != quantifyInfinite) nonGreedyFailures.append(m_jit.branch32(MacroAssembler::Equal, countRegister, MacroAssembler::Imm32(term->quantityMaxCount))); - nonGreedyFailures.append(jumpIfCharNotEquals(ch, op.m_checkedOffset - term->inputPosition, character, term->ignoreCase())); + if (m_direction == Backward && m_decodeSurrogatePairs && !U_IS_BMP(ch)) { + // Backward astral literal: trail at the frontier, then the lead below + // it (see generatePatternCharacterGreedy); a partial take is undone. + Checked offset = op.m_checkedOffset - term->inputPosition; + readCodeUnit(offset, character); + nonGreedyFailures.append(m_jit.branch32(MacroAssembler::NotEqual, character, MacroAssembler::Imm32(U16_TRAIL(ch)))); + m_jit.sub32(MacroAssembler::TrustedImm32(1), m_regs.index); // the trail unit + MacroAssembler::JumpList giveBackTrail; + giveBackTrail.append(m_jit.branchTest32(MacroAssembler::Zero, m_regs.index)); // no lead below + readCodeUnit(offset, character); + giveBackTrail.append(m_jit.branch32(MacroAssembler::NotEqual, character, MacroAssembler::Imm32(U16_LEAD(ch)))); + m_jit.sub32(MacroAssembler::TrustedImm32(1), m_regs.index); // the lead unit + MacroAssembler::Jump matched = m_jit.jump(); + giveBackTrail.link(&m_jit); + m_jit.add32(MacroAssembler::TrustedImm32(1), m_regs.index); + nonGreedyFailures.append(m_jit.jump()); + matched.link(&m_jit); + } else { + nonGreedyFailures.append(jumpIfCharNotEquals(ch, op.m_checkedOffset - term->inputPosition, character, term->ignoreCase())); - m_jit.add32(MacroAssembler::TrustedImm32(1), m_regs.index); + if (m_direction == Backward) + m_jit.sub32(MacroAssembler::TrustedImm32(1), m_regs.index); + else + m_jit.add32(MacroAssembler::TrustedImm32(1), m_regs.index); #if ENABLE(YARR_JIT_UNICODE_EXPRESSIONS) - if (m_decodeSurrogatePairs && !U_IS_BMP(ch)) { - MacroAssembler::Jump surrogatePairOk = notAtEndOfInput(); - m_jit.sub32(MacroAssembler::TrustedImm32(1), m_regs.index); - nonGreedyFailures.append(m_jit.jump()); - surrogatePairOk.link(&m_jit); - m_jit.add32(MacroAssembler::TrustedImm32(1), m_regs.index); - } + if (m_decodeSurrogatePairs && !U_IS_BMP(ch)) { + MacroAssembler::Jump surrogatePairOk = notAtEndOfInput(); + m_jit.sub32(MacroAssembler::TrustedImm32(1), m_regs.index); + nonGreedyFailures.append(m_jit.jump()); + surrogatePairOk.link(&m_jit); + m_jit.add32(MacroAssembler::TrustedImm32(1), m_regs.index); + } #endif + } m_jit.add32(MacroAssembler::TrustedImm32(1), countRegister); m_jit.jump(op.m_reentry); @@ -3121,7 +3722,8 @@ class YarrGenerator final : public YarrJITInfo { m_jit.lshift32(MacroAssembler::TrustedImm32(1), countRegister); } - m_jit.sub32(countRegister, m_regs.index); + // Rewind everything this term consumed. + rewindIndex(countRegister); m_backtrackingState.fallthrough(); } @@ -3139,7 +3741,10 @@ class YarrGenerator final : public YarrJITInfo { } #if ENABLE(YARR_JIT_UNICODE_EXPRESSIONS) - if (m_decodeSurrogatePairs && !term->invert()) { + // The shared-lead fast path loads the (lead, trail) pair forward from the + // term's first unit; a backward frame addresses the term by its last unit + // and uses the general path below. + if (m_decodeSurrogatePairs && !term->invert() && m_direction == Forward) { if (auto sharedLead = term->characterClass->hasSharedLeadSurrogate()) { CharacterClass trailsOnlyClass; buildTrailsOnlyClass(*term->characterClass, trailsOnlyClass); @@ -3149,6 +3754,27 @@ class YarrGenerator final : public YarrJITInfo { } #endif + if (m_direction == Backward && m_decodeSurrogatePairs) { + // Backward + unicode: read the code point that ends at the term's + // position (its last unit is at frame offset checkedOffset - position). + // The class match runs first and leaves the frontier alone, preserving + // `character` for the width test just as the forward path does; + // beginIndex (stored above) restores the frontier on backtrack. + readUnicodeCharBackward(negativeOffsetIndexedAddress(op.m_checkedOffset - term->inputPosition, character), character, scratch); + matchCharacterClassTermInner(term, op.m_jumps, character, scratch); + // A static-width class was laid out at its true width (2 units when its + // members are all astral) and its claim already covers the whole pair. + // Only a variable-width class was budgeted a single unit, so only it + // grows the leftward claim by one when the code point was a pair + // (failing when the frontier is exhausted). + if (!term->isFixedWidthCharacterClass()) { + MacroAssembler::Jump isBMPChar = m_jit.branch32(MacroAssembler::LessThan, character, MacroAssembler::TrustedImm32(0x10000)); + claimBackwardPairLead(op.m_jumps); // claim the pair's lead unit + isBMPChar.link(&m_jit); + } + return; + } + readCharacter(op.m_checkedOffset - term->inputPosition, character); matchCharacterClassTermInner(term, op.m_jumps, character, scratch); @@ -3207,7 +3833,9 @@ class YarrGenerator final : public YarrJITInfo { scaledMaxCount *= 2; nonBMPOnly = true; - if (auto sharedLead = term->characterClass->hasSharedLeadSurrogate()) { + // The shared-lead pair fast path reads (lead, trail) forward from the + // count position; a backward frame uses the general loop below. + if (auto sharedLead = m_direction == Forward ? term->characterClass->hasSharedLeadSurrogate() : std::nullopt) { // Replace tryReadUnicodeCharImpl + full-codepoint class match // with a single 32-bit pair load + lead check + trail-only class match. ASSERT(term->isFixedWidthCharacterClass()); @@ -3225,17 +3853,54 @@ class YarrGenerator final : public YarrJITInfo { } #endif - m_jit.sub32(m_regs.index, MacroAssembler::Imm32(scaledMaxCount), countRegister); + // See generatePatternCharacterFixed for the forward/backward count register walk. + emitFramePosition(scaledMaxCount, countRegister); + + Checked baseOffset = op.m_checkedOffset - term->inputPosition - scaledMaxCount; MacroAssembler::Label loop(&m_jit); - readCharacter(op.m_checkedOffset - term->inputPosition - scaledMaxCount, character, countRegister); + if (m_direction == Backward && m_decodeSurrogatePairs) { + // Backward + unicode. At loop entry countRegister addresses the last + // unit of the next-lower occurrence; decode the code point ending + // there. Width is recovered from the code point itself (>= 0x10000 is + // a pair), after the class match, which preserves `character`. + readUnicodeCharBackward(negativeOffsetIndexedAddress(baseOffset, character, countRegister), character, scratch); + matchCharacterClassTermInner(term, op.m_jumps, character, scratch); + if (nonBMPOnly) { + // Static width 2: budgeted at two units per match. + m_jit.sub32(MacroAssembler::TrustedImm32(2), countRegister); + } else if (term->isFixedWidthCharacterClass()) { + // Static width 1 (all-BMP class). The read still decodes so that a + // unit trailing a lead is presented as its astral code point, exactly + // as the interpreter's tryReadBackward presents it; such a code point + // is never in this class, so no width consequence arises. + m_jit.sub32(MacroAssembler::TrustedImm32(1), countRegister); + } else { + // Variable width: a pair consumed one more unit than budgeted, so + // grow the leftward claim by one (fail if exhausted) and step over + // both units. beginIndex, stored above, restores the frontier when + // this term backtracks. + m_jit.sub32(MacroAssembler::TrustedImm32(1), countRegister); + MacroAssembler::Jump isBMPChar = m_jit.branch32(MacroAssembler::LessThan, character, MacroAssembler::TrustedImm32(0x10000)); + claimBackwardPairLead(op.m_jumps); // claim the pair's lead unit + m_jit.sub32(MacroAssembler::TrustedImm32(1), countRegister); // extra unit of the pair + isBMPChar.link(&m_jit); + } + m_jit.branch32(MacroAssembler::NotEqual, countRegister, m_regs.index).linkTo(loop, &m_jit); + done.link(&m_jit); + return; + } + + readCharacter(baseOffset, character, countRegister); MacroAssembler::Label nonBMPLoop(&m_jit); matchCharacterClassTermInner(term, op.m_jumps, character, scratch); + if (m_direction == Backward) + m_jit.sub32(MacroAssembler::TrustedImm32(1), countRegister); #if ENABLE(YARR_JIT_UNICODE_EXPRESSIONS) - if (m_decodeSurrogatePairs) { + else if (m_decodeSurrogatePairs) { if (term->isFixedWidthCharacterClass()) m_jit.add32(MacroAssembler::TrustedImm32(term->characterClass->hasNonBMPCharacters() ? 2 : 1), countRegister); else { @@ -3246,14 +3911,15 @@ class YarrGenerator final : public YarrJITInfo { m_jit.add32(MacroAssembler::TrustedImm32(1), m_regs.index); isBMPChar.link(&m_jit); } - } else + } #endif + else m_jit.add32(MacroAssembler::TrustedImm32(1), countRegister); #if ENABLE(YARR_JIT_UNICODE_EXPRESSIONS) if (nonBMPOnly) { done.append(m_jit.branch32(MacroAssembler::Equal, countRegister, m_regs.index)); - tryReadNonBMPUnicodeChar(op.m_checkedOffset - term->inputPosition - scaledMaxCount, character, countRegister); + tryReadNonBMPUnicodeChar(baseOffset, character, countRegister); m_jit.jump().linkTo(nonBMPLoop, &m_jit); } else #endif @@ -3295,7 +3961,8 @@ class YarrGenerator final : public YarrJITInfo { m_jit.move(MacroAssembler::TrustedImm32(0), countRegister); #if ENABLE(YARR_JIT_UNICODE_EXPRESSIONS) - if (m_decodeSurrogatePairs && !term->invert()) { + // Forward-only: the shared-lead fast path reads the pair from its first unit. + if (m_decodeSurrogatePairs && !term->invert() && m_direction == Forward) { if (auto sharedLead = term->characterClass->hasSharedLeadSurrogate()) { ASSERT(term->isFixedWidthCharacterClass()); CharacterClass trailsOnlyClass; @@ -3331,16 +3998,38 @@ class YarrGenerator final : public YarrJITInfo { MacroAssembler::Label loop(&m_jit); failures.append(atEndOfInput()); - readCharacter(op.m_checkedOffset - term->inputPosition, character); + if (m_direction == Backward && m_decodeSurrogatePairs) { + // Backward + unicode: decode the code point ending just left of the + // frontier (frame offset 0), class-match it (frontier untouched, so a + // mismatch simply ends the greedy run), then consume it by its width, + // read off the code point exactly as the forward path does. A pair's + // lead is the unit below its trail: once the trail unit is taken and the + // frontier is 0, there is no lead to take, so that path rejoins through + // failuresDecrementIndex, which gives back the trail unit. + readUnicodeCharBackward(negativeOffsetIndexedAddress(op.m_checkedOffset - term->inputPosition, character), character, scratch); + matchCharacterClassTermInner(term, failures, character, scratch); + m_jit.sub32(MacroAssembler::TrustedImm32(1), m_regs.index); // the code point's last unit + MacroAssembler::Jump isBMPChar = m_jit.branch32(MacroAssembler::LessThan, character, MacroAssembler::TrustedImm32(0x10000)); + // The pair's lead is one further unit; mirroring the forward path, test + // for it BEFORE taking it, so a failure here has consumed exactly one unit + // (the trail) -- which failuresDecrementIndex gives back. + failuresDecrementIndex.append(atEndOfInput()); + m_jit.sub32(MacroAssembler::TrustedImm32(1), m_regs.index); // the pair's lead unit + isBMPChar.link(&m_jit); + } else { + readCharacter(op.m_checkedOffset - term->inputPosition, character); - matchCharacterClassTermInner(term, failures, character, scratch); + matchCharacterClassTermInner(term, failures, character, scratch); + if (m_direction == Backward) + m_jit.sub32(MacroAssembler::TrustedImm32(1), m_regs.index); #if ENABLE(YARR_JIT_UNICODE_EXPRESSIONS) - if (m_decodeSurrogatePairs) - advanceIndexAfterCharacterClassTermMatch(term, failuresDecrementIndex, character); - else + else if (m_decodeSurrogatePairs) + advanceIndexAfterCharacterClassTermMatch(term, failuresDecrementIndex, character); #endif - m_jit.add32(MacroAssembler::TrustedImm32(1), m_regs.index); + else + m_jit.add32(MacroAssembler::TrustedImm32(1), m_regs.index); + } m_jit.add32(MacroAssembler::TrustedImm32(1), countRegister); if (term->quantityMaxCount == quantifyInfinite) @@ -3355,7 +4044,12 @@ class YarrGenerator final : public YarrJITInfo { if (!failuresDecrementIndex.empty()) { failuresDecrementIndex.link(&m_jit); - m_jit.sub32(MacroAssembler::TrustedImm32(1), m_regs.index); + // Give back the unit that was already consumed before the failure: the + // frontier moved up in a forward frame, down in a backward frame. + if (m_direction == Backward) + m_jit.add32(MacroAssembler::TrustedImm32(1), m_regs.index); + else + m_jit.sub32(MacroAssembler::TrustedImm32(1), m_regs.index); } failures.link(&m_jit); @@ -3379,7 +4073,7 @@ class YarrGenerator final : public YarrJITInfo { loadFromFrame(term->frameLocation + BackTrackInfoCharacterClass::matchAmountIndex(), countRegister); if (m_decodeSurrogatePairs && term->characterClass->hasNonBMPCharacters()) m_jit.lshift32(MacroAssembler::TrustedImm32(1), countRegister); // 2 code units per match. - m_jit.sub32(countRegister, m_regs.index); + rewindIndex(countRegister); m_backtrackingState.fallthrough(); return; } @@ -3389,12 +4083,20 @@ class YarrGenerator final : public YarrJITInfo { m_jit.sub32(MacroAssembler::TrustedImm32(1), countRegister); storeToFrame(countRegister, term->frameLocation + BackTrackInfoCharacterClass::matchAmountIndex()); - if (!m_decodeSurrogatePairs) + if (m_direction == Backward && !m_decodeSurrogatePairs) + m_jit.add32(MacroAssembler::TrustedImm32(1), m_regs.index); + else if (m_direction == Backward && term->isFixedWidthCharacterClass()) + m_jit.add32(MacroAssembler::TrustedImm32(term->characterClass->hasNonBMPCharacters() ? 2 : 1), m_regs.index); + else if (m_direction != Backward && !m_decodeSurrogatePairs) m_jit.sub32(MacroAssembler::TrustedImm32(1), m_regs.index); - else if (term->isFixedWidthCharacterClass()) + else if (m_direction != Backward && term->isFixedWidthCharacterClass()) m_jit.sub32(MacroAssembler::TrustedImm32(term->characterClass->hasNonBMPCharacters() ? 2 : 1), m_regs.index); else { - // Rematch one less + // Rematch one less. The consumed characters have data-dependent widths, + // so the frontier after count-1 matches is recovered by re-consuming + // that many characters from the recorded begin index: rightward in a + // forward frame, leftward (decoding each code point from its last + // unit) in a backward frame. const MacroAssembler::RegisterID character = m_regs.regT0; loadFromFrame(term->frameLocation + BackTrackInfoCharacterClass::beginIndex(), m_regs.index); @@ -3402,16 +4104,26 @@ class YarrGenerator final : public YarrJITInfo { MacroAssembler::Label rematchLoop(&m_jit); MacroAssembler::Jump doneRematching = m_jit.branchTest32(MacroAssembler::Zero, countRegister); - readCharacter(op.m_checkedOffset - term->inputPosition, character); + if (m_direction == Backward) { + readUnicodeCharBackward(negativeOffsetIndexedAddress(op.m_checkedOffset - term->inputPosition, character), character, m_regs.regT2); + m_usesT2 = true; + m_jit.sub32(MacroAssembler::TrustedImm32(1), countRegister); + m_jit.sub32(MacroAssembler::TrustedImm32(1), m_regs.index); + MacroAssembler::Jump isBMPChar = m_jit.branch32(MacroAssembler::LessThan, character, MacroAssembler::TrustedImm32(0x10000)); + m_jit.sub32(MacroAssembler::TrustedImm32(1), m_regs.index); + isBMPChar.link(&m_jit); + } else { + readCharacter(op.m_checkedOffset - term->inputPosition, character); - m_jit.sub32(MacroAssembler::TrustedImm32(1), countRegister); - m_jit.add32(MacroAssembler::TrustedImm32(1), m_regs.index); + m_jit.sub32(MacroAssembler::TrustedImm32(1), countRegister); + m_jit.add32(MacroAssembler::TrustedImm32(1), m_regs.index); #if ENABLE(YARR_JIT_UNICODE_EXPRESSIONS) - MacroAssembler::Jump isBMPChar = m_jit.branch32(MacroAssembler::LessThan, character, MacroAssembler::TrustedImm32(0x10000)); - m_jit.add32(MacroAssembler::TrustedImm32(1), m_regs.index); - isBMPChar.link(&m_jit); + MacroAssembler::Jump isBMPChar = m_jit.branch32(MacroAssembler::LessThan, character, MacroAssembler::TrustedImm32(0x10000)); + m_jit.add32(MacroAssembler::TrustedImm32(1), m_regs.index); + isBMPChar.link(&m_jit); #endif + } m_jit.jump(rematchLoop); doneRematching.link(&m_jit); @@ -3460,31 +4172,61 @@ class YarrGenerator final : public YarrJITInfo { nonGreedyFailures.append(atEndOfInput()); nonGreedyFailures.append(m_jit.branch32(MacroAssembler::Equal, countRegister, MacroAssembler::Imm32(term->quantityMaxCount))); - readCharacter(op.m_checkedOffset - term->inputPosition, character); + if (m_direction == Backward && m_decodeSurrogatePairs) { + // Backward + unicode: as in the greedy loop, decode the code point + // ending just left of the frontier, class-match it, then consume it by + // the width read off the code point; the pair's lead needs a further + // unit below the frontier or the take is undone via + // nonGreedyFailuresDecrementIndex. + readUnicodeCharBackward(negativeOffsetIndexedAddress(op.m_checkedOffset - term->inputPosition, character), character, scratch); + matchCharacterClassTermInner(term, nonGreedyFailures, character, scratch); + m_jit.sub32(MacroAssembler::TrustedImm32(1), m_regs.index); // the code point's last unit + MacroAssembler::Jump isBMPChar = m_jit.branch32(MacroAssembler::LessThan, character, MacroAssembler::TrustedImm32(0x10000)); + // As in the greedy loop: test for the lead before taking it, so the failure + // path (nonGreedyFailuresDecrementIndex) gives back exactly the trail. + nonGreedyFailuresDecrementIndex.append(atEndOfInput()); + m_jit.sub32(MacroAssembler::TrustedImm32(1), m_regs.index); // the pair's lead unit + isBMPChar.link(&m_jit); + } else { + readCharacter(op.m_checkedOffset - term->inputPosition, character); - matchCharacterClassTermInner(term, nonGreedyFailures, character, scratch); + matchCharacterClassTermInner(term, nonGreedyFailures, character, scratch); + if (m_direction == Backward) + m_jit.sub32(MacroAssembler::TrustedImm32(1), m_regs.index); #if ENABLE(YARR_JIT_UNICODE_EXPRESSIONS) - if (m_decodeSurrogatePairs) - advanceIndexAfterCharacterClassTermMatch(term, nonGreedyFailuresDecrementIndex, character); - else + else if (m_decodeSurrogatePairs) + advanceIndexAfterCharacterClassTermMatch(term, nonGreedyFailuresDecrementIndex, character); #endif - m_jit.add32(MacroAssembler::TrustedImm32(1), m_regs.index); + else + m_jit.add32(MacroAssembler::TrustedImm32(1), m_regs.index); + } m_jit.add32(MacroAssembler::TrustedImm32(1), countRegister); m_jit.jump(op.m_reentry); if (!nonGreedyFailuresDecrementIndex.empty()) { nonGreedyFailuresDecrementIndex.link(&m_jit); - m_jit.sub32(MacroAssembler::TrustedImm32(1), m_regs.index); + // Give back the unit consumed before the failure (frontier moved up in + // a forward frame, down in a backward frame). + if (m_direction == Backward) + m_jit.add32(MacroAssembler::TrustedImm32(1), m_regs.index); + else + m_jit.sub32(MacroAssembler::TrustedImm32(1), m_regs.index); } nonGreedyFailures.link(&m_jit); + // Restore the frontier to where this term began. With per-match widths + // (unicode) the distance is not derivable from the match count, so it is + // reloaded from beginIndex; otherwise each match was exactly one unit. #if ENABLE(YARR_JIT_UNICODE_EXPRESSIONS) if (m_decodeSurrogatePairs) loadFromFrame(term->frameLocation + BackTrackInfoCharacterClass::beginIndex(), m_regs.index); else #endif + if (m_direction == Backward) + m_jit.add32(countRegister, m_regs.index); + else m_jit.sub32(countRegister, m_regs.index); m_backtrackingState.fallthrough(); } @@ -3568,6 +4310,15 @@ class YarrGenerator final : public YarrJITInfo { YarrOp& op = m_ops[opIndex]; PatternTerm* term = op.m_term; +#if ENABLE(YARR_JIT_UNICODE_EXPRESSIONS) && ENABLE(YARR_JIT_UNICODE_CAN_INCREMENT_INDEX_FOR_NON_BMP) + // firstCharacterAdditionalReadSize means "the character at the match START + // was a non-BMP pair, so advancing the start by one code point takes an + // extra unit". Only the term whose first read is at the match start may + // establish that (see YarrOp::m_readsStartCharacter); a non-BMP read + // anywhere else says nothing about the start character. + SetForScope firstCharacterOnly(m_useFirstNonBMPCharacterOptimization, m_canUseFirstNonBMPCharacterOptimization && op.m_readsStartCharacter); +#endif + switch (term->type) { case PatternTerm::Type::PatternCharacter: switch (term->quantityType) { @@ -3734,6 +4485,7 @@ class YarrGenerator final : public YarrJITInfo { m_disassembler->setForGenerate(opIndex, m_jit.label()); YarrOp& op = m_ops[opIndex]; + m_direction = op.m_direction; switch (op.m_op) { case YarrOpCode::Term: @@ -3769,7 +4521,7 @@ class YarrGenerator final : public YarrJITInfo { #if ENABLE(YARR_JIT_UNICODE_EXPRESSIONS) && ENABLE(YARR_JIT_UNICODE_CAN_INCREMENT_INDEX_FOR_NON_BMP) // Initialize before input check to prevent uninitialized data in arithmetic. - if (m_useFirstNonBMPCharacterOptimization) + if (m_canUseFirstNonBMPCharacterOptimization) m_jit.move(MacroAssembler::TrustedImm32(0), m_regs.firstCharacterAdditionalReadSize); #endif @@ -3784,7 +4536,7 @@ class YarrGenerator final : public YarrJITInfo { #if ENABLE(YARR_JIT_UNICODE_EXPRESSIONS) && ENABLE(YARR_JIT_UNICODE_CAN_INCREMENT_INDEX_FOR_NON_BMP) // Initialize after reentry to ensure fresh register state on backtrack retries, // since character reading operations modify this register during execution. - if (m_useFirstNonBMPCharacterOptimization) + if (m_canUseFirstNonBMPCharacterOptimization) m_jit.move(MacroAssembler::TrustedImm32(0), m_regs.firstCharacterAdditionalReadSize); #endif @@ -3994,7 +4746,7 @@ class YarrGenerator final : public YarrJITInfo { // Reset on reentry: a prior alternative may have read a non-BMP codepoint // and left this register set to 1. The next alternative starts a fresh // first-character read. - if (m_useFirstNonBMPCharacterOptimization) + if (m_canUseFirstNonBMPCharacterOptimization) m_jit.move(MacroAssembler::TrustedImm32(0), m_regs.firstCharacterAdditionalReadSize); #endif if (shouldRecordSubpatterns() && priorAlternative->needToCleanupCaptures()) { @@ -4012,7 +4764,7 @@ class YarrGenerator final : public YarrJITInfo { defineReentryLabel(op); #if ENABLE(YARR_JIT_UNICODE_EXPRESSIONS) && ENABLE(YARR_JIT_UNICODE_CAN_INCREMENT_INDEX_FOR_NON_BMP) // Reset on reentry: see BodyAlternativeNext above. - if (m_useFirstNonBMPCharacterOptimization) + if (m_canUseFirstNonBMPCharacterOptimization) m_jit.move(MacroAssembler::TrustedImm32(0), m_regs.firstCharacterAdditionalReadSize); #endif m_jit.sub32(MacroAssembler::Imm32(priorAlternative->m_minimumSize), m_regs.index); @@ -4055,6 +4807,14 @@ class YarrGenerator final : public YarrJITInfo { termMatchTargets.last() = alternative->m_isLastAlternative ? MatchTargets(MatchTargets::PreferredTarget::MatchSuccessFallThrough) : MatchTargets(endOp->m_jumps, op.m_jumps, MatchTargets::PreferredTarget::MatchFailFallThrough); } + // First-character dispatch: read the character once and route to + // the applicable chain, then fall into alternative 0's entry (which + // is now a labelled chain target rather than a fallthrough). + if (op.m_dispatch) { + generateDispatch(op); + bindDispatchEntry(*op.m_dispatch, 0); + } + // Calculate how much input we need to check for, and if non-zero check. op.m_checkAdjust = Checked(alternative->m_minimumSize); if ((term->quantityType == QuantifierType::FixedCount) && (term->quantityMaxCount == 1) && (term->type != PatternTerm::Type::ParentheticalAssertion)) @@ -4108,6 +4868,8 @@ class YarrGenerator final : public YarrJITInfo { // This is the entry point for the next alternative. defineReentryLabel(op); + if (op.m_dispatch) + bindDispatchEntry(*op.m_dispatch, op.m_dispatchAlternativeIndex); // Calculate how much input we need to check for, and if non-zero check. op.m_checkAdjust = alternative->m_minimumSize; @@ -4496,8 +5258,16 @@ class YarrGenerator final : public YarrJITInfo { unsigned parenthesesFrameLocation = term->frameLocation; storeToFrame(m_regs.index, parenthesesFrameLocation + BackTrackInfoParentheticalAssertion::beginIndex()); - if (op.m_checkAdjust) - m_jit.sub32(MacroAssembler::Imm32(op.m_checkAdjust), m_regs.index); + // Realign the index register to the assertion point. In a forward + // frame that point is checkAdjust characters behind the frontier; + // in a backward (mirrored) frame it is checkAdjust characters ahead + // of the leftward cursor. + if (op.m_checkAdjust) { + if (op.m_direction == Backward) + m_jit.add32(MacroAssembler::Imm32(op.m_checkAdjust), m_regs.index); + else + m_jit.sub32(MacroAssembler::Imm32(op.m_checkAdjust), m_regs.index); + } break; } case YarrOpCode::ParentheticalAssertionEnd: { @@ -4543,6 +5313,7 @@ class YarrGenerator final : public YarrJITInfo { m_disassembler->setForBacktrack(opIndex, m_jit.label()); YarrOp& op = m_ops[opIndex]; + m_direction = op.m_direction; switch (op.m_op) { case YarrOpCode::Term: @@ -4624,7 +5395,7 @@ class YarrGenerator final : public YarrJITInfo { if (!m_pattern.m_body->m_hasFixedSize) { if (alternative->m_minimumSize == 1) #if ENABLE(YARR_JIT_UNICODE_EXPRESSIONS) && ENABLE(YARR_JIT_UNICODE_CAN_INCREMENT_INDEX_FOR_NON_BMP) - if (m_useFirstNonBMPCharacterOptimization) { + if (m_canUseFirstNonBMPCharacterOptimization) { m_jit.add32(m_regs.firstCharacterAdditionalReadSize, m_regs.index, m_regs.regT0); setMatchStart(m_regs.regT0); } else @@ -4636,7 +5407,7 @@ class YarrGenerator final : public YarrJITInfo { else m_jit.add32(MacroAssembler::TrustedImm32(1), m_regs.index, m_regs.regT0); #if ENABLE(YARR_JIT_UNICODE_EXPRESSIONS) && ENABLE(YARR_JIT_UNICODE_CAN_INCREMENT_INDEX_FOR_NON_BMP) - if (m_useFirstNonBMPCharacterOptimization) + if (m_canUseFirstNonBMPCharacterOptimization) m_jit.add32(m_regs.firstCharacterAdditionalReadSize, m_regs.regT0); #endif setMatchStart(m_regs.regT0); @@ -4651,7 +5422,7 @@ class YarrGenerator final : public YarrJITInfo { unsigned delta = alternative->m_minimumSize - beginOp->m_alternative->m_minimumSize; ASSERT(delta); #if ENABLE(YARR_JIT_UNICODE_EXPRESSIONS) && ENABLE(YARR_JIT_UNICODE_CAN_INCREMENT_INDEX_FOR_NON_BMP) - if (m_useFirstNonBMPCharacterOptimization) { + if (m_canUseFirstNonBMPCharacterOptimization) { m_jit.add32(m_regs.firstCharacterAdditionalReadSize, m_regs.index); if (delta != 1) m_jit.sub32(MacroAssembler::Imm32(delta - 1), m_regs.index); @@ -4671,7 +5442,7 @@ class YarrGenerator final : public YarrJITInfo { if (delta != 0xFFFFFFFFu) { // We need to check input because we are incrementing the input. #if ENABLE(YARR_JIT_UNICODE_EXPRESSIONS) && ENABLE(YARR_JIT_UNICODE_CAN_INCREMENT_INDEX_FOR_NON_BMP) - if (m_useFirstNonBMPCharacterOptimization) + if (m_canUseFirstNonBMPCharacterOptimization) m_jit.add32(m_regs.firstCharacterAdditionalReadSize, m_regs.index); #endif m_jit.add32(MacroAssembler::Imm32(delta + 1), m_regs.index); @@ -4754,7 +5525,7 @@ class YarrGenerator final : public YarrJITInfo { // If the last alternative had the same minimum size as the disjunction, // just simply increment input pos by 1, no adjustment based on minimum size. #if ENABLE(YARR_JIT_UNICODE_EXPRESSIONS) && ENABLE(YARR_JIT_UNICODE_CAN_INCREMENT_INDEX_FOR_NON_BMP) - if (m_useFirstNonBMPCharacterOptimization) + if (m_canUseFirstNonBMPCharacterOptimization) m_jit.add32(m_regs.firstCharacterAdditionalReadSize, m_regs.index); #endif m_jit.add32(MacroAssembler::TrustedImm32(1), m_regs.index); @@ -4832,6 +5603,36 @@ class YarrGenerator final : public YarrJITInfo { // Treat an input check failure the same as a failed match. m_backtrackingState.append(op.m_jumps); + // Dispatched alternatives (first-character dispatch): a failing + // alternative releases its claim and continues its own chain through + // the resume address the chain stored in the frame, rather than + // falling to a fixed successor. Failures that mean "no alternative can + // match here" (empty routes, exhausted chains) join the group's + // failure flow at the Begin op, where the index is at the group frame. + if (op.m_dispatch) { + if (!m_backtrackingState.isEmpty()) { + m_backtrackingState.link(*this, op); + // Give back the input this alternative claimed on entry. + if (op.m_checkAdjust) + m_jit.sub32(MacroAssembler::Imm32(op.m_checkAdjust), m_regs.index); + loadFromFrameAndJump(op.m_dispatch->frameLocation + BackTrackInfoParenthesesOnce::chainResumeIndex()); + } + if (isBegin) { + // Group-level failures gathered by the dispatch (empty routes, + // exhausted chains) resume at the Begin, where the index is at + // the group frame. (Under dispatch nothing plants jumps on the + // End: Next backtracks jump through chainResume, and dispatch + // requires a FixedCount group, so no zero-length-match check.) + m_backtrackingState.append(op.m_dispatch->groupFailJumps); + } + // For non-simple alternatives, link the alternative's 'return + // address' so we backtrack back out into it correctly. + if (op.m_op == YarrOpCode::NestedAlternativeNext) + m_backtrackingState.append(op.m_returnAddress); + op.m_contentBacktrackEntryLabel = m_jit.label(); + break; + } + // Set the backtracks to jump to the appropriate place. We may need // to link the backtracks in one of three different way depending on // the type of alternative we are dealing with: @@ -4856,7 +5657,11 @@ class YarrGenerator final : public YarrJITInfo { if (!m_backtrackingState.isEmpty()) { // Handle the cases where we need to link the backtracks here. m_backtrackingState.link(*this, op); - m_jit.sub32(MacroAssembler::Imm32(op.m_checkAdjust), m_regs.index); + // Give back the input this alternative claimed on entry. + if (op.m_direction == Backward) + m_jit.add32(MacroAssembler::Imm32(op.m_checkAdjust), m_regs.index); + else + m_jit.sub32(MacroAssembler::Imm32(op.m_checkAdjust), m_regs.index); if (!isLastAlternative) { // An alternative that is not the last should jump to its successor. m_jit.jump(nextOp.m_reentry); @@ -5157,7 +5962,13 @@ class YarrGenerator final : public YarrJITInfo { // This is because NonGreedy already tried with count = 0, failing and reaching here. // So there is no way to proceed further, just propagate a failure. noPreviousIteration.link(&m_jit); - emitClearCapturesForTerm(term); + // Zero iterations of THIS quantified group means its captures are + // undefined -- except for a quantified split's optional copy, whose + // capture ids belong to the mandatory sibling: restoreParenContext has + // already restored the sibling's committed values, and clearing them + // here would erase captures the copy never owned. + if (!term->parentheses.isCopy) + emitClearCapturesForTerm(term); storeToFrame(MacroAssembler::TrustedImm32(-1), parenthesesFrameLocation + BackTrackInfoParentheses::beginIndex()); m_backtrackingState.fallthrough(); break; @@ -5189,7 +6000,9 @@ class YarrGenerator final : public YarrJITInfo { // We need to clear the state, and go back to the further former backtracking. noPreviousIteration.link(&m_jit); op.m_jumps.link(&m_jit); - emitClearCapturesForTerm(term); + // As above: an optional copy owns none of its capture ids. + if (!term->parentheses.isCopy) + emitClearCapturesForTerm(term); } else { // Try fewer iterations. m_jit.jump(endOp.m_reentry); @@ -5254,7 +6067,11 @@ class YarrGenerator final : public YarrJITInfo { exceededMatchLimit.append(m_jit.branch32(MacroAssembler::AboveOrEqual, countTemporary, MacroAssembler::Imm32(term->quantityMaxCount))); } - m_jit.branch32(MacroAssembler::Above, m_regs.index, beginTemporary).linkTo(beginOp.m_reentry, &m_jit); + // Try one more (lazy) iteration only if the last one made progress: + // the frontier moved right in a forward frame, left (down) in a + // mirrored one. Without the direction split a leftward-consuming + // lazy group stops re-entering as soon as its frontier has moved. + m_jit.branch32(op.m_direction == Backward ? MacroAssembler::Below : MacroAssembler::Above, m_regs.index, beginTemporary).linkTo(beginOp.m_reentry, &m_jit); exceededMatchLimit.link(&m_jit); break; @@ -5285,12 +6102,24 @@ class YarrGenerator final : public YarrJITInfo { // We need to handle the backtracks upon backtracking back out // of a parenthetical assertion if either we need to correct - // the input index, or the assertion was inverted. - if (op.m_checkAdjust || term->invert()) { + // the input index, or the assertion was inverted, or the body ran + // in a different direction from this (enclosing) frame -- then the + // body left index on its own coordinate line, and only the position + // saved at entry (beginIndex) is meaningful to the code we back + // out to. An arithmetic undo cannot cross that boundary. + bool bodyDirectionDiffers = term->matchDirection() != op.m_direction; + if (op.m_checkAdjust || term->invert() || bodyDirectionDiffers) { m_backtrackingState.link(*this, op); - if (op.m_checkAdjust) - m_jit.add32(MacroAssembler::Imm32(op.m_checkAdjust), m_regs.index); + if (bodyDirectionDiffers) + loadFromFrame(term->frameLocation + BackTrackInfoParentheticalAssertion::beginIndex(), m_regs.index); + else if (op.m_checkAdjust) { + // Undo the realignment done on entry (see the forward path). + if (op.m_direction == Backward) + m_jit.sub32(MacroAssembler::Imm32(op.m_checkAdjust), m_regs.index); + else + m_jit.add32(MacroAssembler::Imm32(op.m_checkAdjust), m_regs.index); + } // In an inverted assertion failure to match the subpattern // is treated as a successful match - jump to the end of the @@ -5453,12 +6282,25 @@ class YarrGenerator final : public YarrJITInfo { #endif } + // Try first-character dispatch for a plain once-group with several + // alternatives (its ops are the ParenthesesSubpatternOnce + Nested* set). + // The sets must be computed before opCompileAlternative reorders terms. + DispatchInfo* dispatch = nullptr; + if (parenthesesBeginOpCode == YarrOpCode::ParenthesesSubpatternOnceBegin && alternativeBeginOpCode == YarrOpCode::NestedAlternativeBegin) + dispatch = tryPrepareDispatch(term, checkedOffset); + if (Options::verboseRegExpCompilation() && term->parentheses.disjunction->m_alternatives.size() >= alternationDispatchMinAlternatives) { + dumpFirstCharacterSets(*term->parentheses.disjunction); + dataLogLn(dispatch ? "First-character dispatch enabled" : "First-character dispatch not used"); + } + size_t parenBegin = m_ops.size(); appendOp(parenthesesBeginOpCode); appendOp(alternativeBeginOpCode); m_ops.last().m_previousOp = notFound; m_ops.last().m_term = term; + m_ops.last().m_dispatch = dispatch; + m_ops.last().m_dispatchAlternativeIndex = 0; PatternDisjunction* disjunction = term->parentheses.disjunction; auto& alternatives = disjunction->m_alternatives; for (unsigned i = 0; i < alternatives.size(); ++i) { @@ -5494,6 +6336,8 @@ class YarrGenerator final : public YarrJITInfo { lastOp.m_nextOp = thisOpIndex; thisOp.m_previousOp = lastOpIndex; thisOp.m_term = term; + thisOp.m_dispatch = dispatch; + thisOp.m_dispatchAlternativeIndex = i + 1; } YarrOp& lastOp = m_ops.last(); ASSERT(lastOp.m_op == alternativeNextOpCode); @@ -5501,6 +6345,8 @@ class YarrGenerator final : public YarrJITInfo { lastOp.m_alternative = nullptr; lastOp.m_nextOp = notFound; lastOp.m_checkedOffset = checkedOffset; + if (dispatch) + dispatch->endOpIndex = m_ops.size() - 1; size_t parenEnd = m_ops.size(); appendOp(parenthesesEndOpCode); @@ -5515,6 +6361,291 @@ class YarrGenerator final : public YarrJITInfo { m_ops[parenEnd].m_checkedOffset = checkedOffset; } + // Lookbehind body mirroring. + // + // A lookbehind body matches right-to-left ending at the assertion point. + // Matching backward over the input is equivalent to matching a term-reversed + // copy of the body forward over the virtually reversed input, so we build + // such a reversed copy and compile it with the standard forward machinery, + // marking its ops Backward so the input-check/read/end-of-input primitives + // use the leftward cursor variants. + // + // assignAlternativeOffsets numbers a stored term list from a given origin + // using exactly the width rules of + // YarrPatternConstructor::setupAlternativeOffsets. Applied to a mirrored + // copy (stored reversed) it yields the virtual reversed coordinates; applied + // to an ordinary forward body it renumbers that body from the origin. Frame + // locations always stay those the pattern constructor assigned: the + // original lookbehind body is never itself compiled, so a mirrored copy may + // reuse its slots, and a renumbered forward body keeps its own. + // + // Returns false (with m_failureReason set) only on offset overflow. + + bool assignAlternativeOffsets(PatternAlternative* alternative, unsigned initialInputPosition) + { + CheckedUint32 currentInputPosition = initialInputPosition; + + for (auto& term : alternative->m_terms) { + switch (term.type) { + case PatternTerm::Type::AssertionBOL: + case PatternTerm::Type::AssertionEOL: + case PatternTerm::Type::AssertionWordBoundary: + term.inputPosition = currentInputPosition; + break; + + case PatternTerm::Type::NumberedForwardReference: + case PatternTerm::Type::NamedForwardReference: + break; + + case PatternTerm::Type::NumberedBackReference: + case PatternTerm::Type::NamedBackReference: + // Dynamic length: positioned but contributing no width, as in + // setupAlternativeOffsets. Its frame slot (allocated there) is kept. + term.inputPosition = currentInputPosition; + break; + + case PatternTerm::Type::PatternCharacter: + term.inputPosition = currentInputPosition; + if (term.quantityType == QuantifierType::FixedCount) { + if (m_pattern.eitherUnicode()) { + CheckedUint32 tempCount = term.quantityMaxCount; + tempCount *= U16_LENGTH(term.patternCharacter); + currentInputPosition += tempCount; + } else + currentInputPosition += term.quantityMaxCount; + } + break; + + case PatternTerm::Type::CharacterClass: + term.inputPosition = currentInputPosition; + if (term.quantityType == QuantifierType::FixedCount) { + if (m_pattern.eitherUnicode()) { + if (term.characterClass->hasOneCharacterSize() && !term.invert()) { + CheckedUint32 tempCount = term.quantityMaxCount; + tempCount *= term.characterClass->hasNonBMPCharacters() ? 2 : 1; + currentInputPosition += tempCount; + } else + currentInputPosition += term.quantityMaxCount; + } else + currentInputPosition += term.quantityMaxCount; + } + break; + + case PatternTerm::Type::ParenthesesSubpattern: + // Nested group positions continue the enclosing numbering, as in + // setupAlternativeOffsets: the once shape (max count 1, not a copy) + // pre-claims its body minimum and is positioned after it; every + // counted/copied shape claims dynamically and is positioned before it. + if (term.quantityMaxCount == 1 && !term.parentheses.isCopy) { + if (!assignDisjunctionOffsets(term.parentheses.disjunction, currentInputPosition)) + return false; + if (term.quantityType == QuantifierType::FixedCount) + currentInputPosition += term.parentheses.disjunction->m_minimumSize; + term.inputPosition = currentInputPosition; + } else { + term.inputPosition = currentInputPosition; + if (!assignDisjunctionOffsets(term.parentheses.disjunction, currentInputPosition)) + return false; + } + break; + + case PatternTerm::Type::ParentheticalAssertion: + // The assertion's own position is the virtual assertion point. A + // nested lookbehind keeps its original 0-based body (mirrored and + // renumbered on demand when compiled); a nested forward assertion's + // body is numbered from this position, as in + // setupAlternativeOffsets: the enclosing checkedOffset continues into + // the body, so read offsets (checkedOffset - inputPosition) are the + // same as any forward body's, and the body is renumbered from this + // position to stay on that line -- recursively, through further + // nesting. BOL/EOL inside such a body test the real assertion point + // (see generateAssertionBOL/EOL), never the absolute inputPosition. + term.inputPosition = currentInputPosition; + if (term.matchDirection() != Backward) { + if (!assignDisjunctionOffsets(term.parentheses.disjunction, currentInputPosition)) + return false; + } + break; + + default: + RELEASE_ASSERT_NOT_REACHED(); + } + if (currentInputPosition.hasOverflowed()) { + m_failureReason = JITFailureReason::OffsetTooLarge; + return false; + } + } + + // Mirroring reverses term order but must preserve the alternative's width. + RELEASE_ASSERT(alternative->m_minimumSize == (currentInputPosition - initialInputPosition)); + return true; + } + + bool assignDisjunctionOffsets(PatternDisjunction* disjunction, unsigned initialInputPosition) + { + for (auto& alternative : disjunction->m_alternatives) { + if (!assignAlternativeOffsets(alternative.get(), initialInputPosition)) + return false; + } + return true; + } + + // Structurally reverse `source` into `destination`. Nested plain groups are + // deep-copied and reversed as part of the same backward match. + bool mirrorAlternativeInto(PatternAlternative& source, PatternDisjunction& destination) + { + PatternAlternative* mirrored = destination.addNewAlternative(source.m_firstSubpatternId, Backward); + mirrored->m_minimumSize = source.m_minimumSize; + mirrored->m_lastSubpatternId = source.m_lastSubpatternId; + mirrored->m_hasFixedSize = source.m_hasFixedSize; + mirrored->m_containsBOL = source.m_containsBOL; + mirrored->m_isLastAlternative = source.m_isLastAlternative; + + for (size_t i = source.m_terms.size(); i--;) { + PatternTerm term = source.m_terms[i]; + + switch (term.type) { + case PatternTerm::Type::AssertionBOL: + case PatternTerm::Type::AssertionEOL: + case PatternTerm::Type::AssertionWordBoundary: + case PatternTerm::Type::PatternCharacter: + case PatternTerm::Type::CharacterClass: + case PatternTerm::Type::NumberedForwardReference: + case PatternTerm::Type::NamedForwardReference: + break; + + case PatternTerm::Type::NumberedBackReference: + case PatternTerm::Type::NamedBackReference: + break; + + case PatternTerm::Type::ParenthesesSubpattern: + // Groups of any quantifier are mirrored recursively: the paren + // iteration/backtracking machinery keeps positions in saved frame + // slots and per-iteration ParenContexts rather than deriving them + // from counts, so it is direction-neutral. The string-list and + // terminal group forms are only ever produced for the top-level + // pattern's own alternatives, never inside a lookbehind body. + ASSERT(!term.parentheses.isStringList); + ASSERT(!term.parentheses.isEOLStringList); + ASSERT(!term.parentheses.isTerminal); + term.parentheses.disjunction = mirrorDisjunctionForLookbehind(term.parentheses.disjunction); + if (!term.parentheses.disjunction) + return false; + break; + + case PatternTerm::Type::ParentheticalAssertion: + // A nested lookbehind keeps its 0-based body: it is mirrored into its + // own copy on demand when compiled and never mutated in place. A nested + // FORWARD assertion's body is about to be renumbered onto this frame's + // line by assignAlternativeOffsets, so the mirror must own it -- a + // shallow copy would rewrite the pattern's own body, which the + // bytecode interpreter reuses if this JIT compile later bails. + if (term.matchDirection() != Backward) { + term.parentheses.disjunction = copyForwardDisjunctionForMirror(term.parentheses.disjunction); + if (!term.parentheses.disjunction) + return false; + } + break; + + case PatternTerm::Type::DotStarEnclosure: + // Only synthesized for the top-level pattern body (see + // optimizeDotStarWrappedExpressions), never inside a lookbehind. + RELEASE_ASSERT_NOT_REACHED(); + } + + mirrored->m_terms.append(term); + } + return true; + } + + // Build the reversed copy of a lookbehind body disjunction (recursively + // reversing nested groups), then assign virtual input positions from 0. + // Returns null with m_failureReason set on unsupported content. + PatternDisjunction* mirrorDisjunctionForLookbehind(PatternDisjunction* disjunction) + { + if (!isSafeToRecurse()) [[unlikely]] { + m_failureReason = JITFailureReason::ParenthesisNestedTooDeep; + return nullptr; + } + + auto mirroredDisjunction = makeUnique(); + mirroredDisjunction->m_minimumSize = disjunction->m_minimumSize; + mirroredDisjunction->m_callFrameSize = disjunction->m_callFrameSize; + mirroredDisjunction->m_hasFixedSize = disjunction->m_hasFixedSize; + + for (auto& alternative : disjunction->m_alternatives) { + if (!mirrorAlternativeInto(*alternative, *mirroredDisjunction)) + return nullptr; + } + + PatternDisjunction* result = mirroredDisjunction.get(); + m_mirroredDisjunctions.append(WTF::move(mirroredDisjunction)); + return result; + } + + // Structural (in-order) deep copy of a forward body reached from inside a + // mirrored lookbehind body, so that renumbering it (assignAlternativeOffsets) + // touches only mirror-owned terms and never the pattern's own disjunctions. + // Nested groups and nested forward assertions are copied recursively for the + // same reason; a nested lookbehind's body is left shared (it is mirrored into + // its own copy on demand and never renumbered in place). + bool copyForwardAlternativeInto(PatternAlternative& source, PatternDisjunction& destination) + { + PatternAlternative* copy = destination.addNewAlternative(source.m_firstSubpatternId, source.matchDirection()); + copy->m_minimumSize = source.m_minimumSize; + copy->m_lastSubpatternId = source.m_lastSubpatternId; + copy->m_hasFixedSize = source.m_hasFixedSize; + copy->m_containsBOL = source.m_containsBOL; + copy->m_startsWithBOL = source.m_startsWithBOL; + copy->m_onceThrough = source.m_onceThrough; + copy->m_isLastAlternative = source.m_isLastAlternative; + + copy->m_terms.reserveInitialCapacity(source.m_terms.size()); + for (auto& sourceTerm : source.m_terms) { + PatternTerm term = sourceTerm; + switch (term.type) { + case PatternTerm::Type::ParenthesesSubpattern: + term.parentheses.disjunction = copyForwardDisjunctionForMirror(term.parentheses.disjunction); + if (!term.parentheses.disjunction) + return false; + break; + case PatternTerm::Type::ParentheticalAssertion: + if (term.matchDirection() != Backward) { + term.parentheses.disjunction = copyForwardDisjunctionForMirror(term.parentheses.disjunction); + if (!term.parentheses.disjunction) + return false; + } + break; + default: + break; + } + copy->m_terms.append(term); + } + return true; + } + + PatternDisjunction* copyForwardDisjunctionForMirror(PatternDisjunction* disjunction) + { + if (!isSafeToRecurse()) [[unlikely]] { + m_failureReason = JITFailureReason::ParenthesisNestedTooDeep; + return nullptr; + } + + auto copiedDisjunction = makeUnique(); + copiedDisjunction->m_minimumSize = disjunction->m_minimumSize; + copiedDisjunction->m_callFrameSize = disjunction->m_callFrameSize; + copiedDisjunction->m_hasFixedSize = disjunction->m_hasFixedSize; + + for (auto& alternative : disjunction->m_alternatives) { + if (!copyForwardAlternativeInto(*alternative, *copiedDisjunction)) + return nullptr; + } + + PatternDisjunction* result = copiedDisjunction.get(); + m_mirroredDisjunctions.append(WTF::move(copiedDisjunction)); + return result; + } + // opCompileParentheticalAssertion // Emits ops for a parenthetical assertion. These consist of an // YarrOpCode::SimpleNestedAlternativeBegin/Next/End set of nodes wrapping @@ -5530,17 +6661,43 @@ class YarrGenerator final : public YarrJITInfo { return; } + PatternDisjunction* disjunction = term->parentheses.disjunction; + MatchDirection bodyDirection = term->matchDirection(); + if (bodyDirection == Backward) { + // Lookbehind: compile a term-reversed mirrored copy of the body whose + // input positions are in virtual (reversed) coordinates starting at 0. + disjunction = mirrorDisjunctionForLookbehind(disjunction); + if (!disjunction) + return; + if (!assignDisjunctionOffsets(disjunction, 0)) + return; + } + auto originalCheckedOffset = checkedOffset; size_t parenBegin = m_ops.size(); + // The Begin/End ops realign the index register to the assertion point; + // they belong to the enclosing direction (m_direction is unchanged here). appendOp(YarrOpCode::ParentheticalAssertionBegin); m_ops.last().m_checkAdjust = checkedOffset - term->inputPosition; checkedOffset -= m_ops.last().m_checkAdjust; m_ops.last().m_checkedOffset = checkedOffset; + // The body's ops are emitted in the body's own direction. A body starts a + // fresh frame at position 0 whenever its coordinates are not the enclosing + // forward line: a lookbehind body is mirrored and numbered from 0, and a + // forward body nested inside a mirrored (Backward) body is renumbered from + // 0 by assignAlternativeOffsets, since it cannot inherit mirrored offsets. + // A forward body under a forward frame keeps the pattern layer's numbering + // and continues the enclosing checkedOffset. The Begin op has already + // placed the index register at the assertion point in every case. + MatchDirection enclosingDirection = m_direction; + m_direction = bodyDirection; + if (bodyDirection == Backward) + checkedOffset = 0; + appendOp(YarrOpCode::SimpleNestedAlternativeBegin); m_ops.last().m_previousOp = notFound; m_ops.last().m_term = term; - PatternDisjunction* disjunction = term->parentheses.disjunction; auto& alternatives = disjunction->m_alternatives; for (unsigned i = 0; i < alternatives.size(); ++i) { size_t lastOpIndex = m_ops.size() - 1; @@ -5574,6 +6731,9 @@ class YarrGenerator final : public YarrJITInfo { lastOp.m_nextOp = notFound; lastOp.m_checkedOffset = checkedOffset; + // The End op is back in the enclosing direction. + m_direction = enclosingDirection; + size_t parenEnd = m_ops.size(); appendOp(YarrOpCode::ParentheticalAssertionEnd); @@ -5588,25 +6748,49 @@ class YarrGenerator final : public YarrJITInfo { // opCompileAlternative // Called to emit nodes for all terms in an alternative. - void opCompileAlternative(Checked checkedOffset, PatternAlternative* alternative) + // `beginsAtMatchStart` is true only for the pattern's own body alternatives: + // a nested group's or assertion's alternative begins wherever its enclosing + // term sits, so its head term's read says nothing about the character at the + // match start (see YarrOp::m_readsStartCharacter). + void opCompileAlternative(Checked checkedOffset, PatternAlternative* alternative, bool beginsAtMatchStart = false) { optimizeAlternative(alternative); + // The first term reads the match start's character until some term can + // move the frontier at runtime; from then on a read may land anywhere. + bool frontierMayHaveMoved = !beginsAtMatchStart; for (unsigned i = 0; i < alternative->m_terms.size(); ++i) { PatternTerm* term = &alternative->m_terms[i]; switch (term->type) { case PatternTerm::Type::ParenthesesSubpattern: opCompileParenthesesSubpattern(checkedOffset, term); + frontierMayHaveMoved = true; break; case PatternTerm::Type::ParentheticalAssertion: opCompileParentheticalAssertion(checkedOffset, term); break; - default: + default: { appendOp(term); m_ops.last().m_checkedOffset = checkedOffset; + // Only a literal or a class term makes a single fixed-position read of + // the start character; assertions, forward references, and backreferences + // (a variable-length span match, never the start's one code point) do not. + bool readsInput = term->type == PatternTerm::Type::PatternCharacter || term->type == PatternTerm::Type::CharacterClass; + if (readsInput) { + // Only a term that reads exactly once has its single read pinned to + // the match start; a quantified term's later iterations peek beyond + // it. The read variant only records that the start character is + // non-BMP (it never moves the frontier), so a literal or a class head + // that reads once at the start both qualify. + bool readsOnce = term->quantityType == QuantifierType::FixedCount && term->quantityMaxCount == 1; + m_ops.last().m_readsStartCharacter = !frontierMayHaveMoved && readsOnce; + frontierMayHaveMoved = true; + } + break; + } } } } @@ -5644,7 +6828,7 @@ class YarrGenerator final : public YarrJITInfo { PatternAlternative* alternative = alternatives[currentAlternativeIndex].get(); m_ops[lastOpIndex].m_checkedOffset = alternative->m_minimumSize; - opCompileAlternative(alternative->m_minimumSize, alternative); + opCompileAlternative(alternative->m_minimumSize, alternative, /* beginsAtMatchStart */ true); size_t thisOpIndex = m_ops.size(); appendOp(YarrOp(YarrOpCode::BodyAlternativeNext)); @@ -5680,12 +6864,14 @@ class YarrGenerator final : public YarrJITInfo { m_ops.last().m_previousOp = notFound; if (disjunction->m_minimumSize && !m_pattern.sticky()) { - if (!m_pattern.eitherUnicode()) { + if (!m_decodeSurrogatePairs) { // Collect BoyerMooreInfo if it is possible and profitable. BoyerMooreInfo will be used to emit fast skip path with large stride // at the beginning of the body alternatives. - // We do not emit these fast path when RegExp has sticky or unicode flag. Sticky case does not need this since - // it fails when the body alternatives fail to match with the current offset. - // FIXME: Support unicode flag. + // We do not emit these fast path when RegExp has a sticky flag, since it fails when the body alternatives + // fail to match with the current offset. The skip advances by a count of code units, which could land inside + // a surrogate pair, so this is also limited to compiles that decode no pairs: every 8-bit subject (a Latin1 + // string holds no surrogates, whatever the flags), and 16-bit subjects of non-unicode patterns. + // FIXME: Support unicode flag for 16-bit subjects. // https://bugs.webkit.org/show_bug.cgi?id=228611 auto bmInfo = BoyerMooreInfo::create(m_charSize, std::min(disjunction->m_minimumSize, BoyerMooreInfo::maxLength)); if (collectBoyerMooreInfo(disjunction, currentAlternativeIndex, bmInfo.get())) { @@ -5745,7 +6931,7 @@ class YarrGenerator final : public YarrJITInfo { PatternAlternative* alternative = alternatives[currentAlternativeIndex].get(); ASSERT(!alternative->onceThrough()); m_ops[lastOpIndex].m_checkedOffset = alternative->m_minimumSize; - opCompileAlternative(alternative->m_minimumSize, alternative); + opCompileAlternative(alternative->m_minimumSize, alternative, /* beginsAtMatchStart */ true); size_t thisOpIndex = m_ops.size(); appendOp(YarrOp(YarrOpCode::BodyAlternativeNext)); @@ -5813,6 +6999,426 @@ class YarrGenerator final : public YarrJITInfo { } } + // First-character set analysis (see FirstCharacterSet). + // + // Adds the characters that `term`, matched at the start of what remains, + // can begin with. Returns true if the term must consume at least one + // character (so the alternative's first character is now fully accounted + // for), false if the term can match the empty string and the analysis must + // continue into the following term. + bool addFirstCharactersOfTerm(const PatternTerm& term, FirstCharacterSet& set) + { + switch (term.type) { + case PatternTerm::Type::AssertionBOL: + case PatternTerm::Type::AssertionEOL: + case PatternTerm::Type::AssertionWordBoundary: + case PatternTerm::Type::NumberedForwardReference: + case PatternTerm::Type::NamedForwardReference: + // Zero-width: contributes no first character. + return false; + + case PatternTerm::Type::NumberedBackReference: + case PatternTerm::Type::NamedBackReference: + case PatternTerm::Type::DotStarEnclosure: + set.any = true; + return true; + + case PatternTerm::Type::ParentheticalAssertion: + // An assertion consumes no input; a positive lookahead could narrow the + // set but ignoring it keeps the set a valid over-approximation. + return false; + + case PatternTerm::Type::PatternCharacter: { + char32_t ch = term.patternCharacter; + if (term.ignoreCase() && isASCIIAlpha(ch)) { + set.add(toASCIIUpper(ch)); + set.add(toASCIILower(ch)); + } else if (term.ignoreCase() && !isASCII(ch)) + set.any = true; // Non-ASCII case folding: don't try to be clever. + else + set.add(ch); + return !!term.quantityMinCount; + } + + case PatternTerm::Type::CharacterClass: { + const CharacterClass* characterClass = term.characterClass; + if (term.invert() || characterClass->m_anyCharacter || characterClass->m_table || term.ignoreCase()) + set.any = true; + else { + for (char32_t ch : characterClass->m_matches8) + set.add(ch); + for (auto& range : characterClass->m_ranges8) + set.addRange(range.begin, range.end); + if (!characterClass->m_matches32.isEmpty() || !characterClass->m_ranges32.isEmpty() || characterClass->hasStrings()) + set.wide = true; + } + return !!term.quantityMinCount; + } + + case PatternTerm::Type::ParenthesesSubpattern: { + PatternDisjunction* disjunction = term.parentheses.disjunction; + bool allConsumed = true; + for (auto& alternative : disjunction->m_alternatives) { + FirstCharacterSet nested = computeFirstCharacterSet(*alternative); + set.merge(nested); + allConsumed &= nested.consumed; + } + // The group only guarantees a consumed first character if it is + // required (min count >= 1) and every alternative consumes. + return term.quantityMinCount && allConsumed; + } + } + RELEASE_ASSERT_NOT_REACHED(); + } + + // First-character dispatch code generation. + // + // Emitted at the group's alternative Begin op, before any alternative body: + // + // char = input[index - firstCharacterOffset] ; inside claimed input + // [16-bit: char >= 0x100 -> wide chain] + // binary decision tree over char -> jump to chain head, or group failure + // chain stubs: head_C: frame.chainResume = &stub_C[1]; jump entry(alt i0) + // stub_C[1]: frame.chainResume = &stub_C[2]; jump entry(alt i1) + // ... stub_C[n]: jump groupFailure + // + // A failing alternative releases its claim and jumps to frame.chainResume, so + // it continues its own chain rather than a fixed successor. Entry labels of + // later alternatives are bound as they are generated; stubs that target them + // are patched then via DispatchInfo::pendingEntryJumps. + + // Chain stubs are all emitted at the Begin op, before any alternative's + // entry point exists, so every stub target is a forward reference: collected + // per alternative here and linked when the alternative's entry is generated. + void emitJumpToDispatchEntry(DispatchInfo& info, unsigned alternativeIndex) + { + info.pendingEntryJumps[alternativeIndex].append(m_jit.jump()); + } + + void bindDispatchEntry(DispatchInfo& info, unsigned alternativeIndex) + { + info.pendingEntryJumps[alternativeIndex].link(&m_jit); + info.pendingEntryJumps[alternativeIndex].clear(); + } + + // An alternative that is a fixed-length literal string (only fixed-count + // pattern characters) can never be backtracked into: it either matches its + // exact characters or not. Such alternatives are emitted as inline compare + // blocks inside the chain (no frame store/load, no indirect jump), with a + // static fall-through to the next chain element on mismatch. + // Inlining unrolls one compare per character, so it is only for genuinely + // short literals; a counted literal like a{200000000} keeps the entered + // path, whose fixed-count loop runs at match time instead of being unrolled. + static constexpr unsigned maxInlineLiteralLength = 32; + + bool isInlineLiteralAlternative(const PatternAlternative& alternative) + { + if (alternative.m_terms.isEmpty() || !alternative.m_hasFixedSize) + return false; + if (alternative.m_minimumSize > maxInlineLiteralLength) + return false; + for (auto& term : alternative.m_terms) { + if (term.type != PatternTerm::Type::PatternCharacter) + return false; + if (term.quantityType != QuantifierType::FixedCount || !term.quantityMaxCount || term.quantityMinCount != term.quantityMaxCount) + return false; + if (term.matchDirection() != Forward) + return false; + } + return true; + } + + // Emit an inline literal alternative as part of a chain: claim its extra + // input, compare its characters, and on a full match join the group's + // success path; on mismatch release the claim and fall to the next chain + // element. The return-address slot points at an unwind thunk that releases + // the claim and continues the chain, so backtracking into this matched + // alternative from after the group tries the following alternatives. + void emitInlineLiteralAlternative(DispatchInfo& info, const PatternAlternative& alternative, MacroAssembler::JumpList& toNextElement) + { + const MacroAssembler::RegisterID character = m_regs.regT0; + unsigned extraClaim = alternative.m_minimumSize - info.disjunction->m_minimumSize; + Checked frameChecked = info.enclosingCheckedOffset + extraClaim; + + MacroAssembler::JumpList mismatch; + if (extraClaim) { + m_jit.add32(MacroAssembler::Imm32(extraClaim), m_regs.index); + mismatch.append(m_jit.branch32(MacroAssembler::Above, m_regs.index, m_regs.length)); + } + for (auto& term : alternative.m_terms) { + char32_t ch = term.patternCharacter; + for (unsigned repeat = 0; repeat < term.quantityMaxCount; ++repeat) { + Checked offset = frameChecked - term.inputPosition - repeat; + mismatch.append(jumpIfCharNotEquals(ch, offset, character, term.ignoreCase())); + } + } + + // Full match: record where to unwind to, then skip to the group's End. + MacroAssembler::DataLabelPtr returnAddress = storeToFrameWithPatch(info.frameLocation + BackTrackInfoParenthesesOnce::returnAddressIndex()); + m_ops[info.endOpIndex].m_jumps.append(m_jit.jump()); + + // Unwind thunk (reached only through the return-address slot when + // something after the group fails): release the claim and continue the + // chain by falling through to the next element. + MacroAssembler::Label unwind(m_jit); + m_backtrackingState.addReturnAddressRecord(returnAddress, unwind); + mismatch.link(&m_jit); + if (extraClaim) + m_jit.sub32(MacroAssembler::Imm32(extraClaim), m_regs.index); + toNextElement.append(m_jit.jump()); + } + + // Emit one chain: literal alternatives are compared inline; other + // alternatives are entered through a stub that stores the address of the + // following element as the chain-resume point before jumping in. The final + // element is the group failure. Returns the label of the chain head. + MacroAssembler::Label emitDispatchChainStubs(DispatchInfo& info, const Vector& alternativeIndices) + { + JIT_COMMENT(m_jit, "dispatch chain (", alternativeIndices.size(), " alternatives)"); + MacroAssembler::Label head = m_jit.label(); + Vector pendingStores; + MacroAssembler::JumpList toNextElement; + for (unsigned index : alternativeIndices) { + // Bind the previous element's continuations to here. + if (!pendingStores.isEmpty()) + m_backtrackingState.addReturnAddressRecord(pendingStores.takeLast(), m_jit.label()); + toNextElement.link(&m_jit); + toNextElement.clear(); + + const PatternAlternative& alternative = *info.disjunction->m_alternatives[index]; + if (isInlineLiteralAlternative(alternative)) + emitInlineLiteralAlternative(info, alternative, toNextElement); + else { + pendingStores.append(storeToFrameWithPatch(info.frameLocation + BackTrackInfoParenthesesOnce::chainResumeIndex())); + emitJumpToDispatchEntry(info, index); + } + } + // Chain exhausted: no more applicable alternatives at this position. + if (!pendingStores.isEmpty()) + m_backtrackingState.addReturnAddressRecord(pendingStores.takeLast(), m_jit.label()); + toNextElement.link(&m_jit); + info.groupFailJumps.append(m_jit.jump()); + return head; + } + + // Binary decision tree over the character's Latin-1 value. `ranges` are + // maximal [begin,end] intervals with a constant chain id, sorted and + // contiguous over 0..255. Leaves jump to the chain head (or group failure). + struct DispatchRange { + unsigned begin; + unsigned end; + unsigned chainId; + }; + void emitDispatchTree(DispatchInfo& info, MacroAssembler::RegisterID character, const Vector& ranges, unsigned low, unsigned high) + { + if (low == high) { + unsigned chainId = ranges[low].chainId; + if (chainId == DispatchInfo::noChain) + info.groupFailJumps.append(m_jit.jump()); + else + m_jit.jump().linkTo(info.chains[chainId].head, &m_jit); + return; + } + unsigned middle = (low + high + 1) / 2; + MacroAssembler::Jump upper = m_jit.branch32(MacroAssembler::AboveOrEqual, character, MacroAssembler::TrustedImm32(ranges[middle].begin)); + emitDispatchTree(info, character, ranges, low, middle - 1); + upper.link(&m_jit); + emitDispatchTree(info, character, ranges, middle, high); + } + + void generateDispatch(YarrOp& op) + { + DispatchInfo& info = *op.m_dispatch; + const MacroAssembler::RegisterID character = m_regs.regT0; + + JIT_COMMENT(m_jit, "first-character dispatch"); + readCharacter(info.firstCharacterOffset, character); + + MacroAssembler::JumpList wideCharacter; + if (m_charSize == CharSize::Char16) + wideCharacter.append(m_jit.branch32(MacroAssembler::AboveOrEqual, character, MacroAssembler::TrustedImm32(256))); + + // Emit the chain stubs first so tree leaves can jump backward to them; + // the tree itself is entered by jumping over the stubs. + MacroAssembler::Jump toTree = m_jit.jump(); + for (auto& chain : info.chains) + chain.head = emitDispatchChainStubs(info, chain.alternativeIndices); + if (m_charSize == CharSize::Char16) { + if (info.wideChain.alternativeIndices.isEmpty()) { + MacroAssembler::Label failHead(m_jit); + info.wideChain.head = failHead; + info.groupFailJumps.append(m_jit.jump()); + } else + info.wideChain.head = emitDispatchChainStubs(info, info.wideChain.alternativeIndices); + } + toTree.link(&m_jit); + if (m_charSize == CharSize::Char16) + wideCharacter.linkTo(info.wideChain.head, &m_jit); + + // Group the routing table into maximal ranges of equal chain id. + Vector ranges; + for (unsigned ch = 0; ch < 256;) { + unsigned chainId = info.chainForCharacter[ch]; + unsigned begin = ch; + while (ch < 256 && info.chainForCharacter[ch] == chainId) + ++ch; + ranges.append({ begin, ch - 1, chainId }); + } + emitDispatchTree(info, character, ranges, 0, ranges.size() - 1); + } + + // Decide whether a nested group's alternatives are worth dispatching on the + // first character, and if so build the DispatchInfo (chains and character + // routing). Called at op-compile time, before the alternatives' ops are + // emitted (so the first term of each alternative is still in position order). + // First-character dispatch pays for its extra read only from a handful of + // alternatives up; below that the sequential chain is faster. + static constexpr size_t alternationDispatchMinAlternatives = 4; + + DispatchInfo* tryPrepareDispatch(PatternTerm* term, Checked checkedOffset) + { + // Only once-quantified fixed groups take the ParenthesesSubpatternOnce + + // NestedAlternative path whose frame holds BackTrackInfoParenthesesOnce. + if (term->type != PatternTerm::Type::ParenthesesSubpattern) + return nullptr; + if (term->quantityType != QuantifierType::FixedCount || term->quantityMinCount != 1 || term->quantityMaxCount != 1) + return nullptr; + if (term->parentheses.isCopy || term->parentheses.isTerminal || term->parentheses.isStringList) + return nullptr; + if (term->matchDirection() != Forward || m_direction != Forward) + return nullptr; + if (m_decodeSurrogatePairs) + return nullptr; + + PatternDisjunction* disjunction = term->parentheses.disjunction; + auto& alternatives = disjunction->m_alternatives; + if (alternatives.size() < alternationDispatchMinAlternatives) + return nullptr; + // The first character is only guaranteed inside claimed input when every + // alternative must consume at least one character. + if (!disjunction->m_minimumSize) + return nullptr; + + auto info = makeUniqueRef(); + Vector sets; // per alternative; only needed to build the routing + unsigned anyCount = 0; + for (auto& alternative : alternatives) { + FirstCharacterSet set = computeFirstCharacterSet(*alternative); + if (set.any) + ++anyCount; + sets.append(WTF::move(set)); + } + // Dispatch is pointless when most alternatives can start with anything. + if (anyCount * 4 > alternatives.size()) + return nullptr; + + // Route each Latin-1 character to the ordered chain of alternatives whose + // set contains it; identical chains are shared (linear dedupe: chains are + // few in practice). + for (unsigned ch = 0; ch < 256; ++ch) { + Vector members; + for (unsigned i = 0; i < alternatives.size(); ++i) { + if (sets[i].matches(ch)) + members.append(i); + } + if (members.isEmpty()) { + info->chainForCharacter[ch] = DispatchInfo::noChain; + continue; + } + unsigned chainId = DispatchInfo::noChain; + for (unsigned c = 0; c < info->chains.size(); ++c) { + if (info->chains[c].alternativeIndices == members) { + chainId = c; + break; + } + } + if (chainId == DispatchInfo::noChain) { + chainId = info->chains.size(); + DispatchInfo::Chain chain; + chain.alternativeIndices = WTF::move(members); + info->chains.append(WTF::move(chain)); + } + info->chainForCharacter[ch] = chainId; + } + // Bound the emitted stub code. + unsigned stubCount = 0; + for (auto& chain : info->chains) + stubCount += chain.alternativeIndices.size(); + if (info->chains.size() > 96 || stubCount > 768) + return nullptr; + if (m_charSize == CharSize::Char16) { + for (unsigned i = 0; i < alternatives.size(); ++i) { + if (sets[i].matchesWide()) + info->wideChain.alternativeIndices.append(i); + } + } + + // The group's first character is at the frame position where the group + // starts, inside the input the enclosing alternative already claimed. + Checked groupStart = term->inputPosition - disjunction->m_minimumSize; + if (checkedOffset < term->inputPosition) + return nullptr; + info->firstCharacterOffset = checkedOffset - groupStart; + if (info->firstCharacterOffset.hasOverflowed()) + return nullptr; + info->frameLocation = term->frameLocation; + info->disjunction = disjunction; + info->enclosingCheckedOffset = checkedOffset; + info->pendingEntryJumps.resize(alternatives.size()); + + DispatchInfo* result = info.ptr(); + m_dispatchInfos.append(WTF::move(info)); + return result; + } + + void dumpFirstCharacterSets(PatternDisjunction& disjunction) + { + dataLogLn("First-character sets for ", disjunction.m_alternatives.size(), " alternatives:"); + for (size_t i = 0; i < disjunction.m_alternatives.size(); ++i) { + FirstCharacterSet set = computeFirstCharacterSet(*disjunction.m_alternatives[i]); + dataLog(" alt ", i, ": "); + if (set.any) + dataLog("ANY"); + else { + for (unsigned ch = 0; ch < 256; ++ch) { + if (set.latin1.get(ch)) { + if (isASCIIPrintable(ch)) + dataLog("'", static_cast(ch), "'"); + else + dataLog("\\x", hex(ch, 2)); + } + } + if (set.wide) + dataLog(" +wide"); + } + dataLogLn(); + } + } + + FirstCharacterSet computeFirstCharacterSet(const PatternAlternative& alternative) + { + FirstCharacterSet set; + if (!isSafeToRecurse()) [[unlikely]] { + set.any = true; + return set; + } + for (auto& term : alternative.m_terms) { + if (addFirstCharactersOfTerm(term, set)) { + set.consumed = true; + break; + } + if (set.any) { + set.consumed = true; + break; + } + } + // An alternative that can match without consuming a character can start + // at any character (its "first character" is whatever follows). + if (!set.consumed) + set.any = true; + return set; + } + std::optional collectBoyerMooreInfoFromTerm(PatternTerm& term, unsigned cursor, BoyerMooreInfo& bmInfo) { switch (term.type) { @@ -5889,7 +7495,12 @@ class YarrGenerator final : public YarrJITInfo { } case PatternTerm::Type::ParentheticalAssertion: - return std::nullopt; + // An assertion consumes no input, so it contributes no characters at any + // offset of the leading-character sets. A lookbehind constrains input + // before the match start (not tracked here); a lookahead only narrows what + // the following terms may match, so ignoring it keeps the sets a valid + // over-approximation. Conservatively say it just matches, like BOL/EOL/\b. + return cursor; case PatternTerm::Type::DotStarEnclosure: return std::nullopt; @@ -6376,6 +7987,19 @@ class YarrGenerator final : public YarrJITInfo { if (strideLength != 1) return std::nullopt; + // Leave the search scalar once candidates are too common for the vector loop's exit to pay + // for its reads. A sample too short to rate them falls back to counting them, as an absent + // sample already does. + int32_t frequency = 0; + if (m_sampler.canResolveCandidateFrequency()) { + bitmap.forEachSetBit([&](unsigned character) { + frequency += m_sampler.frequency(character); + }); + } else + frequency = bitmap.count(); + if (frequency > SubjectSampler::maxCandidateFrequencyForSIMDSearch) + return std::nullopt; + // Call can clobber SIMD registers if (mayCall()) return std::nullopt; @@ -6867,15 +8491,22 @@ class YarrGenerator final : public YarrJITInfo { } #endif - if (m_pattern.m_containsLookbehinds) { - codeBlock.setFallBackWithFailureReason(JITFailureReason::Lookbehind); - return; - } - + // Lookbehind bodies are JIT-compiled via mirroring (see + // mirrorDisjunctionForLookbehind): the reversed body copy runs through the + // forward machinery with Backward ops, decoding unicode text leftward + // (readUnicodeCharBackward, matching the interpreter's tryReadBackward). + + // The first-non-BMP-character skip advances the next match start past the + // trailing surrogate of an astral start character. That is only sound when + // no match can begin at a mid-pair position -- i.e. when the pattern cannot + // match zero-width (a body minimum size of 0 means some alternative can, and + // a zero-width alternative such as a negative lookahead over . or an + // assertion can then succeed at a dangling-trail position, whose match the + // skip would drop). #if ENABLE(YARR_JIT_UNICODE_EXPRESSIONS) && ENABLE(YARR_JIT_UNICODE_CAN_INCREMENT_INDEX_FOR_NON_BMP) - if (m_decodeSurrogatePairs && m_executionMode != ExecutionMode::InlineTest && !m_pattern.multiline() && !m_pattern.m_containsBOL && !m_pattern.m_containsLookbehinds && !m_pattern.m_containsModifiers) { + if (m_decodeSurrogatePairs && m_executionMode != ExecutionMode::InlineTest && !m_pattern.multiline() && !m_pattern.m_containsBOL && !m_pattern.m_containsLookbehinds && !m_pattern.m_containsModifiers && m_pattern.m_body->m_minimumSize) { ASSERT(m_regs.firstCharacterAdditionalReadSize != InvalidGPRReg); - m_useFirstNonBMPCharacterOptimization = true; + m_canUseFirstNonBMPCharacterOptimization = true; } #endif @@ -7401,6 +9032,7 @@ class YarrGenerator final : public YarrJITInfo { unsigned index = m_ops.size(); m_ops.constructAndAppend(std::forward(args)...); m_ops.last().m_index = index; + m_ops.last().m_direction = m_direction; } private: @@ -7438,6 +9070,12 @@ class YarrGenerator final : public YarrJITInfo { ParenContextSizes m_parenContextSizes; #endif #if ENABLE(YARR_JIT_UNICODE_EXPRESSIONS) && ENABLE(YARR_JIT_UNICODE_CAN_INCREMENT_INDEX_FOR_NON_BMP) + // Whether this pattern may use the index-incrementing first-character read + // (decided once at compile). m_useFirstNonBMPCharacterOptimization is the + // AMBIENT state consulted at emission time: it is off everywhere except while + // generating the one term whose read is the match start (m_readsStartCharacter), + // so loop peeks and backtrack-pass re-reads can never write the register. + bool m_canUseFirstNonBMPCharacterOptimization { false }; bool m_useFirstNonBMPCharacterOptimization { false }; #endif RegisterAtOffsetList m_calleeSaves; @@ -7446,6 +9084,20 @@ class YarrGenerator final : public YarrJITInfo { MacroAssembler::JumpList m_inlinedMatched; MacroAssembler::JumpList m_inlinedFailedMatch; + // The matching direction currently in effect. During op-compilation this is + // the direction ops are being emitted in (stamped onto each YarrOp); during + // code generation and backtracking it is set from the op being processed so + // that the direction-aware primitives (input checks, character addressing, + // end-of-input) emit the leftward variants inside lookbehind bodies. + MatchDirection m_direction { Forward }; + + // Owns the mirrored (term-reversed) copies of lookbehind bodies. The original + // YarrPattern is left untouched so the bytecode fallback remains valid. + Vector> m_mirroredDisjunctions; + + // First-character dispatch state for nested disjunctions (see DispatchInfo). + Vector, 2> m_dispatchInfos; + // The regular expression expressed as a linear sequence of operations. Vector m_ops; Vector, 4> m_bmInfos; @@ -7589,9 +9241,6 @@ static void dumpCompileFailure(JITFailureReason failure) case JITFailureReason::BackReference: dataLog("Can't JIT some patterns containing back references\n"); break; - case JITFailureReason::Lookbehind: - dataLog("Can't JIT a pattern containing lookbehinds\n"); - break; case JITFailureReason::ParenthesizedSubpattern: dataLog("Can't JIT a pattern containing parenthesized subpatterns\n"); break; diff --git a/Source/JavaScriptCore/yarr/YarrJIT.h b/Source/JavaScriptCore/yarr/YarrJIT.h index 4932afa28a40..124a6de19736 100644 --- a/Source/JavaScriptCore/yarr/YarrJIT.h +++ b/Source/JavaScriptCore/yarr/YarrJIT.h @@ -58,7 +58,6 @@ class YarrCodeBlock; enum class JITFailureReason : uint8_t { DecodeSurrogatePair, BackReference, - Lookbehind, ParenthesizedSubpattern, ParenthesisNestedTooDeep, ExecutableMemoryAllocationFailure, diff --git a/Source/JavaScriptCore/yarr/YarrPattern.cpp b/Source/JavaScriptCore/yarr/YarrPattern.cpp index 09bf2a30e9cb..6d7c04553d0a 100644 --- a/Source/JavaScriptCore/yarr/YarrPattern.cpp +++ b/Source/JavaScriptCore/yarr/YarrPattern.cpp @@ -1268,7 +1268,10 @@ class YarrPatternConstructor { void assertionBOL() { - if (!m_alternative->m_terms.size() && !parenthesisInvert() && parenthesisMatchDirection() == Forward) { + // A ^ anywhere inside a lookbehind (even within a lookahead nested in one) + // constrains a position BEHIND the match: it never anchors an alternative to + // the match start, so it must not feed the once-through/loop-copy split. + if (!m_alternative->m_terms.size() && !parenthesisInvert() && parenthesisMatchDirection() == Forward && !insideLookbehind()) { m_alternative->m_startsWithBOL = true; m_alternative->m_containsBOL = true; m_pattern.m_containsBOL = true; @@ -1572,8 +1575,17 @@ class YarrPatternConstructor { if (numBOLAnchoredAlts) { m_alternative->m_containsBOL = true; - // If all the alternatives in parens start with BOL, then so does this one - if (numBOLAnchoredAlts == numParenAlternatives) + // The enclosing alternative can only match at the start of input (what + // m_startsWithBOL means) if this group requires a BOL and nothing can + // match before it: every alternative in the parens starts with BOL, the + // group is the alternative's first term, and it is not an inverted + // assertion. A later {0,n} / ? / * on this group makes the anchor + // optional again; quantifyAtom withdraws the flag in that case. + // A BOL matched behind the match (in a lookbehind body) constrains a + // position to the left, not where the enclosing alternative starts, so + // it never anchors that alternative (copyDisjunction likewise exempts + // backward alternatives from the once-through filter). + if (numBOLAnchoredAlts == numParenAlternatives && m_alternative->m_terms.size() == 1 && !lastTerm.invert() && lastTerm.matchDirection() != Backward) m_alternative->m_startsWithBOL = true; } @@ -1697,20 +1709,37 @@ class YarrPatternConstructor { for (unsigned alt = 0; alt < disjunction->m_alternatives.size(); ++alt) { PatternAlternative* alternative = disjunction->m_alternatives[alt].get(); if (!filterStartsWithBOL || !alternative->m_startsWithBOL || alternative->m_direction == Backward) { + // Copy the terms first: under filterStartsWithBOL a term can turn out to + // be dead (a required group with no satisfiable alternative), which + // kills this whole alternative -- it is then omitted, never emitted + // with the term silently gone. + Vector copiedTerms; + copiedTerms.reserveInitialCapacity(alternative->m_terms.size()); + bool alternativeIsDead = false; + for (auto& term : alternative->m_terms) { + TermCopy copy = copyTerm(term, filterStartsWithBOL); + if (hasError(error())) + return nullptr; + if (copy.isDead) { + alternativeIsDead = true; + break; + } + if (copy.term) + copiedTerms.append(WTF::move(*copy.term)); + } + if (alternativeIsDead) + continue; + if (!newDisjunction) { newDisjunction = makeUnique(); newDisjunction->m_parent = disjunction->m_parent; } PatternAlternative* newAlternative = newDisjunction->addNewAlternative(alternative->m_firstSubpatternId, alternative->matchDirection()); newAlternative->m_lastSubpatternId = alternative->m_lastSubpatternId; - newAlternative->m_terms.reserveCapacity(alternative->m_terms.size()); - for (auto& term : alternative->m_terms) { - if (auto copied = copyTerm(term, filterStartsWithBOL)) - newAlternative->m_terms.append(WTF::move(*copied)); - } + newAlternative->m_terms = WTF::move(copiedTerms); } } - + if (hasError(error())) { newDisjunction = nullptr; return nullptr; @@ -1723,24 +1752,46 @@ class YarrPatternConstructor { m_pattern.m_disjunctions.append(WTF::move(newDisjunction)); return copiedDisjunction; } - - std::optional copyTerm(PatternTerm& term, bool filterStartsWithBOL) + + // A copied term is one of: the term (recursively copied); nothing, when an + // optional group's every alternative was filtered away (matching its empty + // option is exactly the remaining semantics); or dead, when a REQUIRED group + // has no satisfiable alternative left, which makes any alternative + // containing it unmatchable (copyDisjunction drops such an alternative). + struct TermCopy { + std::optional term; + bool isDead { false }; + }; + + TermCopy copyTerm(PatternTerm& term, bool filterStartsWithBOL) { if (!isSafeToRecurse()) [[unlikely]] { m_error = ErrorCode::PatternTooLarge; - return PatternTerm(term); + return { PatternTerm(term), false }; } if ((term.type != PatternTerm::Type::ParenthesesSubpattern) && (term.type != PatternTerm::Type::ParentheticalAssertion)) - return PatternTerm(term); - + return { PatternTerm(term), false }; + if (auto* newDisjunction = copyDisjunction(term.parentheses.disjunction, filterStartsWithBOL)) { PatternTerm termCopy = term; termCopy.parentheses.disjunction = newDisjunction; m_pattern.m_hasCopiedParenSubexpressions = true; - return termCopy; - } - return std::nullopt; + return { WTF::move(termCopy), false }; + } + // copyDisjunction produced nothing. On an error (recursion limit) report + // no term and no death; the caller checks hasError(). Otherwise every + // alternative was filtered away, i.e. the parenthetical construct's body + // can never match here: + // - a group that may match zero times (or a NEGATIVE assertion, which is + // satisfied precisely when its body cannot match) reduces to matching + // empty, so the term is simply gone; + // - a group that must match, or a POSITIVE assertion, can never be + // satisfied, so the term -- and any alternative containing it -- is dead. + if (hasError(error())) + return { std::nullopt, false }; + bool satisfiedByEmpty = term.type == PatternTerm::Type::ParentheticalAssertion ? term.invert() : !term.quantityMinCount; + return { std::nullopt, !satisfiedByEmpty }; } void quantifyAtom(unsigned min, unsigned max, bool greedy) @@ -1748,6 +1799,13 @@ class YarrPatternConstructor { ASSERT(min <= max); ASSERT(m_alternative->m_terms.size()); + // A group or assertion that anchored its alternative to the start of input + // (see atomParenthesesEnd) no longer does so once it may match zero times: + // the anchor is optional, so the alternative can match anywhere. This must + // precede the {0} handling below, which removes the term entirely. + if (!min && m_alternative->m_terms.size() == 1) + m_alternative->m_startsWithBOL = false; + if (!max) { // In a case of backwards parentheses matching, we may have a forward reference that has // been quantified with {0}, meaning that we can elide it. We should check if we added an @@ -1790,23 +1848,35 @@ class YarrPatternConstructor { else if (!min || (term.type == PatternTerm::Type::ParenthesesSubpattern && (m_pattern.m_hasCopiedParenSubexpressions || term.matchDirection() == Forward))) { - // Forward-direction parenthesized subpatterns with non-zero minimum are kept - // as a single PatternTerm with quantityMinCount > 0; YarrJIT compiles these - // natively (see opCompileParenthesesSubpattern) without splitting into - // FixedCount{min} + Greedy/NonGreedy{0,max-min}. The split would deep-copy - // the disjunction subtree, which can hit OffsetTooLarge / pattern-size limits - // for very large bounds (e.g. (?:x){2147483648,...}). Backward parens - // (lookbehinds) still use the expansion path; the JIT's right-to-left - // backtracking machinery hasn't been generalized for single-term VariableMin. + // A single quantified term, compiled natively by YarrJIT + // (opCompileParenthesesSubpattern for groups). Forward parenthesized + // subpatterns keep this form even with a non-zero minimum; so does any + // group once the pattern already contains a split copy + // (m_hasCopiedParenSubexpressions): splitting a body that itself contains + // a copy re-copies it, so nested min>0 groups (e.g. (?:(?:(?:a)+)+)+, in a + // lookbehind or forward) would grow the pattern -- and both the JIT's op + // vector and the interpreter's work -- exponentially in the nesting + // depth. Only the innermost quantification of such a nest splits. This + // also avoids the split's other copy costs (OffsetTooLarge / size limits + // for huge bounds, e.g. (?:x){2147483648,...}). term.quantify(min, max, greedy ? QuantifierType::Greedy : QuantifierType::NonGreedy); } else { + // Split X{min,max} into a mandatory FixedCount piece and an optional + // {0,max-min} copy sharing X's capture ids: reached by non-parenthesis + // atoms and by the innermost quantified group of a backward (lookbehind) + // parenthesized subpattern, whose mirrored body runs through the copy + // machinery. Term order is match + // order: source order forward, reversed by the mirror for backward + // bodies. isCopy marks the optional piece; a copy that ran zero + // iterations must not clear the capture ids it shares with the + // mandatory piece (see the paren Begin backtrack in YarrJIT). if (term.matchDirection() == Forward) { term.quantify(min, min, QuantifierType::FixedCount); - auto copied = copyTerm(term, /* filterStartsWithBOL */ false); + // Unfiltered copies never come back dead; a missing term is the error path. + auto copied = copyTerm(term, /* filterStartsWithBOL */ false).term; if (!copied) [[unlikely]] return; m_alternative->m_terms.append(WTF::move(*copied)); - // NOTE: this term is interesting from an analysis perspective, in that it can be ignored..... m_alternative->lastTerm().quantify((max == quantifyInfinite) ? max : max - min, greedy ? QuantifierType::Greedy : QuantifierType::NonGreedy); if (m_alternative->lastTerm().type == PatternTerm::Type::ParenthesesSubpattern) m_alternative->lastTerm().parentheses.isCopy = true; @@ -1814,7 +1884,8 @@ class YarrPatternConstructor { term.quantify((max == quantifyInfinite) ? max : max - min, greedy ? QuantifierType::Greedy : QuantifierType::NonGreedy); if (term.type == PatternTerm::Type::ParenthesesSubpattern) term.parentheses.isCopy = true; - auto copied = copyTerm(term, /* filterStartsWithBOL */ false); + // Unfiltered copies never come back dead; a missing term is the error path. + auto copied = copyTerm(term, /* filterStartsWithBOL */ false).term; if (!copied) [[unlikely]] return; m_alternative->m_terms.append(WTF::move(*copied)); @@ -2334,11 +2405,340 @@ class YarrPatternConstructor { // Move alternatives from loopDisjunction to disjunction for (unsigned alt = 0; alt < loopDisjunction->m_alternatives.size(); ++alt) disjunction->m_alternatives.append(loopDisjunction->m_alternatives[alt].release()); - + loopDisjunction->m_alternatives.clear(); } } + // Below these alternative counts the plainer code shapes the JIT emits + // for a short alternation (fused literal compares, the two-alternative + // SIMD scan, frame-free inlinable groups) beat the rewrites; the transforms + // pay for themselves only on wide alternations. + static constexpr size_t alternationFactoringMinRun = 8; // prefix factoring / top-level fold + + // Alternation prefix factoring. + // + // Alternatives are tried leftmost-first, so their order is observable -- but + // only between alternatives that can match at the same starting position. + // Two alternatives that must begin with different literal characters have + // disjoint starting points, so a maximal run of consecutive alternatives + // that each begin with a (non-optional, case-sensitive) literal character + // may be stably sorted by that character and then merged on common + // prefixes: /aq|bx|ar|by/ becomes /a(?:q|r)|b(?:x|y)/. Stability keeps + // same-first-character alternatives in source order, and any alternative + // that does not start with such a character (a class, group, anchor, + // optional atom, or the empty alternative) is a barrier that no reordering + // crosses. The rewrite is applied recursively to the factored suffixes. It + // is a pure pattern-level equivalence, so both the JIT and the interpreter + // see the factored form. + + // The leading literal character of an alternative, if its first term is a + // fixed, case-sensitive pattern character that must consume input. + std::optional firstLiteralCharacter(const PatternAlternative& alternative) + { + if (alternative.m_terms.isEmpty()) + return std::nullopt; + const PatternTerm& term = alternative.m_terms[0]; + if (term.type != PatternTerm::Type::PatternCharacter) + return std::nullopt; + if (term.quantityType != QuantifierType::FixedCount || term.quantityMinCount != 1 || term.quantityMaxCount != 1) + return std::nullopt; + if (term.ignoreCase() || term.matchDirection() != Forward) + return std::nullopt; + return term.patternCharacter; + } + + // Two leading terms match for prefix purposes only if they are the same + // fixed, case-sensitive pattern character. + static bool isSameLiteralTerm(const PatternTerm& a, const PatternTerm& b) + { + return a.type == PatternTerm::Type::PatternCharacter + && b.type == PatternTerm::Type::PatternCharacter + && a.quantityType == QuantifierType::FixedCount && b.quantityType == QuantifierType::FixedCount + && a.quantityMinCount == 1 && a.quantityMaxCount == 1 + && b.quantityMinCount == 1 && b.quantityMaxCount == 1 + && !a.ignoreCase() && !b.ignoreCase() + && a.matchDirection() == Forward && b.matchDirection() == Forward + && a.patternCharacter == b.patternCharacter; + } + + // Merge a group of alternatives (already known to share their leading + // literal term) into a single alternative: a X | a Y | a Z -> a (?: X | Y | Z), + // with the longest common literal prefix hoisted and the suffixes factored + // recursively. `members` are removed from their owner and re-parented. + std::unique_ptr mergeSharedPrefix(Vector>&& members) + { + ASSERT(members.size() >= 2); + + // Longest common prefix of literal terms across all members. A member + // that is a prefix of a longer sibling contributes an empty suffix + // alternative, which keeps its own position in the (stable) order. + size_t prefixLength = 1; + for (;; ++prefixLength) { + const auto& first = members[0]->m_terms; + if (prefixLength >= first.size()) + break; + bool allShare = true; + for (auto& member : members) { + if (prefixLength >= member->m_terms.size() || !isSameLiteralTerm(first[prefixLength], member->m_terms[prefixLength])) { + allShare = false; + break; + } + } + if (!allShare) + break; + } + + auto merged = makeUnique(members[0]->m_parent, members[0]->m_firstSubpatternId); + for (size_t i = 0; i < prefixLength; ++i) + merged->m_terms.append(members[0]->m_terms[i]); + + // The suffix disjunction holds each member's remaining terms, in the + // members' (stable, first-character-sorted) order. The group's capture + // span is computed from the terms (see accumulateCaptureRange). + auto suffixDisjunction = makeUnique(); + unsigned firstCaptureId = std::numeric_limits::max(); + unsigned lastCaptureId = 0; + bool containsBOL = false; + for (auto& member : members) { + PatternAlternative* suffix = suffixDisjunction->addNewAlternative(member->m_firstSubpatternId); + suffix->m_lastSubpatternId = member->m_lastSubpatternId; + suffix->m_containsBOL = member->m_containsBOL; + for (size_t i = prefixLength; i < member->m_terms.size(); ++i) + suffix->m_terms.append(member->m_terms[i]); + reparentNestedDisjunctions(*suffix); + clearTerminalMarks(*suffix); + accumulateCaptureRange(*member, firstCaptureId, lastCaptureId); + containsBOL |= member->m_containsBOL; + } + suffixDisjunction->m_alternatives.last()->m_isLastAlternative = true; + + // Factor the suffixes themselves (a shared second character, and so on). + factorAlternatives(*suffixDisjunction); + + bool hasCaptures = firstCaptureId <= lastCaptureId; + merged->m_lastSubpatternId = hasCaptures ? lastCaptureId : 0; + merged->m_containsBOL = containsBOL; + PatternTerm group(PatternTerm::Type::ParenthesesSubpattern, hasCaptures ? firstCaptureId : m_pattern.m_numSubpatterns + 1, suffixDisjunction.get(), m_flags, /* capture */ false); + group.parentheses.lastSubpatternId = hasCaptures ? lastCaptureId : 0; + merged->m_terms.append(group); + suffixDisjunction->m_parent = merged.get(); + m_pattern.m_disjunctions.append(WTF::move(suffixDisjunction)); + return merged; + } + + // A nested group's disjunction points back at its owning alternative; keep + // that consistent when terms move to a new alternative. + void reparentNestedDisjunctions(PatternAlternative& alternative) + { + for (auto& term : alternative.m_terms) { + if ((term.type == PatternTerm::Type::ParenthesesSubpattern || term.type == PatternTerm::Type::ParentheticalAssertion) && term.parentheses.disjunction) + term.parentheses.disjunction->m_parent = &alternative; + } + } + + // The [first, last] capture-subpattern ids actually contained in an + // alternative's terms (recursively). Parser bookkeeping on the alternative + // (m_firstSubpatternId / m_lastSubpatternId) is not reliable enough here: + // m_lastSubpatternId is only set once a following sibling is parsed, and + // sorting reorders which alternative comes first. + static void accumulateCaptureRange(const PatternAlternative& alternative, unsigned& first, unsigned& last) + { + for (auto& term : alternative.m_terms) { + if (term.type != PatternTerm::Type::ParenthesesSubpattern && term.type != PatternTerm::Type::ParentheticalAssertion) + continue; + if (term.capture()) { + first = std::min(first, term.parentheses.subpatternId); + last = std::max(last, term.parentheses.subpatternId); + } + if (auto* nested = term.parentheses.disjunction) { + for (auto& nestedAlternative : nested->m_alternatives) + accumulateCaptureRange(*nestedAlternative, first, last); + } + } + } + + // A "terminal" parenthesis is valid only as the last term of a body + // alternative (nothing after the body can force a backtrack into it). Once + // an alternative moves inside a nested group that guarantee is gone, so + // drop the marks (the general once/greedy path is used instead). + static void clearTerminalMarks(PatternAlternative& alternative) + { + for (auto& term : alternative.m_terms) { + if (term.type == PatternTerm::Type::ParenthesesSubpattern) + term.parentheses.isTerminal = false; + } + } + + // Rewrite `disjunction`'s alternatives in place: sort each barrier-free run + // by leading literal and merge shared prefixes into nested groups. + void factorAlternatives(PatternDisjunction& disjunction) + { + if (!isSafeToRecurse()) [[unlikely]] + return; + + auto& alternatives = disjunction.m_alternatives; + Vector> result; + result.reserveInitialCapacity(alternatives.size()); + + size_t i = 0; + while (i < alternatives.size()) { + // A barrier (no fixed leading literal, or the empty alternative) is + // copied through untouched and never reordered across. + if (!firstLiteralCharacter(*alternatives[i]) || alternatives[i]->onceThrough()) { + result.append(WTF::move(alternatives[i])); + ++i; + continue; + } + + // Gather the maximal run of literal-leading alternatives. + size_t runStart = i; + while (i < alternatives.size() && firstLiteralCharacter(*alternatives[i]) && !alternatives[i]->onceThrough()) + ++i; + size_t runEnd = i; + + // Small alternations are left alone: the sequential JIT path (fused + // compares, the two-alternative SIMD scan, frame-free inlinable code) + // is already optimal there, and factoring would forfeit those. The + // rewrite pays for itself only on large alternations. + if (runEnd - runStart < alternationFactoringMinRun) { + for (size_t j = runStart; j < runEnd; ++j) + result.append(WTF::move(alternatives[j])); + continue; + } + + Vector> run; + for (size_t j = runStart; j < runEnd; ++j) + run.append(WTF::move(alternatives[j])); + std::stable_sort(run.begin(), run.end(), [&](auto& a, auto& b) { + return *firstLiteralCharacter(*a) < *firstLiteralCharacter(*b); + }); + + // Walk the sorted run, merging each maximal group that shares the + // leading literal term. + size_t j = 0; + while (j < run.size()) { + size_t groupEnd = j + 1; + while (groupEnd < run.size() && isSameLiteralTerm(run[j]->m_terms[0], run[groupEnd]->m_terms[0])) + ++groupEnd; + if (groupEnd - j == 1) + result.append(WTF::move(run[j])); + else { + Vector> members; + for (size_t k = j; k < groupEnd; ++k) + members.append(WTF::move(run[k])); + auto merged = mergeSharedPrefix(WTF::move(members)); + merged->m_parent = &disjunction; + result.append(WTF::move(merged)); + } + j = groupEnd; + } + } + + // Every alternative was moved into `result`; always reinstall the list + // (a run that sorted or merged nothing is simply the original order). + for (auto& alternative : result) { + alternative->m_parent = &disjunction; + alternative->m_isLastAlternative = false; + } + result.last()->m_isLastAlternative = true; + alternatives = WTF::move(result); + + // Factor inside pre-existing (source-written) groups too, so that + // /\b(?:about|above|after)\b/ shares its "a" prefix like a top-level + // alternation would. Groups synthesized by mergeSharedPrefix were + // already factored when built; re-running is a harmless no-op. Groups + // that checkForTerminalParentheses already committed to a specialized + // code shape (string lists, terminal parentheses) are left alone: that + // shape is fixed by their alternatives, which restructuring would break. + for (auto& alternative : alternatives) { + for (auto& term : alternative->m_terms) { + if (term.type == PatternTerm::Type::ParenthesesSubpattern && term.parentheses.disjunction && !term.parentheses.isCopy + && !term.parentheses.isStringList && !term.parentheses.isTerminal + && term.parentheses.disjunction->m_alternatives.size() > 1 && term.matchDirection() == Forward) + factorAlternatives(*term.parentheses.disjunction); + } + } + } + + // A large top-level alternation costs one entry attempt per alternative at + // every candidate position when the alternatives are tried in sequence. Fold + // the repeated (non-once-through) alternatives into a single alternative + // holding one non-capturing group -- /X|Y|Z/ is exactly /(?:X|Y|Z)/ -- so + // that the group's alternatives can be dispatched on their first character. + // The rewrite itself is engine-neutral; setupOffsets() (which runs after + // this) lays out the group like any hand-written one. + void factorAndWrapAlternatives() + { + factorAlternatives(*m_pattern.m_body); + wrapAlternativesForDispatch(); + } + + void wrapAlternativesForDispatch() + { + PatternDisjunction* body = m_pattern.m_body; + auto& alternatives = body->m_alternatives; + + // The body is laid out as [onceThrough..., repeated...] (see optimizeBOL). + size_t firstRepeated = 0; + while (firstRepeated < alternatives.size() && alternatives[firstRepeated]->onceThrough()) + ++firstRepeated; + size_t repeatedCount = alternatives.size() - firstRepeated; + if (repeatedCount < alternationFactoringMinRun) + return; + + // A DotStarEnclosure records match bounds through the enclosing body + // alternative; keep such bodies in their existing shape. + for (size_t i = firstRepeated; i < alternatives.size(); ++i) { + for (auto& term : alternatives[i]->m_terms) { + if (term.type == PatternTerm::Type::DotStarEnclosure) + return; + } + } + + // The group's capture span brackets exactly the captures its alternatives + // contain, computed from the terms themselves (sorting reorders which + // alternative is first, and the parser leaves the last body alternative's + // m_lastSubpatternId unset, so per-alternative bookkeeping is not reliable). + unsigned firstCaptureId = std::numeric_limits::max(); + unsigned lastCaptureId = 0; + bool containsBOL = false; + unsigned startsWithBOLCount = 0; + + auto groupDisjunction = makeUnique(); + for (size_t i = firstRepeated; i < alternatives.size(); ++i) { + PatternAlternative* alternative = alternatives[i].get(); + alternative->m_parent = groupDisjunction.get(); + alternative->m_isLastAlternative = false; + clearTerminalMarks(*alternative); + accumulateCaptureRange(*alternative, firstCaptureId, lastCaptureId); + containsBOL |= alternative->m_containsBOL; + if (alternative->m_startsWithBOL) + ++startsWithBOLCount; + groupDisjunction->m_alternatives.append(WTF::move(alternatives[i])); + } + groupDisjunction->m_alternatives.last()->m_isLastAlternative = true; + alternatives.shrink(firstRepeated); + + // With no captures inside, subpatternId > lastSubpatternId makes + // containsAnyCaptures() false (the convention for capture-free groups). + bool hasCaptures = firstCaptureId <= lastCaptureId; + unsigned groupSubpatternId = hasCaptures ? firstCaptureId : m_pattern.m_numSubpatterns + 1; + unsigned groupLastSubpatternId = hasCaptures ? lastCaptureId : 0; + + PatternAlternative* wrapped = body->addNewAlternative(hasCaptures ? firstCaptureId - 1 : m_pattern.m_numSubpatterns); + wrapped->m_lastSubpatternId = groupLastSubpatternId; + wrapped->m_containsBOL = containsBOL; + wrapped->m_startsWithBOL = startsWithBOLCount == groupDisjunction->m_alternatives.size(); + groupDisjunction->m_parent = wrapped; + + PatternTerm group(PatternTerm::Type::ParenthesesSubpattern, groupSubpatternId, groupDisjunction.get(), m_flags, /* capture */ false); + group.parentheses.lastSubpatternId = groupLastSubpatternId; + wrapped->m_terms.append(group); + + m_pattern.m_disjunctions.append(WTF::move(groupDisjunction)); + } + bool containsCapturingTerms(PatternAlternative* alternative, size_t firstTermIndex, size_t endIndex) { if (!isSafeToRecurse()) [[unlikely]] { @@ -2737,19 +3137,21 @@ class YarrPatternConstructor { private: class SavedContext { public: - SavedContext(bool isModifier, bool invert, MatchDirection matchDirection, OptionSet flags) + SavedContext(bool isModifier, bool invert, MatchDirection matchDirection, bool insideLookbehind, OptionSet flags) : m_isModifier(isModifier) , m_invert(invert) , m_matchDirection(matchDirection) + , m_insideLookbehind(insideLookbehind) , m_flags(flags) { } - void NODELETE restore(bool& isModifier, bool& invert, MatchDirection& matchDirection, OptionSet& flags) + void NODELETE restore(bool& isModifier, bool& invert, MatchDirection& matchDirection, bool& insideLookbehind, OptionSet& flags) { isModifier = m_isModifier; invert = m_invert; matchDirection = m_matchDirection; + insideLookbehind = m_insideLookbehind; flags = m_flags; } @@ -2757,6 +3159,7 @@ class YarrPatternConstructor { bool m_isModifier { false }; bool m_invert { false }; MatchDirection m_matchDirection { Forward }; + bool m_insideLookbehind { false }; OptionSet m_flags; }; @@ -2770,9 +3173,10 @@ class YarrPatternConstructor { ASSERT(m_stackDepth < std::numeric_limits::max()); if (m_stackDepth++ > 0) - m_backingStack.append(SavedContext(m_isModifier, m_invert, m_matchDirection, m_flags)); + m_backingStack.append(SavedContext(m_isModifier, m_invert, m_matchDirection, m_insideLookbehind, m_flags)); - // isModifier should only apply to one frame at a time + // isModifier should only apply to one frame at a time. m_insideLookbehind is + // inherited: a nested parenthesis stays inside any enclosing lookbehind. m_isModifier = false; } @@ -2782,11 +3186,12 @@ class YarrPatternConstructor { if (--m_stackDepth > 0) { SavedContext context = m_backingStack.takeLast(); - context.restore(m_isModifier, m_invert, m_matchDirection, m_flags); + context.restore(m_isModifier, m_invert, m_matchDirection, m_insideLookbehind, m_flags); } else { m_isModifier = false; m_invert = false; m_matchDirection = Forward; + m_insideLookbehind = false; m_flags = { }; } } @@ -2814,6 +3219,10 @@ class YarrPatternConstructor { void setMatchDirection(MatchDirection matchDirection) { m_matchDirection = matchDirection; + // Entering a lookbehind puts every deeper context inside one; the bit is + // inherited through push() and cleared only when this frame pops. + if (matchDirection == Backward) + m_insideLookbehind = true; } MatchDirection matchDirection() const @@ -2821,6 +3230,14 @@ class YarrPatternConstructor { return m_matchDirection; } + // True inside a lookbehind at ANY nesting depth (including within a lookahead + // nested in a lookbehind), unlike matchDirection(), which is only the innermost + // assertion's own direction. + bool NODELETE insideLookbehind() const + { + return m_insideLookbehind; + } + void NODELETE setFlags(OptionSet flags) { m_flags = flags; @@ -2839,6 +3256,7 @@ class YarrPatternConstructor { m_isModifier = false; m_invert = false; m_matchDirection = Forward; + m_insideLookbehind = false; m_flags = { }; } @@ -2848,6 +3266,7 @@ class YarrPatternConstructor { bool m_isModifier { false }; bool m_invert { false }; MatchDirection m_matchDirection { Forward }; + bool m_insideLookbehind { false }; OptionSet m_flags; }; @@ -2881,6 +3300,13 @@ class YarrPatternConstructor { return m_parenthesisContext.matchDirection(); } + // Inside a lookbehind at any nesting depth (a lookahead nested within a + // lookbehind still counts). + bool NODELETE insideLookbehind() const + { + return m_parenthesisContext.insideLookbehind(); + } + bool ignoreCase() const { return m_flags.contains(Flags::IgnoreCase); @@ -2930,6 +3356,7 @@ ErrorCode YarrPattern::compile(StringView patternString) constructor.checkForTerminalParentheses(); constructor.optimizeDotStarWrappedExpressions(); constructor.optimizeBOL(); + constructor.factorAndWrapAlternatives(); constructor.optimizePossessiveQuantifiers(); if (hasError(constructor.error())) diff --git a/Source/JavaScriptCore/yarr/YarrPattern.h b/Source/JavaScriptCore/yarr/YarrPattern.h index ade52491a9fa..712c4c4ddc30 100644 --- a/Source/JavaScriptCore/yarr/YarrPattern.h +++ b/Source/JavaScriptCore/yarr/YarrPattern.h @@ -842,10 +842,15 @@ struct YarrPattern { uintptr_t begin; // Not really needed for greedy quantifiers. uintptr_t matchAmount; // Not really needed for fixed quantifiers. uintptr_t backReferenceSize; // Used by greedy quantifiers to backtrack. + // A backward (lookbehind) backreference's span lies left of the cursor; + // this holds that span's left edge across the compare walk (the JIT's + // forward form gets the equivalent from the moving frontier for free). + uintptr_t backwardSpanEdge; static unsigned beginIndex() { return offsetof(BackTrackInfoBackReference, begin) / sizeof(uintptr_t); } static unsigned matchAmountIndex() { return offsetof(BackTrackInfoBackReference, matchAmount) / sizeof(uintptr_t); } static unsigned backReferenceSizeIndex() { return offsetof(BackTrackInfoBackReference, backReferenceSize) / sizeof(uintptr_t); } + static unsigned backwardSpanEdgeIndex() { return offsetof(BackTrackInfoBackReference, backwardSpanEdge) / sizeof(uintptr_t); } }; struct BackTrackInfoAlternative { @@ -863,9 +868,14 @@ struct YarrPattern { struct BackTrackInfoParenthesesOnce { uintptr_t begin; uintptr_t returnAddress; + // Address of the code that continues a first-character dispatch chain + // when the alternative just entered fails (see the dispatched-alternatives + // path in YarrJIT). + uintptr_t chainResume; static unsigned beginIndex() { return offsetof(BackTrackInfoParenthesesOnce, begin) / sizeof(uintptr_t); } static unsigned returnAddressIndex() { return offsetof(BackTrackInfoParenthesesOnce, returnAddress) / sizeof(uintptr_t); } + static unsigned chainResumeIndex() { return offsetof(BackTrackInfoParenthesesOnce, chainResume) / sizeof(uintptr_t); } }; struct BackTrackInfoParenthesesTerminal {