Yarr: lookbehind JIT (incl. unicode), alternation factoring, first-character dispatch, and Boyer-Moore search selection

JSTests/stress/regexp-bm-search-any-character-position.js

@@ -0,0 +1,39 @@
+function shouldBe(actual, expected) {
+    if (actual !== expected)
+        throw new Error("bad value: " + actual + " (expected " + expected + ")");
+}
+
+// A BM search range must never span a position that matches every character. setAll() records
+// such a position in its count alone and leaves its map empty, so merging one drops every
+// character it matches out of the range's union, and the search then skips past positions that
+// can start a match. Every case below therefore expects a MATCH: the failure is a false negative,
+// so an expectation of null could not catch it. Each one is written with single dots rather than
+// `.{n}`, because a quantified count aborts BM collection before any position is recorded and
+// would leave nothing for the search to get wrong.
+for (var i = 0; i < 1e2; ++i) {
+    shouldBe(JSON.stringify("bab".match(/.(a)./)), '["bab","a"]');
+    shouldBe(JSON.stringify("bab".match(/.(a)./u)), '["bab","a"]');
+    shouldBe(JSON.stringify("bab".match(/.(a)./du).indices), "[[0,3],[1,2]]");
+    shouldBe(JSON.stringify("xaz".match(/.a./)), '["xaz"]');
+    shouldBe(JSON.stringify("qqzaaqq".match(/z.a/)), '["zaa"]');
+    shouldBe(JSON.stringify("aaaXb".match(/X./)), '["Xb"]');
+    shouldBe(JSON.stringify("abcQdef".match(/...Q.../)), '["abcQdef"]');
+    shouldBe(JSON.stringify("zzza1b2c".match(/a.b.c/)), '["a1b2c"]');
+    shouldBe(JSON.stringify("zzq12r".match(/q..r/)), '["q12r"]');
+    // A dot that cannot match a newline, and inverted classes, are all-set positions too.
+    shouldBe(JSON.stringify("a\nbXc".match(/b.c/)), '["bXc"]');
+    shouldBe(JSON.stringify("aaaaK".match(/[^0-9]K/)), '["aK"]');
+    shouldBe(JSON.stringify(("z".repeat(37) + "yP").match(/[^P]P/)), '["yP"]');
+    // A range wider than the map turns into an all-set position the same way.
+    shouldBe(JSON.stringify("aaaŐeaaa".match(/[Ā-Ȁ]e/)), '["Őe"]');
+}
+
+// The search is available to unicode patterns whose subject is Latin1, where no surrogate pair
+// exists for a code-unit stride to land inside. These cover that newly reachable path; they too
+// expect matches, so an over-skipping search fails them.
+var subject = "the quick brown fox jumps over the lazy dog. ".repeat(64);
+for (var i = 0; i < 1e2; ++i) {
+    shouldBe(JSON.stringify(("xxqbravo" + subject + "zalpha").match(/zalpha|qbravo|xcharlie/gu)), '["qbravo","zalpha"]');
+    shouldBe(subject.match(/\bfox\b/gu).length, 64);
+    shouldBe(JSON.stringify("abcd".match(new RegExp("[\\q{abc}d]", "gv"))), '["abc","d"]');
+}

JSTests/stress/regexp-lookbehind-jit.js

@@ -0,0 +1,187 @@
+//@ runDefault("--useRegExpJIT=true")
+//@ runNoJIT("--useRegExpJIT=false")
+
+// Lookbehind assertions compiled by the Yarr JIT (mirrored bodies matched
+// right-to-left) must produce results identical to the interpreter.
+
+// Strict structural equality that distinguishes an undefined array element
+// (a non-participating capture group) from null; JSON.stringify would map
+// both to null and let a wrongly-nulled capture pass.
+function isSameValue(actual, expected) {
+  if (Array.isArray(actual) || Array.isArray(expected)) {
+    return Array.isArray(actual) && Array.isArray(expected)
+      && actual.length === expected.length
+      && actual.every((value, i) => isSameValue(value, expected[i]));
+  }
+  return Object.is(actual, expected);
+}
+
+function describe(value) {
+  return JSON.stringify(value, (key, v) => (v === undefined ? "<undefined>" : v));
+}
+
+function shouldBe(actual, expected, message) {
+  if (!isSameValue(actual, expected))
+    throw new Error((message ? message + ": " : "") + "expected " + describe(expected) + " but got " + describe(actual));
+}
+
+function matchOf(re, s) {
+  let m = re.exec(s);
+  return m ? [m.index, [...m]] : null;
+}
+
+// Fixed-width bodies.
+shouldBe(matchOf(/(?<=x)a/, "xa"), [1, ["a"]]);
+shouldBe(matchOf(/(?<=x)a/, "ya"), null);
+shouldBe(matchOf(/(?<!x)a/, "ya"), [1, ["a"]]);
+shouldBe(matchOf(/(?<!x)a/, "xa"), null);
+shouldBe(matchOf(/(?<=abc)d/, "abcd"), [3, ["d"]]);
+shouldBe(matchOf(/(?<=abc)d/, "abd"), null);
+shouldBe(matchOf(/(?<! cu)bot/, "robot"), [2, ["bot"]]);
+shouldBe(matchOf(/(?<! cu)bot/, "a cubot"), null);
+shouldBe(matchOf(/(?<=\d{3})x/, "123x"), [3, ["x"]]);
+shouldBe(matchOf(/(?<=\d{3})x/, "12x"), null);
+
+// Alternation and nested groups inside the body.
+shouldBe(matchOf(/(?<=ab|xyz)q/, "abq"), [2, ["q"]]);
+shouldBe(matchOf(/(?<=ab|xyz)q/, "xyzq"), [3, ["q"]]);
+shouldBe(matchOf(/(?<=ab|xyz)q/, "azq"), null);
+shouldBe(matchOf(/(?<=(?:channel\/|google\/))x/, "channel/x"), [8, ["x"]]);
+shouldBe(matchOf(/(?<=(?:channel\/|google\/))x/, "google/x"), [7, ["x"]]);
+shouldBe(matchOf(/(?<=(?:channel\/|google\/))x/, "chanel/x"), null);
+
+// Variable-width bodies: greedy / lazy quantifiers and optional groups.
+shouldBe(matchOf(/(?<=a+)b/, "aaab"), [3, ["b"]]);
+shouldBe(matchOf(/(?<=a+)b/, "b"), null);
+shouldBe(matchOf(/(?<=ba*)c/, "bc"), [1, ["c"]]);
+shouldBe(matchOf(/(?<=ba*)c/, "baaac"), [4, ["c"]]);
+shouldBe(matchOf(/(?<=ba*?)c/, "baaac"), [4, ["c"]]);
+shouldBe(matchOf(/(?<! ya(?:yandex)?)search/, "yasearch"), [2, ["search"]]);
+shouldBe(matchOf(/(?<! ya(?:yandex)?)search/, " yasearch"), null);
+shouldBe(matchOf(/(?<! ya(?:yandex)?)search/, " yayandexsearch"), null);
+shouldBe(matchOf(/(?<! ya(?:yandex)?)search/, "search"), [0, ["search"]]);
+shouldBe(matchOf(/(?<=[a-z]{2,4})\d/, "ab7"), [2, ["7"]]);
+shouldBe(matchOf(/(?<=[a-z]{2,4})\d/, "a7"), null);
+
+// Captures inside the body record real (forward) start/end.
+shouldBe(matchOf(/(?<=(a)(b))c/, "abc"), [2, ["c", "a", "b"]]);
+shouldBe(matchOf(/(?<=(a+))b/, "xaaab"), [4, ["b", "aaa"]]);
+shouldBe(matchOf(/(?<=(a+?))b/, "xaaab"), [4, ["b", "a"]]);
+shouldBe(matchOf(/(?<!(a))b/, "cb"), [1, ["b", undefined]]);
+
+// Nested lookbehind; lookbehind at start of input; whole-string context.
+shouldBe(matchOf(/(?<=(?<=a)b)c/, "abc"), [2, ["c"]]);
+shouldBe(matchOf(/(?<=(?<=a)b)c/, "xbc"), null);
+shouldBe(matchOf(/(?<=^)a/, "a"), [0, ["a"]]);
+shouldBe(matchOf(/(?<!^)a/, "a"), null);
+shouldBe(matchOf(/(?<!^)a/, "ba"), [1, ["a"]]);
+
+// Anchors inside the body, including multiline.
+shouldBe(matchOf(/(?<=^a)b/m, "xa\nab"), [4, ["b"]]);
+shouldBe(matchOf(/(?<=a$)/m, "a\nb"), [1, [""]]);
+shouldBe(matchOf(/(?<=\bfoo)bar/, "foobar"), [3, ["bar"]]);
+shouldBe(matchOf(/(?<=\bfoo)bar/, "xfoobar"), null);
+
+// The lookbehind sees input before lastIndex / the match start.
+{
+  let re = /(?<=ab)c/g;
+  let s = "abcabc";
+  let m = [];
+  let r;
+  while ((r = re.exec(s)) !== null) m.push(r.index);
+  shouldBe(m, [2, 5]);
+
+  let sticky = /(?<=ab)c/y;
+  sticky.lastIndex = 2;
+  shouldBe(matchOf(sticky, "abc"), [2, ["c"]]);
+}
+
+// Case-insensitive, Latin-1, and true 16-bit (non-Latin-1) strings.
+shouldBe(matchOf(/(?<=ab)c/i, "ABC"), [2, ["C"]]);
+shouldBe(matchOf(/(?<=é)x/, "éx"), [1, ["x"]]);
+shouldBe(matchOf(/(?<=Ā)x/, "Āx"), [1, ["x"]]);
+shouldBe(matchOf(/(?<=ab)c/, "Āabc"), [3, ["c"]]);
+
+// Fixed-count pattern characters and \B inside the body.
+shouldBe(matchOf(/(?<=a{3})b/, "aaab"), [3, ["b"]]);
+shouldBe(matchOf(/(?<=a{3})b/, "aab"), null);
+shouldBe(matchOf(/(?<=xa{2})b/, "xaab"), [3, ["b"]]);
+shouldBe(matchOf(/(?<=\Bb)c/, "abc"), [2, ["c"]]);
+shouldBe(matchOf(/(?<=\Bb)c/, " bc"), null);
+
+// Anchors at the real edges of input inside the body (non-multiline).
+shouldBe(matchOf(/(?<=x$)/, "x"), [1, [""]]);
+shouldBe(matchOf(/(?<=^x)y/, "xy"), [1, ["y"]]);
+shouldBe(matchOf(/(?<=^x)y/, "axy"), null);
+shouldBe(matchOf(/(?<=^)a/m, "\na"), [1, ["a"]]);
+
+// Backreference, quantified group, nested lookahead and unicode surrogate-pair
+// bodies (all compiled natively by the JIT).
+shouldBe(matchOf(/(?<=(a)\1)b/, "aab"), [2, ["b", "a"]]);
+shouldBe(matchOf(/(?<=(?:ab)+)c/, "ababc"), [4, ["c"]]);
+shouldBe(matchOf(/(?<=(?:ab)+)c/, "xc"), null);
+shouldBe(matchOf(/(?<=(?=a)a)b/, "ab"), [1, ["b"]]);
+shouldBe(matchOf(/(?<=(?=a)a)b/, "cb"), null);
+shouldBe(matchOf(/(?<=\u{1F600})x/u, "\u{1F600}x"), [2, ["x"]]);
+shouldBe(matchOf(/(?<=\u{1F600})x/u, "yx"), null);
+
+// Backtracking after a satisfied lookbehind must clear its captures.
+shouldBe(matchOf(/(?<=(a))bz|(?<=(a))b/, "ab"), [1, ["b", undefined, "a"]]);
+
+// A large alternation containing lookbehinds must produce the same set of
+// matches as the same alternation with the assertions verified independently.
+{
+  let re = /(?<! cu)bot|(?<!(?:lib))http|(?<! ya(?:yandex)?)search|crawler|spider/;
+  let lines = [
+    "Googlebot/2.1",
+    "a cubot toy",
+    "libhttp client",
+    "http agent",
+    "yandexsearch",
+    " yasearch",
+    "crawler",
+    "spiderman",
+  ];
+  shouldBe(
+    lines.map(l => re.test(l)),
+    [true, false, false, true, true, false, true, true],
+  );
+}
+
+// Lookbehind patterns whose alternatives contain a group that is BOL-anchored
+// (an optional/quantified/negated group starting with ^) keep the results the
+// interpreter gives: the JIT must not (mis)handle their once-through split.
+shouldBe(JSON.stringify(/a(?<!x)(^)?$/.exec("xa")), '["a",null]');
+shouldBe(JSON.stringify(/(?<=y)a(^)?$/.exec("ya")), '["a",null]');
+shouldBe(JSON.stringify(/c(?<=c)(?!(?=^))/.exec("xc")), '["c"]');
+shouldBe(JSON.stringify(/(?<!q)b(^)*c/.exec("abc")), '["bc",null]');
+shouldBe(JSON.stringify(/\w(?<=\d)(^)?/.exec("a1")), '["1",null]');
+shouldBe(JSON.stringify(/z(?<!q)(?=(^)?)/.exec("yz")), '["z",null]');
+shouldBe(JSON.stringify("qxaqxa".split(/a(?<!x)(^)?$/)), '["qxaqx",null,""]');
+// Optional/quantified group whose first term is ^ (no lookbehind): the anchor is
+// optional, so the alternative is not BOL-anchored and can match anywhere. Once
+// a JIT-only defect (the loop copy dropped the group); both tiers now agree.
+shouldBe(JSON.stringify(/(?:^)?a/.exec("ba")), '["a"]');
+shouldBe(JSON.stringify(/(^)*a/.exec("ba")), '["a",null]');
+shouldBe(JSON.stringify(/\B(?:^)?/.exec("xx")), '[""]');
+shouldBe(JSON.stringify(/(?:^b)?a/.exec("ba")), '["ba"]');
+shouldBe(JSON.stringify("aba".match(/(?:^)?a/g)), '["a","a"]');
+shouldBe(JSON.stringify(/(^){0,2}z/.exec("yz")), '["z",null]');
+shouldBe(JSON.stringify(/(?=^)a/.exec("ba")), "null");
+shouldBe(JSON.stringify(/(?!^)a/.exec("ba")), '["a"]');
+// A REQUIRED all-BOL group still anchors, and a term before it makes the whole
+// alternative unmatchable past position 0 (the loop copy must not over-match).
+shouldBe(/(?:^)a/.exec("ba"), null);
+shouldBe(/(?:^|^)a/.exec("ba"), null);
+shouldBe(/.(^)X/.exec("aX"), null);
+shouldBe(JSON.stringify(/(^)X/.exec("XX")), '["X",""]');
+shouldBe(/y(^\S{2})/.exec("ayzz"), null);
+shouldBe(/\w(?=^[dby]?)/.exec("ab"), null);
+shouldBe(/[^sc](?<a>^)/.exec("qx"), null);
+shouldBe(/c(?:^(?:x))/.exec("acx"), null);
+shouldBe(JSON.stringify(/(\w(^)\s|$)/i.exec("a x")), '["","",null]');
+// Genuine anchoring is unchanged.
+shouldBe(/^a/.exec("ba"), null);
+shouldBe(/(?:^a)/.exec("ba"), null);
+shouldBe(/(^a)+/.exec("ba"), null);
+shouldBe(/(?:^a|^b)/.exec("cb"), null);

Source/JavaScriptCore/runtime/RegExp.cpp

@@ -318,7 +318,6 @@ void RegExp::compile(VM* vm, Yarr::CharSize charSize, std::optional<StringView>
 #if !ENABLE(YARR_JIT_BACKREFERENCES)
         && !pattern.m_containsBackreferences
 #endif
-        && !pattern.m_containsLookbehinds
         ) {
         auto& jitCode = ensureRegExpJITCode();
         Yarr::jitCompile(pattern, m_patternString, charSize, sampleString, vm, jitCode, Yarr::ExecutionMode::IncludeSubpatterns);
@@ -386,7 +385,6 @@ void RegExp::compileMatchOnly(VM* vm, Yarr::CharSize charSize, std::optional<Str
 #if !ENABLE(YARR_JIT_BACKREFERENCES)
         && !pattern.m_containsBackreferences
 #endif
-        && !pattern.m_containsLookbehinds
         ) {
         auto& jitCode = ensureRegExpJITCode();
         Yarr::jitCompile(pattern, m_patternString, charSize, sampleString, vm, jitCode, Yarr::ExecutionMode::MatchOnly);

Source/JavaScriptCore/yarr/Yarr.h

@@ -35,10 +35,10 @@ namespace JSC { namespace Yarr {
 
 #define YarrStackSpaceForBackTrackInfoPatternCharacter 2 // Only for !fixed quantifiers.
 #define YarrStackSpaceForBackTrackInfoCharacterClass 2 // Greedy/NonGreedy, or FixedCount with unicode/unicodeSets flag.
-#define YarrStackSpaceForBackTrackInfoBackReference 3
+#define YarrStackSpaceForBackTrackInfoBackReference 4
 #define YarrStackSpaceForBackTrackInfoAlternative 1 // One per alternative.
 #define YarrStackSpaceForBackTrackInfoParentheticalAssertion 1
-#define YarrStackSpaceForBackTrackInfoParenthesesOnce 2
+#define YarrStackSpaceForBackTrackInfoParenthesesOnce 3
 #define YarrStackSpaceForBackTrackInfoParenthesesTerminal 1
 #define YarrStackSpaceForBackTrackInfoParentheses 4
 #define YarrStackSpaceForDotStarEnclosure 1