From 8b1ad162fc6433d0c567f670f450abb20a07d631 Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 11 Jun 2026 21:24:10 +0000 Subject: [PATCH 1/6] [SER-1312] Improve crash grouping seed normalization in smart preprocessing - Remove symbol/frame offsets as one unit before the standalone-number rule, so partially symbolicated iOS frames normalize cleanly - Remove bare hex sequences (4+ chars, digit required) not covered by the 0x rule, e.g. load addresses and <...> address tokens - Fix custom smart_regexes: flags were passed as the replacement string, so only the first match was replaced (case-sensitively) with the literal text "gim"; now all matches are removed with gim flags as the config help text documents - Collapse whitespace runs and trim as the final smart-preprocessing step so column alignment differences no longer change the seed Crashes that differ only by memory addresses, frame offsets, or padding now produce identical grouping seeds (SER-1312, reported for partially symbolicated iOS SIGTRAP crashes). --- plugins/crashes/api/parts/stacktrace.js | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/plugins/crashes/api/parts/stacktrace.js b/plugins/crashes/api/parts/stacktrace.js index 2845224a0b2..d6ae2dce001 100644 --- a/plugins/crashes/api/parts/stacktrace.js +++ b/plugins/crashes/api/parts/stacktrace.js @@ -69,6 +69,9 @@ var trace = { //seed cleanup if (plugins.getConfig("crashes", overrideSettings || {}, true).smart_preprocessing) { + //remove frame offsets as one unit (MyApp 0x104a3c5d8 0x104a30000 + 50648) + seed = seed.replace(/\s\+\s*(0x)?[0-9a-fA-F]+(?!\S)/g, ""); + //remove stand alone numbers like ids (MongoServerError: cursor id 8983374575113418154 not found) seed = seed.replace(/(?) + seed = seed.replace(/\b(?=[0-9a-fA-F]*\d)[0-9a-fA-F]{4,}\b/g, ""); + //remove additional custom provided regexes var regexes = (plugins.getConfig("crashes", overrideSettings || {}, true).smart_regexes || "").replace(/\r\n|\r|\n/g, "\n").split("\n"); for (let i = 0; i < regexes.length; i++) { if (regexes[i] && regexes[i].length) { try { - seed = seed.replace(new RegExp(regexes[i]), "gim"); + seed = seed.replace(new RegExp(regexes[i], "gim"), ""); } catch (ex) { console.log("Error in smart regex for crash", regexes[i], ex); } } } + + //collapse whitespace runs and trim so alignment differences don't change the seed + seed = seed.replace(/\s+/g, " ").trim(); } callback(seed); From 2ab067e9d9efbc068dc440ac961de94667c2f437 Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 11 Jun 2026 21:26:30 +0000 Subject: [PATCH 2/6] [SER-1312] Add changelog entries for crash grouping improvements --- CHANGELOG.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3c9d9770664..c9cf5943d57 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,9 @@ ## Version 25.03 Fixes: +- [crashes] Improve crash grouping normalization for partially symbolicated stack traces when smart preprocessing is enabled: frame offsets, bare hex addresses and whitespace alignment differences no longer split identical crashes into separate groups +- [crashes] Fixed custom smart_regexes applying only the first match case-sensitively and injecting the literal text "gim" into grouping seeds; all matches are now removed with gim flags as documented in the setting's help text +- [crashes] Note: on apps with smart preprocessing enabled, grouping seeds change after upgrade, so new incoming crashes may form new crash groups instead of joining pre-upgrade groups (one-time regrouping effect; existing groups remain intact and readable) - [journey_engine] resut tab made available for running journeys ## Version 25.03.46 Fixes: From d56105521e0c2783d730763f203fc83e9a74c49c Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 11 Jun 2026 21:30:23 +0000 Subject: [PATCH 3/6] [SER-1312] Add tests for crash grouping seed normalization Unit tests call groupStrategy() directly with explicit override settings: iOS SIGTRAP pair equality, bare-hex normalization, offset removal as a unit, distinct crashes staying distinct, smart_regexes all-match case-insensitive removal without gim injection, invalid regex resilience, smart_preprocessing=false passthrough, stacktrace strategy, and standalone-id regression. Plus an e2e smoke test that two partially symbolicated crashes land in one crash group. --- plugins/crashes/tests.js | 139 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 139 insertions(+) diff --git a/plugins/crashes/tests.js b/plugins/crashes/tests.js index bc46a60e1e0..f9b1039f492 100644 --- a/plugins/crashes/tests.js +++ b/plugins/crashes/tests.js @@ -3173,6 +3173,145 @@ describe('Testing Crashes', function() { }); }); + describe('Crash grouping seed (groupStrategy)', function() { + var stacktrace = require('./api/parts/stacktrace.js'); + var SMART_SETTINGS = {grouping_strategy: 'error_and_file', smart_preprocessing: true, smart_regexes: ''}; + + /** + * Compute the grouping seed produced by groupStrategy for given inputs + * @param {Array} stack - processed stacktrace lines + * @param {Object} crash - crash object (e.g. {_name: "..."}) + * @param {Object} settings - crashes config override; all three keys + * (grouping_strategy, smart_preprocessing, smart_regexes) must be + * passed explicitly because the Mocha process never runs + * plugins.setConfigs("crashes", ...) + * @returns {Promise} resolves with the seed + */ + function getSeed(stack, crash, settings) { + return new Promise(function(resolve) { + stacktrace.groupStrategy(stack, crash, resolve, {crashes: settings}); + }); + } + + it('iOS SIGTRAP pair produces identical seeds', async() => { + var crashName = 'App terminated by SIGTRAP'; + var seedA = await getSeed(['MyApp 0x0000000104a3c5d8 0x104a30000 + 50648'], {_name: crashName}, SMART_SETTINGS); + var seedB = await getSeed(['MyApp 0x000000010472f8a0 0x104728000 + 51360'], {_name: crashName}, SMART_SETTINGS); + should(seedA).equal(seedB); + should(seedA).equal('App terminated by SIGTRAPMyApp'); + seedA.should.not.containEql('0x'); + should(/(?:^|\s)\d+(?:\s|$)/.test(seedA)).equal(false, 'seed should not contain digits-only tokens'); + seedA.should.not.containEql(' '); + should(seedA).equal(seedA.trim()); + }); + + it('bare hex addresses without 0x prefix are normalized', async() => { + var crashName = 'App terminated by SIGTRAP'; + var seedA = await getSeed(['MyApp <0000000104a3c5d8> main + 120'], {_name: crashName}, SMART_SETTINGS); + var seedB = await getSeed(['MyApp <000000010472f8a0> main + 168'], {_name: crashName}, SMART_SETTINGS); + should(seedA).equal(seedB); + should(/[0-9a-f]{4,}/i.test(seedA)).equal(false, 'seed should not contain bare hex sequences'); + should(/[0-9]/.test(seedA)).equal(false, 'seed should not contain digits'); + }); + + it('frame offsets are removed as a unit', async() => { + var crashName = 'App terminated by SIGTRAP'; + var seedA = await getSeed(['fn + 600'], {_name: crashName}, SMART_SETTINGS); + var seedB = await getSeed(['fn +84556'], {_name: crashName}, SMART_SETTINGS); + var seedC = await getSeed(['fn + 0x1a4'], {_name: crashName}, SMART_SETTINGS); + should(seedA).equal(seedB); + should(seedB).equal(seedC); + seedA.should.not.containEql('+'); + }); + + it('genuinely different crashes keep different seeds', async() => { + var seedA = await getSeed(['MyApp 0x0000000104a3c5d8 0x104a30000 + 50648'], {_name: 'App terminated by SIGTRAP'}, SMART_SETTINGS); + var seedB = await getSeed(['CoreFoundation 0x00000001e5669ebc __exceptionPreprocess + 252'], {_name: '-[NSNull length]: unrecognized selector sent to instance'}, SMART_SETTINGS); + should(seedA).not.equal(seedB); + }); + + it('custom smart_regexes remove all matches case-insensitively across lines', async() => { + var settings = {grouping_strategy: 'stacktrace', smart_preprocessing: true, smart_regexes: 'token[a-z]+'}; + var seed = await getSeed(['Error TOKENone happened', 'at tokentwo place'], {}, settings); + should(seed.toLowerCase().indexOf('token')).equal(-1, 'all matches should be removed case-insensitively'); + should(seed.indexOf('gim')).equal(-1, 'the literal string "gim" should never be injected into the seed'); + should(seed).equal('Error happened at place'); + }); + + it('invalid custom regex does not throw and other rules still apply', async() => { + var settings = {grouping_strategy: 'error_and_file', smart_preprocessing: true, smart_regexes: '(['}; + var seed = await getSeed(['fn call here'], {_name: 'Error in module'}, settings); + seed.should.not.containEql(' '); + should(seed).equal('Error in modulefn call here'); + }); + + it('no smart preprocessing → seed passes through unchanged', async() => { + var crashName = 'App terminated by SIGTRAP'; + var stackLine = 'MyApp 0x0000000104a3c5d8 0x104a30000 + 50648'; + var settings = {grouping_strategy: 'error_and_file', smart_preprocessing: false, smart_regexes: ''}; + var seed = await getSeed([stackLine], {_name: crashName}, settings); + should(seed).equal(crashName + stackLine); + }); + + it('stacktrace strategy groups multi-line traces differing only by addresses', async() => { + var settings = {grouping_strategy: 'stacktrace', smart_preprocessing: true, smart_regexes: ''}; + var seedA = await getSeed([ + 'MyApp 0x0000000104a3c5d8 0x104a30000 + 50648', + 'CoreFoundation 0x00000001e55f40e0 CFRunLoopRunSpecific + 436', + 'libdyld.dylib 0x00000001e50b2bb4 start + 4' + ], {}, settings); + var seedB = await getSeed([ + 'MyApp 0x000000010472f8a0 0x104728000 + 51360', + 'CoreFoundation 0x00000001e55f4112 CFRunLoopRunSpecific + 412', + 'libdyld.dylib 0x00000001e50b2aa0 start + 16' + ], {}, settings); + should(seedA).equal(seedB); + seedA.should.not.containEql('\n'); + }); + + it('existing standalone-id rule still works', async() => { + var seedA = await getSeed([], {_name: 'MongoServerError: cursor id 8983374575113418154 not found'}, SMART_SETTINGS); + var seedB = await getSeed([], {_name: 'MongoServerError: cursor id 1234567890123456789 not found'}, SMART_SETTINGS); + should(seedA).equal(seedB); + seedA.should.containEql('cursor id'); + }); + + it('two partially symbolicated iOS crashes land in one crash group', async() => { + // _os "tvOS" bypasses the ios-only masking branch in preprocessCrash, + // so raw addresses reach groupStrategy like in the customer case; + // depends on server-level config smart_preprocessing=true (default) + const crashDataA = { + '_name': 'App terminated by SIGTRAP', + '_error': 'MyApp 0x0000000104a3c5d8 0x104a30000 + 50648', + '_os_version': '17.4', + '_os': 'tvOS', + '_app_version': '79.0.0', + }; + const crashDataB = { + '_name': 'App terminated by SIGTRAP', + '_error': 'MyApp 0x000000010472f8a0 0x104728000 + 51360', + '_os_version': '17.4', + '_os': 'tvOS', + '_app_version': '79.0.0', + }; + + await request.get(`/i?app_key=${APP_KEY}&device_id=${DEVICE_ID}&crash=${JSON.stringify(crashDataA)}`).expect(200); + await request.get(`/i?app_key=${APP_KEY}&device_id=${DEVICE_ID}&crash=${JSON.stringify(crashDataB)}`).expect(200); + + const crashGroupQuery = JSON.stringify({ + os: crashDataA._os, + latest_version: crashDataA._app_version, + }); + let crashGroupResponse = await request + .get(`/o?method=crashes&api_key=${API_KEY_ADMIN}&app_id=${APP_ID}&query=${crashGroupQuery}`); + crashGroupResponse.body.aaData.should.have.lengthOf(1); + const crashGroup = crashGroupResponse.body.aaData[0]; + + await request + .get('/i/crashes/delete?args=' + JSON.stringify({ crash_id: crashGroup._id }) + '&app_id=' + APP_ID + '&api_key=' + API_KEY_ADMIN); + }); + }); + describe('Reset app', function() { it('should reset data', function(done) { var params = {app_id: APP_ID, period: "reset"}; From eecb35fe26b989b991ac7267c50e8f2a1c5e950a Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 11 Jun 2026 22:13:02 +0000 Subject: [PATCH 4/6] [SER-1312] Fix race condition in crash grouping e2e test Crash ingestion is fire-and-forget: /i returns 200 before the crash group upsert lands, so asserting immediately after submission is flaky on slow runners. Wait 100*testScalingFactor ms after each submission (suite convention) and poll the crashes query until the group count is stable before asserting. Also URL-encode the crash payloads so the literal '+ 50648' frame offset reaches the server instead of decoding to a space, exercising the offset-removal rule as intended. The strict lengthOf(1) assertion is unchanged. --- plugins/crashes/tests.js | 29 +++++++++++++++++++++++++---- 1 file changed, 25 insertions(+), 4 deletions(-) diff --git a/plugins/crashes/tests.js b/plugins/crashes/tests.js index f9b1039f492..7fac17f839e 100644 --- a/plugins/crashes/tests.js +++ b/plugins/crashes/tests.js @@ -3295,15 +3295,36 @@ describe('Testing Crashes', function() { '_app_version': '79.0.0', }; - await request.get(`/i?app_key=${APP_KEY}&device_id=${DEVICE_ID}&crash=${JSON.stringify(crashDataA)}`).expect(200); - await request.get(`/i?app_key=${APP_KEY}&device_id=${DEVICE_ID}&crash=${JSON.stringify(crashDataB)}`).expect(200); + const wait = function(ms) { + return new Promise(function(resolve) { + setTimeout(resolve, ms); + }); + }; + + await request.get(`/i?app_key=${APP_KEY}&device_id=${DEVICE_ID}&crash=${encodeURIComponent(JSON.stringify(crashDataA))}`).expect(200); + await wait(100 * testUtils.testScalingFactor); + await request.get(`/i?app_key=${APP_KEY}&device_id=${DEVICE_ID}&crash=${encodeURIComponent(JSON.stringify(crashDataB))}`).expect(200); + await wait(100 * testUtils.testScalingFactor); const crashGroupQuery = JSON.stringify({ os: crashDataA._os, latest_version: crashDataA._app_version, }); - let crashGroupResponse = await request - .get(`/o?method=crashes&api_key=${API_KEY_ADMIN}&app_id=${APP_ID}&query=${crashGroupQuery}`); + // crash ingestion is fire-and-forget (/i returns 200 before the + // group upsert lands in app_crashgroups), so poll until at least + // one group is visible and the count is stable across two polls + let crashGroupResponse; + let previousLength = -1; + for (let attempt = 0; attempt < 10; attempt++) { + crashGroupResponse = await request + .get(`/o?method=crashes&api_key=${API_KEY_ADMIN}&app_id=${APP_ID}&query=${crashGroupQuery}`); + const currentLength = (crashGroupResponse.body.aaData || []).length; + if (currentLength >= 1 && currentLength === previousLength) { + break; + } + previousLength = currentLength; + await wait(200 * testUtils.testScalingFactor); + } crashGroupResponse.body.aaData.should.have.lengthOf(1); const crashGroup = crashGroupResponse.body.aaData[0]; From b107e2200288c0d178b65746c2cb31a7b1059e8f Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 11 Jun 2026 22:21:45 +0000 Subject: [PATCH 5/6] [SER-1312] Re-trigger CI to assess test flakiness window From 7d65cf30a585d0631a7730845a1af248e718ea7c Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 12 Jun 2026 06:27:29 +0000 Subject: [PATCH 6/6] [SER-1312] Re-trigger CI