diff --git a/queries/cdmq/cdm.js b/queries/cdmq/cdm.js index 9a624b95..28defef7 100644 --- a/queries/cdmq/cdm.js +++ b/queries/cdmq/cdm.js @@ -2651,48 +2651,92 @@ getMetricGroupTermsFromAgg = function (agg, terms) { }; exports.getMetricGroupTermsFromAgg = getMetricGroupTermsFromAgg; +// -------------------------------------------------------------------------------------------------------------- +// Parse a breakout entry from either structured object or legacy string format. +// Returns { name, values (array|null), regex (string|null), aggregate (bool) } +function parseBreakoutEntry(entry) { + if (typeof entry === 'object' && entry !== null && entry.name) { + return { + name: entry.name, + values: entry.values || null, + regex: entry.regex || null, + aggregate: !!entry.aggregate + }; + } + var parsed = { name: String(entry), values: null, regex: null, aggregate: false }; + var eqMatch = /^([^=]+)=(.+)$/.exec(entry); + if (eqMatch) { + parsed.name = eqMatch[1]; + var val = eqMatch[2]; + var regexMatch = /^([rR])(.)(.+)\2$/.exec(val); + if (regexMatch) { + parsed.regex = regexMatch[3]; + parsed.aggregate = regexMatch[1] === 'R'; + } else { + parsed.values = val.split('+'); + } + } + return parsed; +} +exports.parseBreakoutEntry = parseBreakoutEntry; + +function buildAggregateLabel(bp, maxLen) { + maxLen = maxLen || 30; + if (bp.values && bp.values.length > 0) { + var vals = bp.values.slice().sort(function (a, b) { + var na = Number(a), nb = Number(b); + if (!isNaN(na) && !isNaN(nb)) return na - nb; + return a < b ? -1 : a > b ? 1 : 0; + }); + var allNumeric = vals.every(function (v) { return !isNaN(Number(v)); }); + if (allNumeric) { + var nums = vals.map(Number); + var ranges = []; + var start = nums[0], end = nums[0]; + for (var i = 1; i < nums.length; i++) { + if (nums[i] === end + 1) { + end = nums[i]; + } else { + ranges.push(start === end ? String(start) : start + '-' + end); + start = end = nums[i]; + } + } + ranges.push(start === end ? String(start) : start + '-' + end); + var rangeStr = ranges.join(','); + if (rangeStr.length <= maxLen) return rangeStr; + } + var joined = vals.join(','); + if (joined.length <= maxLen) return joined; + } + if (bp.regex) { + if (bp.regex.length <= maxLen) return '/' + bp.regex + '/'; + } + var count = bp.values ? bp.values.length : '?'; + return count + ' values'; +} + // -------------------------------------------------------------------------------------------------------------- getBreakoutAggregation = function (source, type, breakout) { var agg_str = '{'; agg_str += '"metric_desc.source": { "terms": { "field": "metric_desc.source"}'; agg_str += ',"aggs": { "metric_desc.type": { "terms": { "field": "metric_desc.type"}'; - // More nested aggregations are added, one per field found in the broeakout + // More nested aggregations are added, one per field found in the breakout var field_count = 0; - var regExp = /([^\=]+)\=([^\=]+)/; - //var matches = regExp.exec(""); if (Array.isArray(breakout)) { - breakout.forEach((field) => { - //if (/([^\=]+)\=([^\=]+)/.exec(field)) { - var matches = regExp.exec(field); - var shouldAggregate = true; // default: include in aggregation - - if (matches) { - //field = $1; - var fieldName = matches[1]; - var value = matches[2]; - - // Check if this is an aggregated regex pattern (R/pattern/) - // If uppercase R, we should NOT add this field to the aggregation - // (all matches will be combined into a single metric) - if (/^R./.test(value)) { - shouldAggregate = false; - } - - field = fieldName; - } + breakout.forEach((entry) => { + var bp = parseBreakoutEntry(entry); - // Only add to aggregation if shouldAggregate is true - if (shouldAggregate) { + if (!bp.aggregate) { agg_str += ',"aggs": { "metric_desc.names.' + - field + + bp.name + '": { "terms": ' + '{ "show_term_doc_count_error": true, "size": ' + bigQuerySize + ',' + '"field": "metric_desc.names.' + - field + + bp.name + '" }'; field_count++; } @@ -2871,48 +2915,22 @@ getMetricGroupsFromBreakouts = async function (instance, sets, yearDotMonth) { if (set.run != null) { q.query.bool.filter.push(JSON.parse('{"term": {"run.run-uuid": "' + set.run + '"}}')); } - // If the breakout contains a match requirement (something like "host=myhost"), then we must add a term filter for it. - // Multiple values can be specified with commas: "host=a,b,c" which will match any of those values. - // Regex patterns can be specified with r/pattern/ (separate metrics) or R/pattern/ (aggregated metric). - var regExp = /([^\=]+)\=([^\=]+)/; - set.breakout.forEach((field) => { - var matches = regExp.exec(field); - if (matches) { - field = matches[1]; - value = matches[2]; - - // Check if it's a regex pattern: r/pattern/ or R/pattern/ - // Group 1: r or R (lowercase = separate metrics, uppercase = aggregated) - // Group 2: delimiter character (usually /, but can be any char) - // Group 3: the actual regex pattern - // \2: backreference to ensure matching closing delimiter - var regexMatch = /^([rR])(.)(.+)\2$/.exec(value); - - if (regexMatch) { - // It's a regex pattern - var isAggregated = regexMatch[1] === 'R'; - var delimiter = regexMatch[2]; - var pattern = regexMatch[3]; - - // Add regexp filter to OpenSearch query - // Both r/pattern/ and R/pattern/ use the same filter, - // the difference is in the aggregation (handled in getBreakoutAggregation) + // Add filters for breakout entries that specify values or regex patterns + set.breakout.forEach((entry) => { + var bp = parseBreakoutEntry(entry); + if (bp.regex) { + q.query.bool.filter.push( + JSON.parse('{"regexp": {"metric_desc.names.' + bp.name + '": ' + JSON.stringify(bp.regex) + '}}') + ); + } else if (bp.values) { + if (bp.values.length > 1) { q.query.bool.filter.push( - JSON.parse('{"regexp": {"metric_desc.names.' + field + '": ' + JSON.stringify(pattern) + '}}') + JSON.parse('{"terms": {"metric_desc.names.' + bp.name + '": ' + JSON.stringify(bp.values) + '}}') ); } else { - // Not a regex pattern, handle as literal value(s) - // Multiple values are separated by '+': field=value1+value2 - var values = value.split('+'); - if (values.length > 1) { - // Multiple values: use "terms" query (note the plural) - q.query.bool.filter.push( - JSON.parse('{"terms": {"metric_desc.names.' + field + '": ' + JSON.stringify(values) + '}}') - ); - } else { - // Single value: use "term" query (singular) - q.query.bool.filter.push(JSON.parse('{"term": {"metric_desc.names.' + field + '": "' + value + '"}}')); - } + q.query.bool.filter.push( + JSON.parse('{"term": {"metric_desc.names.' + bp.name + '": "' + bp.values[0] + '"}}') + ); } } }); @@ -2938,26 +2956,34 @@ getMetricGroupsFromBreakouts = async function (instance, sets, yearDotMonth) { // Derive the label from each group and organize into a dict, key = label, value = the filter terms var metricGroupTermsByLabel = getMetricGroupTermsByLabel(metricGroupTerms); - // Extract regexp filters that were excluded from aggregation (R/pattern/) - // These need to be preserved when querying for metric IDs + // For aggregated breakouts, insert a synthetic label segment at the correct position var regexpFilters = []; - var regExp = /([^\=]+)\=([^\=]+)/; - sets[idx].breakout.forEach((field) => { - var matches = regExp.exec(field); - if (matches) { - var fieldName = matches[1]; - var value = matches[2]; - var regexMatch = /^([rR])(.)(.+)\2$/.exec(value); - if (regexMatch) { - var isAggregated = regexMatch[1] === 'R'; - var pattern = regexMatch[3]; - if (isAggregated) { - // This field was excluded from aggregation, need to preserve the regexp filter - regexpFilters.push({ field: fieldName, pattern: pattern }); - } - } - } + var aggregatedPositions = []; + sets[idx].breakout.forEach((entry, bpIdx) => { + var bp = parseBreakoutEntry(entry); + if (!bp.aggregate) return; + if (bp.regex) { + regexpFilters.push({ field: bp.name, pattern: bp.regex }); + } + aggregatedPositions.push({ position: bpIdx, segment: '<' + buildAggregateLabel(bp) + '>' }); }); + if (aggregatedPositions.length > 0) { + var oldLabels = Object.keys(metricGroupTermsByLabel); + if (oldLabels.length === 0) { + var synLabel = aggregatedPositions.map(function (ap) { return ap.segment; }).join('-'); + metricGroupTermsByLabel[synLabel] = ''; + } else { + var updated = {}; + oldLabels.forEach(function (oldLabel) { + var segments = oldLabel.match(/<[^>]*>/g) || []; + aggregatedPositions.forEach(function (ap) { + segments.splice(ap.position, 0, ap.segment); + }); + updated[segments.join('-')] = metricGroupTermsByLabel[oldLabel]; + }); + metricGroupTermsByLabel = updated; + } + } var thisLabelSet = { run: sets[idx].run, @@ -3519,14 +3545,7 @@ getMetricDataSets = async function (instance, sets, yearDotMonth) { for (var i = 0; i < sets.length; i++) { if (sets[i].breakout != 'undefined') { for (var j = 0; j < sets[i].breakout.length; j++) { - var breakout = sets[i].breakout[j]; - // The breakout requested might have a match included, for example, csid=1. We only - // want the string before the '=' - var regExp = /([^\=]+)\=([^\=]+)/; - var matches = regExp.exec(breakout); - if (matches) { - breakout = matches[1]; - } + var breakout = parseBreakoutEntry(sets[i].breakout[j]).name; if (!setBreakouts[i].includes(breakout)) { retMsg += 'ERROR: the breakout [' + @@ -3561,18 +3580,14 @@ getMetricDataSets = async function (instance, sets, yearDotMonth) { // Check if any regex filters resulted in zero matches for (var idx = 0; idx < metricGroupIdsByLabelSets.length; idx++) { if (Object.keys(metricGroupIdsByLabelSets[idx]).length === 0) { - // This set has no metric groups - check if it was due to a regex filter + // This set has no metric groups - check if it was due to a regex or value filter var regexFilters = []; - var regExp = /([^\=]+)\=([^\=]+)/; - sets[idx].breakout.forEach((field) => { - var matches = regExp.exec(field); - if (matches) { - var fieldName = matches[1]; - var value = matches[2]; - // Check if it's a regex pattern - if (/^[rR]./.test(value)) { - regexFilters.push({ field: fieldName, pattern: value }); - } + sets[idx].breakout.forEach((entry) => { + var bp = parseBreakoutEntry(entry); + if (bp.regex) { + regexFilters.push({ field: bp.name, pattern: bp.regex }); + } else if (bp.values) { + regexFilters.push({ field: bp.name, pattern: bp.values.join('+') }); } }); @@ -3622,19 +3637,14 @@ getMetricDataSets = async function (instance, sets, yearDotMonth) { // Build the label-decoder and the remaining breakouts dataSets[i].usedBreakouts = sets[i].breakout; dataSets[i].valueSeriesLabelDecoder = ''; - var regExp = /([^\=]+)\=([^\=]+)/; - dataSets[i].usedBreakouts.forEach((field) => { - var matches = regExp.exec(field); - if (matches) { - field = matches[1]; - value = matches[2]; - } - dataSets[i].valueSeriesLabelDecoder += '-' + '<' + field + '>'; - //TODO: validate if user's breakouts are available by checking against data.breakouts + var usedNames = []; + dataSets[i].usedBreakouts.forEach((entry) => { + var bp = parseBreakoutEntry(entry); + usedNames.push(bp.name); + dataSets[i].valueSeriesLabelDecoder += '-' + '<' + bp.name + '>'; }); dataSets[i].valueSeriesLabelDecoder = dataSets[i].valueSeriesLabelDecoder.replace('-', ''); - // Breakouts already used should not show up in the list of avauilable breakouts - dataSets[i].remainingBreakouts = setBreakouts[i].filter((n) => !dataSets[i].usedBreakouts.includes(n)); + dataSets[i].remainingBreakouts = setBreakouts[i].filter((n) => !usedNames.includes(n)); } for (var i = 0; i < sets.length; i++) { diff --git a/queries/cdmq/server.js b/queries/cdmq/server.js index 81dae62f..ad9defaf 100755 --- a/queries/cdmq/server.js +++ b/queries/cdmq/server.js @@ -1559,7 +1559,7 @@ app.post('/api/v1/metric-data', async (req, res) => { var { run, period, begin, end, source, type, resolution, breakout, filter, instances: reqInstances } = req.body; var reqStart = Date.now(); - var breakoutStr = Array.isArray(breakout) ? breakout.join(',') : (breakout || 'none'); + var breakoutStr = Array.isArray(breakout) ? breakout.map(function (b) { return typeof b === 'object' && b.name ? b.name : b; }).join(',') : (breakout || 'none'); serverLog('POST /api/v1/metric-data: ' + source + '::' + type + ' resolution=' + resolution + ' breakout=[' + breakoutStr + ']' + (filter ? ' filter=' + filter : '') + ' run=' + (run || 'none').toString().substring(0, 8) + '... period=' + (period || 'none').toString().substring(0, 8) + '...', req.reqId); //serverLog(' curl: curl -s -X POST http://localhost:3000/api/v1/metric-data -H "Content-Type: application/json" -d \'' + JSON.stringify({ run: run, period: period, begin: begin, end: end, source: source, type: type, resolution: resolution, breakout: breakout, filter: filter }) + '\'', req.reqId); diff --git a/queries/cdmq/web-ui/src/components/CompareView.jsx b/queries/cdmq/web-ui/src/components/CompareView.jsx index 54eba53e..fe557d74 100644 --- a/queries/cdmq/web-ui/src/components/CompareView.jsx +++ b/queries/cdmq/web-ui/src/components/CompareView.jsx @@ -365,7 +365,8 @@ function renderGroupedBreakouts(items, depth, breakoutNames) { var headers = []; var commonSuffixes = []; for (var h = 0; h < numCols; h++) { - var name = (breakoutNames && h < breakoutNames.length) ? breakoutNames[h] : ''; + var nameEntry = (breakoutNames && h < breakoutNames.length) ? breakoutNames[h] : ''; + var name = (typeof nameEntry === 'object' && nameEntry !== null && nameEntry.name) ? nameEntry.name : String(nameEntry); if (name.indexOf('=') >= 0) name = name.substring(0, name.indexOf('=')); // Collect unique values for this column @@ -520,6 +521,8 @@ const CompareView = forwardRef(function CompareView({ selected, groupByList, set var [breakoutValueCache, setBreakoutValueCache] = useState({}); // { "source::type": { "hostname": ["h1","h2"], ... } } var [openBreakoutDropdown, setOpenBreakoutDropdown] = useState(null); // index of metric with open dropdown var [breakoutSelections, setBreakoutSelections] = useState({}); // { "dimName": Set of selected values } + var [breakoutRegex, setBreakoutRegex] = useState({}); // { "dimName": "regexString" } + var [breakoutAggregate, setBreakoutAggregate] = useState({}); // { "dimName": bool } var breakoutDropdownRef = useRef(null); // Close breakout dropdown on outside click @@ -1188,9 +1191,13 @@ const CompareView = forwardRef(function CompareView({ selected, groupByList, set if (isOpen) { setOpenBreakoutDropdown(null); setBreakoutSelections({}); + setBreakoutRegex({}); + setBreakoutAggregate({}); } else { setOpenBreakoutDropdown(si); setBreakoutSelections({}); + setBreakoutRegex({}); + setBreakoutAggregate({}); fetchBreakoutValues(sm.source, sm.type, sm.remainingBreakouts); } }}>+ Breakout @@ -1207,12 +1214,53 @@ const CompareView = forwardRef(function CompareView({ selected, groupByList, set
{b} + {!isSingle && vals && vals.length > 1 && ( + 0) { next[dimName] = matching; } else { delete next[dimName]; } + return next; + }); + } catch (e) { /* invalid regex — leave selections unchanged */ } + }} + /> + )} all {vals && vals.map(function (v) { @@ -1234,13 +1282,28 @@ const CompareView = forwardRef(function CompareView({ selected, groupByList, set ); })} {hasSelection && !allSelected && ( + <> + + )}
@@ -1292,36 +1355,31 @@ const CompareView = forwardRef(function CompareView({ selected, groupByList, set )} - {deepDiveMetrics && (function () { - var metricKey = sm.source + '::' + sm.type; - return ( - - ); - })()} {sm.breakouts.length > 0 && (
{sm.breakouts.map(function (b, bi) { - var eqIdx = b.indexOf('='); - var fieldName = eqIdx >= 0 ? b.substring(0, eqIdx) : b; - var filterVal = eqIdx >= 0 ? b.substring(eqIdx + 1) : ''; + var isObj = typeof b === 'object' && b !== null && b.name; + var fieldName = isObj ? b.name : (b.indexOf('=') >= 0 ? b.substring(0, b.indexOf('=')) : b); + var filterVal = isObj ? (b.values ? b.values.join('+') : (b.regex || '')) : (b.indexOf('=') >= 0 ? b.substring(b.indexOf('=') + 1) : ''); + var isAggregate = isObj && b.aggregate; return ( - - {fieldName} + + {fieldName}{isAggregate ? ' (sum)' : ''} @@ -1436,7 +1494,22 @@ const CompareView = forwardRef(function CompareView({ selected, groupByList, set {renderMetricControls(sm, si)}
-
{sm.source}::{sm.type}
+
+ {sm.source}::{sm.type} + {deepDiveMetrics && ( + + )} +
@@ -1642,7 +1715,22 @@ const CompareView = forwardRef(function CompareView({ selected, groupByList, set
-
{sm.source}::{sm.type}
+
+ {sm.source}::{sm.type} + {deepDiveMetrics && ( + + )} +
@@ -1823,7 +1911,21 @@ const CompareView = forwardRef(function CompareView({ selected, groupByList, set })}
-
{chart.metricName}
+
+ {chart.metricName} + {deepDiveMetrics && ( + + )} +
{/* Toolbar: hidden dims, add, auto, clear — above headers */} @@ -2181,18 +2283,6 @@ const CompareView = forwardRef(function CompareView({ selected, groupByList, set Refine {pmStr} )} - {deepDiveMetrics && ( - - )}
); })()} @@ -2219,7 +2309,22 @@ const CompareView = forwardRef(function CompareView({ selected, groupByList, set
-
{sm.source}::{sm.type}
+
+ {sm.source}::{sm.type} + {deepDiveMetrics && ( + + )} +
diff --git a/queries/cdmq/web-ui/src/components/DeepDiveView.jsx b/queries/cdmq/web-ui/src/components/DeepDiveView.jsx index 194c9c1a..c87aea63 100644 --- a/queries/cdmq/web-ui/src/components/DeepDiveView.jsx +++ b/queries/cdmq/web-ui/src/components/DeepDiveView.jsx @@ -214,10 +214,11 @@ export default function DeepDiveView({ selected, deepDiveMetrics, metricConfigs: }); }).then(function (data) { if (abortRef.current) return; + var mk = metricKey; setMetricData(function (prev) { var next = Object.assign({}, prev); - if (!next[metricKey]) next[metricKey] = {}; - next[metricKey][it.iterationId] = { + if (!next[mk]) next[mk] = {}; + next[mk][it.iterationId] = { values: data.values || {}, periodBegin: String(queryBegin), periodEnd: String(queryEnd), @@ -233,6 +234,8 @@ export default function DeepDiveView({ selected, deepDiveMetrics, metricConfigs: // Wait for all iterations of this metric to complete before starting next metric await Promise.all(promises); + // Yield to let React render this metric's data before fetching the next + await new Promise(function (resolve) { setTimeout(resolve, 0); }); } })(); }).catch(function (err) { @@ -459,6 +462,7 @@ export default function DeepDiveView({ selected, deepDiveMetrics, metricConfigs: // Get breakout dimension names from config var config = configLookup[metricKey] || {}; var breakoutNames = (config.breakouts || []).map(function (b) { + if (typeof b === 'object' && b !== null && b.name) return b.name; var eqIdx = b.indexOf('='); return eqIdx >= 0 ? b.substring(0, eqIdx) : b; }); diff --git a/queries/cdmq/web-ui/src/index.css b/queries/cdmq/web-ui/src/index.css index cba039e5..502a91a6 100644 --- a/queries/cdmq/web-ui/src/index.css +++ b/queries/cdmq/web-ui/src/index.css @@ -1771,6 +1771,50 @@ a.run-id:hover { padding: 2px 8px !important; } +.breakout-aggregate-check { + display: inline-flex; + align-items: center; + gap: 2px; + font-size: 10px; + cursor: pointer; + color: var(--text-secondary); + white-space: nowrap; +} + +.breakout-aggregate-check input { + margin: 0; +} + +.compare-breakout-aggregate { + border-color: rgba(91, 141, 239, 0.4); + background: rgba(91, 141, 239, 0.05); +} + +.breakout-regex-input { + width: 90px; + padding: 1px 5px; + border: 1px solid var(--border); + border-radius: var(--radius-sm); + font-size: 10px; + font-family: 'SF Mono', ui-monospace, Consolas, monospace; + background: var(--surface); + color: var(--text); + outline: none; + flex-shrink: 0; +} + +.breakout-regex-input:focus { + border-color: var(--accent); +} + +.breakout-regex-input::placeholder { + color: var(--text-muted); +} + +.breakout-regex-invalid { + border-color: #ef4444; +} + .compare-filter-group { display: inline-flex; align-items: center; @@ -1976,6 +2020,26 @@ a.run-id:hover { transform: rotate(180deg); } +.compare-yaxis-dive { + display: flex; + align-items: center; + gap: 2px; + cursor: pointer; + font-size: 10px; + font-weight: 600; + color: var(--text-muted); + margin-top: 8px; + white-space: nowrap; +} + +.compare-yaxis-dive input { + margin: 0; +} + +.compare-yaxis-dive:hover { + color: var(--accent); +} + .compare-yaxis-right { /* natural vertical-lr direction */ }