From aee2ab9a23b546d71e952c8944d619e1cc4c5e99 Mon Sep 17 00:00:00 2001 From: Aaron Knudtson <87577305+knudtty@users.noreply.github.com> Date: Thu, 11 Jun 2026 18:45:06 -0400 Subject: [PATCH 1/4] feat: add pattern column selector for event pattern matching on any column --- .changeset/event-patterns-column-selector.md | 5 + packages/app/src/DBSearchPage.tsx | 7 ++ packages/app/src/components/PatternTable.tsx | 28 ++++- .../Patterns/PatternColumnSelector.tsx | 92 ++++++++++++++ .../__tests__/PatternColumnSelector.test.tsx | 114 ++++++++++++++++++ 5 files changed, 244 insertions(+), 2 deletions(-) create mode 100644 .changeset/event-patterns-column-selector.md create mode 100644 packages/app/src/components/Patterns/PatternColumnSelector.tsx create mode 100644 packages/app/src/components/Patterns/__tests__/PatternColumnSelector.test.tsx diff --git a/.changeset/event-patterns-column-selector.md b/.changeset/event-patterns-column-selector.md new file mode 100644 index 0000000000..ae84bf3264 --- /dev/null +++ b/.changeset/event-patterns-column-selector.md @@ -0,0 +1,5 @@ +--- +'@hyperdx/app': patch +--- + +feat: Allow selecting the column used for event pattern grouping with URL state diff --git a/packages/app/src/DBSearchPage.tsx b/packages/app/src/DBSearchPage.tsx index 2862d3b857..3587a47cae 100644 --- a/packages/app/src/DBSearchPage.tsx +++ b/packages/app/src/DBSearchPage.tsx @@ -853,6 +853,11 @@ export function DBSearchPage() { ]).withDefault('results'), ); + const [patternColumn, setPatternColumn] = useQueryState( + 'patternColumn', + parseAsString, + ); + const [isLive, setIsLive] = useQueryState( 'isLive', parseAsBoolean.withDefault(true), @@ -2181,6 +2186,8 @@ export function DBSearchPage() { ? (getEventBody(searchedSource) ?? '') : (chartConfig.implicitColumnExpression ?? '') } + patternColumn={patternColumn} + onPatternColumnChange={setPatternColumn} totalCountConfig={histogramTimeChartConfig} totalCountQueryKeyPrefix={QUERY_KEY_PREFIX} /> diff --git a/packages/app/src/components/PatternTable.tsx b/packages/app/src/components/PatternTable.tsx index e147243dc4..9082d42fcd 100644 --- a/packages/app/src/components/PatternTable.tsx +++ b/packages/app/src/components/PatternTable.tsx @@ -12,6 +12,10 @@ import { RawLogTable } from '@/components/DBRowTable'; import { useSearchTotalCount } from '@/components/SearchTotalCountChart'; import { Pattern, useGroupedPatterns } from '@/hooks/usePatterns'; +import { + PatternColumnSelector, + usePatternColumnExpression, +} from './Patterns/PatternColumnSelector'; import PatternSidePanel from './PatternSidePanel'; const emptyMap = new Map(); @@ -21,11 +25,15 @@ export default function PatternTable({ totalCountConfig, totalCountQueryKeyPrefix, bodyValueExpression, + patternColumn, + onPatternColumnChange, source, }: { config: BuilderChartConfigWithDateRange; totalCountConfig: BuilderChartConfigWithDateRange; bodyValueExpression: string; + patternColumn?: string | null; + onPatternColumnChange?: (column: string | null) => void; totalCountQueryKeyPrefix: string; source?: TSource; }) { @@ -33,6 +41,12 @@ export default function PatternTable({ const [selectedPattern, setSelectedPattern] = useState(null); + const effectiveBodyValueExpression = usePatternColumnExpression({ + sourceId: source?.id, + patternColumn, + fallback: bodyValueExpression, + }); + const { error: totalCountError, isLoading: isTotalCountLoading, @@ -48,7 +62,7 @@ export default function PatternTable({ } = useGroupedPatterns({ config, samples: SAMPLES, - bodyValueExpression, + bodyValueExpression: effectiveBodyValueExpression, severityTextExpression: (source?.kind === SourceKind.Log && source.severityTextExpression) || '', statusCodeExpression: @@ -69,6 +83,11 @@ export default function PatternTable({ return error ? ( + Error Message: @@ -100,6 +119,11 @@ export default function PatternTable({ ) : ( <> + setSelectedPattern(null)} /> )} diff --git a/packages/app/src/components/Patterns/PatternColumnSelector.tsx b/packages/app/src/components/Patterns/PatternColumnSelector.tsx new file mode 100644 index 0000000000..eebfbf210e --- /dev/null +++ b/packages/app/src/components/Patterns/PatternColumnSelector.tsx @@ -0,0 +1,92 @@ +import { useMemo } from 'react'; +import { + ColumnMeta, + convertCHDataTypeToJSType, + JSDataType, +} from '@hyperdx/common-utils/dist/clickhouse'; +import { tcFromSource } from '@hyperdx/common-utils/dist/core/metadata'; +import { Box, Group, Select, Text } from '@mantine/core'; + +import { useColumns } from '@/hooks/useMetadata'; +import { useSource } from '@/source'; + +export function buildPatternColumnExpression({ + patternColumn, + fallback, + columns, +}: { + patternColumn: string | null | undefined; + fallback: string; + columns: ColumnMeta[] | undefined; +}): string { + if (!patternColumn) return fallback; + const col = columns?.find(c => c.name === patternColumn); + if (!col) return patternColumn; + const jsType = convertCHDataTypeToJSType(col.type); + if (jsType === JSDataType.String) return patternColumn; + return `toString(${patternColumn})`; +} + +function useSourceColumns(sourceId: string | undefined) { + const { data: source } = useSource({ id: sourceId }); + const tc = tcFromSource(source); + return useColumns(tc, { + enabled: !!tc.databaseName && !!tc.tableName && !!tc.connectionId, + }); +} + +export function usePatternColumnExpression({ + sourceId, + patternColumn, + fallback, +}: { + sourceId: string | undefined; + patternColumn: string | null | undefined; + fallback: string; +}): string { + const { data: columns } = useSourceColumns(sourceId); + return useMemo( + () => buildPatternColumnExpression({ patternColumn, fallback, columns }), + [patternColumn, fallback, columns], + ); +} + +export function PatternColumnSelector({ + sourceId, + patternColumn, + onChange, +}: { + sourceId: string | undefined; + patternColumn: string | null | undefined; + onChange?: (column: string | null) => void; +}) { + const { data: columns } = useSourceColumns(sourceId); + const options = useMemo( + () => columns?.map(col => ({ value: col.name, label: col.name })) ?? [], + [columns], + ); + + if (!onChange) return null; + + return ( + + + + Pattern Column + + - + + ); } diff --git a/packages/app/src/components/Patterns/__tests__/PatternColumnSelector.test.tsx b/packages/app/src/components/Patterns/__tests__/PatternColumnSelector.test.tsx index acb2f0079f..5b9a7ac5e4 100644 --- a/packages/app/src/components/Patterns/__tests__/PatternColumnSelector.test.tsx +++ b/packages/app/src/components/Patterns/__tests__/PatternColumnSelector.test.tsx @@ -1,114 +1,42 @@ -import { ColumnMeta } from '@hyperdx/common-utils/dist/clickhouse'; - import { buildPatternColumnExpression } from '../PatternColumnSelector'; -function makeColumn(name: string, type: string): ColumnMeta { - return { - name, - type, - codec_expression: '', - comment: '', - default_expression: '', - default_type: '', - ttl_expression: '', - }; -} - describe('buildPatternColumnExpression', () => { const fallback = 'Body'; - it('returns the fallback when no pattern column is selected', () => { + it('returns the fallback when no expression is provided', () => { expect( - buildPatternColumnExpression({ - patternColumn: null, - fallback, - columns: [makeColumn('Body', 'String')], - }), + buildPatternColumnExpression({ patternColumn: null, fallback }), ).toBe(fallback); - expect( - buildPatternColumnExpression({ - patternColumn: undefined, - fallback, - columns: [], - }), - ).toBe(fallback); - - expect( - buildPatternColumnExpression({ - patternColumn: '', - fallback, - columns: [], - }), + buildPatternColumnExpression({ patternColumn: undefined, fallback }), ).toBe(fallback); + expect(buildPatternColumnExpression({ patternColumn: '', fallback })).toBe( + fallback, + ); }); - it('uses the column directly when it is a String type', () => { - expect( - buildPatternColumnExpression({ - patternColumn: 'ServiceName', - fallback, - columns: [makeColumn('ServiceName', 'String')], - }), - ).toBe('ServiceName'); - }); - - it('uses the column directly when it is a LowCardinality(String) type', () => { - expect( - buildPatternColumnExpression({ - patternColumn: 'SeverityText', - fallback, - columns: [makeColumn('SeverityText', 'LowCardinality(String)')], - }), - ).toBe('SeverityText'); - }); - - it('wraps non-string columns in toString()', () => { + it('wraps a plain column reference in toString()', () => { expect( buildPatternColumnExpression({ patternColumn: 'ResourceAttributes', fallback, - columns: [ - makeColumn( - 'ResourceAttributes', - 'Map(LowCardinality(String), String)', - ), - ], }), ).toBe('toString(ResourceAttributes)'); - - expect( - buildPatternColumnExpression({ - patternColumn: 'Timestamp', - fallback, - columns: [makeColumn('Timestamp', 'DateTime64(9)')], - }), - ).toBe('toString(Timestamp)'); - - expect( - buildPatternColumnExpression({ - patternColumn: 'SpanCount', - fallback, - columns: [makeColumn('SpanCount', 'UInt32')], - }), - ).toBe('toString(SpanCount)'); }); - it('falls back to the raw column name when columns metadata is not loaded', () => { + it('wraps an arbitrary SQL expression in toString()', () => { expect( buildPatternColumnExpression({ - patternColumn: 'CustomColumn', + patternColumn: "concatWithSeparator(' ', Body, LogAttributes)", fallback, - columns: undefined, }), - ).toBe('CustomColumn'); + ).toBe("toString(concatWithSeparator(' ', Body, LogAttributes))"); expect( buildPatternColumnExpression({ - patternColumn: 'MissingColumn', + patternColumn: "JSONExtractString(Body, 'message')", fallback, - columns: [makeColumn('OtherColumn', 'String')], }), - ).toBe('MissingColumn'); + ).toBe("toString(JSONExtractString(Body, 'message'))"); }); }); diff --git a/packages/app/src/components/Patterns/__tests__/reconstructTemplate.test.ts b/packages/app/src/components/Patterns/__tests__/reconstructTemplate.test.ts new file mode 100644 index 0000000000..93f89cbfe8 --- /dev/null +++ b/packages/app/src/components/Patterns/__tests__/reconstructTemplate.test.ts @@ -0,0 +1,59 @@ +import { reconstructTemplate } from '../reconstructTemplate'; + +describe('reconstructTemplate', () => { + it('returns the original log when the template is empty', () => { + expect(reconstructTemplate('hello world', '')).toBe('hello world'); + }); + + it('restores JSON separators around stable and variable tokens', () => { + expect( + reconstructTemplate( + `{"hostname":"foo","pid":12345,"time":1700000000}`, + 'hostname foo pid <*> time <*>', + ), + ).toBe(`{"hostname":"foo","pid":<*>,"time":<*>}`); + }); + + it('restores ClickHouse Map (single-quoted) separators', () => { + expect( + reconstructTemplate( + `{'hostname':'Aarons-MacBook-Pro.local','pid':12345,'time':1700000000}`, + 'hostname Aarons MacBook Pro local pid <*> time <*>', + ), + ).toBe(`{'hostname':'Aarons-MacBook-Pro.local','pid':<*>,'time':<*>}`); + }); + + it('restores key=value separators', () => { + expect( + reconstructTemplate( + 'level=info msg=hello user_id=42', + 'level info msg hello user id <*>', + ), + ).toBe('level=info msg=hello user_id=<*>'); + }); + + it('keeps the original token when the template runs short', () => { + expect(reconstructTemplate('alpha beta gamma delta', 'alpha beta')).toBe( + 'alpha beta gamma delta', + ); + }); + + it('preserves leading and trailing separators', () => { + expect(reconstructTemplate('[INFO] hello world', 'INFO hello world')).toBe( + '[INFO] hello world', + ); + }); + + it('collapses newlines and tabs in the original log to single spaces', () => { + expect( + reconstructTemplate( + 'Error:\n message: "failed"\n code: 500', + 'Error message failed code <*>', + ), + ).toBe('Error: message: "failed" code: <*>'); + + expect(reconstructTemplate('foo\n\n\nbar', 'foo bar')).toBe('foo bar'); + + expect(reconstructTemplate('foo\tbar', 'foo bar')).toBe('foo bar'); + }); +}); diff --git a/packages/app/src/components/Patterns/reconstructTemplate.ts b/packages/app/src/components/Patterns/reconstructTemplate.ts new file mode 100644 index 0000000000..53f07f8eed --- /dev/null +++ b/packages/app/src/components/Patterns/reconstructTemplate.ts @@ -0,0 +1,24 @@ +const TOKEN_OR_SEPARATOR = /([A-Za-z0-9]+)|([^A-Za-z0-9]+)/g; + +export function reconstructTemplate( + originalLog: string, + templateMined: string, +): string { + const normalized = originalLog.replace(/\s+/g, ' '); + const tokens = templateMined.split(' ').filter(t => t.length > 0); + if (tokens.length === 0) return normalized; + + let result = ''; + let tokenIdx = 0; + TOKEN_OR_SEPARATOR.lastIndex = 0; + let match: RegExpExecArray | null; + while ((match = TOKEN_OR_SEPARATOR.exec(normalized)) !== null) { + if (match[1] !== undefined) { + result += tokens[tokenIdx] ?? match[1]; + tokenIdx++; + } else { + result += match[2]; + } + } + return result; +} diff --git a/packages/app/src/hooks/usePatterns.tsx b/packages/app/src/hooks/usePatterns.tsx index 6911e347f9..2d0d92641d 100644 --- a/packages/app/src/hooks/usePatterns.tsx +++ b/packages/app/src/hooks/usePatterns.tsx @@ -6,6 +6,7 @@ import { useQuery } from '@tanstack/react-query'; import { timeBucketByGranularity, toStartOfInterval } from '@/ChartUtils'; import { useConfigWithAdditionalSelect } from '@/components/DBRowTable'; +import { reconstructTemplate } from '@/components/Patterns/reconstructTemplate'; import { useQueriedChartConfig } from '@/hooks/useChartConfig'; import { getFirstTimestampValueExpression } from '@/source'; @@ -54,10 +55,14 @@ class Miner { await this.pyodide.runPythonAsync(` import js import json +import string from drain3 import TemplateMiner from drain3.template_miner_config import TemplateMinerConfig -${this.minerVariableName} = TemplateMiner(None, TemplateMinerConfig()) +_config = TemplateMinerConfig() +_config.drain_extra_delimiters = list(string.punctuation) + +${this.minerVariableName} = TemplateMiner(None, _config) `); } @@ -168,7 +173,7 @@ function usePatterns({ } = usePyodide({ enabled }); const query = useQuery({ - queryKey: ['patterns', config], + queryKey: ['patterns', config, bodyValueExpression], queryFn: () => { if (configWithPrimaryAndPartitionKey == null) { throw new Error('Unexpected configWithPrimaryAndPartitionKey is null'); @@ -290,10 +295,16 @@ export function useGroupedPatterns({ // return at least 1 const count = Math.max(Math.round(rows.length * sampleMultiplier), 1); const lastRow = rows.at(-1); + const reconstructedPattern = lastRow + ? reconstructTemplate( + stripAnsi((lastRow[PATTERN_COLUMN_ALIAS] ?? '') as string), + (lastRow.__hdx_pattern ?? '') as string, + ) + : undefined; fullPatternGroups[patternId] = { id: patternId, - pattern: lastRow?.__hdx_pattern, // last pattern is usually the most up to date templated pattern + pattern: reconstructedPattern, // last pattern is usually the most up to date templated pattern count, countStr: `~${count}`, severityText: lastRow?.[SEVERITY_TEXT_COLUMN_ALIAS], // last severitytext is usually representative of the entire pattern set From 304df2702a6fc36bc4934473468e294b33f6037b Mon Sep 17 00:00:00 2001 From: Aaron Knudtson <87577305+knudtty@users.noreply.github.com> Date: Fri, 12 Jun 2026 16:00:10 -0400 Subject: [PATCH 3/4] Update packages/app/src/components/Patterns/reconstructTemplate.ts Co-authored-by: greptile-apps[bot] <165735046+greptile-apps[bot]@users.noreply.github.com> --- packages/app/src/components/Patterns/reconstructTemplate.ts | 2 -- 1 file changed, 2 deletions(-) diff --git a/packages/app/src/components/Patterns/reconstructTemplate.ts b/packages/app/src/components/Patterns/reconstructTemplate.ts index 53f07f8eed..e65753dcda 100644 --- a/packages/app/src/components/Patterns/reconstructTemplate.ts +++ b/packages/app/src/components/Patterns/reconstructTemplate.ts @@ -1,5 +1,3 @@ -const TOKEN_OR_SEPARATOR = /([A-Za-z0-9]+)|([^A-Za-z0-9]+)/g; - export function reconstructTemplate( originalLog: string, templateMined: string, From b20d0b492b8c0d335cbf6f7150ef4b5ace3b2bdd Mon Sep 17 00:00:00 2001 From: Aaron Knudtson <87577305+knudtty@users.noreply.github.com> Date: Fri, 12 Jun 2026 16:00:25 -0400 Subject: [PATCH 4/4] Update packages/app/src/components/Patterns/reconstructTemplate.ts Co-authored-by: greptile-apps[bot] <165735046+greptile-apps[bot]@users.noreply.github.com> --- packages/app/src/components/Patterns/reconstructTemplate.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/app/src/components/Patterns/reconstructTemplate.ts b/packages/app/src/components/Patterns/reconstructTemplate.ts index e65753dcda..e8c6710665 100644 --- a/packages/app/src/components/Patterns/reconstructTemplate.ts +++ b/packages/app/src/components/Patterns/reconstructTemplate.ts @@ -6,11 +6,11 @@ export function reconstructTemplate( const tokens = templateMined.split(' ').filter(t => t.length > 0); if (tokens.length === 0) return normalized; + const tokenOrSeparator = /([A-Za-z0-9]+)|([^A-Za-z0-9]+)/g; let result = ''; let tokenIdx = 0; - TOKEN_OR_SEPARATOR.lastIndex = 0; let match: RegExpExecArray | null; - while ((match = TOKEN_OR_SEPARATOR.exec(normalized)) !== null) { + while ((match = tokenOrSeparator.exec(normalized)) !== null) { if (match[1] !== undefined) { result += tokens[tokenIdx] ?? match[1]; tokenIdx++;