diff --git a/.changeset/event-patterns-column-selector.md b/.changeset/event-patterns-column-selector.md new file mode 100644 index 0000000000..a6b2f1a4fc --- /dev/null +++ b/.changeset/event-patterns-column-selector.md @@ -0,0 +1,5 @@ +--- +'@hyperdx/app': patch +--- + +feat: Allow selecting the column or SQL expression used for event pattern grouping (with shareable URL state) diff --git a/packages/app/src/DBSearchPage.tsx b/packages/app/src/DBSearchPage.tsx index 2862d3b857..a481e4458b 100644 --- a/packages/app/src/DBSearchPage.tsx +++ b/packages/app/src/DBSearchPage.tsx @@ -853,6 +853,17 @@ export function DBSearchPage() { ]).withDefault('results'), ); + const [patternColumn, setPatternColumn] = useQueryState( + 'patternColumn', + parseAsString, + ); + const [draftPatternColumn, setDraftPatternColumn] = useState( + patternColumn ?? '', + ); + useEffect(() => { + setDraftPatternColumn(patternColumn ?? ''); + }, [patternColumn]); + const [isLive, setIsLive] = useQueryState( 'isLive', parseAsBoolean.withDefault(true), @@ -1051,6 +1062,7 @@ export function DBSearchPage() { }); }, )(); + setPatternColumn(draftPatternColumn || null); // clear query errors setQueryErrors({}); }, [ @@ -1059,6 +1071,8 @@ export function DBSearchPage() { displayedTimeInputValue, onSearch, setQueryErrors, + draftPatternColumn, + setPatternColumn, ]); const debouncedSubmit = useDebouncedCallback(onSubmit, 1000); @@ -2181,6 +2195,10 @@ export function DBSearchPage() { ? (getEventBody(searchedSource) ?? '') : (chartConfig.implicitColumnExpression ?? '') } + patternColumn={patternColumn} + draftPatternColumn={draftPatternColumn} + onDraftPatternColumnChange={setDraftPatternColumn} + onSubmit={onSubmit} totalCountConfig={histogramTimeChartConfig} totalCountQueryKeyPrefix={QUERY_KEY_PREFIX} /> diff --git a/packages/app/src/components/PatternTable.tsx b/packages/app/src/components/PatternTable.tsx index e147243dc4..45bdea4b8d 100644 --- a/packages/app/src/components/PatternTable.tsx +++ b/packages/app/src/components/PatternTable.tsx @@ -12,6 +12,10 @@ import { RawLogTable } from '@/components/DBRowTable'; import { useSearchTotalCount } from '@/components/SearchTotalCountChart'; import { Pattern, useGroupedPatterns } from '@/hooks/usePatterns'; +import { + buildPatternColumnExpression, + PatternColumnSelector, +} from './Patterns/PatternColumnSelector'; import PatternSidePanel from './PatternSidePanel'; const emptyMap = new Map(); @@ -21,11 +25,19 @@ export default function PatternTable({ totalCountConfig, totalCountQueryKeyPrefix, bodyValueExpression, + patternColumn, + draftPatternColumn, + onDraftPatternColumnChange, + onSubmit, source, }: { config: BuilderChartConfigWithDateRange; totalCountConfig: BuilderChartConfigWithDateRange; bodyValueExpression: string; + patternColumn?: string | null; + draftPatternColumn?: string; + onDraftPatternColumnChange?: (value: string) => void; + onSubmit?: () => void; totalCountQueryKeyPrefix: string; source?: TSource; }) { @@ -33,6 +45,11 @@ export default function PatternTable({ const [selectedPattern, setSelectedPattern] = useState(null); + const effectiveBodyValueExpression = buildPatternColumnExpression({ + patternColumn, + fallback: bodyValueExpression, + }); + const { error: totalCountError, isLoading: isTotalCountLoading, @@ -48,7 +65,7 @@ export default function PatternTable({ } = useGroupedPatterns({ config, samples: SAMPLES, - bodyValueExpression, + bodyValueExpression: effectiveBodyValueExpression, severityTextExpression: (source?.kind === SourceKind.Log && source.severityTextExpression) || '', statusCodeExpression: @@ -69,6 +86,13 @@ export default function PatternTable({ return error ? ( + Error Message: @@ -100,6 +124,13 @@ export default function PatternTable({ ) : ( <> + setSelectedPattern(null)} /> )} diff --git a/packages/app/src/components/Patterns/PatternColumnSelector.tsx b/packages/app/src/components/Patterns/PatternColumnSelector.tsx new file mode 100644 index 0000000000..7b7789d3b3 --- /dev/null +++ b/packages/app/src/components/Patterns/PatternColumnSelector.tsx @@ -0,0 +1,53 @@ +import { tcFromSource } from '@hyperdx/common-utils/dist/core/metadata'; +import { Box } from '@mantine/core'; + +import SQLInlineEditor from '@/components/SQLEditor/SQLInlineEditor'; +import { useSource } from '@/source'; + +export function buildPatternColumnExpression({ + patternColumn, + fallback, +}: { + patternColumn: string | null | undefined; + fallback: string; +}): string { + if (!patternColumn) return fallback; + return `toString(${patternColumn})`; +} + +export function PatternColumnSelector({ + sourceId, + value, + onChange, + onSubmit, + dateRange, +}: { + sourceId: string | undefined; + value: string; + onChange?: (value: string) => void; + onSubmit?: () => void; + dateRange?: [Date, Date]; +}) { + const { data: source } = useSource({ id: sourceId }); + const tableConnection = tcFromSource(source); + + if (!onChange) return null; + + return ( + + + + ); +} diff --git a/packages/app/src/components/Patterns/__tests__/PatternColumnSelector.test.tsx b/packages/app/src/components/Patterns/__tests__/PatternColumnSelector.test.tsx new file mode 100644 index 0000000000..5b9a7ac5e4 --- /dev/null +++ b/packages/app/src/components/Patterns/__tests__/PatternColumnSelector.test.tsx @@ -0,0 +1,42 @@ +import { buildPatternColumnExpression } from '../PatternColumnSelector'; + +describe('buildPatternColumnExpression', () => { + const fallback = 'Body'; + + it('returns the fallback when no expression is provided', () => { + expect( + buildPatternColumnExpression({ patternColumn: null, fallback }), + ).toBe(fallback); + expect( + buildPatternColumnExpression({ patternColumn: undefined, fallback }), + ).toBe(fallback); + expect(buildPatternColumnExpression({ patternColumn: '', fallback })).toBe( + fallback, + ); + }); + + it('wraps a plain column reference in toString()', () => { + expect( + buildPatternColumnExpression({ + patternColumn: 'ResourceAttributes', + fallback, + }), + ).toBe('toString(ResourceAttributes)'); + }); + + it('wraps an arbitrary SQL expression in toString()', () => { + expect( + buildPatternColumnExpression({ + patternColumn: "concatWithSeparator(' ', Body, LogAttributes)", + fallback, + }), + ).toBe("toString(concatWithSeparator(' ', Body, LogAttributes))"); + + expect( + buildPatternColumnExpression({ + patternColumn: "JSONExtractString(Body, 'message')", + fallback, + }), + ).toBe("toString(JSONExtractString(Body, 'message'))"); + }); +}); diff --git a/packages/app/src/components/Patterns/__tests__/reconstructTemplate.test.ts b/packages/app/src/components/Patterns/__tests__/reconstructTemplate.test.ts new file mode 100644 index 0000000000..93f89cbfe8 --- /dev/null +++ b/packages/app/src/components/Patterns/__tests__/reconstructTemplate.test.ts @@ -0,0 +1,59 @@ +import { reconstructTemplate } from '../reconstructTemplate'; + +describe('reconstructTemplate', () => { + it('returns the original log when the template is empty', () => { + expect(reconstructTemplate('hello world', '')).toBe('hello world'); + }); + + it('restores JSON separators around stable and variable tokens', () => { + expect( + reconstructTemplate( + `{"hostname":"foo","pid":12345,"time":1700000000}`, + 'hostname foo pid <*> time <*>', + ), + ).toBe(`{"hostname":"foo","pid":<*>,"time":<*>}`); + }); + + it('restores ClickHouse Map (single-quoted) separators', () => { + expect( + reconstructTemplate( + `{'hostname':'Aarons-MacBook-Pro.local','pid':12345,'time':1700000000}`, + 'hostname Aarons MacBook Pro local pid <*> time <*>', + ), + ).toBe(`{'hostname':'Aarons-MacBook-Pro.local','pid':<*>,'time':<*>}`); + }); + + it('restores key=value separators', () => { + expect( + reconstructTemplate( + 'level=info msg=hello user_id=42', + 'level info msg hello user id <*>', + ), + ).toBe('level=info msg=hello user_id=<*>'); + }); + + it('keeps the original token when the template runs short', () => { + expect(reconstructTemplate('alpha beta gamma delta', 'alpha beta')).toBe( + 'alpha beta gamma delta', + ); + }); + + it('preserves leading and trailing separators', () => { + expect(reconstructTemplate('[INFO] hello world', 'INFO hello world')).toBe( + '[INFO] hello world', + ); + }); + + it('collapses newlines and tabs in the original log to single spaces', () => { + expect( + reconstructTemplate( + 'Error:\n message: "failed"\n code: 500', + 'Error message failed code <*>', + ), + ).toBe('Error: message: "failed" code: <*>'); + + expect(reconstructTemplate('foo\n\n\nbar', 'foo bar')).toBe('foo bar'); + + expect(reconstructTemplate('foo\tbar', 'foo bar')).toBe('foo bar'); + }); +}); diff --git a/packages/app/src/components/Patterns/reconstructTemplate.ts b/packages/app/src/components/Patterns/reconstructTemplate.ts new file mode 100644 index 0000000000..e8c6710665 --- /dev/null +++ b/packages/app/src/components/Patterns/reconstructTemplate.ts @@ -0,0 +1,22 @@ +export function reconstructTemplate( + originalLog: string, + templateMined: string, +): string { + const normalized = originalLog.replace(/\s+/g, ' '); + const tokens = templateMined.split(' ').filter(t => t.length > 0); + if (tokens.length === 0) return normalized; + + const tokenOrSeparator = /([A-Za-z0-9]+)|([^A-Za-z0-9]+)/g; + let result = ''; + let tokenIdx = 0; + let match: RegExpExecArray | null; + while ((match = tokenOrSeparator.exec(normalized)) !== null) { + if (match[1] !== undefined) { + result += tokens[tokenIdx] ?? match[1]; + tokenIdx++; + } else { + result += match[2]; + } + } + return result; +} diff --git a/packages/app/src/hooks/usePatterns.tsx b/packages/app/src/hooks/usePatterns.tsx index 6911e347f9..2d0d92641d 100644 --- a/packages/app/src/hooks/usePatterns.tsx +++ b/packages/app/src/hooks/usePatterns.tsx @@ -6,6 +6,7 @@ import { useQuery } from '@tanstack/react-query'; import { timeBucketByGranularity, toStartOfInterval } from '@/ChartUtils'; import { useConfigWithAdditionalSelect } from '@/components/DBRowTable'; +import { reconstructTemplate } from '@/components/Patterns/reconstructTemplate'; import { useQueriedChartConfig } from '@/hooks/useChartConfig'; import { getFirstTimestampValueExpression } from '@/source'; @@ -54,10 +55,14 @@ class Miner { await this.pyodide.runPythonAsync(` import js import json +import string from drain3 import TemplateMiner from drain3.template_miner_config import TemplateMinerConfig -${this.minerVariableName} = TemplateMiner(None, TemplateMinerConfig()) +_config = TemplateMinerConfig() +_config.drain_extra_delimiters = list(string.punctuation) + +${this.minerVariableName} = TemplateMiner(None, _config) `); } @@ -168,7 +173,7 @@ function usePatterns({ } = usePyodide({ enabled }); const query = useQuery({ - queryKey: ['patterns', config], + queryKey: ['patterns', config, bodyValueExpression], queryFn: () => { if (configWithPrimaryAndPartitionKey == null) { throw new Error('Unexpected configWithPrimaryAndPartitionKey is null'); @@ -290,10 +295,16 @@ export function useGroupedPatterns({ // return at least 1 const count = Math.max(Math.round(rows.length * sampleMultiplier), 1); const lastRow = rows.at(-1); + const reconstructedPattern = lastRow + ? reconstructTemplate( + stripAnsi((lastRow[PATTERN_COLUMN_ALIAS] ?? '') as string), + (lastRow.__hdx_pattern ?? '') as string, + ) + : undefined; fullPatternGroups[patternId] = { id: patternId, - pattern: lastRow?.__hdx_pattern, // last pattern is usually the most up to date templated pattern + pattern: reconstructedPattern, // last pattern is usually the most up to date templated pattern count, countStr: `~${count}`, severityText: lastRow?.[SEVERITY_TEXT_COLUMN_ALIAS], // last severitytext is usually representative of the entire pattern set