feat: add pattern column selector for event pattern matching on any column

.changeset/event-patterns-column-selector.md

@@ -0,0 +1,5 @@
+---
+'@hyperdx/app': patch
+---
+
+feat: Allow selecting the column or SQL expression used for event pattern grouping (with shareable URL state)

packages/app/src/DBSearchPage.tsx

@@ -853,6 +853,17 @@ export function DBSearchPage() {
     ]).withDefault('results'),
   );
 
+  const [patternColumn, setPatternColumn] = useQueryState(
+    'patternColumn',
+    parseAsString,
+  );
+  const [draftPatternColumn, setDraftPatternColumn] = useState(
+    patternColumn ?? '',
+  );
+  useEffect(() => {
+    setDraftPatternColumn(patternColumn ?? '');
+  }, [patternColumn]);
+
   const [isLive, setIsLive] = useQueryState(
     'isLive',
     parseAsBoolean.withDefault(true),
@@ -1051,6 +1062,7 @@ export function DBSearchPage() {
         });
       },
     )();
+    setPatternColumn(draftPatternColumn || null);
     // clear query errors
     setQueryErrors({});
   }, [
@@ -1059,6 +1071,8 @@ export function DBSearchPage() {
     displayedTimeInputValue,
     onSearch,
     setQueryErrors,
+    draftPatternColumn,
+    setPatternColumn,
   ]);
 
   const debouncedSubmit = useDebouncedCallback(onSubmit, 1000);
@@ -2181,6 +2195,10 @@ export function DBSearchPage() {
                             ? (getEventBody(searchedSource) ?? '')
                             : (chartConfig.implicitColumnExpression ?? '')
                         }
+                        patternColumn={patternColumn}
+                        draftPatternColumn={draftPatternColumn}
+                        onDraftPatternColumnChange={setDraftPatternColumn}
+                        onSubmit={onSubmit}
                         totalCountConfig={histogramTimeChartConfig}
                         totalCountQueryKeyPrefix={QUERY_KEY_PREFIX}
                       />

packages/app/src/components/PatternTable.tsx

@@ -12,6 +12,10 @@ import { RawLogTable } from '@/components/DBRowTable';
 import { useSearchTotalCount } from '@/components/SearchTotalCountChart';
 import { Pattern, useGroupedPatterns } from '@/hooks/usePatterns';
 
+import {
+  buildPatternColumnExpression,
+  PatternColumnSelector,
+} from './Patterns/PatternColumnSelector';
 import PatternSidePanel from './PatternSidePanel';
 
 const emptyMap = new Map();
@@ -21,18 +25,31 @@ export default function PatternTable({
   totalCountConfig,
   totalCountQueryKeyPrefix,
   bodyValueExpression,
+  patternColumn,
+  draftPatternColumn,
+  onDraftPatternColumnChange,
+  onSubmit,
   source,
 }: {
   config: BuilderChartConfigWithDateRange;
   totalCountConfig: BuilderChartConfigWithDateRange;
   bodyValueExpression: string;
+  patternColumn?: string | null;
+  draftPatternColumn?: string;
+  onDraftPatternColumnChange?: (value: string) => void;
+  onSubmit?: () => void;
   totalCountQueryKeyPrefix: string;
   source?: TSource;
 }) {
   const SAMPLES = 10_000;
 
   const [selectedPattern, setSelectedPattern] = useState<Pattern | null>(null);
 
+  const effectiveBodyValueExpression = buildPatternColumnExpression({
+    patternColumn,
+    fallback: bodyValueExpression,
+  });
+
   const {
     error: totalCountError,
     isLoading: isTotalCountLoading,
@@ -48,7 +65,7 @@ export default function PatternTable({
   } = useGroupedPatterns({
     config,
     samples: SAMPLES,
-    bodyValueExpression,
+    bodyValueExpression: effectiveBodyValueExpression,
     severityTextExpression:
       (source?.kind === SourceKind.Log && source.severityTextExpression) || '',
     statusCodeExpression:
@@ -69,6 +86,13 @@ export default function PatternTable({
 
   return error ? (
     <Container style={{ overflow: 'auto' }}>
+      <PatternColumnSelector
+        sourceId={source?.id}
+        value={draftPatternColumn ?? ''}
+        onChange={onDraftPatternColumnChange}
+        onSubmit={onSubmit}
+        dateRange={config.dateRange}
+      />
       <Box mt="lg">
         <Text my="sm" size="sm">
           Error Message:
@@ -100,6 +124,13 @@ export default function PatternTable({
     </Container>
   ) : (
     <>
+      <PatternColumnSelector
+        sourceId={source?.id}
+        value={draftPatternColumn ?? ''}
+        onChange={onDraftPatternColumnChange}
+        onSubmit={onSubmit}
+        dateRange={config.dateRange}
+      />
       <RawLogTable
         isLive={false}
         wrapLines={true}
@@ -131,7 +162,7 @@ export default function PatternTable({
           isOpen
           source={source}
           pattern={selectedPattern}
-          bodyValueExpression={bodyValueExpression}
+          bodyValueExpression={effectiveBodyValueExpression}
           onClose={() => setSelectedPattern(null)}
         />
       )}

packages/app/src/components/Patterns/PatternColumnSelector.tsx

@@ -0,0 +1,53 @@
+import { tcFromSource } from '@hyperdx/common-utils/dist/core/metadata';
+import { Box } from '@mantine/core';
+
+import SQLInlineEditor from '@/components/SQLEditor/SQLInlineEditor';
+import { useSource } from '@/source';
+
+export function buildPatternColumnExpression({
+  patternColumn,
+  fallback,
+}: {
+  patternColumn: string | null | undefined;
+  fallback: string;
+}): string {
+  if (!patternColumn) return fallback;
+  return `toString(${patternColumn})`;
+}
+
+export function PatternColumnSelector({
+  sourceId,
+  value,
+  onChange,
+  onSubmit,
+  dateRange,
+}: {
+  sourceId: string | undefined;
+  value: string;
+  onChange?: (value: string) => void;
+  onSubmit?: () => void;
+  dateRange?: [Date, Date];
+}) {
+  const { data: source } = useSource({ id: sourceId });
+  const tableConnection = tcFromSource(source);
+
+  if (!onChange) return null;
+
+  return (
+    <Box py="xs" maw={600}>
+      <SQLInlineEditor
+        tableConnection={tableConnection}
+        value={value}
+        onChange={onChange}
+        onSubmit={onSubmit}
+        enableHotkey
+        label="Pattern Expression"
+        placeholder="Default (body) — column name or expression"
+        size="xs"
+        allowMultiline={false}
+        sourceId={sourceId}
+        dateRange={dateRange}
+      />
+    </Box>
+  );
+}

packages/app/src/components/Patterns/__tests__/PatternColumnSelector.test.tsx

@@ -0,0 +1,42 @@
+import { buildPatternColumnExpression } from '../PatternColumnSelector';
+
+describe('buildPatternColumnExpression', () => {
+  const fallback = 'Body';
+
+  it('returns the fallback when no expression is provided', () => {
+    expect(
+      buildPatternColumnExpression({ patternColumn: null, fallback }),
+    ).toBe(fallback);
+    expect(
+      buildPatternColumnExpression({ patternColumn: undefined, fallback }),
+    ).toBe(fallback);
+    expect(buildPatternColumnExpression({ patternColumn: '', fallback })).toBe(
+      fallback,
+    );
+  });
+
+  it('wraps a plain column reference in toString()', () => {
+    expect(
+      buildPatternColumnExpression({
+        patternColumn: 'ResourceAttributes',
+        fallback,
+      }),
+    ).toBe('toString(ResourceAttributes)');
+  });
+
+  it('wraps an arbitrary SQL expression in toString()', () => {
+    expect(
+      buildPatternColumnExpression({
+        patternColumn: "concatWithSeparator(' ', Body, LogAttributes)",
+        fallback,
+      }),
+    ).toBe("toString(concatWithSeparator(' ', Body, LogAttributes))");
+
+    expect(
+      buildPatternColumnExpression({
+        patternColumn: "JSONExtractString(Body, 'message')",
+        fallback,
+      }),
+    ).toBe("toString(JSONExtractString(Body, 'message'))");
+  });
+});

packages/app/src/components/Patterns/__tests__/reconstructTemplate.test.ts

@@ -0,0 +1,59 @@
+import { reconstructTemplate } from '../reconstructTemplate';
+
+describe('reconstructTemplate', () => {
+  it('returns the original log when the template is empty', () => {
+    expect(reconstructTemplate('hello world', '')).toBe('hello world');
+  });
+
+  it('restores JSON separators around stable and variable tokens', () => {
+    expect(
+      reconstructTemplate(
+        `{"hostname":"foo","pid":12345,"time":1700000000}`,
+        'hostname foo pid <*> time <*>',
+      ),
+    ).toBe(`{"hostname":"foo","pid":<*>,"time":<*>}`);
+  });
+
+  it('restores ClickHouse Map (single-quoted) separators', () => {
+    expect(
+      reconstructTemplate(
+        `{'hostname':'Aarons-MacBook-Pro.local','pid':12345,'time':1700000000}`,
+        'hostname Aarons MacBook Pro local pid <*> time <*>',
+      ),
+    ).toBe(`{'hostname':'Aarons-MacBook-Pro.local','pid':<*>,'time':<*>}`);
+  });
+
+  it('restores key=value separators', () => {
+    expect(
+      reconstructTemplate(
+        'level=info msg=hello user_id=42',
+        'level info msg hello user id <*>',
+      ),
+    ).toBe('level=info msg=hello user_id=<*>');
+  });
+
+  it('keeps the original token when the template runs short', () => {
+    expect(reconstructTemplate('alpha beta gamma delta', 'alpha beta')).toBe(
+      'alpha beta gamma delta',
+    );
+  });
+
+  it('preserves leading and trailing separators', () => {
+    expect(reconstructTemplate('[INFO] hello world', 'INFO hello world')).toBe(
+      '[INFO] hello world',
+    );
+  });
+
+  it('collapses newlines and tabs in the original log to single spaces', () => {
+    expect(
+      reconstructTemplate(
+        'Error:\n  message: "failed"\n  code: 500',
+        'Error message failed code <*>',
+      ),
+    ).toBe('Error: message: "failed" code: <*>');
+
+    expect(reconstructTemplate('foo\n\n\nbar', 'foo bar')).toBe('foo bar');
+
+    expect(reconstructTemplate('foo\tbar', 'foo bar')).toBe('foo bar');
+  });
+});

packages/app/src/components/Patterns/reconstructTemplate.ts

@@ -0,0 +1,22 @@
+export function reconstructTemplate(
+  originalLog: string,
+  templateMined: string,
+): string {
+  const normalized = originalLog.replace(/\s+/g, ' ');
+  const tokens = templateMined.split(' ').filter(t => t.length > 0);
+  if (tokens.length === 0) return normalized;
+
+  const tokenOrSeparator = /([A-Za-z0-9]+)|([^A-Za-z0-9]+)/g;
+  let result = '';
+  let tokenIdx = 0;
+  let match: RegExpExecArray | null;
+  while ((match = tokenOrSeparator.exec(normalized)) !== null) {
+    if (match[1] !== undefined) {
+      result += tokens[tokenIdx] ?? match[1];
+      tokenIdx++;
+    } else {
+      result += match[2];
+    }
+  }
+  return result;
+}

packages/app/src/hooks/usePatterns.tsx

@@ -6,6 +6,7 @@ import { useQuery } from '@tanstack/react-query';
 
 import { timeBucketByGranularity, toStartOfInterval } from '@/ChartUtils';
 import { useConfigWithAdditionalSelect } from '@/components/DBRowTable';
+import { reconstructTemplate } from '@/components/Patterns/reconstructTemplate';
 import { useQueriedChartConfig } from '@/hooks/useChartConfig';
 import { getFirstTimestampValueExpression } from '@/source';
 
@@ -54,10 +55,14 @@ class Miner {
     await this.pyodide.runPythonAsync(`
 import js
 import json
+import string
 from drain3 import TemplateMiner
 from drain3.template_miner_config import TemplateMinerConfig
 
-${this.minerVariableName} = TemplateMiner(None, TemplateMinerConfig())
+_config = TemplateMinerConfig()
+_config.drain_extra_delimiters = list(string.punctuation)
+
+${this.minerVariableName} = TemplateMiner(None, _config)
     `);
   }
 
@@ -168,7 +173,7 @@ function usePatterns({
   } = usePyodide({ enabled });
 
   const query = useQuery({
-    queryKey: ['patterns', config],
+    queryKey: ['patterns', config, bodyValueExpression],
     queryFn: () => {
       if (configWithPrimaryAndPartitionKey == null) {
         throw new Error('Unexpected configWithPrimaryAndPartitionKey is null');
@@ -290,10 +295,16 @@ export function useGroupedPatterns({
       // return at least 1
       const count = Math.max(Math.round(rows.length * sampleMultiplier), 1);
       const lastRow = rows.at(-1);
+      const reconstructedPattern = lastRow
+        ? reconstructTemplate(
+            stripAnsi((lastRow[PATTERN_COLUMN_ALIAS] ?? '') as string),
+            (lastRow.__hdx_pattern ?? '') as string,
+          )
+        : undefined;
 
       fullPatternGroups[patternId] = {
         id: patternId,
-        pattern: lastRow?.__hdx_pattern, // last pattern is usually the most up to date templated pattern
+        pattern: reconstructedPattern, // last pattern is usually the most up to date templated pattern
         count,
         countStr: `~${count}`,
         severityText: lastRow?.[SEVERITY_TEXT_COLUMN_ALIAS], // last severitytext is usually representative of the entire pattern set