From da543398d42a844e7be49e8fd95e2dc2f31eb63a Mon Sep 17 00:00:00 2001 From: Jeffrey Melloy Date: Fri, 12 Jun 2026 09:30:46 -0700 Subject: [PATCH] feat(importer): add repository attribution from session metadata --- backend/internal/importer/claude.go | 249 ++++++++++++++---- backend/internal/importer/codex.go | 107 +++++--- backend/internal/importer/importer_test.go | 127 ++++++++++ backend/internal/importer/repos.go | 281 +++++++++++++++++++++ 4 files changed, 674 insertions(+), 90 deletions(-) create mode 100644 backend/internal/importer/repos.go diff --git a/backend/internal/importer/claude.go b/backend/internal/importer/claude.go index aee4478..8616177 100644 --- a/backend/internal/importer/claude.go +++ b/backend/internal/importer/claude.go @@ -105,7 +105,7 @@ func (p *ClaudeParser) FindSessionFiles(ctx context.Context) ([]string, error) { // ClaudeJSONLEntry represents a single line in Claude Code JSONL files type ClaudeJSONLEntry struct { - Type string `json:"type,omitempty"` // Root type: "assistant", "user", "queue-operation", etc. + Type string `json:"type,omitempty"` // Root type: "assistant", "user", "pr-link", etc. Timestamp string `json:"timestamp"` SessionID string `json:"sessionId,omitempty"` Version string `json:"version,omitempty"` @@ -113,6 +113,13 @@ type ClaudeJSONLEntry struct { RequestID string `json:"requestId,omitempty"` CostUSD *float64 `json:"costUSD,omitempty"` Message *ClaudeMessage `json:"message,omitempty"` + GitBranch string `json:"gitBranch,omitempty"` + // PR link fields (present when type == "pr-link") + PRNumber int `json:"prNumber,omitempty"` + PRUrl string `json:"prUrl,omitempty"` + PRRepository string `json:"prRepository,omitempty"` + // Worktree-state fields (present when type == "worktree-state") + WorktreeSession *claudeWorktreeSession `json:"worktreeSession,omitempty"` } type ClaudeMessage struct { @@ -140,6 +147,81 @@ type ClaudeUsage struct { CacheReadInputTokens int `json:"cache_read_input_tokens"` } +// claudeWorktreeSession holds fields from a "worktree-state" entry's worktreeSession object. +type claudeWorktreeSession struct { + OriginalCwd string `json:"originalCwd,omitempty"` +} + +// claudeSessionMeta holds session-level metadata collected in a first pass +type claudeSessionMeta struct { + GitBranch string + Cwd string + OriginalCwd string // from worktree-state.worktreeSession.originalCwd + CandidateRepos []string // owner/repo references from pr-link entries (most-frequent first) + PRNumber int + PRUrl string + PRRepository string + PRTimestamp time.Time +} + +// collectSessionMeta does a first pass over raw JSONL lines to collect session-level metadata +func (p *ClaudeParser) collectSessionMeta(lines []string) claudeSessionMeta { + meta := claudeSessionMeta{} + seenPRs := make(map[int]bool) + seenRepos := make(map[string]bool) + + for _, line := range lines { + var entry ClaudeJSONLEntry + if err := json.Unmarshal([]byte(line), &entry); err != nil { + continue + } + if entry.GitBranch != "" && meta.GitBranch == "" { + meta.GitBranch = entry.GitBranch + } + if entry.Cwd != "" && meta.Cwd == "" { + meta.Cwd = entry.Cwd + } + if entry.Type == "worktree-state" && entry.WorktreeSession != nil { + if oc := entry.WorktreeSession.OriginalCwd; oc != "" && meta.OriginalCwd == "" { + meta.OriginalCwd = oc + } + } + if entry.Type == "pr-link" && entry.PRNumber != 0 && !seenPRs[entry.PRNumber] { + seenPRs[entry.PRNumber] = true + // Collect all pr-link repositories as candidates for resolution. + if r := entry.PRRepository; r != "" && !seenRepos[r] { + seenRepos[r] = true + meta.CandidateRepos = append(meta.CandidateRepos, r) + } + // Use the first PR linked in the session for structured PR metadata. + if meta.PRNumber == 0 { + meta.PRNumber = entry.PRNumber + meta.PRUrl = entry.PRUrl + meta.PRRepository = entry.PRRepository + if ts, err := time.Parse(time.RFC3339Nano, entry.Timestamp); err == nil { + meta.PRTimestamp = ts + } else if ts, err := time.Parse(time.RFC3339, entry.Timestamp); err == nil { + meta.PRTimestamp = ts + } + } + } + } + return meta +} + +// extractRepository returns the best repository identifier for a session. +// +// Priority: +// 1. PRRepository from a structured pr-link entry (authoritative - this is +// exactly where the linked PR lives, not just a text-mined reference). +// 2. resolveSessionRepository: on-disk git remote -> matching candidate -> cwd name. +func extractRepository(meta claudeSessionMeta) string { + if meta.PRRepository != "" { + return meta.PRRepository + } + return resolveSessionRepository(meta.CandidateRepos, "", meta.OriginalCwd, meta.Cwd) +} + // ParseFile parses a Claude Code JSONL file func (p *ClaudeParser) ParseFile(ctx context.Context, path string) (*ImportResult, error) { file, err := os.Open(path) @@ -161,24 +243,35 @@ func (p *ClaudeParser) ParseFile(ctx context.Context, path string) (*ImportResul buf := make([]byte, 0, 64*1024) scanner.Buffer(buf, 1024*1024) - lineNum := 0 - messageIndex := 0 // Track message order for transcripts - seenRequests := make(map[string]bool) // For deduplication of metrics - + // Collect all non-empty lines + var lines []string for scanner.Scan() { if ctx.Err() != nil { return nil, ctx.Err() } + if line := scanner.Text(); strings.TrimSpace(line) != "" { + lines = append(lines, line) + } + } + if err := scanner.Err(); err != nil { + return nil, fmt.Errorf("reading file: %w", err) + } - lineNum++ - line := scanner.Text() - if strings.TrimSpace(line) == "" { - continue + // First pass: collect session-level metadata (gitBranch, PR info, cwd) + meta := p.collectSessionMeta(lines) + repository := extractRepository(meta) + + messageIndex := 0 + seenRequests := make(map[string]bool) // For deduplication of metrics + + // Second pass: process records + for _, line := range lines { + if ctx.Err() != nil { + return nil, ctx.Err() } var entry ClaudeJSONLEntry if err := json.Unmarshal([]byte(line), &entry); err != nil { - // Skip malformed lines continue } @@ -219,7 +312,7 @@ func (p *ClaudeParser) ParseFile(ctx context.Context, path string) (*ImportResul } // Create transcript log records from message content - transcriptLogs := p.CreateTranscriptLogs(entry, ts, sessionID, &messageIndex) + transcriptLogs := p.createTranscriptLogs(entry, ts, sessionID, meta, &messageIndex) result.Logs = append(result.Logs, transcriptLogs...) // For assistant entries with usage data, also create metrics @@ -233,7 +326,7 @@ func (p *ClaudeParser) ParseFile(ctx context.Context, path string) (*ImportResul seenRequests[dedupKey] = true } - // Create api_request log record (existing behavior) + // Create api_request log record logRecord := api.LogRecord{ Timestamp: ts, ServiceName: SourceClaude.ServiceName(), @@ -253,27 +346,32 @@ func (p *ClaudeParser) ParseFile(ctx context.Context, path string) (*ImportResul if entry.RequestID != "" { logRecord.LogAttributes["request_id"] = entry.RequestID } + if meta.GitBranch != "" { + logRecord.LogAttributes["git_branch"] = meta.GitBranch + } + if repository != "" { + logRecord.LogAttributes["repository"] = repository + } result.Logs = append(result.Logs, logRecord) - // Create metrics + // Token usage metrics usage := entry.Message.Usage model := entry.Message.Model - // Token usage metrics (creates both regular and user-facing variants) if usage.InputTokens > 0 { - result.Metrics = append(result.Metrics, CreateTokenMetrics(ts, model, "input", float64(usage.InputTokens))...) + result.Metrics = append(result.Metrics, createTokenMetrics(ts, model, "input", float64(usage.InputTokens), meta, repository)...) } if usage.OutputTokens > 0 { - result.Metrics = append(result.Metrics, CreateTokenMetrics(ts, model, "output", float64(usage.OutputTokens))...) + result.Metrics = append(result.Metrics, createTokenMetrics(ts, model, "output", float64(usage.OutputTokens), meta, repository)...) } if usage.CacheCreationInputTokens > 0 { - result.Metrics = append(result.Metrics, CreateTokenMetrics(ts, model, "cacheCreation", float64(usage.CacheCreationInputTokens))...) + result.Metrics = append(result.Metrics, createTokenMetrics(ts, model, "cacheCreation", float64(usage.CacheCreationInputTokens), meta, repository)...) } if usage.CacheReadInputTokens > 0 { - result.Metrics = append(result.Metrics, CreateTokenMetrics(ts, model, "cacheRead", float64(usage.CacheReadInputTokens))...) + result.Metrics = append(result.Metrics, createTokenMetrics(ts, model, "cacheRead", float64(usage.CacheReadInputTokens), meta, repository)...) } - // Cost metrics using pricing mode (creates both regular and user-facing variants) + // Cost metrics tokenUsage := pricing.ClaudeTokenUsage{ InputTokens: int64(usage.InputTokens), OutputTokens: int64(usage.OutputTokens), @@ -282,22 +380,24 @@ func (p *ClaudeParser) ParseFile(ctx context.Context, path string) (*ImportResul } cost := pricing.GetClaudeCostWithMode(p.pricingMode, model, tokenUsage, entry.CostUSD) if cost > 0 { - result.Metrics = append(result.Metrics, CreateCostMetrics(ts, model, cost)...) + result.Metrics = append(result.Metrics, createCostMetrics(ts, model, cost, meta, repository)...) } } result.RecordCount++ } - if err := scanner.Err(); err != nil { - return nil, fmt.Errorf("reading file: %w", err) - } - return result, nil } -// CreateTranscriptLogs creates transcript log records from message content +// CreateTranscriptLogs creates transcript log records from message content. +// This exported variant is used by the file watcher for incremental processing. func (p *ClaudeParser) CreateTranscriptLogs(entry ClaudeJSONLEntry, ts time.Time, sessionID string, messageIndex *int) []api.LogRecord { + return p.createTranscriptLogs(entry, ts, sessionID, claudeSessionMeta{}, messageIndex) +} + +// createTranscriptLogs creates transcript log records with session metadata for repository attribution. +func (p *ClaudeParser) createTranscriptLogs(entry ClaudeJSONLEntry, ts time.Time, sessionID string, meta claudeSessionMeta, messageIndex *int) []api.LogRecord { var logs []api.LogRecord if entry.Message == nil || len(entry.Message.Content) == 0 { @@ -309,13 +409,12 @@ func (p *ClaudeParser) CreateTranscriptLogs(entry ClaudeJSONLEntry, ts time.Time for _, content := range entry.Message.Content { var body string - var role string attrs := map[string]string{ - "event.name": "transcript.message", - "session.id": sessionID, - "message.index": strconv.Itoa(*messageIndex), - "message.role": baseRole, - "import_source": "local_jsonl", + "event.name": "transcript.message", + "session.id": sessionID, + "message.index": strconv.Itoa(*messageIndex), + "message.role": baseRole, + "import_source": "local_jsonl", } if entry.Message.Model != "" { @@ -324,14 +423,23 @@ func (p *ClaudeParser) CreateTranscriptLogs(entry ClaudeJSONLEntry, ts time.Time if entry.Message.ID != "" { attrs["message.id"] = entry.Message.ID } + if meta.GitBranch != "" { + attrs["git_branch"] = meta.GitBranch + } + if repo := extractRepository(meta); repo != "" { + attrs["repository"] = repo + } + if meta.PRNumber != 0 { + attrs["pr_number"] = strconv.Itoa(meta.PRNumber) + attrs["pr_repository"] = meta.PRRepository + attrs["pr_url"] = meta.PRUrl + } switch content.Type { case "text": body = content.Text - role = baseRole case "tool_use": - role = "tool_use" - attrs["message.role"] = role + attrs["message.role"] = "tool_use" attrs["tool.name"] = content.Name if content.Input != nil { if inputBytes, err := json.Marshal(content.Input); err == nil { @@ -340,8 +448,7 @@ func (p *ClaudeParser) CreateTranscriptLogs(entry ClaudeJSONLEntry, ts time.Time } body = fmt.Sprintf("Tool call: %s", content.Name) case "tool_result": - role = "tool_result" - attrs["message.role"] = role + attrs["message.role"] = "tool_result" if content.ToolUseID != "" { attrs["tool.use_id"] = content.ToolUseID } @@ -383,53 +490,85 @@ const ( ClaudeUserFacingCostMetric = "claude_code.cost.usage_user_facing" ) -// CreateTokenMetric creates a token usage metric with the specified name +// sessionAttrs builds the common session-level attributes (git_branch, repository) +func sessionAttrs(meta claudeSessionMeta, repository string) map[string]string { + attrs := make(map[string]string) + if meta.GitBranch != "" { + attrs["git_branch"] = meta.GitBranch + } + if repository != "" { + attrs["repository"] = repository + } + return attrs +} + +// CreateTokenMetric creates a token usage metric with the specified name. +// This exported variant is used by the file watcher. func CreateTokenMetric(ts time.Time, metricName, model, tokenType string, value float64) api.MetricDataPoint { + return createTokenMetric(ts, metricName, model, tokenType, value, claudeSessionMeta{}, "") +} + +// createTokenMetric creates a token usage metric with the specified name +func createTokenMetric(ts time.Time, metricName, model, tokenType string, value float64, meta claudeSessionMeta, repository string) api.MetricDataPoint { + attrs := sessionAttrs(meta, repository) + attrs["type"] = tokenType + attrs["model"] = model + attrs["import_source"] = "local_jsonl" return api.MetricDataPoint{ Timestamp: ts, ServiceName: SourceClaude.ServiceName(), MetricName: metricName, MetricType: "sum", Value: &value, - Attributes: map[string]string{ - "type": tokenType, - "model": model, - "import_source": "local_jsonl", - }, + Attributes: attrs, } } // CreateTokenMetrics creates both regular and user-facing token usage metrics. -// JSONL data is already user-facing (only assistant messages with cache tokens), -// so both metrics have identical values for consistency with OTLP-derived metrics. +// This exported variant is used by the file watcher (no repository attribution). func CreateTokenMetrics(ts time.Time, model, tokenType string, value float64) []api.MetricDataPoint { + return createTokenMetrics(ts, model, tokenType, value, claudeSessionMeta{}, "") +} + +// createTokenMetrics creates both regular and user-facing token usage metrics. +func createTokenMetrics(ts time.Time, model, tokenType string, value float64, meta claudeSessionMeta, repository string) []api.MetricDataPoint { return []api.MetricDataPoint{ - CreateTokenMetric(ts, ClaudeTokenUsageMetric, model, tokenType, value), - CreateTokenMetric(ts, ClaudeUserFacingTokenUsageMetric, model, tokenType, value), + createTokenMetric(ts, ClaudeTokenUsageMetric, model, tokenType, value, meta, repository), + createTokenMetric(ts, ClaudeUserFacingTokenUsageMetric, model, tokenType, value, meta, repository), } } -// CreateCostMetric creates a cost metric with the specified name +// CreateCostMetric creates a cost metric with the specified name. +// This exported variant is used by the file watcher. func CreateCostMetric(ts time.Time, metricName, model string, value float64) api.MetricDataPoint { + return createCostMetric(ts, metricName, model, value, claudeSessionMeta{}, "") +} + +// createCostMetric creates a cost metric with the specified name +func createCostMetric(ts time.Time, metricName, model string, value float64, meta claudeSessionMeta, repository string) api.MetricDataPoint { + attrs := sessionAttrs(meta, repository) + attrs["model"] = model + attrs["import_source"] = "local_jsonl" return api.MetricDataPoint{ Timestamp: ts, ServiceName: SourceClaude.ServiceName(), MetricName: metricName, MetricType: "sum", Value: &value, - Attributes: map[string]string{ - "model": model, - "import_source": "local_jsonl", - }, + Attributes: attrs, } } // CreateCostMetrics creates both regular and user-facing cost metrics. -// JSONL data is already user-facing (only assistant messages with cache tokens), -// so both metrics have identical values for consistency with OTLP-derived metrics. +// This exported variant is used by the file watcher (no repository attribution). func CreateCostMetrics(ts time.Time, model string, value float64) []api.MetricDataPoint { + return createCostMetrics(ts, model, value, claudeSessionMeta{}, "") +} + +// createCostMetrics creates both regular and user-facing cost metrics. +func createCostMetrics(ts time.Time, model string, value float64, meta claudeSessionMeta, repository string) []api.MetricDataPoint { return []api.MetricDataPoint{ - CreateCostMetric(ts, ClaudeCostMetric, model, value), - CreateCostMetric(ts, ClaudeUserFacingCostMetric, model, value), + createCostMetric(ts, ClaudeCostMetric, model, value, meta, repository), + createCostMetric(ts, ClaudeUserFacingCostMetric, model, value, meta, repository), } } diff --git a/backend/internal/importer/codex.go b/backend/internal/importer/codex.go index 816b3e3..961784e 100644 --- a/backend/internal/importer/codex.go +++ b/backend/internal/importer/codex.go @@ -95,8 +95,8 @@ type CodexJSONLEntry struct { type CodexResponseItem struct { Type string `json:"type"` // For "message" type - Role string `json:"role,omitempty"` - Content []CodexContentItem `json:"content,omitempty"` + Role string `json:"role,omitempty"` + Content []CodexContentItem `json:"content,omitempty"` // For "function_call" type Name string `json:"name,omitempty"` Arguments string `json:"arguments,omitempty"` @@ -119,22 +119,28 @@ type CodexReasoningSummary struct { Text string `json:"text,omitempty"` } +// codexGitInfo contains git metadata from session_meta +type codexGitInfo struct { + RepositoryURL string `json:"repository_url"` +} + // CodexSessionMeta represents session metadata type CodexSessionMeta struct { - ID string `json:"id"` - Timestamp string `json:"timestamp"` - Cwd string `json:"cwd"` - Originator string `json:"originator"` - CliVersion string `json:"cli_version"` - ModelProvider string `json:"model_provider"` - Model string `json:"model"` + ID string `json:"id"` + Timestamp string `json:"timestamp"` + Cwd string `json:"cwd"` + Originator string `json:"originator"` + CliVersion string `json:"cli_version"` + ModelProvider string `json:"model_provider"` + Model string `json:"model"` + Git *codexGitInfo `json:"git,omitempty"` } // CodexEventMsg represents an event message payload // The actual JSON structure has "info" at the same level as "type", not nested in another "payload" type CodexEventMsg struct { - Type string `json:"type"` - Info *CodexTokenInfo `json:"info"` // Present for token_count events + Type string `json:"type"` + Info *CodexTokenInfo `json:"info"` // Present for token_count events } // CodexTokenInfo contains token usage information @@ -176,6 +182,7 @@ func (p *CodexParser) ParseFile(ctx context.Context, path string) (*ImportResult var sessionMeta *CodexSessionMeta var currentModel string var lastTokenCount *CodexTokenCount + var repository string messageIndex := 0 // Track message order for transcripts for scanner.Scan() { @@ -222,6 +229,13 @@ func (p *CodexParser) ParseFile(ctx context.Context, path string) (*ImportResult currentModel = meta.Model } + // Resolve repository using git URL from session metadata (authoritative for Codex). + gitRepoURL := "" + if meta.Git != nil { + gitRepoURL = meta.Git.RepositoryURL + } + repository = resolveSessionRepository(nil, gitRepoURL, "", meta.Cwd) + // Create session start log logRecord := api.LogRecord{ Timestamp: ts, @@ -230,17 +244,20 @@ func (p *CodexParser) ParseFile(ctx context.Context, path string) (*ImportResult SeverityNumber: 9, Body: "conversation_starts", LogAttributes: map[string]string{ - "event.name": "codex.conversation_starts", - "session.id": meta.ID, - "model": meta.Model, - "model_provider": meta.ModelProvider, - "cli_version": meta.CliVersion, - "import_source": "local_jsonl", + "event.name": "codex.conversation_starts", + "session.id": meta.ID, + "model": meta.Model, + "model_provider": meta.ModelProvider, + "cli_version": meta.CliVersion, + "import_source": "local_jsonl", }, } if meta.Cwd != "" { logRecord.LogAttributes["cwd"] = meta.Cwd } + if repository != "" { + logRecord.LogAttributes["repository"] = repository + } result.Logs = append(result.Logs, logRecord) result.RecordCount++ } @@ -288,29 +305,29 @@ func (p *CodexParser) ParseFile(ctx context.Context, path string) (*ImportResult // Create metrics for non-zero deltas if deltaInput > 0 { - result.Metrics = append(result.Metrics, CreateCodexTokenMetric(ts, currentModel, "input", float64(deltaInput))) + result.Metrics = append(result.Metrics, createCodexTokenMetric(ts, currentModel, "input", float64(deltaInput), repository)) } if deltaOutput > 0 { - result.Metrics = append(result.Metrics, CreateCodexTokenMetric(ts, currentModel, "output", float64(deltaOutput))) + result.Metrics = append(result.Metrics, createCodexTokenMetric(ts, currentModel, "output", float64(deltaOutput), repository)) } if deltaCacheCreation > 0 { - result.Metrics = append(result.Metrics, CreateCodexTokenMetric(ts, currentModel, "cache_creation", float64(deltaCacheCreation))) + result.Metrics = append(result.Metrics, createCodexTokenMetric(ts, currentModel, "cache_creation", float64(deltaCacheCreation), repository)) } if deltaCacheRead > 0 { - result.Metrics = append(result.Metrics, CreateCodexTokenMetric(ts, currentModel, "cache_read", float64(deltaCacheRead))) + result.Metrics = append(result.Metrics, createCodexTokenMetric(ts, currentModel, "cache_read", float64(deltaCacheRead), repository)) } if deltaReasoning > 0 { - result.Metrics = append(result.Metrics, CreateCodexTokenMetric(ts, currentModel, "reasoning", float64(deltaReasoning))) + result.Metrics = append(result.Metrics, createCodexTokenMetric(ts, currentModel, "reasoning", float64(deltaReasoning), repository)) } if deltaTool > 0 { - result.Metrics = append(result.Metrics, CreateCodexTokenMetric(ts, currentModel, "tool", float64(deltaTool))) + result.Metrics = append(result.Metrics, createCodexTokenMetric(ts, currentModel, "tool", float64(deltaTool), repository)) } // Calculate and add cost metric // Note: cache_read is used for cost calculation (cache_creation tokens are billed at input rate) cost := pricing.CalculateCodexCost(currentModel, int64(deltaInput), int64(deltaCacheRead), int64(deltaOutput)) if cost != nil && *cost > 0 { - result.Metrics = append(result.Metrics, CreateCodexCostMetric(ts, currentModel, *cost)) + result.Metrics = append(result.Metrics, createCodexCostMetric(ts, currentModel, *cost, repository)) } lastTokenCount = tokenCount @@ -518,33 +535,53 @@ func (p *CodexParser) ParseFile(ctx context.Context, path string) (*ImportResult return result, nil } -// CreateCodexTokenMetric creates a token usage metric for Codex +// CreateCodexTokenMetric creates a token usage metric for Codex. +// This exported variant is used by the file watcher. func CreateCodexTokenMetric(ts time.Time, model, tokenType string, value float64) api.MetricDataPoint { + return createCodexTokenMetric(ts, model, tokenType, value, "") +} + +// createCodexTokenMetric creates a token usage metric for Codex +func createCodexTokenMetric(ts time.Time, model, tokenType string, value float64, repository string) api.MetricDataPoint { + attrs := map[string]string{ + "type": tokenType, + "model": model, + "import_source": "local_jsonl", + } + if repository != "" { + attrs["repository"] = repository + } return api.MetricDataPoint{ Timestamp: ts, ServiceName: SourceCodex.ServiceName(), MetricName: "codex_cli_rs.token.usage", MetricType: "sum", Value: &value, - Attributes: map[string]string{ - "type": tokenType, - "model": model, - "import_source": "local_jsonl", - }, + Attributes: attrs, } } -// CreateCodexCostMetric creates a cost usage metric for Codex +// CreateCodexCostMetric creates a cost usage metric for Codex. +// This exported variant is used by the file watcher. func CreateCodexCostMetric(ts time.Time, model string, cost float64) api.MetricDataPoint { + return createCodexCostMetric(ts, model, cost, "") +} + +// createCodexCostMetric creates a cost usage metric for Codex +func createCodexCostMetric(ts time.Time, model string, cost float64, repository string) api.MetricDataPoint { + attrs := map[string]string{ + "model": model, + "import_source": "local_jsonl", + } + if repository != "" { + attrs["repository"] = repository + } return api.MetricDataPoint{ Timestamp: ts, ServiceName: SourceCodex.ServiceName(), MetricName: "codex_cli_rs.cost.usage", MetricType: "sum", Value: &cost, - Attributes: map[string]string{ - "model": model, - "import_source": "local_jsonl", - }, + Attributes: attrs, } } diff --git a/backend/internal/importer/importer_test.go b/backend/internal/importer/importer_test.go index 3dfaf6a..83501e6 100644 --- a/backend/internal/importer/importer_test.go +++ b/backend/internal/importer/importer_test.go @@ -846,6 +846,133 @@ func TestImportUnregisteredParser(t *testing.T) { } } +// TestClaudeParserGitBranchAndPR tests that gitBranch and pr-link metadata are captured. +func TestClaudeParserGitBranchAndPR(t *testing.T) { + tmpDir, err := os.MkdirTemp("", "claude-meta-test-*") + if err != nil { + t.Fatalf("failed to create temp dir: %v", err) + } + defer os.RemoveAll(tmpDir) + + cwd := "/home/user/my-repo" + + // Raw JSONL lines mixing entry types + lines := []string{ + // user entry with gitBranch + `{"type":"user","timestamp":"2025-01-02T10:00:00.000Z","sessionId":"s1","gitBranch":"feat/my-branch","cwd":"` + cwd + `","message":{"role":"user","content":[{"type":"text","text":"hello"}]}}`, + // pr-link entry (no message field) + `{"type":"pr-link","sessionId":"s1","prNumber":42,"prUrl":"https://github.com/org/repo/pull/42","prRepository":"org/repo","timestamp":"2025-01-02T10:00:30.000Z"}`, + // assistant entry with usage + `{"type":"assistant","timestamp":"2025-01-02T10:01:00.000Z","sessionId":"s1","requestId":"r1","gitBranch":"feat/my-branch","message":{"id":"m1","model":"claude-sonnet-4-20250514","role":"assistant","usage":{"input_tokens":100,"output_tokens":50},"content":[{"type":"text","text":"done"}]}}`, + } + + testFile := filepath.Join(tmpDir, "s1.jsonl") + if err := os.WriteFile(testFile, []byte(joinLines(lines)), 0644); err != nil { + t.Fatalf("failed to write test file: %v", err) + } + + os.Setenv("AI_OBSERVER_CLAUDE_PATH", tmpDir) + defer os.Unsetenv("AI_OBSERVER_CLAUDE_PATH") + + parser := NewClaudeParser() + ctx := context.Background() + result, err := parser.ParseFile(ctx, testFile) + if err != nil { + t.Fatalf("ParseFile failed: %v", err) + } + + // Verify git_branch and repository appear on existing token metrics. + foundTokenMetric := false + for _, m := range result.Metrics { + if m.MetricName != ClaudeTokenUsageMetric { + continue + } + foundTokenMetric = true + if m.Attributes["git_branch"] != "feat/my-branch" { + t.Errorf("expected git_branch on token metric, got %q", m.Attributes["git_branch"]) + } + if m.Attributes["repository"] != "org/repo" { + t.Errorf("expected repository on token metric, got %q", m.Attributes["repository"]) + } + } + if !foundTokenMetric { + t.Fatal("expected token metric") + } + + // Verify repository metadata appears in transcript and API request logs. + foundTranscript := false + foundAPIRequest := false + for _, log := range result.Logs { + switch log.LogAttributes["event.name"] { + case "transcript.message": + foundTranscript = true + if log.LogAttributes["pr_number"] != "42" { + t.Errorf("expected pr_number=42 in log attributes, got %q", log.LogAttributes["pr_number"]) + } + case "claude_code.api_request": + foundAPIRequest = true + if log.LogAttributes["git_branch"] != "feat/my-branch" { + t.Errorf("expected git_branch on API request log, got %q", log.LogAttributes["git_branch"]) + } + if log.LogAttributes["repository"] != "org/repo" { + t.Errorf("expected repository on API request log, got %q", log.LogAttributes["repository"]) + } + } + } + if !foundTranscript { + t.Fatal("expected transcript log") + } + if !foundAPIRequest { + t.Fatal("expected API request log") + } +} + +// TestCollectSessionMeta tests the session metadata collection first pass +func TestCollectSessionMeta(t *testing.T) { + parser := NewClaudeParser() + + lines := []string{ + `{"type":"user","timestamp":"2025-01-01T00:00:00Z","gitBranch":"main","cwd":"/home/user/project"}`, + `{"type":"pr-link","prNumber":7,"prUrl":"https://github.com/a/b/pull/7","prRepository":"a/b","timestamp":"2025-01-01T00:01:00Z"}`, + // duplicate pr-link should not overwrite the first + `{"type":"pr-link","prNumber":8,"prUrl":"https://github.com/a/b/pull/8","prRepository":"a/b","timestamp":"2025-01-01T00:02:00Z"}`, + } + + meta := parser.collectSessionMeta(lines) + + if meta.GitBranch != "main" { + t.Errorf("expected GitBranch=main, got %q", meta.GitBranch) + } + if meta.Cwd != "/home/user/project" { + t.Errorf("expected Cwd, got %q", meta.Cwd) + } + if meta.PRNumber != 7 { + t.Errorf("expected PRNumber=7 (first pr-link), got %d", meta.PRNumber) + } + if meta.PRRepository != "a/b" { + t.Errorf("expected PRRepository=a/b, got %q", meta.PRRepository) + } +} + +// TestExtractRepository tests repository name derivation +func TestExtractRepository(t *testing.T) { + tests := []struct { + meta claudeSessionMeta + expected string + }{ + {claudeSessionMeta{PRRepository: "org/repo"}, "org/repo"}, + {claudeSessionMeta{Cwd: "/home/user/my-project"}, "my-project"}, + {claudeSessionMeta{PRRepository: "org/repo", Cwd: "/home/user/other"}, "org/repo"}, // PR takes precedence + {claudeSessionMeta{}, ""}, + } + for _, tc := range tests { + got := extractRepository(tc.meta) + if got != tc.expected { + t.Errorf("extractRepository(%+v) = %q, want %q", tc.meta, got, tc.expected) + } + } +} + // Helper functions func floatPtr(f float64) *float64 { return &f diff --git a/backend/internal/importer/repos.go b/backend/internal/importer/repos.go new file mode 100644 index 0000000..ccd73eb --- /dev/null +++ b/backend/internal/importer/repos.go @@ -0,0 +1,281 @@ +package importer + +import ( + "os" + "os/exec" + "path/filepath" + "regexp" + "strconv" + "strings" + "sync" +) + +// Repository attribution logic. +// +// Ported from ducktrace/claude_analysis/repos.py and prmatch.py. +// Maps a session to a canonical "owner/repo" (or bare repo name) using the +// strongest available signal. +// +// Signal priority (best first): +// 1. Codex session_meta.git.repository_url (authoritative) +// 2. On-disk git remote from cwd (real remote, authoritative) +// 3. Candidate repos whose name matches cwd name or path segment +// 4. Bare cwd directory name +// 5. First candidate repository reference + +var ( + githubURLRe = regexp.MustCompile(`(?i)(?:^|://|@)(?:[^/@]+@)?github\.com[:/]+([^/]+/[^/?#]+)`) + schemeRe = regexp.MustCompile(`(?i)^[a-z][a-z0-9+.-]*://`) + userAtRe = regexp.MustCompile(`^[^@/]+@`) + + ghPRRe = regexp.MustCompile(`(?i)\bgh\s+pr\s+(create|merge|close|reopen|checkout|view|diff|review|list|edit|comment|ready|status)\b`) + repoFlagRe = regexp.MustCompile(`(?i)--repo[ =]\s*['"]?([^\s'"]+)`) + pullURLRe = regexp.MustCompile(`(?i)https?://github\.com/([^/\s]+/[^/\s]+?)/pull/(\d+)`) + // repoURLRe catches any github.com/owner/repo reference not already matched by pullURLRe. + // The lazy second component combined with the required terminator stops at path separators. + repoURLRe = regexp.MustCompile(`(?i)github\.com[:/]+([^/\s]+/[^/\s]+?)(?:\.git)?(?:[/\s)"',` + "`" + `]|$)`) +) + +var placeholderRepos = map[string]bool{ + "owner/repo": true, "owner/name": true, "org/repo": true, + "user/repo": true, "your-org/your-repo": true, + "username/repo": true, "owner/repository": true, +} + +var gitRemoteCache sync.Map + +// githubRepoFromURL extracts owner/repo from any GitHub URL (https or ssh), +// stripping embedded credentials and a trailing .git. Returns "" on no match. +func githubRepoFromURL(url string) string { + if url == "" { + return "" + } + m := githubURLRe.FindStringSubmatch(url) + if m == nil { + return "" + } + return strings.TrimSuffix(m[1], ".git") +} + +// normalizeGitURL normalizes any git remote URL to owner/repo when possible. +// Handles git@host:owner/repo.git, https://host/owner/repo.git, and +// https://@host/owner/repo.git (credentials are dropped). Falls back +// to the last two path segments for non-GitHub hosts. +func normalizeGitURL(url string) string { + if gh := githubRepoFromURL(url); gh != "" { + return gh + } + if url == "" { + return "" + } + u := schemeRe.ReplaceAllString(strings.TrimSpace(url), "") + u = userAtRe.ReplaceAllString(u, "") + u = strings.Replace(u, ":", "/", 1) // ssh host:path -> host/path + u = strings.TrimSuffix(u, ".git") + var parts []string + for _, p := range strings.Split(u, "/") { + if p != "" { + parts = append(parts, p) + } + } + if len(parts) >= 3 { // host/owner/repo... + return parts[len(parts)-2] + "/" + parts[len(parts)-1] + } + return u +} + +// resolveRepoName returns the repo name for a working dir, resolving git +// worktrees to the main repo's directory name. If the path no longer exists +// we just return its basename. +func resolveRepoName(cwd string) string { + if cwd == "" { + return "" + } + // A worktree's .git is a file (not a dir) containing "gitdir: ". + content, err := os.ReadFile(filepath.Join(cwd, ".git")) + if err == nil { + text := strings.TrimSpace(string(content)) + if strings.HasPrefix(text, "gitdir:") { + gitdir := strings.TrimSpace(strings.TrimPrefix(text, "gitdir:")) + parts := strings.Split(filepath.ToSlash(gitdir), "/") + for i, seg := range parts { + if seg == ".git" && i > 0 { + return filepath.Base(filepath.Join(parts[:i]...)) + } + } + } + } + return filepath.Base(cwd) +} + +// cwdGitRemote returns the owner/repo of the cwd's origin remote, cached per +// path. This is authoritative when the directory still exists locally - the +// real remote is correct even when the local dir name differs from the repo name. +func cwdGitRemote(path string) string { + if path == "" { + return "" + } + if cached, ok := gitRemoteCache.Load(path); ok { + return cached.(string) + } + out, err := exec.Command("git", "-C", path, "config", "--get", "remote.origin.url").Output() + result := "" + if err == nil { + result = normalizeGitURL(strings.TrimSpace(string(out))) + } + gitRemoteCache.Store(path, result) + return result +} + +// resolveSessionRepository returns the best repository id for a session. +// +// candidateRepos are owner/repo references seen for the session - from pr-link +// entries and text-mined references - ideally most-frequent first. +// +// The working directory says which repo we're in; candidates supply the owner/ +// prefix the cwd alone can't. A candidate whose repo-part matches the cwd name +// or a path segment is the highest-confidence answer. A candidate that doesn't +// match is treated as a stray and used only when there's no cwd at all. +func resolveSessionRepository(candidateRepos []string, gitRepoURL, originalCwd, cwd string) string { + // Codex carries the cwd's actual git remote - authoritative. + if gitRepoURL != "" { + if norm := normalizeGitURL(gitRepoURL); norm != "" { + return norm + } + } + + path := originalCwd + if path == "" { + path = cwd + } + + // On-disk git remote is authoritative (handles dir name != repo name). + if onDisk := cwdGitRemote(path); onDisk != "" { + return onDisk + } + + cwdName := resolveRepoName(path) + var candidates []string + for _, r := range candidateRepos { + if r != "" { + candidates = append(candidates, r) + } + } + + if len(candidates) > 0 && path != "" { + segments := make(map[string]bool) + for _, seg := range strings.Split(filepath.ToSlash(path), "/") { + if seg != "" { + segments[strings.ToLower(seg)] = true + } + } + for _, r := range candidates { + parts := strings.Split(r, "/") + name := strings.ToLower(parts[len(parts)-1]) + if name == strings.ToLower(cwdName) || segments[name] { + return r + } + } + } + + if cwdName != "" { + return cwdName + } + if len(candidates) > 0 { + return candidates[0] + } + return "" +} + +// isPlaceholder returns true if repo is a documentation/example placeholder. +func isPlaceholder(repo string) bool { + return placeholderRepos[strings.ToLower(repo)] || + strings.Contains(repo, "<") || strings.Contains(repo, ">") +} + +func normalizeRepoFlag(val string) string { + if gh := githubRepoFromURL(val); gh != "" { + if isPlaceholder(gh) { + return "" + } + return gh + } + val = strings.TrimRight(strings.TrimSpace(val), "/") + var parts []string + for _, p := range strings.Split(val, "/") { + if p != "" { + parts = append(parts, p) + } + } + if len(parts) >= 2 && !strings.Contains(parts[len(parts)-2], ".") { + repo := strings.TrimSuffix(parts[len(parts)-2]+"/"+parts[len(parts)-1], ".git") + if isPlaceholder(repo) { + return "" + } + return repo + } + return "" +} + +// extractedRefs holds PR/repo references mined from free text. +type extractedRefs struct { + PRActions []string + PRURLs []string + PRNumbers []int + Repos []string +} + +// extractRefs scans one or more strings for gh pr commands, PR URLs, +// --repo flags, and GitHub repo references. +func extractRefs(texts ...string) extractedRefs { + result := extractedRefs{} + seenActions := map[string]bool{} + seenURLs := map[string]bool{} + seenNumbers := map[int]bool{} + seenRepos := map[string]bool{} + + for _, text := range texts { + if text == "" { + continue + } + for _, m := range ghPRRe.FindAllStringSubmatch(text, -1) { + action := "gh pr " + strings.ToLower(m[1]) + if !seenActions[action] { + seenActions[action] = true + result.PRActions = append(result.PRActions, action) + } + } + for _, m := range pullURLRe.FindAllStringSubmatch(text, -1) { + repo := strings.TrimSuffix(m[1], ".git") + if isPlaceholder(repo) { + continue + } + if !seenRepos[repo] { + seenRepos[repo] = true + result.Repos = append(result.Repos, repo) + } + if num, err := strconv.Atoi(m[2]); err == nil && !seenNumbers[num] { + seenNumbers[num] = true + result.PRNumbers = append(result.PRNumbers, num) + } + if url := m[0]; !seenURLs[url] { + seenURLs[url] = true + result.PRURLs = append(result.PRURLs, url) + } + } + for _, m := range repoFlagRe.FindAllStringSubmatch(text, -1) { + if r := normalizeRepoFlag(m[1]); r != "" && !seenRepos[r] { + seenRepos[r] = true + result.Repos = append(result.Repos, r) + } + } + for _, m := range repoURLRe.FindAllStringSubmatch(text, -1) { + r := strings.TrimSuffix(m[1], ".git") + if !isPlaceholder(r) && !seenRepos[r] { + seenRepos[r] = true + result.Repos = append(result.Repos, r) + } + } + } + return result +}