diff --git a/backend/internal/importer/claude.go b/backend/internal/importer/claude.go index aee4478..6d75aba 100644 --- a/backend/internal/importer/claude.go +++ b/backend/internal/importer/claude.go @@ -105,7 +105,7 @@ func (p *ClaudeParser) FindSessionFiles(ctx context.Context) ([]string, error) { // ClaudeJSONLEntry represents a single line in Claude Code JSONL files type ClaudeJSONLEntry struct { - Type string `json:"type,omitempty"` // Root type: "assistant", "user", "queue-operation", etc. + Type string `json:"type,omitempty"` // Root type: "assistant", "user", "pr-link", "system", etc. Timestamp string `json:"timestamp"` SessionID string `json:"sessionId,omitempty"` Version string `json:"version,omitempty"` @@ -113,6 +113,16 @@ type ClaudeJSONLEntry struct { RequestID string `json:"requestId,omitempty"` CostUSD *float64 `json:"costUSD,omitempty"` Message *ClaudeMessage `json:"message,omitempty"` + GitBranch string `json:"gitBranch,omitempty"` + // PR link fields (present when type == "pr-link") + PRNumber int `json:"prNumber,omitempty"` + PRUrl string `json:"prUrl,omitempty"` + PRRepository string `json:"prRepository,omitempty"` + // System entry fields (present when type == "system") + Subtype string `json:"subtype,omitempty"` + DurationMs float64 `json:"durationMs,omitempty"` + // Worktree-state fields (present when type == "worktree-state") + WorktreeSession *claudeWorktreeSession `json:"worktreeSession,omitempty"` } type ClaudeMessage struct { @@ -140,6 +150,111 @@ type ClaudeUsage struct { CacheReadInputTokens int `json:"cache_read_input_tokens"` } +// claudeEditInput holds the input for Edit tool calls +type claudeEditInput struct { + FilePath string `json:"file_path"` + OldString string `json:"old_string"` + NewString string `json:"new_string"` +} + +// claudeWriteInput holds the input for Write tool calls +type claudeWriteInput struct { + FilePath string `json:"file_path"` + Content string `json:"content"` +} + +// claudeMultiEditInput holds the input for MultiEdit tool calls +type claudeMultiEditInput struct { + FilePath string `json:"file_path"` + Edits []struct { + OldString string `json:"old_string"` + NewString string `json:"new_string"` + } `json:"edits"` +} + +// claudeWorktreeSession holds fields from a "worktree-state" entry's worktreeSession object. +type claudeWorktreeSession struct { + OriginalCwd string `json:"originalCwd,omitempty"` +} + +// claudeSessionMeta holds session-level metadata collected in a first pass +type claudeSessionMeta struct { + GitBranch string + Cwd string + OriginalCwd string // from worktree-state.worktreeSession.originalCwd + CandidateRepos []string // owner/repo references from pr-link entries (most-frequent first) + PRNumber int + PRUrl string + PRRepository string + PRTimestamp time.Time +} + +// collectSessionMeta does a first pass over raw JSONL lines to collect session-level metadata +func (p *ClaudeParser) collectSessionMeta(lines []string) claudeSessionMeta { + meta := claudeSessionMeta{} + seenPRs := make(map[int]bool) + seenRepos := make(map[string]bool) + + for _, line := range lines { + var entry ClaudeJSONLEntry + if err := json.Unmarshal([]byte(line), &entry); err != nil { + continue + } + if entry.GitBranch != "" && meta.GitBranch == "" { + meta.GitBranch = entry.GitBranch + } + if entry.Cwd != "" && meta.Cwd == "" { + meta.Cwd = entry.Cwd + } + if entry.Type == "worktree-state" && entry.WorktreeSession != nil { + if oc := entry.WorktreeSession.OriginalCwd; oc != "" && meta.OriginalCwd == "" { + meta.OriginalCwd = oc + } + } + if entry.Type == "pr-link" && entry.PRNumber != 0 && !seenPRs[entry.PRNumber] { + seenPRs[entry.PRNumber] = true + // Collect all pr-link repositories as candidates for resolution. + if r := entry.PRRepository; r != "" && !seenRepos[r] { + seenRepos[r] = true + meta.CandidateRepos = append(meta.CandidateRepos, r) + } + // Use the first PR linked in the session for structured PR metadata. + if meta.PRNumber == 0 { + meta.PRNumber = entry.PRNumber + meta.PRUrl = entry.PRUrl + meta.PRRepository = entry.PRRepository + if ts, err := time.Parse(time.RFC3339Nano, entry.Timestamp); err == nil { + meta.PRTimestamp = ts + } else if ts, err := time.Parse(time.RFC3339, entry.Timestamp); err == nil { + meta.PRTimestamp = ts + } + } + } + } + return meta +} + +// extractRepository returns the best repository identifier for a session. +// +// Priority: +// 1. PRRepository from a structured pr-link entry (authoritative - this is +// exactly where the linked PR lives, not just a text-mined reference). +// 2. resolveSessionRepository: on-disk git remote -> matching candidate -> cwd name. +func extractRepository(meta claudeSessionMeta) string { + if meta.PRRepository != "" { + return meta.PRRepository + } + return resolveSessionRepository(meta.CandidateRepos, "", meta.OriginalCwd, meta.Cwd) +} + +// countLines returns the number of lines in s (at least 1 for non-empty strings) +func countLines(s string) int { + if s == "" { + return 0 + } + return strings.Count(s, "\n") + 1 +} + // ParseFile parses a Claude Code JSONL file func (p *ClaudeParser) ParseFile(ctx context.Context, path string) (*ImportResult, error) { file, err := os.Open(path) @@ -161,24 +276,59 @@ func (p *ClaudeParser) ParseFile(ctx context.Context, path string) (*ImportResul buf := make([]byte, 0, 64*1024) scanner.Buffer(buf, 1024*1024) - lineNum := 0 - messageIndex := 0 // Track message order for transcripts - seenRequests := make(map[string]bool) // For deduplication of metrics - + // Collect all non-empty lines + var lines []string for scanner.Scan() { if ctx.Err() != nil { return nil, ctx.Err() } + if line := scanner.Text(); strings.TrimSpace(line) != "" { + lines = append(lines, line) + } + } + if err := scanner.Err(); err != nil { + return nil, fmt.Errorf("reading file: %w", err) + } - lineNum++ - line := scanner.Text() - if strings.TrimSpace(line) == "" { - continue + // First pass: collect session-level metadata (gitBranch, PR info, cwd) + meta := p.collectSessionMeta(lines) + repository := extractRepository(meta) + + // Emit pull_request.count metric if a PR was linked + if meta.PRNumber != 0 && !meta.PRTimestamp.IsZero() { + result.Metrics = append(result.Metrics, createPRMetric(meta, repository)) + } + + messageIndex := 0 + seenRequests := make(map[string]bool) // For deduplication of metrics + + // Second pass: process records + for _, line := range lines { + if ctx.Err() != nil { + return nil, ctx.Err() } var entry ClaudeJSONLEntry if err := json.Unmarshal([]byte(line), &entry); err != nil { - // Skip malformed lines + continue + } + + // Handle system/turn_duration entries for active time (no message field) + if entry.Type == "system" && entry.Subtype == "turn_duration" && entry.DurationMs > 0 { + ts, err := time.Parse(time.RFC3339Nano, entry.Timestamp) + if err != nil { + ts, err = time.Parse(time.RFC3339, entry.Timestamp) + } + if err == nil { + if result.FirstTime.IsZero() || ts.Before(result.FirstTime) { + result.FirstTime = ts + } + if ts.After(result.LastTime) { + result.LastTime = ts + } + secs := entry.DurationMs / 1000.0 + result.Metrics = append(result.Metrics, createActiveTimeMetric(ts, secs, meta, repository)) + } continue } @@ -219,11 +369,11 @@ func (p *ClaudeParser) ParseFile(ctx context.Context, path string) (*ImportResul } // Create transcript log records from message content - transcriptLogs := p.CreateTranscriptLogs(entry, ts, sessionID, &messageIndex) + transcriptLogs := p.createTranscriptLogs(entry, ts, sessionID, meta, &messageIndex) result.Logs = append(result.Logs, transcriptLogs...) - // For assistant entries with usage data, also create metrics - if entry.Type == "assistant" && entry.Message.Usage != nil { + // For assistant entries: create metrics and process file-editing tool calls + if entry.Type == "assistant" { // Deduplication using messageId:requestId for metrics only dedupKey := fmt.Sprintf("%s:%s", entry.Message.ID, entry.RequestID) if entry.Message.ID != "" && entry.RequestID != "" { @@ -233,71 +383,204 @@ func (p *ClaudeParser) ParseFile(ctx context.Context, path string) (*ImportResul seenRequests[dedupKey] = true } - // Create api_request log record (existing behavior) - logRecord := api.LogRecord{ - Timestamp: ts, - ServiceName: SourceClaude.ServiceName(), - SeverityText: "INFO", - SeverityNumber: 9, - Body: "api_request", - LogAttributes: map[string]string{ - "event.name": "claude_code.api_request", - "session.id": sessionID, - "model": entry.Message.Model, - "import_source": "local_jsonl", - }, - } - if entry.Cwd != "" { - logRecord.LogAttributes["cwd"] = entry.Cwd - } - if entry.RequestID != "" { - logRecord.LogAttributes["request_id"] = entry.RequestID + // Lines of code from file-editing tool calls + locMetrics := p.extractLinesOfCodeMetrics(entry, ts, meta, repository) + result.Metrics = append(result.Metrics, locMetrics...) + + // Commits from Bash tool calls + commitMetrics := p.extractCommitMetrics(entry, ts, meta, repository) + result.Metrics = append(result.Metrics, commitMetrics...) + + if entry.Message.Usage != nil { + // Create api_request log record + logRecord := api.LogRecord{ + Timestamp: ts, + ServiceName: SourceClaude.ServiceName(), + SeverityText: "INFO", + SeverityNumber: 9, + Body: "api_request", + LogAttributes: map[string]string{ + "event.name": "claude_code.api_request", + "session.id": sessionID, + "model": entry.Message.Model, + "import_source": "local_jsonl", + }, + } + if entry.Cwd != "" { + logRecord.LogAttributes["cwd"] = entry.Cwd + } + if entry.RequestID != "" { + logRecord.LogAttributes["request_id"] = entry.RequestID + } + if meta.GitBranch != "" { + logRecord.LogAttributes["git_branch"] = meta.GitBranch + } + if repository != "" { + logRecord.LogAttributes["repository"] = repository + } + result.Logs = append(result.Logs, logRecord) + + // Token usage metrics + usage := entry.Message.Usage + model := entry.Message.Model + + if usage.InputTokens > 0 { + result.Metrics = append(result.Metrics, createTokenMetrics(ts, model, "input", float64(usage.InputTokens), meta, repository)...) + } + if usage.OutputTokens > 0 { + result.Metrics = append(result.Metrics, createTokenMetrics(ts, model, "output", float64(usage.OutputTokens), meta, repository)...) + } + if usage.CacheCreationInputTokens > 0 { + result.Metrics = append(result.Metrics, createTokenMetrics(ts, model, "cacheCreation", float64(usage.CacheCreationInputTokens), meta, repository)...) + } + if usage.CacheReadInputTokens > 0 { + result.Metrics = append(result.Metrics, createTokenMetrics(ts, model, "cacheRead", float64(usage.CacheReadInputTokens), meta, repository)...) + } + + // Cost metrics + tokenUsage := pricing.ClaudeTokenUsage{ + InputTokens: int64(usage.InputTokens), + OutputTokens: int64(usage.OutputTokens), + CacheCreationInputTokens: int64(usage.CacheCreationInputTokens), + CacheReadInputTokens: int64(usage.CacheReadInputTokens), + } + cost := pricing.GetClaudeCostWithMode(p.pricingMode, model, tokenUsage, entry.CostUSD) + if cost > 0 { + result.Metrics = append(result.Metrics, createCostMetrics(ts, model, cost, meta, repository)...) + } } - result.Logs = append(result.Logs, logRecord) + } + + result.RecordCount++ + } + + // Session count metric - one per file, timestamped at first activity + if !result.FirstTime.IsZero() { + result.Metrics = append(result.Metrics, createSessionMetric(result.FirstTime, meta, repository)) + } + + return result, nil +} + +// CreateTranscriptLogs creates transcript log records from message content. +// This public form is used by the file watcher; ParseFile uses createTranscriptLogs +// (private) which also enriches records with session metadata. +func (p *ClaudeParser) CreateTranscriptLogs(entry ClaudeJSONLEntry, ts time.Time, sessionID string, messageIndex *int) []api.LogRecord { + return p.createTranscriptLogs(entry, ts, sessionID, claudeSessionMeta{}, messageIndex) +} - // Create metrics - usage := entry.Message.Usage - model := entry.Message.Model +// extractLinesOfCodeMetrics parses file-editing tool calls in an assistant entry and returns +// claude_code.lines_of_code.count metrics broken down by file type and path. +func (p *ClaudeParser) extractLinesOfCodeMetrics(entry ClaudeJSONLEntry, ts time.Time, meta claudeSessionMeta, repository string) []api.MetricDataPoint { + var metrics []api.MetricDataPoint - // Token usage metrics (creates both regular and user-facing variants) - if usage.InputTokens > 0 { - result.Metrics = append(result.Metrics, CreateTokenMetrics(ts, model, "input", float64(usage.InputTokens))...) + for _, content := range entry.Message.Content { + if content.Type != "tool_use" { + continue + } + + // Marshal input back to JSON so we can decode into typed structs + inputBytes, err := json.Marshal(content.Input) + if err != nil { + continue + } + + switch content.Name { + case "Edit": + var inp claudeEditInput + if err := json.Unmarshal(inputBytes, &inp); err != nil || inp.FilePath == "" { + continue } - if usage.OutputTokens > 0 { - result.Metrics = append(result.Metrics, CreateTokenMetrics(ts, model, "output", float64(usage.OutputTokens))...) + removed := countLines(inp.OldString) + added := countLines(inp.NewString) + fileType := filepath.Ext(inp.FilePath) + relPath := relativeFilePath(inp.FilePath, meta.Cwd) + if removed > 0 { + metrics = append(metrics, createLOCMetric(ts, "removed", relPath, fileType, float64(removed), meta, repository)) } - if usage.CacheCreationInputTokens > 0 { - result.Metrics = append(result.Metrics, CreateTokenMetrics(ts, model, "cacheCreation", float64(usage.CacheCreationInputTokens))...) + if added > 0 { + metrics = append(metrics, createLOCMetric(ts, "added", relPath, fileType, float64(added), meta, repository)) } - if usage.CacheReadInputTokens > 0 { - result.Metrics = append(result.Metrics, CreateTokenMetrics(ts, model, "cacheRead", float64(usage.CacheReadInputTokens))...) + + case "Write": + var inp claudeWriteInput + if err := json.Unmarshal(inputBytes, &inp); err != nil || inp.FilePath == "" { + continue + } + added := countLines(inp.Content) + fileType := filepath.Ext(inp.FilePath) + relPath := relativeFilePath(inp.FilePath, meta.Cwd) + if added > 0 { + metrics = append(metrics, createLOCMetric(ts, "added", relPath, fileType, float64(added), meta, repository)) } - // Cost metrics using pricing mode (creates both regular and user-facing variants) - tokenUsage := pricing.ClaudeTokenUsage{ - InputTokens: int64(usage.InputTokens), - OutputTokens: int64(usage.OutputTokens), - CacheCreationInputTokens: int64(usage.CacheCreationInputTokens), - CacheReadInputTokens: int64(usage.CacheReadInputTokens), + case "MultiEdit": + var inp claudeMultiEditInput + if err := json.Unmarshal(inputBytes, &inp); err != nil || inp.FilePath == "" { + continue + } + fileType := filepath.Ext(inp.FilePath) + relPath := relativeFilePath(inp.FilePath, meta.Cwd) + var totalRemoved, totalAdded int + for _, edit := range inp.Edits { + totalRemoved += countLines(edit.OldString) + totalAdded += countLines(edit.NewString) } - cost := pricing.GetClaudeCostWithMode(p.pricingMode, model, tokenUsage, entry.CostUSD) - if cost > 0 { - result.Metrics = append(result.Metrics, CreateCostMetrics(ts, model, cost)...) + if totalRemoved > 0 { + metrics = append(metrics, createLOCMetric(ts, "removed", relPath, fileType, float64(totalRemoved), meta, repository)) + } + if totalAdded > 0 { + metrics = append(metrics, createLOCMetric(ts, "added", relPath, fileType, float64(totalAdded), meta, repository)) } } - - result.RecordCount++ } - if err := scanner.Err(); err != nil { - return nil, fmt.Errorf("reading file: %w", err) + return metrics +} + +// claudeBashInput holds the input for Bash tool calls +type claudeBashInput struct { + Command string `json:"command"` +} + +// extractCommitMetrics parses Bash tool calls in an assistant entry and returns +// claude_code.commit.count metrics for git commit commands. +func (p *ClaudeParser) extractCommitMetrics(entry ClaudeJSONLEntry, ts time.Time, meta claudeSessionMeta, repository string) []api.MetricDataPoint { + var metrics []api.MetricDataPoint + + for _, content := range entry.Message.Content { + if content.Type != "tool_use" || content.Name != "Bash" { + continue + } + inputBytes, err := json.Marshal(content.Input) + if err != nil { + continue + } + var inp claudeBashInput + if err := json.Unmarshal(inputBytes, &inp); err != nil { + continue + } + if strings.Contains(inp.Command, "git commit") { + metrics = append(metrics, createCommitMetric(ts, meta, repository)) + } } - return result, nil + return metrics } -// CreateTranscriptLogs creates transcript log records from message content -func (p *ClaudeParser) CreateTranscriptLogs(entry ClaudeJSONLEntry, ts time.Time, sessionID string, messageIndex *int) []api.LogRecord { +// relativeFilePath returns a path relative to baseDir, falling back to the base filename +func relativeFilePath(filePath, baseDir string) string { + if baseDir == "" { + return filepath.Base(filePath) + } + if rel, err := filepath.Rel(baseDir, filePath); err == nil && !strings.HasPrefix(rel, "..") { + return rel + } + return filepath.Base(filePath) +} + +// createTranscriptLogs creates transcript log records with optional session metadata enrichment. +func (p *ClaudeParser) createTranscriptLogs(entry ClaudeJSONLEntry, ts time.Time, sessionID string, meta claudeSessionMeta, messageIndex *int) []api.LogRecord { var logs []api.LogRecord if entry.Message == nil || len(entry.Message.Content) == 0 { @@ -311,11 +594,11 @@ func (p *ClaudeParser) CreateTranscriptLogs(entry ClaudeJSONLEntry, ts time.Time var body string var role string attrs := map[string]string{ - "event.name": "transcript.message", - "session.id": sessionID, - "message.index": strconv.Itoa(*messageIndex), - "message.role": baseRole, - "import_source": "local_jsonl", + "event.name": "transcript.message", + "session.id": sessionID, + "message.index": strconv.Itoa(*messageIndex), + "message.role": baseRole, + "import_source": "local_jsonl", } if entry.Message.Model != "" { @@ -324,6 +607,17 @@ func (p *ClaudeParser) CreateTranscriptLogs(entry ClaudeJSONLEntry, ts time.Time if entry.Message.ID != "" { attrs["message.id"] = entry.Message.ID } + if meta.GitBranch != "" { + attrs["git_branch"] = meta.GitBranch + } + if repo := extractRepository(meta); repo != "" { + attrs["repository"] = repo + } + if meta.PRNumber != 0 { + attrs["pr_number"] = strconv.Itoa(meta.PRNumber) + attrs["pr_repository"] = meta.PRRepository + attrs["pr_url"] = meta.PRUrl + } switch content.Type { case "text": @@ -381,9 +675,16 @@ const ( ClaudeUserFacingTokenUsageMetric = "claude_code.token.usage_user_facing" ClaudeCostMetric = "claude_code.cost.usage" ClaudeUserFacingCostMetric = "claude_code.cost.usage_user_facing" + claudeLinesOfCodeMetric = "claude_code.lines_of_code.count" + claudePullRequestMetric = "claude_code.pull_request.count" + claudeCommitMetric = "claude_code.commit.count" + claudeSessionMetric = "claude_code.session.count" + claudeActiveTimeMetric = "claude_code.active_time.total" ) -// CreateTokenMetric creates a token usage metric with the specified name +// CreateTokenMetric creates a token usage metric with the specified name. +// This public form is used by the file watcher; ParseFile uses createTokenMetric +// (private) which also attaches session attributes. func CreateTokenMetric(ts time.Time, metricName, model, tokenType string, value float64) api.MetricDataPoint { return api.MetricDataPoint{ Timestamp: ts, @@ -399,6 +700,34 @@ func CreateTokenMetric(ts time.Time, metricName, model, tokenType string, value } } +// sessionAttrs builds the common session-level attributes (git_branch, repository). +func sessionAttrs(meta claudeSessionMeta, repository string) map[string]string { + attrs := make(map[string]string) + if meta.GitBranch != "" { + attrs["git_branch"] = meta.GitBranch + } + if repository != "" { + attrs["repository"] = repository + } + return attrs +} + +// createTokenMetric creates a token usage metric with the specified name, including session attributes. +func createTokenMetric(ts time.Time, metricName, model, tokenType string, value float64, meta claudeSessionMeta, repository string) api.MetricDataPoint { + attrs := sessionAttrs(meta, repository) + attrs["type"] = tokenType + attrs["model"] = model + attrs["import_source"] = "local_jsonl" + return api.MetricDataPoint{ + Timestamp: ts, + ServiceName: SourceClaude.ServiceName(), + MetricName: metricName, + MetricType: "sum", + Value: &value, + Attributes: attrs, + } +} + // CreateTokenMetrics creates both regular and user-facing token usage metrics. // JSONL data is already user-facing (only assistant messages with cache tokens), // so both metrics have identical values for consistency with OTLP-derived metrics. @@ -409,7 +738,17 @@ func CreateTokenMetrics(ts time.Time, model, tokenType string, value float64) [] } } -// CreateCostMetric creates a cost metric with the specified name +// createTokenMetrics creates both regular and user-facing token usage metrics with session attributes. +func createTokenMetrics(ts time.Time, model, tokenType string, value float64, meta claudeSessionMeta, repository string) []api.MetricDataPoint { + return []api.MetricDataPoint{ + createTokenMetric(ts, ClaudeTokenUsageMetric, model, tokenType, value, meta, repository), + createTokenMetric(ts, ClaudeUserFacingTokenUsageMetric, model, tokenType, value, meta, repository), + } +} + +// CreateCostMetric creates a cost metric with the specified name. +// This public form is used by the file watcher; ParseFile uses createCostMetric +// (private) which also attaches session attributes. func CreateCostMetric(ts time.Time, metricName, model string, value float64) api.MetricDataPoint { return api.MetricDataPoint{ Timestamp: ts, @@ -424,6 +763,21 @@ func CreateCostMetric(ts time.Time, metricName, model string, value float64) api } } +// createCostMetric creates a cost metric with the specified name, including session attributes. +func createCostMetric(ts time.Time, metricName, model string, value float64, meta claudeSessionMeta, repository string) api.MetricDataPoint { + attrs := sessionAttrs(meta, repository) + attrs["model"] = model + attrs["import_source"] = "local_jsonl" + return api.MetricDataPoint{ + Timestamp: ts, + ServiceName: SourceClaude.ServiceName(), + MetricName: metricName, + MetricType: "sum", + Value: &value, + Attributes: attrs, + } +} + // CreateCostMetrics creates both regular and user-facing cost metrics. // JSONL data is already user-facing (only assistant messages with cache tokens), // so both metrics have identical values for consistency with OTLP-derived metrics. @@ -433,3 +787,90 @@ func CreateCostMetrics(ts time.Time, model string, value float64) []api.MetricDa CreateCostMetric(ts, ClaudeUserFacingCostMetric, model, value), } } + +// createCostMetrics creates both regular and user-facing cost metrics with session attributes. +func createCostMetrics(ts time.Time, model string, value float64, meta claudeSessionMeta, repository string) []api.MetricDataPoint { + return []api.MetricDataPoint{ + createCostMetric(ts, ClaudeCostMetric, model, value, meta, repository), + createCostMetric(ts, ClaudeUserFacingCostMetric, model, value, meta, repository), + } +} + +// createLOCMetric creates a lines_of_code metric for a file edit +func createLOCMetric(ts time.Time, lineType, filePath, fileType string, value float64, meta claudeSessionMeta, repository string) api.MetricDataPoint { + attrs := sessionAttrs(meta, repository) + attrs["type"] = lineType + attrs["file_type"] = fileType + attrs["file_path"] = filePath + attrs["import_source"] = "local_jsonl" + return api.MetricDataPoint{ + Timestamp: ts, + ServiceName: SourceClaude.ServiceName(), + MetricName: claudeLinesOfCodeMetric, + MetricType: "sum", + Value: &value, + Attributes: attrs, + } +} + +// createPRMetric creates a pull_request.count metric for a linked PR +func createPRMetric(meta claudeSessionMeta, repository string) api.MetricDataPoint { + one := 1.0 + attrs := sessionAttrs(meta, repository) + attrs["pr_number"] = strconv.Itoa(meta.PRNumber) + attrs["pr_url"] = meta.PRUrl + attrs["import_source"] = "local_jsonl" + return api.MetricDataPoint{ + Timestamp: meta.PRTimestamp, + ServiceName: SourceClaude.ServiceName(), + MetricName: claudePullRequestMetric, + MetricType: "sum", + Value: &one, + Attributes: attrs, + } +} + +// createCommitMetric creates a commit.count metric for a git commit tool call +func createCommitMetric(ts time.Time, meta claudeSessionMeta, repository string) api.MetricDataPoint { + one := 1.0 + attrs := sessionAttrs(meta, repository) + attrs["import_source"] = "local_jsonl" + return api.MetricDataPoint{ + Timestamp: ts, + ServiceName: SourceClaude.ServiceName(), + MetricName: claudeCommitMetric, + MetricType: "sum", + Value: &one, + Attributes: attrs, + } +} + +// createSessionMetric creates a session.count metric (one per JSONL file) +func createSessionMetric(ts time.Time, meta claudeSessionMeta, repository string) api.MetricDataPoint { + one := 1.0 + attrs := sessionAttrs(meta, repository) + attrs["import_source"] = "local_jsonl" + return api.MetricDataPoint{ + Timestamp: ts, + ServiceName: SourceClaude.ServiceName(), + MetricName: claudeSessionMetric, + MetricType: "sum", + Value: &one, + Attributes: attrs, + } +} + +// createActiveTimeMetric creates an active_time.total metric from a turn_duration entry. +// Each turn emits its own data point (consistent with OTLP telemetry). +func createActiveTimeMetric(ts time.Time, secs float64, meta claudeSessionMeta, repository string) api.MetricDataPoint { + attrs := sessionAttrs(meta, repository) + attrs["import_source"] = "local_jsonl" + return api.MetricDataPoint{ + Timestamp: ts, + ServiceName: SourceClaude.ServiceName(), + MetricName: claudeActiveTimeMetric, + MetricType: "sum", + Value: &secs, + Attributes: attrs, + } +} diff --git a/backend/internal/importer/codex.go b/backend/internal/importer/codex.go index 816b3e3..07f8860 100644 --- a/backend/internal/importer/codex.go +++ b/backend/internal/importer/codex.go @@ -95,8 +95,8 @@ type CodexJSONLEntry struct { type CodexResponseItem struct { Type string `json:"type"` // For "message" type - Role string `json:"role,omitempty"` - Content []CodexContentItem `json:"content,omitempty"` + Role string `json:"role,omitempty"` + Content []CodexContentItem `json:"content,omitempty"` // For "function_call" type Name string `json:"name,omitempty"` Arguments string `json:"arguments,omitempty"` @@ -119,22 +119,28 @@ type CodexReasoningSummary struct { Text string `json:"text,omitempty"` } +// codexGitInfo contains git metadata from session_meta +type codexGitInfo struct { + RepositoryURL string `json:"repository_url"` +} + // CodexSessionMeta represents session metadata type CodexSessionMeta struct { - ID string `json:"id"` - Timestamp string `json:"timestamp"` - Cwd string `json:"cwd"` - Originator string `json:"originator"` - CliVersion string `json:"cli_version"` - ModelProvider string `json:"model_provider"` - Model string `json:"model"` + ID string `json:"id"` + Timestamp string `json:"timestamp"` + Cwd string `json:"cwd"` + Originator string `json:"originator"` + CliVersion string `json:"cli_version"` + ModelProvider string `json:"model_provider"` + Model string `json:"model"` + Git *codexGitInfo `json:"git,omitempty"` } // CodexEventMsg represents an event message payload // The actual JSON structure has "info" at the same level as "type", not nested in another "payload" type CodexEventMsg struct { - Type string `json:"type"` - Info *CodexTokenInfo `json:"info"` // Present for token_count events + Type string `json:"type"` + Info *CodexTokenInfo `json:"info"` // Present for token_count events } // CodexTokenInfo contains token usage information @@ -176,6 +182,7 @@ func (p *CodexParser) ParseFile(ctx context.Context, path string) (*ImportResult var sessionMeta *CodexSessionMeta var currentModel string var lastTokenCount *CodexTokenCount + var repository string messageIndex := 0 // Track message order for transcripts for scanner.Scan() { @@ -222,6 +229,13 @@ func (p *CodexParser) ParseFile(ctx context.Context, path string) (*ImportResult currentModel = meta.Model } + // Resolve repository using git URL from session metadata (authoritative for Codex). + gitRepoURL := "" + if meta.Git != nil { + gitRepoURL = meta.Git.RepositoryURL + } + repository = resolveSessionRepository(nil, gitRepoURL, "", meta.Cwd) + // Create session start log logRecord := api.LogRecord{ Timestamp: ts, @@ -230,17 +244,20 @@ func (p *CodexParser) ParseFile(ctx context.Context, path string) (*ImportResult SeverityNumber: 9, Body: "conversation_starts", LogAttributes: map[string]string{ - "event.name": "codex.conversation_starts", - "session.id": meta.ID, - "model": meta.Model, - "model_provider": meta.ModelProvider, - "cli_version": meta.CliVersion, - "import_source": "local_jsonl", + "event.name": "codex.conversation_starts", + "session.id": meta.ID, + "model": meta.Model, + "model_provider": meta.ModelProvider, + "cli_version": meta.CliVersion, + "import_source": "local_jsonl", }, } if meta.Cwd != "" { logRecord.LogAttributes["cwd"] = meta.Cwd } + if repository != "" { + logRecord.LogAttributes["repository"] = repository + } result.Logs = append(result.Logs, logRecord) result.RecordCount++ } @@ -288,29 +305,29 @@ func (p *CodexParser) ParseFile(ctx context.Context, path string) (*ImportResult // Create metrics for non-zero deltas if deltaInput > 0 { - result.Metrics = append(result.Metrics, CreateCodexTokenMetric(ts, currentModel, "input", float64(deltaInput))) + result.Metrics = append(result.Metrics, createCodexTokenMetric(ts, currentModel, "input", float64(deltaInput), repository)) } if deltaOutput > 0 { - result.Metrics = append(result.Metrics, CreateCodexTokenMetric(ts, currentModel, "output", float64(deltaOutput))) + result.Metrics = append(result.Metrics, createCodexTokenMetric(ts, currentModel, "output", float64(deltaOutput), repository)) } if deltaCacheCreation > 0 { - result.Metrics = append(result.Metrics, CreateCodexTokenMetric(ts, currentModel, "cache_creation", float64(deltaCacheCreation))) + result.Metrics = append(result.Metrics, createCodexTokenMetric(ts, currentModel, "cache_creation", float64(deltaCacheCreation), repository)) } if deltaCacheRead > 0 { - result.Metrics = append(result.Metrics, CreateCodexTokenMetric(ts, currentModel, "cache_read", float64(deltaCacheRead))) + result.Metrics = append(result.Metrics, createCodexTokenMetric(ts, currentModel, "cache_read", float64(deltaCacheRead), repository)) } if deltaReasoning > 0 { - result.Metrics = append(result.Metrics, CreateCodexTokenMetric(ts, currentModel, "reasoning", float64(deltaReasoning))) + result.Metrics = append(result.Metrics, createCodexTokenMetric(ts, currentModel, "reasoning", float64(deltaReasoning), repository)) } if deltaTool > 0 { - result.Metrics = append(result.Metrics, CreateCodexTokenMetric(ts, currentModel, "tool", float64(deltaTool))) + result.Metrics = append(result.Metrics, createCodexTokenMetric(ts, currentModel, "tool", float64(deltaTool), repository)) } // Calculate and add cost metric // Note: cache_read is used for cost calculation (cache_creation tokens are billed at input rate) cost := pricing.CalculateCodexCost(currentModel, int64(deltaInput), int64(deltaCacheRead), int64(deltaOutput)) if cost != nil && *cost > 0 { - result.Metrics = append(result.Metrics, CreateCodexCostMetric(ts, currentModel, *cost)) + result.Metrics = append(result.Metrics, createCodexCostMetric(ts, currentModel, *cost, repository)) } lastTokenCount = tokenCount @@ -518,33 +535,55 @@ func (p *CodexParser) ParseFile(ctx context.Context, path string) (*ImportResult return result, nil } -// CreateCodexTokenMetric creates a token usage metric for Codex -func CreateCodexTokenMetric(ts time.Time, model, tokenType string, value float64) api.MetricDataPoint { +// createCodexTokenMetric creates a token usage metric for Codex with optional repository attribution. +func createCodexTokenMetric(ts time.Time, model, tokenType string, value float64, repository string) api.MetricDataPoint { + attrs := map[string]string{ + "type": tokenType, + "model": model, + "import_source": "local_jsonl", + } + if repository != "" { + attrs["repository"] = repository + } return api.MetricDataPoint{ Timestamp: ts, ServiceName: SourceCodex.ServiceName(), MetricName: "codex_cli_rs.token.usage", MetricType: "sum", Value: &value, - Attributes: map[string]string{ - "type": tokenType, - "model": model, - "import_source": "local_jsonl", - }, + Attributes: attrs, } } -// CreateCodexCostMetric creates a cost usage metric for Codex -func CreateCodexCostMetric(ts time.Time, model string, cost float64) api.MetricDataPoint { +// CreateCodexTokenMetric creates a token usage metric for Codex. +// This public form is used by the file watcher; ParseFile uses createCodexTokenMetric +// (private) which also attaches repository attribution. +func CreateCodexTokenMetric(ts time.Time, model, tokenType string, value float64) api.MetricDataPoint { + return createCodexTokenMetric(ts, model, tokenType, value, "") +} + +// createCodexCostMetric creates a cost usage metric for Codex with optional repository attribution. +func createCodexCostMetric(ts time.Time, model string, cost float64, repository string) api.MetricDataPoint { + attrs := map[string]string{ + "model": model, + "import_source": "local_jsonl", + } + if repository != "" { + attrs["repository"] = repository + } return api.MetricDataPoint{ Timestamp: ts, ServiceName: SourceCodex.ServiceName(), MetricName: "codex_cli_rs.cost.usage", MetricType: "sum", Value: &cost, - Attributes: map[string]string{ - "model": model, - "import_source": "local_jsonl", - }, + Attributes: attrs, } } + +// CreateCodexCostMetric creates a cost usage metric for Codex. +// This public form is used by the file watcher; ParseFile uses createCodexCostMetric +// (private) which also attaches repository attribution. +func CreateCodexCostMetric(ts time.Time, model string, cost float64) api.MetricDataPoint { + return createCodexCostMetric(ts, model, cost, "") +} diff --git a/backend/internal/importer/importer_test.go b/backend/internal/importer/importer_test.go index 3dfaf6a..535a959 100644 --- a/backend/internal/importer/importer_test.go +++ b/backend/internal/importer/importer_test.go @@ -846,6 +846,374 @@ func TestImportUnregisteredParser(t *testing.T) { } } +// TestClaudeParserGitBranchAndPR tests that gitBranch, pr-link, and file-edit data are captured +func TestClaudeParserGitBranchAndPR(t *testing.T) { + tmpDir, err := os.MkdirTemp("", "claude-meta-test-*") + if err != nil { + t.Fatalf("failed to create temp dir: %v", err) + } + defer os.RemoveAll(tmpDir) + + cwd := "/home/user/my-repo" + + // Raw JSONL lines mixing entry types + lines := []string{ + // user entry with gitBranch + `{"type":"user","timestamp":"2025-01-02T10:00:00.000Z","sessionId":"s1","gitBranch":"feat/my-branch","cwd":"` + cwd + `","message":{"role":"user","content":[{"type":"text","text":"hello"}]}}`, + // pr-link entry (no message field) + `{"type":"pr-link","sessionId":"s1","prNumber":42,"prUrl":"https://github.com/org/repo/pull/42","prRepository":"org/repo","timestamp":"2025-01-02T10:00:30.000Z"}`, + // assistant entry with usage and an Edit tool call + `{"type":"assistant","timestamp":"2025-01-02T10:01:00.000Z","sessionId":"s1","requestId":"r1","gitBranch":"feat/my-branch","message":{"id":"m1","model":"claude-sonnet-4-20250514","role":"assistant","usage":{"input_tokens":100,"output_tokens":50},"content":[{"type":"tool_use","name":"Edit","input":{"file_path":"/home/user/my-repo/main.go","old_string":"line1\nline2","new_string":"line1\nline2\nline3"}}]}}`, + // assistant entry with a Write tool call + `{"type":"assistant","timestamp":"2025-01-02T10:02:00.000Z","sessionId":"s1","requestId":"r2","gitBranch":"feat/my-branch","message":{"id":"m2","model":"claude-sonnet-4-20250514","role":"assistant","usage":{"input_tokens":50,"output_tokens":20},"content":[{"type":"tool_use","name":"Write","input":{"file_path":"/home/user/my-repo/README.md","content":"# Title\nLine two\nLine three\n"}}]}}`, + } + + testFile := filepath.Join(tmpDir, "s1.jsonl") + if err := os.WriteFile(testFile, []byte(joinLines(lines)), 0644); err != nil { + t.Fatalf("failed to write test file: %v", err) + } + + os.Setenv("AI_OBSERVER_CLAUDE_PATH", tmpDir) + defer os.Unsetenv("AI_OBSERVER_CLAUDE_PATH") + + parser := NewClaudeParser() + ctx := context.Background() + result, err := parser.ParseFile(ctx, testFile) + if err != nil { + t.Fatalf("ParseFile failed: %v", err) + } + + // Verify PR metric was emitted + var prMetrics []api.MetricDataPoint + var locMetrics []api.MetricDataPoint + for _, m := range result.Metrics { + switch m.MetricName { + case claudePullRequestMetric: + prMetrics = append(prMetrics, m) + case claudeLinesOfCodeMetric: + locMetrics = append(locMetrics, m) + } + } + + if len(prMetrics) != 1 { + t.Errorf("expected 1 PR metric, got %d", len(prMetrics)) + } else { + pr := prMetrics[0] + if pr.Attributes["pr_number"] != "42" { + t.Errorf("expected pr_number=42, got %s", pr.Attributes["pr_number"]) + } + if pr.Attributes["repository"] != "org/repo" { + t.Errorf("expected repository=org/repo, got %s", pr.Attributes["repository"]) + } + if pr.Attributes["git_branch"] != "feat/my-branch" { + t.Errorf("expected git_branch=feat/my-branch, got %s", pr.Attributes["git_branch"]) + } + } + + // Verify lines_of_code metrics: + // Edit: old_string has 2 lines (removed), new_string has 3 lines (added) + // Write: content has 4 lines (added, counting trailing newline as +1 line) + var locByType = map[string]float64{} + var locFileTypes = map[string]bool{} + for _, m := range locMetrics { + locByType[m.Attributes["type"]] += *m.Value + locFileTypes[m.Attributes["file_type"]] = true + } + if locByType["removed"] != 2 { + t.Errorf("expected 2 lines removed, got %v", locByType["removed"]) + } + if locByType["added"] != 7 { // 3 from Edit + 4 from Write + t.Errorf("expected 7 lines added, got %v", locByType["added"]) + } + if !locFileTypes[".go"] { + t.Error("expected .go file type in loc metrics") + } + if !locFileTypes[".md"] { + t.Error("expected .md file type in loc metrics") + } + + // Verify git_branch on token metrics + for _, m := range result.Metrics { + if m.MetricName == ClaudeTokenUsageMetric { + if m.Attributes["git_branch"] != "feat/my-branch" { + t.Errorf("expected git_branch on token metric, got %q", m.Attributes["git_branch"]) + } + if m.Attributes["repository"] != "org/repo" { + t.Errorf("expected repository on token metric, got %q", m.Attributes["repository"]) + } + break + } + } + + // Verify pr_number appears in transcript log attributes + for _, log := range result.Logs { + if log.LogAttributes["event.name"] == "transcript.message" { + if log.LogAttributes["pr_number"] != "42" { + t.Errorf("expected pr_number=42 in log attributes, got %q", log.LogAttributes["pr_number"]) + } + break + } + } +} + +// TestClaudeParserSessionCommitActiveTime tests session, commit, and active time metric generation +func TestClaudeParserSessionCommitActiveTime(t *testing.T) { + tmpDir, err := os.MkdirTemp("", "claude-sca-test-*") + if err != nil { + t.Fatalf("failed to create temp dir: %v", err) + } + defer os.RemoveAll(tmpDir) + + lines := []string{ + // user entry (session starts) + `{"type":"user","timestamp":"2025-01-02T10:00:00.000Z","sessionId":"s2","gitBranch":"main","cwd":"/home/user/proj","message":{"role":"user","content":[{"type":"text","text":"do stuff"}]}}`, + // assistant with a git commit bash call + `{"type":"assistant","timestamp":"2025-01-02T10:01:00.000Z","sessionId":"s2","requestId":"r1","gitBranch":"main","message":{"id":"m1","model":"claude-sonnet-4-20250514","role":"assistant","usage":{"input_tokens":100,"output_tokens":50},"content":[{"type":"tool_use","name":"Bash","input":{"command":"git add . && git commit -m \"feat: add thing\""}}]}}`, + // another assistant with two git commits in separate tool calls + `{"type":"assistant","timestamp":"2025-01-02T10:02:00.000Z","sessionId":"s2","requestId":"r2","gitBranch":"main","message":{"id":"m2","model":"claude-sonnet-4-20250514","role":"assistant","usage":{"input_tokens":50,"output_tokens":20},"content":[{"type":"tool_use","name":"Bash","input":{"command":"git commit -m \"fix: typo\""}},{"type":"tool_use","name":"Bash","input":{"command":"echo hello"}}]}}`, + // system/turn_duration entries + `{"type":"system","subtype":"turn_duration","durationMs":30000,"timestamp":"2025-01-02T10:01:30.000Z","sessionId":"s2","gitBranch":"main","cwd":"/home/user/proj"}`, + `{"type":"system","subtype":"turn_duration","durationMs":15000,"timestamp":"2025-01-02T10:02:15.000Z","sessionId":"s2","gitBranch":"main","cwd":"/home/user/proj"}`, + } + + testFile := filepath.Join(tmpDir, "s2.jsonl") + if err := os.WriteFile(testFile, []byte(joinLines(lines)), 0644); err != nil { + t.Fatalf("failed to write test file: %v", err) + } + + os.Setenv("AI_OBSERVER_CLAUDE_PATH", tmpDir) + defer os.Unsetenv("AI_OBSERVER_CLAUDE_PATH") + + parser := NewClaudeParser() + ctx := context.Background() + result, err := parser.ParseFile(ctx, testFile) + if err != nil { + t.Fatalf("ParseFile failed: %v", err) + } + + counts := map[string]int{} + var totalActiveSecs float64 + for _, m := range result.Metrics { + counts[m.MetricName]++ + if m.MetricName == claudeActiveTimeMetric && m.Value != nil { + totalActiveSecs += *m.Value + } + } + + // One session metric per file + if counts[claudeSessionMetric] != 1 { + t.Errorf("expected 1 session metric, got %d", counts[claudeSessionMetric]) + } + + // Two commits: one from r1, one from r2 (the echo call is not a commit) + if counts[claudeCommitMetric] != 2 { + t.Errorf("expected 2 commit metrics, got %d", counts[claudeCommitMetric]) + } + + // Two turn_duration entries -> two active_time metrics, total 45s + if counts[claudeActiveTimeMetric] != 2 { + t.Errorf("expected 2 active_time metrics, got %d", counts[claudeActiveTimeMetric]) + } + if totalActiveSecs != 45.0 { + t.Errorf("expected 45s total active time, got %v", totalActiveSecs) + } + + // All metrics should have git_branch and repository set + for _, m := range result.Metrics { + if m.MetricName == claudeSessionMetric || m.MetricName == claudeCommitMetric || m.MetricName == claudeActiveTimeMetric { + if m.Attributes["git_branch"] != "main" { + t.Errorf("%s: expected git_branch=main, got %q", m.MetricName, m.Attributes["git_branch"]) + } + if m.Attributes["repository"] != "proj" { + t.Errorf("%s: expected repository=proj, got %q", m.MetricName, m.Attributes["repository"]) + } + } + } +} + +// TestClaudeParserMultiEdit tests that MultiEdit tool calls produce lines_of_code metrics +func TestClaudeParserMultiEdit(t *testing.T) { + tmpDir, err := os.MkdirTemp("", "claude-multiedit-test-*") + if err != nil { + t.Fatalf("failed to create temp dir: %v", err) + } + defer os.RemoveAll(tmpDir) + + multiEditInput := `{"file_path":"/home/user/proj/app.go","edits":[{"old_string":"line1\nline2","new_string":"line1\nline2\nline3"},{"old_string":"foo","new_string":""}]}` + lines := []string{ + `{"type":"user","timestamp":"2025-01-02T10:00:00.000Z","sessionId":"s3","cwd":"/home/user/proj","message":{"role":"user","content":[{"type":"text","text":"edit"}]}}`, + `{"type":"assistant","timestamp":"2025-01-02T10:01:00.000Z","sessionId":"s3","requestId":"r1","message":{"id":"m1","model":"claude-sonnet-4-20250514","role":"assistant","usage":{"input_tokens":50,"output_tokens":20},"content":[{"type":"tool_use","name":"MultiEdit","input":` + multiEditInput + `}]}}`, + } + + testFile := filepath.Join(tmpDir, "s3.jsonl") + if err := os.WriteFile(testFile, []byte(joinLines(lines)), 0644); err != nil { + t.Fatalf("failed to write test file: %v", err) + } + + os.Setenv("AI_OBSERVER_CLAUDE_PATH", tmpDir) + defer os.Unsetenv("AI_OBSERVER_CLAUDE_PATH") + + parser := NewClaudeParser() + result, err := parser.ParseFile(context.Background(), testFile) + if err != nil { + t.Fatalf("ParseFile failed: %v", err) + } + + var added, removed float64 + for _, m := range result.Metrics { + if m.MetricName != claudeLinesOfCodeMetric { + continue + } + switch m.Attributes["type"] { + case "added": + added += *m.Value + case "removed": + removed += *m.Value + } + if m.Attributes["file_type"] != ".go" { + t.Errorf("expected file_type=.go, got %q", m.Attributes["file_type"]) + } + } + // edit[0]: old=2 lines removed, new=3 lines added + // edit[1]: old=1 line removed, new=0 lines added (empty new_string) + if removed != 3 { + t.Errorf("expected 3 lines removed, got %v", removed) + } + if added != 3 { + t.Errorf("expected 3 lines added, got %v", added) + } +} + +// TestClaudeParserSessionMetric tests that exactly one session metric is emitted per file +// and that duplicate pr-link entries produce only one PR metric +func TestClaudeParserSessionMetric(t *testing.T) { + tmpDir, err := os.MkdirTemp("", "claude-session-test-*") + if err != nil { + t.Fatalf("failed to create temp dir: %v", err) + } + defer os.RemoveAll(tmpDir) + + lines := []string{ + `{"type":"user","timestamp":"2025-01-02T10:00:00.000Z","sessionId":"s4","gitBranch":"main","cwd":"/home/user/proj","message":{"role":"user","content":[{"type":"text","text":"hi"}]}}`, + // same PR linked twice - only one PR metric should be emitted + `{"type":"pr-link","sessionId":"s4","prNumber":99,"prUrl":"https://github.com/org/repo/pull/99","prRepository":"org/repo","timestamp":"2025-01-02T10:00:05.000Z"}`, + `{"type":"pr-link","sessionId":"s4","prNumber":99,"prUrl":"https://github.com/org/repo/pull/99","prRepository":"org/repo","timestamp":"2025-01-02T10:00:10.000Z"}`, + `{"type":"assistant","timestamp":"2025-01-02T10:01:00.000Z","sessionId":"s4","requestId":"r1","message":{"id":"m1","model":"claude-sonnet-4-20250514","role":"assistant","usage":{"input_tokens":100,"output_tokens":50},"content":[]}}`, + } + + testFile := filepath.Join(tmpDir, "s4.jsonl") + if err := os.WriteFile(testFile, []byte(joinLines(lines)), 0644); err != nil { + t.Fatalf("failed to write test file: %v", err) + } + + os.Setenv("AI_OBSERVER_CLAUDE_PATH", tmpDir) + defer os.Unsetenv("AI_OBSERVER_CLAUDE_PATH") + + parser := NewClaudeParser() + result, err := parser.ParseFile(context.Background(), testFile) + if err != nil { + t.Fatalf("ParseFile failed: %v", err) + } + + counts := map[string]int{} + for _, m := range result.Metrics { + counts[m.MetricName]++ + } + + if counts[claudeSessionMetric] != 1 { + t.Errorf("expected 1 session metric, got %d", counts[claudeSessionMetric]) + } + if counts[claudePullRequestMetric] != 1 { + t.Errorf("expected 1 PR metric for duplicate pr-link entries, got %d", counts[claudePullRequestMetric]) + } +} + +// TestCollectSessionMeta tests the session metadata collection first pass +func TestCollectSessionMeta(t *testing.T) { + parser := NewClaudeParser() + + lines := []string{ + `{"type":"user","timestamp":"2025-01-01T00:00:00Z","gitBranch":"main","cwd":"/home/user/project"}`, + `{"type":"pr-link","prNumber":7,"prUrl":"https://github.com/a/b/pull/7","prRepository":"a/b","timestamp":"2025-01-01T00:01:00Z"}`, + // duplicate pr-link should not overwrite the first + `{"type":"pr-link","prNumber":8,"prUrl":"https://github.com/a/b/pull/8","prRepository":"a/b","timestamp":"2025-01-01T00:02:00Z"}`, + } + + meta := parser.collectSessionMeta(lines) + + if meta.GitBranch != "main" { + t.Errorf("expected GitBranch=main, got %q", meta.GitBranch) + } + if meta.Cwd != "/home/user/project" { + t.Errorf("expected Cwd, got %q", meta.Cwd) + } + if meta.PRNumber != 7 { + t.Errorf("expected PRNumber=7 (first pr-link), got %d", meta.PRNumber) + } + if meta.PRRepository != "a/b" { + t.Errorf("expected PRRepository=a/b, got %q", meta.PRRepository) + } +} + +// TestExtractRepository tests repository name derivation +func TestExtractRepository(t *testing.T) { + tests := []struct { + meta claudeSessionMeta + expected string + }{ + {claudeSessionMeta{PRRepository: "org/repo"}, "org/repo"}, + {claudeSessionMeta{Cwd: "/home/user/my-project"}, "my-project"}, + {claudeSessionMeta{PRRepository: "org/repo", Cwd: "/home/user/other"}, "org/repo"}, // PR takes precedence + {claudeSessionMeta{}, ""}, + } + for _, tc := range tests { + got := extractRepository(tc.meta) + if got != tc.expected { + t.Errorf("extractRepository(%+v) = %q, want %q", tc.meta, got, tc.expected) + } + } +} + +// TestCountLines tests line counting +func TestCountLines(t *testing.T) { + tests := []struct { + input string + expected int + }{ + {"", 0}, + {"single line", 1}, + {"line1\nline2", 2}, + {"line1\nline2\nline3", 3}, + {"line1\n", 2}, // trailing newline counts as an extra line + } + for _, tc := range tests { + got := countLines(tc.input) + if got != tc.expected { + t.Errorf("countLines(%q) = %d, want %d", tc.input, got, tc.expected) + } + } +} + +// TestRelativeFilePath tests relative path extraction +func TestRelativeFilePath(t *testing.T) { + tests := []struct { + filePath string + baseDir string + expected string + }{ + {"/home/user/project/main.go", "/home/user/project", "main.go"}, + {"/home/user/project/sub/file.ts", "/home/user/project", "sub/file.ts"}, + {"/home/user/project/main.go", "", "main.go"}, + {"/other/path/file.py", "/home/user/project", "file.py"}, // outside base -> basename + } + for _, tc := range tests { + got := relativeFilePath(tc.filePath, tc.baseDir) + if got != tc.expected { + t.Errorf("relativeFilePath(%q, %q) = %q, want %q", tc.filePath, tc.baseDir, got, tc.expected) + } + } +} + // Helper functions func floatPtr(f float64) *float64 { return &f diff --git a/backend/internal/importer/repos.go b/backend/internal/importer/repos.go new file mode 100644 index 0000000..ccd73eb --- /dev/null +++ b/backend/internal/importer/repos.go @@ -0,0 +1,281 @@ +package importer + +import ( + "os" + "os/exec" + "path/filepath" + "regexp" + "strconv" + "strings" + "sync" +) + +// Repository attribution logic. +// +// Ported from ducktrace/claude_analysis/repos.py and prmatch.py. +// Maps a session to a canonical "owner/repo" (or bare repo name) using the +// strongest available signal. +// +// Signal priority (best first): +// 1. Codex session_meta.git.repository_url (authoritative) +// 2. On-disk git remote from cwd (real remote, authoritative) +// 3. Candidate repos whose name matches cwd name or path segment +// 4. Bare cwd directory name +// 5. First candidate repository reference + +var ( + githubURLRe = regexp.MustCompile(`(?i)(?:^|://|@)(?:[^/@]+@)?github\.com[:/]+([^/]+/[^/?#]+)`) + schemeRe = regexp.MustCompile(`(?i)^[a-z][a-z0-9+.-]*://`) + userAtRe = regexp.MustCompile(`^[^@/]+@`) + + ghPRRe = regexp.MustCompile(`(?i)\bgh\s+pr\s+(create|merge|close|reopen|checkout|view|diff|review|list|edit|comment|ready|status)\b`) + repoFlagRe = regexp.MustCompile(`(?i)--repo[ =]\s*['"]?([^\s'"]+)`) + pullURLRe = regexp.MustCompile(`(?i)https?://github\.com/([^/\s]+/[^/\s]+?)/pull/(\d+)`) + // repoURLRe catches any github.com/owner/repo reference not already matched by pullURLRe. + // The lazy second component combined with the required terminator stops at path separators. + repoURLRe = regexp.MustCompile(`(?i)github\.com[:/]+([^/\s]+/[^/\s]+?)(?:\.git)?(?:[/\s)"',` + "`" + `]|$)`) +) + +var placeholderRepos = map[string]bool{ + "owner/repo": true, "owner/name": true, "org/repo": true, + "user/repo": true, "your-org/your-repo": true, + "username/repo": true, "owner/repository": true, +} + +var gitRemoteCache sync.Map + +// githubRepoFromURL extracts owner/repo from any GitHub URL (https or ssh), +// stripping embedded credentials and a trailing .git. Returns "" on no match. +func githubRepoFromURL(url string) string { + if url == "" { + return "" + } + m := githubURLRe.FindStringSubmatch(url) + if m == nil { + return "" + } + return strings.TrimSuffix(m[1], ".git") +} + +// normalizeGitURL normalizes any git remote URL to owner/repo when possible. +// Handles git@host:owner/repo.git, https://host/owner/repo.git, and +// https://@host/owner/repo.git (credentials are dropped). Falls back +// to the last two path segments for non-GitHub hosts. +func normalizeGitURL(url string) string { + if gh := githubRepoFromURL(url); gh != "" { + return gh + } + if url == "" { + return "" + } + u := schemeRe.ReplaceAllString(strings.TrimSpace(url), "") + u = userAtRe.ReplaceAllString(u, "") + u = strings.Replace(u, ":", "/", 1) // ssh host:path -> host/path + u = strings.TrimSuffix(u, ".git") + var parts []string + for _, p := range strings.Split(u, "/") { + if p != "" { + parts = append(parts, p) + } + } + if len(parts) >= 3 { // host/owner/repo... + return parts[len(parts)-2] + "/" + parts[len(parts)-1] + } + return u +} + +// resolveRepoName returns the repo name for a working dir, resolving git +// worktrees to the main repo's directory name. If the path no longer exists +// we just return its basename. +func resolveRepoName(cwd string) string { + if cwd == "" { + return "" + } + // A worktree's .git is a file (not a dir) containing "gitdir: ". + content, err := os.ReadFile(filepath.Join(cwd, ".git")) + if err == nil { + text := strings.TrimSpace(string(content)) + if strings.HasPrefix(text, "gitdir:") { + gitdir := strings.TrimSpace(strings.TrimPrefix(text, "gitdir:")) + parts := strings.Split(filepath.ToSlash(gitdir), "/") + for i, seg := range parts { + if seg == ".git" && i > 0 { + return filepath.Base(filepath.Join(parts[:i]...)) + } + } + } + } + return filepath.Base(cwd) +} + +// cwdGitRemote returns the owner/repo of the cwd's origin remote, cached per +// path. This is authoritative when the directory still exists locally - the +// real remote is correct even when the local dir name differs from the repo name. +func cwdGitRemote(path string) string { + if path == "" { + return "" + } + if cached, ok := gitRemoteCache.Load(path); ok { + return cached.(string) + } + out, err := exec.Command("git", "-C", path, "config", "--get", "remote.origin.url").Output() + result := "" + if err == nil { + result = normalizeGitURL(strings.TrimSpace(string(out))) + } + gitRemoteCache.Store(path, result) + return result +} + +// resolveSessionRepository returns the best repository id for a session. +// +// candidateRepos are owner/repo references seen for the session - from pr-link +// entries and text-mined references - ideally most-frequent first. +// +// The working directory says which repo we're in; candidates supply the owner/ +// prefix the cwd alone can't. A candidate whose repo-part matches the cwd name +// or a path segment is the highest-confidence answer. A candidate that doesn't +// match is treated as a stray and used only when there's no cwd at all. +func resolveSessionRepository(candidateRepos []string, gitRepoURL, originalCwd, cwd string) string { + // Codex carries the cwd's actual git remote - authoritative. + if gitRepoURL != "" { + if norm := normalizeGitURL(gitRepoURL); norm != "" { + return norm + } + } + + path := originalCwd + if path == "" { + path = cwd + } + + // On-disk git remote is authoritative (handles dir name != repo name). + if onDisk := cwdGitRemote(path); onDisk != "" { + return onDisk + } + + cwdName := resolveRepoName(path) + var candidates []string + for _, r := range candidateRepos { + if r != "" { + candidates = append(candidates, r) + } + } + + if len(candidates) > 0 && path != "" { + segments := make(map[string]bool) + for _, seg := range strings.Split(filepath.ToSlash(path), "/") { + if seg != "" { + segments[strings.ToLower(seg)] = true + } + } + for _, r := range candidates { + parts := strings.Split(r, "/") + name := strings.ToLower(parts[len(parts)-1]) + if name == strings.ToLower(cwdName) || segments[name] { + return r + } + } + } + + if cwdName != "" { + return cwdName + } + if len(candidates) > 0 { + return candidates[0] + } + return "" +} + +// isPlaceholder returns true if repo is a documentation/example placeholder. +func isPlaceholder(repo string) bool { + return placeholderRepos[strings.ToLower(repo)] || + strings.Contains(repo, "<") || strings.Contains(repo, ">") +} + +func normalizeRepoFlag(val string) string { + if gh := githubRepoFromURL(val); gh != "" { + if isPlaceholder(gh) { + return "" + } + return gh + } + val = strings.TrimRight(strings.TrimSpace(val), "/") + var parts []string + for _, p := range strings.Split(val, "/") { + if p != "" { + parts = append(parts, p) + } + } + if len(parts) >= 2 && !strings.Contains(parts[len(parts)-2], ".") { + repo := strings.TrimSuffix(parts[len(parts)-2]+"/"+parts[len(parts)-1], ".git") + if isPlaceholder(repo) { + return "" + } + return repo + } + return "" +} + +// extractedRefs holds PR/repo references mined from free text. +type extractedRefs struct { + PRActions []string + PRURLs []string + PRNumbers []int + Repos []string +} + +// extractRefs scans one or more strings for gh pr commands, PR URLs, +// --repo flags, and GitHub repo references. +func extractRefs(texts ...string) extractedRefs { + result := extractedRefs{} + seenActions := map[string]bool{} + seenURLs := map[string]bool{} + seenNumbers := map[int]bool{} + seenRepos := map[string]bool{} + + for _, text := range texts { + if text == "" { + continue + } + for _, m := range ghPRRe.FindAllStringSubmatch(text, -1) { + action := "gh pr " + strings.ToLower(m[1]) + if !seenActions[action] { + seenActions[action] = true + result.PRActions = append(result.PRActions, action) + } + } + for _, m := range pullURLRe.FindAllStringSubmatch(text, -1) { + repo := strings.TrimSuffix(m[1], ".git") + if isPlaceholder(repo) { + continue + } + if !seenRepos[repo] { + seenRepos[repo] = true + result.Repos = append(result.Repos, repo) + } + if num, err := strconv.Atoi(m[2]); err == nil && !seenNumbers[num] { + seenNumbers[num] = true + result.PRNumbers = append(result.PRNumbers, num) + } + if url := m[0]; !seenURLs[url] { + seenURLs[url] = true + result.PRURLs = append(result.PRURLs, url) + } + } + for _, m := range repoFlagRe.FindAllStringSubmatch(text, -1) { + if r := normalizeRepoFlag(m[1]); r != "" && !seenRepos[r] { + seenRepos[r] = true + result.Repos = append(result.Repos, r) + } + } + for _, m := range repoURLRe.FindAllStringSubmatch(text, -1) { + r := strings.TrimSuffix(m[1], ".git") + if !isPlaceholder(r) && !seenRepos[r] { + seenRepos[r] = true + result.Repos = append(result.Repos, r) + } + } + } + return result +}