diff --git a/.github/workflows/benchmark-processor.yml b/.github/workflows/benchmark-processor.yml index b8bd2945..641554b2 100644 --- a/.github/workflows/benchmark-processor.yml +++ b/.github/workflows/benchmark-processor.yml @@ -3,6 +3,15 @@ name: benchmark-processor on: workflow_dispatch: inputs: + language: + description: 'Symbolication language: js (5 impls incl. Honeycomb), dart (traceway-dart-mem/disk), or both' + required: true + default: 'js' + type: choice + options: + - js + - dart + - both scenarios: description: 'Scenarios to run' required: true @@ -84,6 +93,7 @@ jobs: ./bin/builder --config manifest-traceway.yaml --skip-compilation cd build-traceway && CGO_ENABLED=1 go build -tags oxc -o otelcol-bench-traceway . - name: Build honeycomb collector + if: inputs.language != 'dart' working-directory: benchmarks/processor run: | GOBIN=$PWD/bin go install go.opentelemetry.io/collector/cmd/builder@v0.143.0 @@ -97,6 +107,7 @@ jobs: cd loadgen && go mod tidy && go build -o loadgen . && cd .. cd corpusgen && go build -o corpusgen . - name: Build node-app bundle + if: inputs.language != 'dart' working-directory: testing/symbolication/node-app run: npm install && npm run build - name: Assemble artifacts @@ -104,25 +115,44 @@ jobs: run: | mkdir -p artifacts cp build-traceway/otelcol-bench-traceway artifacts/ - cp build-honeycomb/otelcol-bench-honeycomb artifacts/ - cp "$(find ~/go/pkg/mod/github.com/honeycombio -name 'libsymbolic_cabi.so' -path '*linux_x86_64*' | head -1)" artifacts/ cp drain/target/release/drain artifacts/ cp loadgen/loadgen corpusgen/corpusgen artifacts/ - cp config-traceway.yaml config-honeycomb.yaml rss-sampler.sh artifacts/ - cp ../../testing/symbolication/node-app/dist/app.mjs ../../testing/symbolication/node-app/dist/app.mjs.map artifacts/ + cp config-traceway.yaml rss-sampler.sh artifacts/ + cp -r seeds artifacts/ + if [ -f build-honeycomb/otelcol-bench-honeycomb ]; then + cp build-honeycomb/otelcol-bench-honeycomb artifacts/ + cp "$(find ~/go/pkg/mod/github.com/honeycombio -name 'libsymbolic_cabi.so' -path '*linux_x86_64*' | head -1)" artifacts/ + cp config-honeycomb.yaml artifacts/ + cp ../../testing/symbolication/node-app/dist/app.mjs ../../testing/symbolication/node-app/dist/app.mjs.map artifacts/ + fi - uses: actions/upload-artifact@v4 with: name: bench-artifacts path: benchmarks/processor/artifacts/ + setup: + runs-on: ubuntu-latest + outputs: + impls: ${{ steps.m.outputs.impls }} + steps: + - id: m + run: | + case "${{ inputs.language }}" in + js) impls='["honeycomb","traceway-oxc-mem","traceway-oxc-disk","traceway-goja-mem","traceway-goja-disk"]' ;; + dart) impls='["traceway-dart-mem","traceway-dart-disk"]' ;; + both) impls='["honeycomb","traceway-oxc-mem","traceway-oxc-disk","traceway-goja-mem","traceway-goja-disk","traceway-dart-mem","traceway-dart-disk"]' ;; + *) echo "unknown language ${{ inputs.language }}" >&2; exit 1 ;; + esac + echo "impls=$impls" >> "$GITHUB_OUTPUT" + bench: - needs: build + needs: [build, setup] runs-on: ubuntu-latest strategy: fail-fast: false max-parallel: 1 matrix: - impl: [honeycomb, traceway-oxc-mem, traceway-oxc-disk, traceway-goja-mem, traceway-goja-disk] + impl: ${{ fromJson(needs.setup.outputs.impls) }} steps: - uses: actions/checkout@v4 - uses: actions/download-artifact@v4 diff --git a/backend/app/controllers/clientcontrollers/client.controller.go b/backend/app/controllers/clientcontrollers/client.controller.go index 47b18a53..a18620f9 100644 --- a/backend/app/controllers/clientcontrollers/client.controller.go +++ b/backend/app/controllers/clientcontrollers/client.controller.go @@ -27,9 +27,6 @@ import ( type clientController struct{} -// isEmptyRaw reports whether a json.RawMessage carries no meaningful payload — -// nil, blank, `null`, `[]`, or `{}` all count as empty. Used to drop session -// recordings that would otherwise just be wasted S3 writes. func isEmptyRaw(r json.RawMessage) bool { if len(r) == 0 { return true @@ -98,10 +95,6 @@ func (e clientController) Report(c *gin.Context) { var recordingsWork []recordings.Job - // Map frontend sessionRecordingId → backend-generated exception UUID. - // Used only for the legacy exception-bound recording path; the always-on - // session linkage uses the SDK-supplied sessionId UUID directly (it IS - // the session row id by design, so no in-request map is needed). recordingIdToExceptionId := map[string]uuid.UUID{} convertSpan := traceway.StartSpan(c, "report.convert_frames") @@ -109,9 +102,7 @@ func (e clientController) Report(c *gin.Context) { for _, cs := range cf.Sessions { s := cs.ToSession(request.AppVersion, request.ServerName) s.ProjectId = projectId - // The SDK can't see the public-facing IP; we stamp it server-side - // into the attributes blob so the dashboard surfaces it alongside - // browser/url/viewport collected by the SDK. + if clientIP := c.ClientIP(); clientIP != "" { if s.Attributes == nil { s.Attributes = map[string]string{} @@ -142,13 +133,16 @@ func (e clientController) Report(c *gin.Context) { spansToInsert = append(spansToInsert, span) } } - resolveJs := project != nil && isJsFramework(project.Framework) && project.SourceMapToken != nil + resolveJs := project != nil && project.SourceMapToken != nil && jsFrameworks[project.Framework] + resolveDart := project != nil && project.SourceMapToken != nil && project.Framework == "flutter" resolveSpan := traceway.StartSpan(c, "report.resolve_stack_traces") for _, cst := range cf.StackTraces { resolvedStackTrace := cst.StackTrace if resolveJs { resolvedStackTrace = services.ResolveStackTrace(c, projectId, cst.StackTrace, cst.DebugIds) + } else if resolveDart { + resolvedStackTrace = services.ResolveDartStackTrace(c, projectId, cst.StackTrace) } est := cst.ToExceptionStackTrace(ComputeExceptionHash(resolvedStackTrace, cst.IsMessage), request.AppVersion, request.ServerName) est.StackTrace = resolvedStackTrace @@ -157,10 +151,7 @@ func (e clientController) Report(c *gin.Context) { if cst.SessionRecordingId != nil { recordingIdToExceptionId[*cst.SessionRecordingId] = est.Id } - // The SDK-provided session UID is the session row id by design, so - // parse it directly. The parent `sessions` row may have been - // upserted in an earlier request — we don't require it in this - // batch. + if cst.SessionId != nil { if parsed, err := uuid.Parse(*cst.SessionId); err == nil { est.SessionId = &parsed @@ -177,10 +168,7 @@ func (e clientController) Report(c *gin.Context) { } for _, sr := range cf.SessionRecordings { - // A recording can be exception-bound (legacy path), session-bound - // (always-on path), or both. Exception linkage requires the - // exception to be in this batch; session linkage doesn't, since - // the SDK-provided sessionId is the session row id by design. + var exceptionId uuid.UUID if sr.ExceptionId != "" { if id, ok := recordingIdToExceptionId[sr.ExceptionId]; ok { @@ -323,27 +311,24 @@ func (e clientController) Report(c *gin.Context) { } var ( - errorMessageRe = regexp.MustCompile(`(?m)^(\*?[\w.]+):\s*.+`) - causedByRe = regexp.MustCompile(`(?m)^(Caused by:\s*[\w.$]+):\s*.+`) - jsFuncLineRe = regexp.MustCompile(`(?m)^( {0,4})(.+)\(\)(\n {4}.+:\d+:\d+)$`) - urlOriginRe = regexp.MustCompile(`[a-zA-Z][a-zA-Z0-9+.\-]*://[^/\s]*`) - absolutePathRe = regexp.MustCompile(`/[^\s:]+/([^/\s:]+:\d+)`) - // Engines disagree on column conventions for the same throw, so columns - // only disambiguate when everything sits on line 1 (minified bundles with - // no matching source map). For any other line the column is dropped from - // the hash so resolved frames group across engines. + errorMessageRe = regexp.MustCompile(`(?m)^(\*?[\w.]+):\s*.+`) + causedByRe = regexp.MustCompile(`(?m)^(Caused by:\s*[\w.$]+):\s*.+`) + jsFuncLineRe = regexp.MustCompile(`(?m)^( {0,4})(.+)\(\)(\n {4}.+:\d+:\d+)$`) + urlOriginRe = regexp.MustCompile(`[a-zA-Z][a-zA-Z0-9+.\-]*://[^/\s]*`) + absolutePathRe = regexp.MustCompile(`/[^\s:]+/([^/\s:]+:\d+)`) + laterLineColRe = regexp.MustCompile(`(?m)^(\s*.+:(?:[2-9]|[1-9]\d+)):\d+$`) - versionRe = regexp.MustCompile(`@v[\d.]+`) - hexRe = regexp.MustCompile(`0x[0-9a-fA-F]+`) - uuidRe = regexp.MustCompile(`[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}`) - largeNumberRe = regexp.MustCompile(`(^|[^:\d])(\d{5,})($|[^\d])`) - emailRe = regexp.MustCompile(`[\w.\-]+@[\w.\-]+\.\w+`) - ipRe = regexp.MustCompile(`\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}(:\d+)?`) - goroutineRe = regexp.MustCompile(`goroutine \d+`) - javaLineNumRe = regexp.MustCompile(`\((\w[\w.$]*\.(?:java|kt|scala)):\d+\)`) - javaEllipsisRe = regexp.MustCompile(`\.\.\. \d+ more`) - spacesRe = regexp.MustCompile(`[ \t]+`) - newlinesRe = regexp.MustCompile(`\n+`) + versionRe = regexp.MustCompile(`@v[\d.]+`) + hexRe = regexp.MustCompile(`0x[0-9a-fA-F]+`) + uuidRe = regexp.MustCompile(`[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}`) + largeNumberRe = regexp.MustCompile(`(^|[^:\d])(\d{5,})($|[^\d])`) + emailRe = regexp.MustCompile(`[\w.\-]+@[\w.\-]+\.\w+`) + ipRe = regexp.MustCompile(`\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}(:\d+)?`) + goroutineRe = regexp.MustCompile(`goroutine \d+`) + javaLineNumRe = regexp.MustCompile(`\((\w[\w.$]*\.(?:java|kt|scala)):\d+\)`) + javaEllipsisRe = regexp.MustCompile(`\.\.\. \d+ more`) + spacesRe = regexp.MustCompile(`[ \t]+`) + newlinesRe = regexp.MustCompile(`\n+`) ) func ComputeExceptionHash(stackTrace string, isMessage bool) string { @@ -353,10 +338,7 @@ func ComputeExceptionHash(stackTrace string, isMessage bool) string { normalized = causedByRe.ReplaceAllString(normalized, "$1") normalized = errorMessageRe.ReplaceAllString(normalized, "$1") normalized = jsFuncLineRe.ReplaceAllString(normalized, "${1}${3}") - // Bundle URLs (https://host/assets/app.js, file:///srv/app.mjs, - // webpack://app/./src/x.js) must group with the same frames reported - // as bare filenames by the JS SDK, so the origin goes first and the - // remaining /path is reduced by absolutePathRe like any other path. + normalized = urlOriginRe.ReplaceAllString(normalized, "") normalized = absolutePathRe.ReplaceAllString(normalized, "$1") normalized = laterLineColRe.ReplaceAllString(normalized, "$1") @@ -390,15 +372,6 @@ var jsFrameworks = map[string]bool{ "react-native": true, } -func isJsFramework(framework string) bool { - return jsFrameworks[framework] -} - -// Browser-only frameworks. Spans arriving for these projects are page-load -// noise (web vitals, fetch spans), never server endpoints, so OTel trace -// ingest only extracts exceptions for them. Fullstack frameworks (nextjs, -// remix) and JS backends (nestjs, express, hono, cloudflare) are excluded -// because they emit legitimate SERVER spans. var frontendJsFrameworks = map[string]bool{ "react": true, "svelte": true, diff --git a/backend/app/controllers/clientcontrollers/dart_hash_test.go b/backend/app/controllers/clientcontrollers/dart_hash_test.go new file mode 100644 index 00000000..75e75adb --- /dev/null +++ b/backend/app/controllers/clientcontrollers/dart_hash_test.go @@ -0,0 +1,38 @@ +package clientcontrollers + +import "testing" + +func TestDartSymbolicatedHashIsMachineIndependent(t *testing.T) { + alice := "PaymentDeclinedException: card declined for $30.59\n" + + "#0 chargeCard (/Users/alice/app/lib/main.dart:20:3)\n" + + "#1 applyTax (/Users/alice/app/lib/main.dart:30:10)\n" + + "#2 checkout (/Users/alice/app/lib/main.dart:35:10)" + + bob := "PaymentDeclinedException: card declined for $42.00\n" + + "#0 chargeCard (/home/bob/ci/workspace/lib/main.dart:20:3)\n" + + "#1 applyTax (/home/bob/ci/workspace/lib/main.dart:30:10)\n" + + "#2 checkout (/home/bob/ci/workspace/lib/main.dart:35:10)" + + if ComputeExceptionHash(alice, false) != ComputeExceptionHash(bob, false) { + t.Errorf("same crash on different machines should hash the same") + } + + other := "PaymentDeclinedException: card declined for $30.59\n" + + "#0 refund (/Users/alice/app/lib/main.dart:88:3)" + if ComputeExceptionHash(alice, false) == ComputeExceptionHash(other, false) { + t.Errorf("different stacks should not share a hash") + } +} + +func TestDartOffsetFrameHashStable(t *testing.T) { + + report1 := "PaymentDeclinedException: card declined for $30.59\n" + + "#0 _kDartIsolateSnapshotInstructions+141e6b\n" + + "#1 _kDartIsolateSnapshotInstructions+141d9b" + report2 := "PaymentDeclinedException: card declined for $99.10\n" + + "#0 _kDartIsolateSnapshotInstructions+141e6b\n" + + "#1 _kDartIsolateSnapshotInstructions+141d9b" + if ComputeExceptionHash(report1, false) != ComputeExceptionHash(report2, false) { + t.Errorf("same crash (same offsets, different message values) should hash the same") + } +} diff --git a/backend/app/controllers/otelcontrollers/otel.controller.go b/backend/app/controllers/otelcontrollers/otel.controller.go index 58de7bbf..ad10f05e 100644 --- a/backend/app/controllers/otelcontrollers/otel.controller.go +++ b/backend/app/controllers/otelcontrollers/otel.controller.go @@ -15,7 +15,7 @@ import ( "github.com/tracewayapp/traceway/backend/app/repositories" "github.com/tracewayapp/traceway/backend/app/services" "github.com/tracewayapp/traceway/backend/app/storage" - "github.com/tracewayapp/traceway/backend/app/symbolicator/jsstack" + "github.com/tracewayapp/traceway/backend/app/symbolicator/sourcemap/jsstack" traceway "go.tracewayapp.com" ) diff --git a/backend/app/controllers/routes.go b/backend/app/controllers/routes.go index a4dc51a0..43277914 100644 --- a/backend/app/controllers/routes.go +++ b/backend/app/controllers/routes.go @@ -32,7 +32,6 @@ func RegisterControllers(router *gin.RouterGroup) { router.OPTIONS("/report", middleware.CORSReport) router.POST("/report", middleware.CORSReport, middleware.UseClientAuth, middleware.UseGzip, clientcontrollers.ClientController.Report) - // OTLP/HTTP ingestion otelGroup := router.Group("/otel") otelGroup.OPTIONS("/v1/traces", middleware.CORSReport) otelGroup.OPTIONS("/v1/metrics", middleware.CORSReport) @@ -41,7 +40,6 @@ func RegisterControllers(router *gin.RouterGroup) { otelGroup.POST("/v1/metrics", middleware.CORSReport, middleware.UseClientAuth, otelcontrollers.OtelController.ExportMetrics) otelGroup.POST("/v1/logs", middleware.CORSReport, middleware.UseClientAuth, otelcontrollers.OtelController.ExportLogs) - // Project management router.GET("/projects", middleware.UseAppAuth, ProjectController.ListProjects) router.POST("/projects", middleware.UseAppAuth, middleware.RequireProjectAccess, middleware.RequireWriteAccess, ProjectController.CreateProject) router.PUT("/projects", middleware.UseAppAuth, middleware.RequireProjectAccess, middleware.RequireWriteAccess, ProjectController.UpdateProject) @@ -49,23 +47,19 @@ func RegisterControllers(router *gin.RouterGroup) { router.GET("/health/deep", middleware.UseAppAuth, HealthDeepController.Get) - // Dashboard endpoints (projectId in query param) router.POST("/stats", middleware.UseAppAuth, middleware.RequireProjectAccess, MetricRecordController.FindHomepageStats) router.GET("/dashboard", middleware.UseAppAuth, middleware.RequireProjectAccess, DashboardController.GetDashboard) router.GET("/dashboard/overview", middleware.UseAppAuth, middleware.RequireProjectAccess, DashboardController.GetDashboardOverview) - // Metrics endpoints (projectId in query param) router.GET("/metrics/application", middleware.UseAppAuth, middleware.RequireProjectAccess, MetricsController.GetApplicationMetrics) router.GET("/metrics/stats", middleware.UseAppAuth, middleware.RequireProjectAccess, MetricsController.GetStatsMetrics) router.GET("/metrics/server", middleware.UseAppAuth, middleware.RequireProjectAccess, MetricsController.GetServerMetrics) - // New metrics query API router.POST("/metrics/query", middleware.UseAppAuth, middleware.RequireProjectAccess, MetricQueryController.Query) router.GET("/metrics/discover", middleware.UseAppAuth, middleware.RequireProjectAccess, MetricQueryController.Discover) router.GET("/metrics/discover/tags", middleware.UseAppAuth, middleware.RequireProjectAccess, MetricQueryController.DiscoverTags) router.PUT("/metrics/registry", middleware.UseAppAuth, middleware.RequireProjectAccess, middleware.RequireWriteAccess, MetricQueryController.UpdateRegistry) - // Widget groups router.GET("/widget-groups", middleware.UseAppAuth, middleware.RequireProjectAccess, middleware.Transactional, WidgetGroupController.List) router.POST("/widget-groups", middleware.UseAppAuth, middleware.RequireProjectAccess, middleware.RequireWriteAccess, middleware.Transactional, WidgetGroupController.Create) router.POST("/widget-groups/populate-defaults", middleware.UseAppAuth, middleware.RequireProjectAccess, middleware.RequireWriteAccess, middleware.Transactional, WidgetGroupController.PopulateDefaults) @@ -74,14 +68,12 @@ func RegisterControllers(router *gin.RouterGroup) { router.PUT("/widget-groups/:id", middleware.UseAppAuth, middleware.RequireProjectAccess, middleware.RequireWriteAccess, middleware.Transactional, WidgetGroupController.Update) router.DELETE("/widget-groups/:id", middleware.UseAppAuth, middleware.RequireProjectAccess, middleware.RequireWriteAccess, middleware.Transactional, WidgetGroupController.Delete) - // Widgets (within widget groups) router.POST("/widget-groups/:id/widgets", middleware.UseAppAuth, middleware.RequireProjectAccess, middleware.RequireWriteAccess, middleware.Transactional, WidgetController.Add) router.PUT("/widget-groups/:id/widgets/:wid", middleware.UseAppAuth, middleware.RequireProjectAccess, middleware.RequireWriteAccess, middleware.Transactional, WidgetController.Update) router.PUT("/widget-groups/:id/widgets/:wid/move", middleware.UseAppAuth, middleware.RequireProjectAccess, middleware.RequireWriteAccess, middleware.Transactional, WidgetController.Move) router.PUT("/widget-groups/:id/widgets/:wid/star", middleware.UseAppAuth, middleware.RequireProjectAccess, middleware.RequireWriteAccess, middleware.Transactional, WidgetController.ToggleStar) router.DELETE("/widget-groups/:id/widgets/:wid", middleware.UseAppAuth, middleware.RequireProjectAccess, middleware.RequireWriteAccess, middleware.Transactional, WidgetController.Delete) - // Endpoints (projectId in body) router.POST("/endpoints", middleware.UseAppAuth, middleware.RequireProjectAccess, EndpointController.FindAllEndpoints) router.POST("/endpoints/grouped", middleware.UseAppAuth, middleware.RequireProjectAccess, EndpointController.FindGroupedByEndpoint) router.POST("/endpoints/endpoint", middleware.UseAppAuth, middleware.RequireProjectAccess, EndpointController.FindByEndpoint) @@ -90,41 +82,33 @@ func RegisterControllers(router *gin.RouterGroup) { router.POST("/endpoints/slow", middleware.UseAppAuth, middleware.RequireProjectAccess, middleware.RequireWriteAccess, EndpointController.SetSlowEndpoint) router.POST("/endpoints/:endpointId", middleware.UseAppAuth, middleware.RequireProjectAccess, EndpointDetailController.GetEndpointDetail) - // Tasks (projectId in body) router.POST("/tasks", middleware.UseAppAuth, middleware.RequireProjectAccess, TaskController.FindAllTasks) router.POST("/tasks/grouped", middleware.UseAppAuth, middleware.RequireProjectAccess, TaskController.FindGroupedByTaskName) router.POST("/tasks/task", middleware.UseAppAuth, middleware.RequireProjectAccess, TaskController.FindByTaskName) router.POST("/tasks/:taskId", middleware.UseAppAuth, middleware.RequireProjectAccess, TaskDetailController.GetTaskDetail) - // Sessions (projectId in body / query) router.POST("/sessions", middleware.UseAppAuth, middleware.RequireProjectAccess, SessionController.FindAllSessions) router.POST("/sessions/:sessionId", middleware.UseAppAuth, middleware.RequireProjectAccess, SessionDetailController.GetSessionDetail) router.GET("/sessions/:sessionId/recording", middleware.UseAppAuth, middleware.RequireProjectAccess, SessionDetailController.GetSessionRecording) - // AI Traces (projectId in body) router.POST("/ai-traces/grouped", middleware.UseAppAuth, middleware.RequireProjectAccess, AiTraceController.FindGroupedByTraceName) router.POST("/ai-traces/trace", middleware.UseAppAuth, middleware.RequireProjectAccess, AiTraceController.FindByTraceName) router.POST("/ai-traces/:traceId", middleware.UseAppAuth, middleware.RequireProjectAccess, AiTraceController.GetAiTraceDetail) - // Distributed traces router.POST("/distributed-traces/:distributedTraceId", middleware.UseAppAuth, DistributedTraceController.GetDistributedTrace) - // Logs (projectId in body) router.POST("/logs", middleware.UseAppAuth, middleware.RequireProjectAccess, LogController.List) - // Exceptions (projectId in body) router.POST("/exception-stack-traces", middleware.UseAppAuth, middleware.RequireProjectAccess, ExceptionStackTraceController.FindGrouppedExceptionStackTraces) router.POST("/exception-stack-traces/archive", middleware.UseAppAuth, middleware.RequireProjectAccess, middleware.RequireWriteAccess, ExceptionStackTraceController.ArchiveExceptions) router.POST("/exception-stack-traces/unarchive", middleware.UseAppAuth, middleware.RequireProjectAccess, middleware.RequireWriteAccess, ExceptionStackTraceController.UnarchiveExceptions) router.POST("/exception-stack-traces/by-id/:exceptionId", middleware.UseAppAuth, middleware.RequireProjectAccess, ExceptionStackTraceController.FindById) router.POST("/exception-stack-traces/:hash", middleware.UseAppAuth, middleware.RequireProjectAccess, ExceptionStackTraceController.FindByHash) - // Auth router.POST("/login", middleware.Transactional, AuthController.Login) router.POST("/register", middleware.Transactional, AuthController.Register) router.GET("/me/login-bundle", middleware.UseAppAuth, middleware.Transactional, AuthController.LoginBundle) - // OAuth (Google, GitHub via markbates/goth) router.GET("/auth/providers", OAuthController.ListProviders) router.GET("/auth/start/:provider", middleware.Transactional, OAuthController.Begin) router.GET("/auth/callback/:provider", middleware.Transactional, OAuthController.Callback) @@ -134,42 +118,35 @@ func RegisterControllers(router *gin.RouterGroup) { router.GET("/has-organizations", middleware.Transactional, AuthController.HasOrganizations) } - // Password reset router.POST("/forgot-password", middleware.Transactional, PasswordResetController.ForgotPassword) router.GET("/password-reset/:token", PasswordResetController.ValidateToken) router.POST("/password-reset/:token", middleware.Transactional, PasswordResetController.ResetPassword) - // Organization settings (admin/owner access) router.GET("/organizations/:organizationId/settings", middleware.UseAppAuth, middleware.RequireAdminAccess, OrganizationController.GetSettings) router.PUT("/organizations/:organizationId/settings", middleware.UseAppAuth, middleware.RequireAdminAccess, middleware.Transactional, OrganizationController.UpdateSettings) router.GET("/organizations/:organizationId/members", middleware.UseAppAuth, middleware.RequireAdminAccess, OrganizationController.GetMembers) - // Member management (admin/owner) - TRANSACTIONAL router.PUT("/organizations/:organizationId/members/:userId", middleware.UseAppAuth, middleware.RequireAdminAccess, middleware.Transactional, MemberController.UpdateRole) router.DELETE("/organizations/:organizationId/members/:userId", middleware.UseAppAuth, middleware.RequireAdminAccess, middleware.Transactional, MemberController.RemoveMember) - // Invitations management (admin/owner) - TRANSACTIONAL router.POST("/organizations/:organizationId/invitations", middleware.UseAppAuth, middleware.RequireAdminAccess, middleware.Transactional, InvitationController.InviteUser) router.GET("/organizations/:organizationId/invitations", middleware.UseAppAuth, middleware.RequireAdminAccess, InvitationController.ListInvitations) router.DELETE("/organizations/:organizationId/invitations/:id", middleware.UseAppAuth, middleware.RequireAdminAccess, middleware.Transactional, InvitationController.RevokeInvitation) - // Public invitation endpoints - TRANSACTIONAL router.GET("/invitations/:token", InvitationController.GetInvitationInfo) router.POST("/invitations/:token/accept", middleware.Transactional, InvitationController.AcceptInvitation) router.POST("/invitations/:token/accept-existing", middleware.UseAppAuth, middleware.Transactional, InvitationController.AcceptExistingUser) - // Source map management router.POST("/projects/source-map-token", middleware.UseAppAuth, middleware.RequireProjectAccess, middleware.RequireWriteAccess, ProjectController.GenerateSourceMapToken) router.POST("/sourcemaps/upload", middleware.UseSourceMapAuth, SourceMapController.Upload) + router.POST("/symbols/upload", middleware.UseSourceMapAuth, SymbolsController.Upload) - // Notification channels router.GET("/notification-channels", middleware.UseAppAuth, middleware.RequireProjectAccess, middleware.Transactional, NotificationChannelController.List) router.POST("/notification-channels", middleware.UseAppAuth, middleware.RequireProjectAccess, middleware.RequireWriteAccess, middleware.Transactional, NotificationChannelController.Create) router.PUT("/notification-channels/:id", middleware.UseAppAuth, middleware.RequireProjectAccess, middleware.RequireWriteAccess, middleware.Transactional, NotificationChannelController.Update) router.DELETE("/notification-channels/:id", middleware.UseAppAuth, middleware.RequireProjectAccess, middleware.RequireWriteAccess, middleware.Transactional, NotificationChannelController.Delete) router.POST("/notification-channels/:id/test", middleware.UseAppAuth, middleware.RequireProjectAccess, middleware.RequireWriteAccess, NotificationChannelController.Test) - // Notification rules router.GET("/notification-rules", middleware.UseAppAuth, middleware.RequireProjectAccess, middleware.Transactional, NotificationRuleController.List) router.POST("/notification-rules", middleware.UseAppAuth, middleware.RequireProjectAccess, middleware.RequireWriteAccess, middleware.Transactional, NotificationRuleController.Create) router.PUT("/notification-rules/:id", middleware.UseAppAuth, middleware.RequireProjectAccess, middleware.RequireWriteAccess, middleware.Transactional, NotificationRuleController.Update) @@ -177,7 +154,6 @@ func RegisterControllers(router *gin.RouterGroup) { router.POST("/notification-rules/:id/toggle", middleware.UseAppAuth, middleware.RequireProjectAccess, middleware.RequireWriteAccess, middleware.Transactional, NotificationRuleController.Toggle) router.POST("/notification-rules/:id/snooze", middleware.UseAppAuth, middleware.RequireProjectAccess, middleware.RequireWriteAccess, middleware.Transactional, NotificationRuleController.Snooze) - // Notification history router.POST("/notification-history", middleware.UseAppAuth, middleware.RequireProjectAccess, NotificationHistoryController.List) for _, register := range ExtensionRoutes { diff --git a/backend/app/controllers/symbols.controller.go b/backend/app/controllers/symbols.controller.go new file mode 100644 index 00000000..75115d05 --- /dev/null +++ b/backend/app/controllers/symbols.controller.go @@ -0,0 +1,114 @@ +package controllers + +import ( + "fmt" + "io" + "net/http" + "path/filepath" + "strings" + + "github.com/tracewayapp/traceway/backend/app/middleware" + "github.com/tracewayapp/traceway/backend/app/services" + "github.com/tracewayapp/traceway/backend/app/storage" + "github.com/tracewayapp/traceway/backend/app/symbolicator/dart" + + "github.com/gin-gonic/gin" + traceway "go.tracewayapp.com" +) + +type symbolsController struct{} + +func (s symbolsController) Upload(c *gin.Context) { + projectId, err := middleware.GetProjectId(c) + if err != nil { + c.AbortWithError(http.StatusInternalServerError, traceway.NewStackTraceErrorf("UseSourceMapAuth middleware must be applied: %w", err)) + return + } + + if err := c.Request.ParseMultipartForm(200 << 20); err != nil { + c.JSON(http.StatusBadRequest, gin.H{"error": "Failed to parse multipart form"}) + return + } + + files := c.Request.MultipartForm.File["files"] + if len(files) == 0 { + c.JSON(http.StatusBadRequest, gin.H{"error": "No files uploaded"}) + return + } + + uploaded := 0 + for _, fileHeader := range files { + if filepath.Ext(fileHeader.Filename) != ".symbols" { + continue + } + if fileHeader.Size > 200<<20 { + c.JSON(http.StatusBadRequest, gin.H{"error": fmt.Sprintf("File %s exceeds 200MB limit", fileHeader.Filename)}) + return + } + + f, err := fileHeader.Open() + if err != nil { + c.AbortWithError(http.StatusInternalServerError, traceway.NewStackTraceErrorf("failed to open uploaded file %s: %w", fileHeader.Filename, err)) + return + } + data, err := io.ReadAll(f) + f.Close() + if err != nil { + c.AbortWithError(http.StatusInternalServerError, traceway.NewStackTraceErrorf("failed to read uploaded file %s: %w", fileHeader.Filename, err)) + return + } + + arch := c.PostForm("arch") + if arch == "" { + arch = archFromSymbolsFilename(fileHeader.Filename) + } + if arch == "" { + c.JSON(http.StatusBadRequest, gin.H{"error": fmt.Sprintf("cannot determine arch for %s; pass an 'arch' field", fileHeader.Filename)}) + return + } + if !dart.IsValidArch(arch) { + c.JSON(http.StatusBadRequest, gin.H{"error": fmt.Sprintf("invalid arch %q for %s; expected an architecture token like arm64, x64, arm, or ia32", arch, fileHeader.Filename)}) + return + } + + buildID, err := dart.ReadBuildID(data) + if err != nil { + c.JSON(http.StatusUnprocessableEntity, gin.H{"error": fmt.Sprintf("%s is not a valid Dart symbols file: %v", fileHeader.Filename, err)}) + return + } + + debugId := services.NormalizeDartDebugId(c.PostForm("debug_id")) + if note := services.NormalizeDartDebugId(buildID); note != "" { + if debugId != "" && debugId != note { + c.JSON(http.StatusUnprocessableEntity, gin.H{"error": fmt.Sprintf("debug_id %s does not match the build-id note %s in %s", debugId, note, fileHeader.Filename)}) + return + } + debugId = note + } + if debugId == "" { + c.JSON(http.StatusBadRequest, gin.H{"error": fmt.Sprintf("no debug_id for %s; the file has no build-id note, so pass a 'debug_id' field (the Mach-O UUID)", fileHeader.Filename)}) + return + } + + key := services.DartSymbolsKey(projectId, debugId, arch) + if err := storage.Store.Write(c, key, data); err != nil { + c.AbortWithError(http.StatusInternalServerError, traceway.NewStackTraceErrorf("failed to write symbols to storage: %w", err)) + return + } + services.InvalidateDartSymbols(key) + + uploaded++ + } + + c.JSON(http.StatusOK, gin.H{"uploaded": uploaded}) +} + +func archFromSymbolsFilename(name string) string { + base := strings.TrimSuffix(filepath.Base(name), ".symbols") + if i := strings.LastIndex(base, "-"); i != -1 { + return base[i+1:] + } + return "" +} + +var SymbolsController = symbolsController{} diff --git a/backend/app/services/dart_arch_security_test.go b/backend/app/services/dart_arch_security_test.go new file mode 100644 index 00000000..22635ddc --- /dev/null +++ b/backend/app/services/dart_arch_security_test.go @@ -0,0 +1,36 @@ +package services + +import ( + "path/filepath" + "strings" + "testing" + + "github.com/google/uuid" +) + +func TestDartSymbolsKeyConfinesArch(t *testing.T) { + pid := uuid.New() + prefix := "dartsymbols/" + pid.String() + "/" + hostile := []string{ + "../../../../../../tmp/pwned", + "../../" + uuid.New().String() + "/aaaa-arm64", + "arm64/../../../etc/cron.d/x", + "..", + "a/b", + `back\slash`, + } + for _, arch := range hostile { + key := DartSymbolsKey(pid, fixtureBuildID, arch) + seg, ok := strings.CutPrefix(key, prefix) + if !ok { + t.Errorf("arch %q escaped the project prefix: key=%q", arch, key) + continue + } + if strings.ContainsAny(seg, `/\`) || strings.Contains(seg, "..") { + t.Errorf("arch %q produced traversal chars in key segment %q (full key %q)", arch, seg, key) + } + if cleaned := filepath.ToSlash(filepath.Clean(key)); cleaned != key { + t.Errorf("arch %q produced a non-clean key %q (clean=%q)", arch, key, cleaned) + } + } +} diff --git a/backend/app/services/dart_symbolicator.go b/backend/app/services/dart_symbolicator.go new file mode 100644 index 00000000..e43e6c27 --- /dev/null +++ b/backend/app/services/dart_symbolicator.go @@ -0,0 +1,142 @@ +package services + +import ( + "context" + "errors" + "fmt" + "strings" + + "github.com/tracewayapp/traceway/backend/app/storage" + "github.com/tracewayapp/traceway/backend/app/symbolicator/dart" + "github.com/tracewayapp/traceway/backend/app/symbolicator/twcache" + + "github.com/google/uuid" + traceway "go.tracewayapp.com" +) + +const maxDartFrames = 50 + +func DartSymbolsKey(projectId uuid.UUID, debugId, arch string) string { + return fmt.Sprintf("dartsymbols/%s/%s-%s.symbols", projectId, NormalizeDartDebugId(debugId), NormalizeDartArch(arch)) +} + +func NormalizeDartDebugId(debugId string) string { return dart.NormalizeDebugID(debugId) } +func NormalizeDartArch(arch string) string { return dart.NormalizeArch(arch) } + +func ResolveDartStackTrace(ctx context.Context, projectId uuid.UUID, rawTrace string) string { + if !dart.IsNonSymbolic(rawTrace) { + return rawTrace + } + trace := dart.ParseTrace(rawTrace) + if len(trace.Frames) == 0 { + return rawTrace + } + + data, done := loadDartData(ctx, projectId, trace) + defer done() + + var b strings.Builder + if preamble := dartErrorPreamble(rawTrace); preamble != "" { + b.WriteString(preamble) + b.WriteByte('\n') + } + + n := 0 + for _, f := range trace.Frames { + if n >= maxDartFrames { + break + } + var resolved []dart.SymFrame + if data != nil { + resolved = dart.LookupFlat(data, f) + } + if len(resolved) == 0 { + + fmt.Fprintf(&b, "#%d %s+%x\n", n, dart.InstructionSymbol(f.Section), f.Offset) + n++ + continue + } + for _, sf := range resolved { + if n >= maxDartFrames { + break + } + fmt.Fprintf(&b, "#%d %s (%s)\n", n, sf.Function, sf.Location()) + n++ + } + } + + out := strings.TrimRight(b.String(), "\n") + if out == "" { + return rawTrace + } + return out +} + +func dartErrorPreamble(raw string) string { + lines := strings.Split(raw, "\n") + cut := len(lines) + for i, ln := range lines { + if strings.HasPrefix(strings.TrimSpace(ln), "***") { + cut = i + break + } + } + return strings.TrimRight(strings.Join(lines[:cut], "\n"), "\n") +} + +func loadDartData(ctx context.Context, projectId uuid.UUID, trace dart.StackTrace) ([]byte, func()) { + + if trace.BuildID == "" || trace.Arch == "" { + return nil, noop + } + cacheKey := dartFlatKey(DartSymbolsKey(projectId, trace.BuildID, trace.Arch)) + if sharedCache.IsNegative(cacheKey) { + return nil, noop + } + data, done, err := sharedCache.Get(ctx, cacheKey, loadDartBlob(cacheKey)) + if err != nil { + return nil, noop + } + return data, done +} + +func dartFlatKey(symbolsKey string) string { + return strings.TrimSuffix(symbolsKey, ".symbols") + ".tw" +} + +func InvalidateDartSymbols(keys ...string) { + for _, k := range keys { + cacheKey := dartFlatKey(k) + sharedCache.Invalidate(cacheKey) + _ = storage.Store.Delete(context.Background(), cacheKey) + } +} + +func loadDartBlob(cacheKey string) twcache.LoadFunc { + symbolsKey := strings.TrimSuffix(cacheKey, ".tw") + ".symbols" + return func(ctx context.Context) ([]byte, error) { + base := context.WithoutCancel(ctx) + + if twBytes, err := readWithTimeout(base, cacheKey); err == nil { + if dart.ValidFlat(twBytes) { + return twBytes, nil + } + + } else if !errors.Is(err, storage.ErrNotFound) { + traceway.CaptureException(fmt.Errorf("failed to read dart flat artifact, rebuilding (key=%s): %w", cacheKey, err)) + } + + elf, err := readWithTimeout(base, symbolsKey) + if err != nil { + return nil, err + } + blob, buildErr := dart.BuildFlat(elf) + if buildErr != nil { + return nil, fmt.Errorf("failed to build dart flat artifact (key=%s): %w", symbolsKey, buildErr) + } + if werr := storage.Store.Write(base, cacheKey, blob); werr != nil { + traceway.CaptureException(fmt.Errorf("failed to persist dart flat artifact (key=%s): %w", cacheKey, werr)) + } + return blob, nil + } +} diff --git a/backend/app/services/dart_symbolicator_test.go b/backend/app/services/dart_symbolicator_test.go new file mode 100644 index 00000000..f35d2ef7 --- /dev/null +++ b/backend/app/services/dart_symbolicator_test.go @@ -0,0 +1,247 @@ +package services + +import ( + "context" + "os" + "path/filepath" + "regexp" + "strings" + "sync" + "testing" + + "github.com/tracewayapp/traceway/backend/app/storage" + + "github.com/google/uuid" +) + +const dartFixtureDir = "../symbolicator/dart/fixtures/flutter-macos-arm64-dart3.10.1" + +const fixtureBuildID = "fe664295997135e7b67b648ba66ca9eb" + +type dartMemStore struct { + mu sync.Mutex + data map[string][]byte +} + +func newDartMemStore() *dartMemStore { return &dartMemStore{data: map[string][]byte{}} } + +func (s *dartMemStore) Write(_ context.Context, key string, data []byte) error { + s.mu.Lock() + defer s.mu.Unlock() + s.data[key] = data + return nil +} + +func (s *dartMemStore) Read(_ context.Context, key string) ([]byte, error) { + s.mu.Lock() + defer s.mu.Unlock() + d, ok := s.data[key] + if !ok { + return nil, storage.ErrNotFound + } + return d, nil +} + +func (s *dartMemStore) Delete(_ context.Context, key string) error { + s.mu.Lock() + defer s.mu.Unlock() + delete(s.data, key) + return nil +} + +func readDartFixture(t *testing.T, name string) []byte { + t.Helper() + b, err := os.ReadFile(filepath.Join(dartFixtureDir, name)) + if err != nil { + t.Fatal(err) + } + return b +} + +var frameLineRe = regexp.MustCompile(`^#\d+\s+(.*)$`) + +func resolvedFrameLines(out string) []string { + var lines []string + for _, ln := range strings.Split(out, "\n") { + if m := frameLineRe.FindStringSubmatch(ln); m != nil { + lines = append(lines, strings.TrimSpace(m[1])) + } + } + return lines +} + +func expectedLines(b []byte) []string { + var out []string + for _, ln := range strings.Split(string(b), "\n") { + if strings.TrimSpace(ln) != "" { + out = append(out, strings.TrimSpace(ln)) + } + } + return out +} + +func TestResolveDartStackTraceArchFallback(t *testing.T) { + useMemCache(t) + prev := storage.Store + defer func() { storage.Store = prev }() + store := newDartMemStore() + storage.Store = store + + projectId := uuid.New() + symbols := readDartFixture(t, "app.darwin-arm64.symbols") + + store.data[DartSymbolsKey(projectId, fixtureBuildID, "arm64")] = symbols + + raw := string(readDartFixture(t, "trace.txt")) + out := ResolveDartStackTrace(context.Background(), projectId, raw) + + got := resolvedFrameLines(out) + want := expectedLines(readDartFixture(t, "expected.txt")) + if len(got) != len(want) { + t.Fatalf("frame count: got %d want %d\nout:\n%s", len(got), len(want), out) + } + for i := range want { + if got[i] != want[i] { + t.Errorf("frame %d: got %q want %q", i, got[i], want[i]) + } + } + if !strings.Contains(out, "chargeCard (") || !strings.Contains(out, "main.dart:20:3") { + t.Errorf("missing chargeCard frame in:\n%s", out) + } +} + +func TestResolveDartStackTraceNoSymbolsStable(t *testing.T) { + useMemCache(t) + prev := storage.Store + defer func() { storage.Store = prev }() + storage.Store = newDartMemStore() + + raw := string(readDartFixture(t, "trace.txt")) + + rawShifted := shiftAbsAddresses(raw) + if rawShifted == raw { + t.Fatal("test setup: abs addresses were not shifted") + } + + pid := uuid.New() + out1 := ResolveDartStackTrace(context.Background(), pid, raw) + out2 := ResolveDartStackTrace(context.Background(), uuid.New(), rawShifted) + + if out1 != out2 { + t.Errorf("same crash, different abs -> different output:\n--- out1 ---\n%s\n--- out2 ---\n%s", out1, out2) + } + if strings.Contains(out1, "build_id:") || strings.Contains(out1, "abs ") { + t.Errorf("volatile header/abs leaked into normalized output:\n%s", out1) + } + if !strings.Contains(out1, "_kDartIsolateSnapshotInstructions+") { + t.Errorf("expected stable offset frames in:\n%s", out1) + } +} + +func shiftAbsAddresses(raw string) string { + re := regexp.MustCompile(`abs [0-9a-fA-F]+`) + return re.ReplaceAllString(raw, "abs 00000007abcdef00") +} + +func TestDartSymbolsColdLoadsFromFlatArtifact(t *testing.T) { + useMemCache(t) + prev := storage.Store + defer func() { storage.Store = prev }() + store := newDartMemStore() + storage.Store = store + + projectId := uuid.New() + archKey := DartSymbolsKey(projectId, fixtureBuildID, "arm64") + symbols := readDartFixture(t, "app.darwin-arm64.symbols") + store.data[archKey] = symbols + raw := string(readDartFixture(t, "trace.txt")) + + if out := ResolveDartStackTrace(context.Background(), projectId, raw); !strings.Contains(out, "chargeCard (") { + t.Fatalf("first resolve did not symbolicate:\n%s", out) + } + twKey := dartFlatKey(archKey) + store.mu.Lock() + flat, hasFlat := store.data[twKey] + store.mu.Unlock() + if !hasFlat { + t.Fatalf("expected a flat artifact at %s", twKey) + } + t.Logf("flat artifact %d bytes vs symbols %d bytes", len(flat), len(symbols)) + + sharedCache = newSymbolicatorCache() + store.mu.Lock() + delete(store.data, archKey) + store.mu.Unlock() + + out := ResolveDartStackTrace(context.Background(), projectId, raw) + if !strings.Contains(out, "chargeCard (") || !strings.Contains(out, "main.dart:20:3") { + t.Errorf("cold load from .tw failed:\n%s", out) + } +} + +func TestDartSymbolsLocalDiskTier(t *testing.T) { + prev := storage.Store + defer func() { storage.Store = prev }() + store := newDartMemStore() + storage.Store = store + + dir := t.TempDir() + useDiskCache(t, dir, 64<<20) + + projectId := uuid.New() + archKey := DartSymbolsKey(projectId, fixtureBuildID, "arm64") + store.data[archKey] = readDartFixture(t, "app.darwin-arm64.symbols") + raw := string(readDartFixture(t, "trace.txt")) + + if out := ResolveDartStackTrace(context.Background(), projectId, raw); !strings.Contains(out, "chargeCard (") { + t.Fatalf("first resolve failed:\n%s", out) + } + + useDiskCache(t, dir, 64<<20) + + store.mu.Lock() + delete(store.data, archKey) + delete(store.data, dartFlatKey(archKey)) + store.mu.Unlock() + + out := ResolveDartStackTrace(context.Background(), projectId, raw) + if !strings.Contains(out, "chargeCard (") || !strings.Contains(out, "main.dart:20:3") { + t.Errorf("local disk tier did not serve the artifact:\n%s", out) + } +} + +func TestDartSymbolsKeyMatchesAcrossForms(t *testing.T) { + pid := uuid.New() + machoUUID := "FE664295-9971-35E7-B67B-648BA66CA9EB" + headerID := "fe664295997135e7b67b648ba66ca9eb" + if DartSymbolsKey(pid, machoUUID, "arm64") != DartSymbolsKey(pid, headerID, "arm64") { + t.Fatal("Mach-O UUID and trace-header id must produce the same key") + } + if DartSymbolsKey(pid, headerID, "x86_64") != DartSymbolsKey(pid, headerID, "x64") { + t.Error("arch synonyms should collapse to the same key") + } +} + +func TestResolveDartStackTraceInvalidationParity(t *testing.T) { + useMemCache(t) + prev := storage.Store + defer func() { storage.Store = prev }() + store := newDartMemStore() + storage.Store = store + + projectId := uuid.New() + raw := string(readDartFixture(t, "trace.txt")) + + if out := ResolveDartStackTrace(context.Background(), projectId, raw); strings.Contains(out, "chargeCard (") { + t.Fatal("did not expect symbolication before upload") + } + + archKey := DartSymbolsKey(projectId, fixtureBuildID, "arm64") + store.data[archKey] = readDartFixture(t, "app.darwin-arm64.symbols") + InvalidateDartSymbols(archKey) + + out := ResolveDartStackTrace(context.Background(), projectId, raw) + if !strings.Contains(out, "chargeCard (") { + t.Errorf("symbols not picked up after invalidation:\n%s", out) + } +} diff --git a/backend/app/services/jwt.service.go b/backend/app/services/jwt.service.go index a394499b..a17bd954 100644 --- a/backend/app/services/jwt.service.go +++ b/backend/app/services/jwt.service.go @@ -1,9 +1,9 @@ package services import ( - "github.com/tracewayapp/traceway/backend/app/config" "errors" "fmt" + "github.com/tracewayapp/traceway/backend/app/config" "time" "github.com/golang-jwt/jwt/v5" @@ -34,7 +34,7 @@ func GenerateToken(userId int, email string) (string, error) { UserId: userId, Email: email, RegisteredClaims: jwt.RegisteredClaims{ - ExpiresAt: jwt.NewNumericDate(time.Now().Add(7 * 24 * time.Hour)), // 7 days + ExpiresAt: jwt.NewNumericDate(time.Now().Add(7 * 24 * time.Hour)), IssuedAt: jwt.NewNumericDate(time.Now()), NotBefore: jwt.NewNumericDate(time.Now()), }, diff --git a/backend/app/services/sourcemap_cache_test.go b/backend/app/services/sourcemap_cache_test.go index 680fbd3f..d7b4a962 100644 --- a/backend/app/services/sourcemap_cache_test.go +++ b/backend/app/services/sourcemap_cache_test.go @@ -1,7 +1,6 @@ package services import ( - "container/list" "context" "errors" "fmt" @@ -11,36 +10,10 @@ import ( "time" "github.com/tracewayapp/traceway/backend/app/storage" - "github.com/tracewayapp/traceway/backend/app/symbolicator" "github.com/google/uuid" ) -func newTestSourceMapCache(maxEntries int, maxBytes int64) *sourceMapCache { - return &sourceMapCache{ - items: make(map[string]*list.Element), - order: list.New(), - loading: make(map[string]*resolverLoad), - negative: make(map[string]*negativeEntry), - maxEntries: maxEntries, - maxBytes: maxBytes, - } -} - -func storageResolverBuild(key string) resolverBuild { - return func(ctx context.Context) (*symbolicator.Resolver, int64, error) { - data, err := storage.Store.Read(ctx, key) - if err != nil { - return nil, 0, err - } - r, err := symbolicator.NewResolver(data, nil) - if err != nil { - return nil, 0, err - } - return r, int64(len(data)), nil - } -} - type countingStorage struct { mu sync.Mutex reads map[string]int @@ -73,222 +46,8 @@ func (c *countingStorage) Read(_ context.Context, key string) ([]byte, error) { return d, nil } -func TestSourceMapCacheSingleflight(t *testing.T) { - prev := storage.Store - defer func() { storage.Store = prev }() - cs := &countingStorage{reads: map[string]int{}, data: map[string][]byte{ - "singleflight-test.js.map": []byte(`{"version":3,"sources":["a.js"],"names":[],"mappings":"AAAA"}`), - }} - storage.Store = cs - c := newTestSourceMapCache(10, 1<<20) - - const n = 16 - results := make([]*symbolicator.Resolver, n) - var wg sync.WaitGroup - start := make(chan struct{}) - for i := range n { - wg.Go(func() { - <-start - r, err := c.getOrBuild(context.Background(), "singleflight-test.js.map", storageResolverBuild("singleflight-test.js.map")) - if err != nil { - t.Error(err) - return - } - results[i] = r - }) - } - close(start) - wg.Wait() - - if got := cs.reads["singleflight-test.js.map"]; got != 1 { - t.Errorf("expected 1 storage read for concurrent lookups, got %d", got) - } - for i := 1; i < n; i++ { - if results[i] != results[0] { - t.Fatal("expected all callers to share a single resolver instance") - } - } - - if _, err := c.getOrBuild(context.Background(), "singleflight-test.js.map", storageResolverBuild("singleflight-test.js.map")); err != nil { - t.Fatal(err) - } - if got := cs.reads["singleflight-test.js.map"]; got != 1 { - t.Errorf("expected cached lookup to not hit storage, got %d reads", got) - } -} - -func TestSourceMapCacheDistinctKeysConcurrent(t *testing.T) { - prev := storage.Store - defer func() { storage.Store = prev }() - cs := &countingStorage{reads: map[string]int{}, data: map[string][]byte{}} - const keys = 8 - for i := range keys { - cs.data[fmt.Sprintf("distinct-%d.js.map", i)] = []byte(`{"version":3,"sources":["a.js"],"names":[],"mappings":"AAAA"}`) - } - storage.Store = cs - c := newTestSourceMapCache(keys, 1<<20) - - var wg sync.WaitGroup - start := make(chan struct{}) - results := make([]*symbolicator.Resolver, keys*4) - for i := range keys * 4 { - wg.Go(func() { - <-start - key := fmt.Sprintf("distinct-%d.js.map", i%keys) - r, err := c.getOrBuild(context.Background(), key, storageResolverBuild(key)) - if err != nil { - t.Error(err) - return - } - results[i] = r - }) - } - close(start) - wg.Wait() - - for i := range keys { - key := fmt.Sprintf("distinct-%d.js.map", i) - if got := cs.reads[key]; got != 1 { - t.Errorf("key %s: expected 1 storage read, got %d", key, got) - } - } - for i := range keys * 4 { - if results[i] != results[i%keys] { - t.Fatal("callers of the same key should share one instance") - } - } -} - -type flakyStorage struct { - mu sync.Mutex - reads int - failures int - data []byte -} - -func (f *flakyStorage) Write(_ context.Context, _ string, _ []byte) error { return nil } - -func (f *flakyStorage) Delete(_ context.Context, _ string) error { return nil } - -func (f *flakyStorage) Read(_ context.Context, _ string) ([]byte, error) { - f.mu.Lock() - defer f.mu.Unlock() - f.reads++ - if f.reads <= f.failures { - return nil, errors.New("storage down") - } - return f.data, nil -} - -func TestSourceMapCacheFailedLoadRetries(t *testing.T) { - prev := storage.Store - defer func() { storage.Store = prev }() - fs := &flakyStorage{failures: 1, data: []byte(`{"version":3,"sources":["a.js"],"names":[],"mappings":"AAAA"}`)} - storage.Store = fs - c := newTestSourceMapCache(10, 1<<20) - - if _, err := c.getOrBuild(context.Background(), "flaky.js.map", storageResolverBuild("flaky.js.map")); err == nil { - t.Fatal("expected first load to fail") - } - c.mu.Lock() - if c.failures != 1 { - t.Errorf("expected 1 recorded failure, got %d", c.failures) - } - c.mu.Unlock() - r, err := c.getOrBuild(context.Background(), "flaky.js.map", storageResolverBuild("flaky.js.map")) - if err != nil || r == nil { - t.Fatalf("expected retry to succeed, got %v", err) - } - if fs.reads != 2 { - t.Errorf("expected 2 storage reads (fail then retry), got %d", fs.reads) - } - if _, err := c.getOrBuild(context.Background(), "flaky.js.map", storageResolverBuild("flaky.js.map")); err != nil { - t.Fatal(err) - } - if fs.reads != 2 { - t.Errorf("expected cached lookup after retry, got %d reads", fs.reads) - } -} - -type blockingPanicStorage struct { - release chan struct{} -} - -func (b *blockingPanicStorage) Write(_ context.Context, _ string, _ []byte) error { return nil } - -func (b *blockingPanicStorage) Delete(_ context.Context, _ string) error { return nil } - -func (b *blockingPanicStorage) Read(_ context.Context, _ string) ([]byte, error) { - <-b.release - panic("storage exploded") -} - -func TestSourceMapCachePanicRecovery(t *testing.T) { - prev := storage.Store - defer func() { storage.Store = prev }() - release := make(chan struct{}) - storage.Store = &blockingPanicStorage{release: release} - c := newTestSourceMapCache(10, 1<<20) - - type result struct { - resolver *symbolicator.Resolver - err error - } - leader := make(chan result, 1) - go func() { - r, err := c.getOrBuild(context.Background(), "boom.js.map", storageResolverBuild("boom.js.map")) - leader <- result{r, err} - }() - - deadline := time.Now().Add(2 * time.Second) - for { - c.mu.Lock() - _, loading := c.loading["boom.js.map"] - c.mu.Unlock() - if loading { - break - } - if time.Now().After(deadline) { - t.Fatal("load never started") - } - time.Sleep(time.Millisecond) - } - - waiters := make(chan result, 2) - for range 2 { - go func() { - r, err := c.getOrBuild(context.Background(), "boom.js.map", storageResolverBuild("boom.js.map")) - waiters <- result{r, err} - }() - } - - close(release) - - r := <-leader - if r.err == nil || r.resolver != nil { - t.Fatalf("expected leader to receive panic error, got resolver=%v err=%v", r.resolver, r.err) - } - for range 2 { - r := <-waiters - if r.err == nil || r.resolver != nil { - t.Fatalf("expected waiter to receive an error, got resolver=%v err=%v", r.resolver, r.err) - } - } - - c.mu.Lock() - defer c.mu.Unlock() - if len(c.loading) != 0 { - t.Error("loading entry leaked after panic") - } - if c.order.Len() != 0 { - t.Error("nothing should be cached after a failed load") - } - if c.failures == 0 { - t.Error("expected at least one recorded failure") - } -} - func TestResolveStackTraceFailedMapAttemptedOncePerTrace(t *testing.T) { + useMemCache(t) prev := storage.Store defer func() { storage.Store = prev }() cs := &countingStorage{reads: map[string]int{}, data: map[string][]byte{}} @@ -307,52 +66,26 @@ func TestResolveStackTraceFailedMapAttemptedOncePerTrace(t *testing.T) { } } -func TestSourceMapNegativeBackoffEscalates(t *testing.T) { - c := newTestSourceMapCache(10, 1<<20) - - c.mu.Lock() - c.markNegativeLocked("k", sourceMapNegativeBaseTTL) - first := time.Until(c.negative["k"].expiresAt) - c.mu.Unlock() - if first > sourceMapNegativeBaseTTL || first < sourceMapNegativeBaseTTL-time.Second { - t.Errorf("first failure should use the base TTL, got %v", first) - } - - c.mu.Lock() - c.markNegativeLocked("k", sourceMapNegativeBaseTTL) - second := time.Until(c.negative["k"].expiresAt) - for range 20 { - c.markNegativeLocked("k", sourceMapNegativeBaseTTL) - } - capped := time.Until(c.negative["k"].expiresAt) - c.mu.Unlock() - - if second < first { - t.Errorf("TTL should escalate on repeat failures: first %v, second %v", first, second) - } - if capped > sourceMapNegativeMaxTTL || capped < sourceMapNegativeMaxTTL-time.Second { - t.Errorf("TTL should cap at %v, got %v", sourceMapNegativeMaxTTL, capped) - } +type flakyStorage struct { + mu sync.Mutex + reads int } -func TestSourceMapNegativeMapPrunesAtCap(t *testing.T) { - c := newTestSourceMapCache(10, 1<<20) - c.mu.Lock() - for i := range sourceMapNegativeMaxKeys { - c.markNegativeLocked(fmt.Sprintf("k-%d", i), sourceMapNegativeBaseTTL) - } - c.markNegativeLocked("one-more", sourceMapNegativeBaseTTL) - size := len(c.negative) - c.mu.Unlock() - if size > sourceMapNegativeMaxKeys { - t.Errorf("negative map should stay at or below %d keys, got %d", sourceMapNegativeMaxKeys, size) - } +func (f *flakyStorage) Write(_ context.Context, _ string, _ []byte) error { return nil } +func (f *flakyStorage) Delete(_ context.Context, _ string) error { return nil } + +func (f *flakyStorage) Read(_ context.Context, _ string) ([]byte, error) { + f.mu.Lock() + defer f.mu.Unlock() + f.reads++ + return nil, errors.New("storage down") } func TestResolveStackTraceTransientFailureNegativeCached(t *testing.T) { + useMemCache(t) prev := storage.Store defer func() { storage.Store = prev }() - fs := &flakyStorage{failures: 1 << 30} + fs := &flakyStorage{} storage.Store = fs projectId := uuid.New() @@ -368,7 +101,7 @@ func TestResolveStackTraceTransientFailureNegativeCached(t *testing.T) { } func TestInvalidateSourceMapClearsNegative(t *testing.T) { - InitSourceMapCache(100, 64<<20) + useMemCache(t) prev := storage.Store defer func() { storage.Store = prev }() cs := &countingStorage{reads: map[string]int{}, data: map[string][]byte{}} @@ -392,7 +125,7 @@ func TestInvalidateSourceMapClearsNegative(t *testing.T) { } func TestInvalidateSourceMapEvictsStaleResolver(t *testing.T) { - InitSourceMapCache(100, 64<<20) + useMemCache(t) prev := storage.Store defer func() { storage.Store = prev }() cs := &countingStorage{reads: map[string]int{}, data: map[string][]byte{}} @@ -414,7 +147,7 @@ func TestInvalidateSourceMapEvictsStaleResolver(t *testing.T) { GenerateTWArtifacts(context.Background(), projectId, []string{"stable.js.map"}) if got := ResolveStackTrace(context.Background(), projectId, trace, nil); !strings.Contains(got, "b.js:1:1") { - t.Errorf("upload must evict both the cached resolver and the stale tw artifact, got %q", got) + t.Errorf("upload must evict both the cached artifact and the stale tw, got %q", got) } } @@ -431,7 +164,7 @@ func (b *bundleFailStorage) Write(_ context.Context, key string, data []byte) er } func (b *bundleFailStorage) Read(_ context.Context, key string) ([]byte, error) { - if b.failBundle && !strings.HasSuffix(key, ".map") { + if b.failBundle && !strings.HasSuffix(key, ".map") && !strings.HasSuffix(key, ".tw") { return nil, errors.New("storage down") } d, ok := b.data[key] @@ -442,7 +175,7 @@ func (b *bundleFailStorage) Read(_ context.Context, key string) ([]byte, error) } func TestTransientBundleReadFailureFailsBuild(t *testing.T) { - InitSourceMapCache(100, 64<<20) + useMemCache(t) prev := storage.Store defer func() { storage.Store = prev }() projectId := uuid.New() @@ -456,7 +189,7 @@ func TestTransientBundleReadFailureFailsBuild(t *testing.T) { trace := "Error: boom\n fn()\n app.js:1:1" if got := ResolveStackTrace(context.Background(), projectId, trace, nil); got != trace { - t.Errorf("a transient bundle read failure must not cache a names-less resolver, got %q", got) + t.Errorf("a transient bundle read failure must not cache a names-less artifact, got %q", got) } bs.failBundle = false @@ -466,39 +199,3 @@ func TestTransientBundleReadFailureFailsBuild(t *testing.T) { t.Errorf("expected full resolution once the bundle read recovers, got %q", got) } } - -func TestSourceMapCacheByteCapEviction(t *testing.T) { - prev := storage.Store - defer func() { storage.Store = prev }() - cs := &countingStorage{reads: map[string]int{}, data: map[string][]byte{}} - storage.Store = cs - - c := newTestSourceMapCache(10, 2000) - - content := make([]byte, 700) - for i := range content { - content[i] = 'x' - } - for i := range 3 { - key := fmt.Sprintf("evict-%d.js.map", i) - cs.data[key] = fmt.Appendf(nil, `{"version":3,"sources":["a.js"],"sourcesContent":[%q],"names":[],"mappings":"AAAA"}`, content) - if _, err := c.getOrBuild(context.Background(), key, storageResolverBuild(key)); err != nil { - t.Fatal(err) - } - } - - c.mu.Lock() - defer c.mu.Unlock() - if c.curBytes > c.maxBytes { - t.Errorf("curBytes %d exceeds maxBytes %d", c.curBytes, c.maxBytes) - } - if c.evictions == 0 { - t.Error("expected at least one eviction") - } - if _, ok := c.items["evict-2.js.map"]; !ok { - t.Error("most recent entry should still be cached") - } - if _, ok := c.items["evict-0.js.map"]; ok { - t.Error("oldest entry should have been evicted") - } -} diff --git a/backend/app/services/sourcemap_debugid_test.go b/backend/app/services/sourcemap_debugid_test.go index 47d09d54..cca4fd33 100644 --- a/backend/app/services/sourcemap_debugid_test.go +++ b/backend/app/services/sourcemap_debugid_test.go @@ -39,7 +39,7 @@ func TestExtractDebugIdFromBundle(t *testing.T) { } func TestResolveStackTracePrefersDebugId(t *testing.T) { - InitSourceMapCache(100, 64<<20) + useMemCache(t) prev := storage.Store defer func() { storage.Store = prev }() cs := &countingStorage{reads: map[string]int{}, data: map[string][]byte{}} @@ -67,7 +67,7 @@ func TestResolveStackTracePrefersDebugId(t *testing.T) { } func TestResolveStackTraceFallsBackToFilenameWhenDebugIdMissing(t *testing.T) { - InitSourceMapCache(100, 64<<20) + useMemCache(t) prev := storage.Store defer func() { storage.Store = prev }() cs := &countingStorage{reads: map[string]int{}, data: map[string][]byte{}} @@ -88,21 +88,29 @@ func TestResolveStackTraceFallsBackToFilenameWhenDebugIdMissing(t *testing.T) { } func TestInvalidateSourceMapKeepsDebugIdDir(t *testing.T) { - InitSourceMapCache(100, 64<<20) + useMemCache(t) + prev := storage.Store + defer func() { storage.Store = prev }() + storage.Store = &countingStorage{reads: map[string]int{}, data: map[string][]byte{}} + projectId := uuid.New() debugId := "85314830-023f-4cf1-a267-535f4e37bb17" - - smCache.mu.Lock() - key := SourceMapStorageKey(projectId, DebugIdMapName(debugId)) - smCache.negative[key] = &negativeEntry{} - smCache.mu.Unlock() + mapKey := SourceMapStorageKey(projectId, DebugIdMapName(debugId)) + bundleKey := SourceMapStorageKey(projectId, DebugIdBundleName(debugId)) + twKey := twKeyFor(mapKey) + + _, done, err := sharedCache.Get(context.Background(), twKey, loadSourceMapBlob(mapKey, bundleKey)) + done() + if err == nil { + t.Fatal("expected the missing map to fail to load") + } + if !sharedCache.IsNegative(twKey) { + t.Fatal("expected a negative entry after the failed load") + } InvalidateSourceMap(projectId, DebugIdBundleName(debugId)) - smCache.mu.Lock() - _, stillThere := smCache.negative[key] - smCache.mu.Unlock() - if stillThere { + if sharedCache.IsNegative(twKey) { t.Error("expected debug id map key to be invalidated") } } diff --git a/backend/app/services/sourcemap_disk_cache.go b/backend/app/services/sourcemap_disk_cache.go deleted file mode 100644 index 33dd5c6b..00000000 --- a/backend/app/services/sourcemap_disk_cache.go +++ /dev/null @@ -1,77 +0,0 @@ -package services - -import ( - "context" - "fmt" - "strings" - - "github.com/tracewayapp/traceway/backend/app/symbolicator" - "github.com/tracewayapp/traceway/backend/app/symbolicator/twcache" - - traceway "go.tracewayapp.com" -) - -type sourceMapDiskCache struct { - mem *sourceMapCache - disk *twcache.Cache -} - -func EnableSourceMapDiskCache(dir string, maxBytes int64) error { - disk, err := twcache.New(dir, maxBytes, func(err error) { - traceway.CaptureException(err) - }) - if err != nil { - return fmt.Errorf("source map disk cache: %w", err) - } - activeSMCache = &sourceMapDiskCache{mem: smCache, disk: disk} - return nil -} - -func twKeyFor(mapKey string) string { - return strings.TrimSuffix(mapKey, ".map") + ".tw" -} - -func (d *sourceMapDiskCache) getOrBuild(ctx context.Context, key string, build resolverBuild) (*symbolicator.Resolver, error) { - return d.mem.getOrBuild(ctx, key, func(ctx context.Context) (*symbolicator.Resolver, int64, error) { - return d.load(ctx, key, build) - }) -} - -func (d *sourceMapDiskCache) load(ctx context.Context, mapKey string, build resolverBuild) (*symbolicator.Resolver, int64, error) { - name := twKeyFor(mapKey) - if r, err := d.disk.Open(name); err == nil { - return r, r.ApproxSize(), nil - } - - resolver, size, err := build(ctx) - if err != nil { - return nil, 0, err - } - if r, werr := d.disk.Write(name, resolver.MarshalTW()); werr == nil { - return r, r.ApproxSize(), nil - } else { - traceway.CaptureException(fmt.Errorf("failed to write tw cache file (key=%s): %w", mapKey, werr)) - } - return resolver, size, nil -} - -func (d *sourceMapDiskCache) isNegative(key string) bool { - return d.mem.isNegative(key) -} - -func (d *sourceMapDiskCache) invalidate(key string) { - d.mem.invalidate(key) - d.disk.Remove(twKeyFor(key)) -} - -func (d *sourceMapDiskCache) stats() SourceMapCacheStats { - s := d.mem.stats() - ds := d.disk.Stats() - s.DiskEnabled = true - s.DiskEntries = ds.Entries - s.DiskBytes = ds.Bytes - s.DiskMaxBytes = ds.MaxBytes - s.DiskHits = ds.Hits - s.DiskEvictions = ds.Evictions - return s -} diff --git a/backend/app/services/sourcemap_disk_cache_test.go b/backend/app/services/sourcemap_disk_cache_test.go index 20ed749c..380e8397 100644 --- a/backend/app/services/sourcemap_disk_cache_test.go +++ b/backend/app/services/sourcemap_disk_cache_test.go @@ -9,31 +9,42 @@ import ( "testing" "github.com/tracewayapp/traceway/backend/app/storage" - "github.com/tracewayapp/traceway/backend/app/symbolicator" + "github.com/tracewayapp/traceway/backend/app/symbolicator/sourcemap" "github.com/tracewayapp/traceway/backend/app/symbolicator/twcache" "github.com/google/uuid" ) -func newTestDiskCache(t *testing.T, maxBytes int64) *sourceMapDiskCache { +func useMemCache(t *testing.T) { t.Helper() - return newTestDiskCacheAt(t, t.TempDir(), maxBytes) + prev := sharedCache + sharedCache = newSymbolicatorCache() + t.Cleanup(func() { sharedCache = prev }) } -func newTestDiskCacheAt(t *testing.T, dir string, maxBytes int64) *sourceMapDiskCache { +func useDiskCache(t *testing.T, dir string, maxBytes int64) *twcache.Cache { t.Helper() - disk, err := twcache.New(dir, maxBytes, nil) - if err != nil { - t.Fatalf("twcache.New: %v", err) + prev := sharedCache + if err := EnableSymbolicatorDiskCache(dir, maxBytes); err != nil { + t.Fatalf("EnableSymbolicatorDiskCache: %v", err) } - return &sourceMapDiskCache{ - mem: newTestSourceMapCache(100, 64<<20), - disk: disk, + t.Cleanup(func() { sharedCache = prev }) + return sharedCache +} + +func resolveSMFrame(t *testing.T, ctx context.Context, mapKey, bundleKey string) (sourcemap.StackTraceFrame, bool, error) { + t.Helper() + data, done, err := sharedCache.Get(ctx, twKeyFor(mapKey), loadSourceMapBlob(mapKey, bundleKey)) + if err != nil { + return sourcemap.StackTraceFrame{}, false, err } + defer done() + frame, ok := sourcemap.LookupTW(data, 0, 10) + return frame, ok, nil } -func twPathFor(d *sourceMapDiskCache, mapKey string) string { - return filepath.Join(d.disk.Dir(), filepath.FromSlash(twKeyFor(mapKey))) +func twPathFor(disk *twcache.Cache, mapKey string) string { + return filepath.Join(disk.Dir(), filepath.FromSlash(twKeyFor(mapKey))) } func swapStorage(t *testing.T) *countingStorage { @@ -47,9 +58,8 @@ func swapStorage(t *testing.T) *countingStorage { return cs } -func assertSimpleLookup(t *testing.T, r *symbolicator.Resolver) { +func assertSimpleLookup(t *testing.T, frame sourcemap.StackTraceFrame, ok bool) { t.Helper() - frame, ok := r.Lookup(0, 10) if !ok { t.Fatal("expected lookup to resolve") } @@ -60,7 +70,7 @@ func assertSimpleLookup(t *testing.T, r *symbolicator.Resolver) { func TestDiskCacheBuildsThenServesFromLocalFile(t *testing.T) { cs := swapStorage(t) - d := newTestDiskCache(t, 64<<20) + disk := useDiskCache(t, t.TempDir(), 64<<20) projectId := uuid.New() prefix := fmt.Sprintf("sourcemaps/%s/", projectId) @@ -70,28 +80,27 @@ func TestDiskCacheBuildsThenServesFromLocalFile(t *testing.T) { mapKey := prefix + "minified.js.map" bundleKey := prefix + "minified.js" - r, err := d.getOrBuild(context.Background(), mapKey, buildResolver(mapKey, bundleKey)) + frame, ok, err := resolveSMFrame(t, context.Background(), mapKey, bundleKey) if err != nil { - t.Fatalf("getOrBuild: %v", err) + t.Fatalf("resolve: %v", err) } - assertSimpleLookup(t, r) + assertSimpleLookup(t, frame, ok) - if _, err := os.Stat(twPathFor(d, mapKey)); err != nil { + if _, err := os.Stat(twPathFor(disk, mapKey)); err != nil { t.Fatalf("expected tw file on disk: %v", err) } if smBuilds.Load() != 1 { t.Fatalf("builds: got %d, want 1", smBuilds.Load()) } - restarted := newTestDiskCacheAt(t, d.disk.Dir(), 64<<20) - + restarted := useDiskCache(t, disk.Dir(), 64<<20) mapReads := cs.reads[mapKey] - r2, err := restarted.getOrBuild(context.Background(), mapKey, buildResolver(mapKey, bundleKey)) + frame2, ok2, err := resolveSMFrame(t, context.Background(), mapKey, bundleKey) if err != nil { - t.Fatalf("getOrBuild after restart: %v", err) + t.Fatalf("resolve after restart: %v", err) } - assertSimpleLookup(t, r2) - if hits := restarted.disk.Stats().Hits; hits != 1 { + assertSimpleLookup(t, frame2, ok2) + if hits := restarted.Stats().Hits; hits != 1 { t.Fatalf("disk hits: got %d, want 1", hits) } if cs.reads[mapKey] != mapReads { @@ -101,7 +110,7 @@ func TestDiskCacheBuildsThenServesFromLocalFile(t *testing.T) { func TestDiskCachePullsTWFromStorage(t *testing.T) { cs := swapStorage(t) - d := newTestDiskCache(t, 64<<20) + disk := useDiskCache(t, t.TempDir(), 64<<20) projectId := uuid.New() prefix := fmt.Sprintf("sourcemaps/%s/", projectId) @@ -116,17 +125,17 @@ func TestDiskCachePullsTWFromStorage(t *testing.T) { if err != nil { t.Fatal(err) } - resolver, err := symbolicator.NewResolver(mapBytes, bundleBytes) + tw, err := sourcemap.BuildTW(mapBytes, bundleBytes) if err != nil { t.Fatal(err) } - cs.data[prefix+"minified.js.tw"] = resolver.MarshalTW() + cs.data[prefix+"minified.js.tw"] = tw - r, err := d.getOrBuild(context.Background(), mapKey, buildResolver(mapKey, bundleKey)) + frame, ok, err := resolveSMFrame(t, context.Background(), mapKey, bundleKey) if err != nil { - t.Fatalf("getOrBuild: %v", err) + t.Fatalf("resolve: %v", err) } - assertSimpleLookup(t, r) + assertSimpleLookup(t, frame, ok) if smStoreHits.Load() != 1 { t.Fatalf("storeHits: got %d, want 1", smStoreHits.Load()) @@ -134,21 +143,21 @@ func TestDiskCachePullsTWFromStorage(t *testing.T) { if cs.reads[mapKey] != 0 || cs.reads[bundleKey] != 0 { t.Fatal("tw artifact in storage should make map and bundle reads unnecessary") } - if !fileExists(twPathFor(d, mapKey)) { + if !fileExists(twPathFor(disk, mapKey)) { t.Fatal("tw pulled from storage should be cached on local disk") } } func TestDiskCacheCorruptLocalFileFallsBack(t *testing.T) { cs := swapStorage(t) - d := newTestDiskCache(t, 64<<20) + disk := useDiskCache(t, t.TempDir(), 64<<20) projectId := uuid.New() prefix := fmt.Sprintf("sourcemaps/%s/", projectId) seedFixture(t, cs, prefix+"minified.js.map", "testdata/sourcemapcache/simple/minified.js.map") mapKey := prefix + "minified.js.map" - twPath := twPathFor(d, mapKey) + twPath := twPathFor(disk, mapKey) if err := os.MkdirAll(filepath.Dir(twPath), 0o755); err != nil { t.Fatal(err) } @@ -156,13 +165,14 @@ func TestDiskCacheCorruptLocalFileFallsBack(t *testing.T) { t.Fatal(err) } - r, err := d.getOrBuild(context.Background(), mapKey, buildResolver(mapKey, prefix+"minified.js")) + frame, ok, err := resolveSMFrame(t, context.Background(), mapKey, prefix+"minified.js") if err != nil { - t.Fatalf("getOrBuild: %v", err) + t.Fatalf("resolve: %v", err) } - if _, ok := r.Lookup(0, 10); !ok { + if !ok { t.Fatal("expected lookup to resolve after rebuilding from source map") } + _ = frame if smBuilds.Load() != 1 { t.Fatalf("builds: got %d, want 1", smBuilds.Load()) } @@ -170,8 +180,8 @@ func TestDiskCacheCorruptLocalFileFallsBack(t *testing.T) { if err != nil { t.Fatalf("expected regenerated tw file: %v", err) } - if _, err := symbolicator.OpenTW(data); err != nil { - t.Fatalf("regenerated tw file should be valid: %v", err) + if !sourcemap.ValidTW(data) { + t.Fatal("regenerated tw file should be valid") } } @@ -182,13 +192,13 @@ func TestDiskCacheCapacityEviction(t *testing.T) { if err != nil { t.Fatal(err) } - resolver, err := symbolicator.NewResolver(mapBytes, nil) + tw, err := sourcemap.BuildTW(mapBytes, nil) if err != nil { t.Fatal(err) } - twSize := int64(len(resolver.MarshalTW())) + twSize := int64(len(tw)) - d := newTestDiskCache(t, twSize+twSize/2) + disk := useDiskCache(t, t.TempDir(), twSize+twSize/2) projectId := uuid.New() prefix := fmt.Sprintf("sourcemaps/%s/", projectId) @@ -198,20 +208,20 @@ func TestDiskCacheCapacityEviction(t *testing.T) { firstKey := prefix + "first.js.map" secondKey := prefix + "second.js.map" - if _, err := d.getOrBuild(context.Background(), firstKey, buildResolver(firstKey, prefix+"first.js")); err != nil { - t.Fatalf("getOrBuild first: %v", err) + if _, _, err := resolveSMFrame(t, context.Background(), firstKey, prefix+"first.js"); err != nil { + t.Fatalf("resolve first: %v", err) } - if _, err := d.getOrBuild(context.Background(), secondKey, buildResolver(secondKey, prefix+"second.js")); err != nil { - t.Fatalf("getOrBuild second: %v", err) + if _, _, err := resolveSMFrame(t, context.Background(), secondKey, prefix+"second.js"); err != nil { + t.Fatalf("resolve second: %v", err) } - if fileExists(twPathFor(d, firstKey)) { + if fileExists(twPathFor(disk, firstKey)) { t.Fatal("oldest tw file should be evicted when over capacity") } - if !fileExists(twPathFor(d, secondKey)) { + if !fileExists(twPathFor(disk, secondKey)) { t.Fatal("newest tw file should survive eviction") } - stats := d.disk.Stats() + stats := disk.Stats() if stats.Evictions != 1 { t.Fatalf("disk evictions: got %d, want 1", stats.Evictions) } @@ -222,33 +232,33 @@ func TestDiskCacheCapacityEviction(t *testing.T) { func TestDiskCacheInvalidateRemovesFile(t *testing.T) { cs := swapStorage(t) - d := newTestDiskCache(t, 64<<20) + disk := useDiskCache(t, t.TempDir(), 64<<20) projectId := uuid.New() prefix := fmt.Sprintf("sourcemaps/%s/", projectId) seedFixture(t, cs, prefix+"minified.js.map", "testdata/sourcemapcache/simple/minified.js.map") mapKey := prefix + "minified.js.map" - if _, err := d.getOrBuild(context.Background(), mapKey, buildResolver(mapKey, prefix+"minified.js")); err != nil { - t.Fatalf("getOrBuild: %v", err) + if _, _, err := resolveSMFrame(t, context.Background(), mapKey, prefix+"minified.js"); err != nil { + t.Fatalf("resolve: %v", err) } - twPath := twPathFor(d, mapKey) + twPath := twPathFor(disk, mapKey) if !fileExists(twPath) { t.Fatal("expected tw file before invalidate") } - d.invalidate(mapKey) + InvalidateSourceMap(projectId, "minified.js.map") if fileExists(twPath) { t.Fatal("invalidate should remove the local tw file") } - if stats := d.disk.Stats(); stats.Entries != 0 || stats.Bytes != 0 { + if stats := disk.Stats(); stats.Entries != 0 || stats.Bytes != 0 { t.Fatalf("expected empty disk index, got %d entries / %d bytes", stats.Entries, stats.Bytes) } } func TestDiskCacheResolveStackTraceEndToEnd(t *testing.T) { cs := swapStorage(t) - swapActiveCache(t, newTestDiskCache(t, 64<<20)) + useDiskCache(t, t.TempDir(), 64<<20) projectId := uuid.New() prefix := fmt.Sprintf("sourcemaps/%s/", projectId) @@ -266,16 +276,9 @@ func TestDiskCacheResolveStackTraceEndToEnd(t *testing.T) { } } -func swapActiveCache(t *testing.T, c resolverCache) { - t.Helper() - prev := activeSMCache - activeSMCache = c - t.Cleanup(func() { activeSMCache = prev }) -} - func TestGenerateTWArtifacts(t *testing.T) { cs := swapStorage(t) - swapActiveCache(t, newTestDiskCache(t, 64<<20)) + useDiskCache(t, t.TempDir(), 64<<20) projectId := uuid.New() prefix := fmt.Sprintf("sourcemaps/%s/", projectId) @@ -288,16 +291,13 @@ func TestGenerateTWArtifacts(t *testing.T) { if !ok { t.Fatal("expected tw artifact in storage after generation") } - r, err := symbolicator.OpenTW(twBytes) - if err != nil { - t.Fatalf("OpenTW: %v", err) - } - assertSimpleLookup(t, r) + frame, lok := sourcemap.LookupTW(twBytes, 0, 10) + assertSimpleLookup(t, frame, lok) } func TestGenerateTWArtifactsDeletesStaleOnFailure(t *testing.T) { cs := swapStorage(t) - swapActiveCache(t, newTestDiskCache(t, 64<<20)) + useDiskCache(t, t.TempDir(), 64<<20) projectId := uuid.New() prefix := fmt.Sprintf("sourcemaps/%s/", projectId) @@ -325,7 +325,7 @@ func (s *twErrStorage) Read(ctx context.Context, key string) ([]byte, error) { func TestDiskCacheTransientTwErrorFallsBackToBuild(t *testing.T) { cs := swapStorage(t) storage.Store = &twErrStorage{countingStorage: cs} - d := newTestDiskCache(t, 64<<20) + useDiskCache(t, t.TempDir(), 64<<20) projectId := uuid.New() prefix := fmt.Sprintf("sourcemaps/%s/", projectId) @@ -333,11 +333,11 @@ func TestDiskCacheTransientTwErrorFallsBackToBuild(t *testing.T) { seedFixture(t, cs, prefix+"minified.js", "testdata/sourcemapcache/simple/minified.js") mapKey := prefix + "minified.js.map" - r, err := d.getOrBuild(context.Background(), mapKey, buildResolver(mapKey, prefix+"minified.js")) + frame, ok, err := resolveSMFrame(t, context.Background(), mapKey, prefix+"minified.js") if err != nil { - t.Fatalf("getOrBuild should fall back to building from the map, got: %v", err) + t.Fatalf("resolve should fall back to building from the map, got: %v", err) } - assertSimpleLookup(t, r) + assertSimpleLookup(t, frame, ok) if smBuilds.Load() != 1 { t.Fatalf("builds: got %d, want 1", smBuilds.Load()) } @@ -348,7 +348,7 @@ func TestDiskCacheTransientTwErrorFallsBackToBuild(t *testing.T) { func TestDiskCacheRefreshesCorruptStoreTw(t *testing.T) { cs := swapStorage(t) - d := newTestDiskCache(t, 64<<20) + useDiskCache(t, t.TempDir(), 64<<20) projectId := uuid.New() prefix := fmt.Sprintf("sourcemaps/%s/", projectId) @@ -356,17 +356,16 @@ func TestDiskCacheRefreshesCorruptStoreTw(t *testing.T) { cs.data[prefix+"minified.js.tw"] = []byte("corrupt artifact") mapKey := prefix + "minified.js.map" - r, err := d.getOrBuild(context.Background(), mapKey, buildResolver(mapKey, prefix+"minified.js")) + _, ok, err := resolveSMFrame(t, context.Background(), mapKey, prefix+"minified.js") if err != nil { - t.Fatalf("getOrBuild: %v", err) + t.Fatalf("resolve: %v", err) } - if _, ok := r.Lookup(0, 10); !ok { + if !ok { t.Fatal("expected lookup to resolve after rebuild") } - refreshed := cs.data[prefix+"minified.js.tw"] - if _, err := symbolicator.OpenTW(refreshed); err != nil { - t.Fatalf("corrupt storage tw should be replaced with a valid artifact: %v", err) + if !sourcemap.ValidTW(cs.data[prefix+"minified.js.tw"]) { + t.Fatal("corrupt storage tw should be replaced with a valid artifact") } } @@ -386,7 +385,7 @@ func TestDiskCacheScanSkipsUnreadableEntries(t *testing.T) { } t.Cleanup(func() { _ = os.Chmod(locked, 0o755) }) - if _, err := twcache.New(dir, 64<<20, nil); err != nil { + if _, err := twcache.NewDisk(dir, 64<<20, nil); err != nil { t.Fatalf("scan must tolerate unreadable entries, got: %v", err) } } diff --git a/backend/app/services/sourcemap_resolver.go b/backend/app/services/sourcemap_resolver.go index 580468e1..2815e4ea 100644 --- a/backend/app/services/sourcemap_resolver.go +++ b/backend/app/services/sourcemap_resolver.go @@ -1,7 +1,6 @@ package services import ( - "container/list" "context" "errors" "fmt" @@ -9,199 +8,21 @@ import ( "regexp" "strconv" "strings" - "sync" "sync/atomic" "time" "github.com/tracewayapp/traceway/backend/app/storage" - "github.com/tracewayapp/traceway/backend/app/symbolicator" + "github.com/tracewayapp/traceway/backend/app/symbolicator/sourcemap" + "github.com/tracewayapp/traceway/backend/app/symbolicator/twcache" "github.com/google/uuid" traceway "go.tracewayapp.com" ) const sourceMapLoadTimeout = 5 * time.Second -const sourceMapFailReportInterval = time.Minute -const sourceMapNegativeBaseTTL = time.Minute -const sourceMapTransientNegativeBaseTTL = 15 * time.Second -const sourceMapNegativeMaxTTL = 15 * time.Minute -const sourceMapNegativeMaxKeys = 10000 - -type resolverBuild func(context.Context) (*symbolicator.Resolver, int64, error) - -type resolverCache interface { - getOrBuild(ctx context.Context, key string, build resolverBuild) (*symbolicator.Resolver, error) - isNegative(key string) bool - invalidate(key string) - stats() SourceMapCacheStats -} - -type sourceMapCache struct { - mu sync.Mutex - items map[string]*list.Element - order *list.List - loading map[string]*resolverLoad - negative map[string]*negativeEntry - maxEntries int - maxBytes int64 - curBytes int64 - hits uint64 - misses uint64 - evictions uint64 - failures uint64 - notFound uint64 - negativeHits uint64 - lastParseMs float64 - failuresSinceReport uint64 - lastFailAt time.Time -} - -type negativeEntry struct { - expiresAt time.Time - failures uint32 -} - -type sourceMapCacheEntry struct { - key string - resolver *symbolicator.Resolver - size int64 -} - -type resolverLoad struct { - done chan struct{} - resolver *symbolicator.Resolver - err error -} - -var smCache = &sourceMapCache{ - items: make(map[string]*list.Element), - order: list.New(), - loading: make(map[string]*resolverLoad), - negative: make(map[string]*negativeEntry), - maxEntries: 200, - maxBytes: 500 << 20, -} - -var activeSMCache resolverCache = smCache func InitSourceMapCache(maxEntries int, maxBytes int64) { - smCache.mu.Lock() - defer smCache.mu.Unlock() - smCache.maxEntries = maxEntries - smCache.maxBytes = maxBytes - smCache.evictLocked() -} - -func (c *sourceMapCache) getOrBuild(ctx context.Context, key string, build resolverBuild) (resolver *symbolicator.Resolver, err error) { - c.mu.Lock() - if el, ok := c.items[key]; ok { - c.hits++ - c.order.MoveToFront(el) - cached := el.Value.(*sourceMapCacheEntry).resolver - c.mu.Unlock() - return cached, nil - } - if l, ok := c.loading[key]; ok { - c.mu.Unlock() - <-l.done - if l.err == nil { - c.mu.Lock() - c.hits++ - c.mu.Unlock() - } - return l.resolver, l.err - } - c.misses++ - l := &resolverLoad{done: make(chan struct{})} - c.loading[key] = l - c.mu.Unlock() - - var size int64 - var buildMs float64 - defer func() { - if r := recover(); r != nil { - l.resolver = nil - l.err = fmt.Errorf("source map resolver build panicked (key=%s): %v", key, r) - c.reportLoadFailure(l.err) - resolver, err = nil, l.err - } - c.mu.Lock() - delete(c.loading, key) - if l.err == nil && l.resolver != nil { - c.lastParseMs = buildMs - c.insertLocked(key, l.resolver, size) - delete(c.negative, key) - } else if l.err != nil { - if errors.Is(l.err, storage.ErrNotFound) { - c.notFound++ - c.markNegativeLocked(key, sourceMapNegativeBaseTTL) - } else { - c.failures++ - c.markNegativeLocked(key, sourceMapTransientNegativeBaseTTL) - } - } - c.mu.Unlock() - close(l.done) - }() - - start := time.Now() - l.resolver, size, l.err = build(ctx) - buildMs = float64(time.Since(start).Microseconds()) / 1000.0 - if l.err != nil && !errors.Is(l.err, storage.ErrNotFound) { - c.reportLoadFailure(fmt.Errorf("failed to build source map resolver (key=%s): %w", key, l.err)) - } - return l.resolver, l.err -} - -func (c *sourceMapCache) isNegative(key string) bool { - c.mu.Lock() - defer c.mu.Unlock() - e, ok := c.negative[key] - if !ok || time.Now().After(e.expiresAt) { - return false - } - c.negativeHits++ - return true -} - -func (c *sourceMapCache) markNegativeLocked(key string, base time.Duration) { - e := c.negative[key] - if e == nil { - if len(c.negative) >= sourceMapNegativeMaxKeys { - c.pruneNegativeLocked() - } - e = &negativeEntry{} - c.negative[key] = e - } - ttl := min(base<= sourceMapFailReportInterval { - report = c.failuresSinceReport - c.failuresSinceReport = 0 - c.lastFailAt = time.Now() - } - c.mu.Unlock() - if report > 0 { - traceway.CaptureException(fmt.Errorf("source map resolver builds failed %d time(s) since last report: %w", report, err)) - } -} - -func (c *sourceMapCache) insertLocked(key string, resolver *symbolicator.Resolver, size int64) { - el := c.order.PushFront(&sourceMapCacheEntry{key: key, resolver: resolver, size: size}) - c.items[key] = el - c.curBytes += size - c.evictLocked() -} - -func (c *sourceMapCache) evictLocked() { - for c.order.Len() > c.maxEntries || c.curBytes > c.maxBytes { - back := c.order.Back() - if back == nil { - break - } - evicted := c.order.Remove(back).(*sourceMapCacheEntry) - delete(c.items, evicted.key) - c.curBytes -= evicted.size - c.evictions++ - } + sharedCache.Invalidate(twKeyFor(SourceMapStorageKey(projectId, name))) } type SourceMapCacheStats struct { @@ -279,32 +65,41 @@ type SourceMapCacheStats struct { } func SourceMapStats() SourceMapCacheStats { - return activeSMCache.stats() -} - -func (c *sourceMapCache) stats() SourceMapCacheStats { - c.mu.Lock() - defer c.mu.Unlock() - return SourceMapCacheStats{ - Entries: c.order.Len(), - Bytes: c.curBytes, - MaxEntries: c.maxEntries, - MaxBytes: c.maxBytes, - Hits: c.hits, - Misses: c.misses, - Evictions: c.evictions, - Failures: c.failures, - NotFound: c.notFound, - NegativeHits: c.negativeHits, - NegativeEntries: len(c.negative), - LastParseMs: c.lastParseMs, + s := sharedCache.Stats() + out := SourceMapCacheStats{ + Entries: s.Entries, + Bytes: s.Bytes, + MaxEntries: s.MaxEntries, + MaxBytes: s.MaxBytes, + Hits: s.Hits, + Misses: s.Misses, + Evictions: s.Evictions, + Failures: s.Failures, + NotFound: s.NotFound, + NegativeHits: s.NegativeHits, + NegativeEntries: s.NegativeEntries, + LastParseMs: s.LastParseMs, StoreHits: smStoreHits.Load(), Builds: smBuilds.Load(), } + if s.Mode == "disk" { + out.DiskEnabled = true + out.DiskEntries = s.Entries + out.DiskBytes = s.Bytes + out.DiskMaxBytes = s.MaxBytes + out.DiskHits = s.Hits + out.DiskEvictions = s.Evictions + } + return out } var stackFrameRe = regexp.MustCompile(`^(\s{4})(.+):(\d+):(\d+)$`) +type borrow struct { + data []byte + done func() +} + func ResolveStackTrace(ctx context.Context, projectId uuid.UUID, stackTrace string, debugIds map[string]string) string { prefix := SourceMapStorageKey(projectId, "") @@ -313,7 +108,14 @@ func ResolveStackTrace(ctx context.Context, projectId uuid.UUID, stackTrace stri framesResolved := 0 maxFrames := 50 - localResolvers := make(map[string]*symbolicator.Resolver) + local := map[string]borrow{} + defer func() { + for _, br := range local { + if br.done != nil { + br.done() + } + } + }() for _, line := range lines { if framesResolved >= maxFrames { @@ -338,13 +140,13 @@ func ResolveStackTrace(ctx context.Context, projectId uuid.UUID, stackTrace stri } base := filepath.Base(clean) - resolver := frameResolver(ctx, prefix, fileName, base, debugIds, localResolvers) - if resolver == nil { + data := frameData(ctx, prefix, fileName, base, debugIds, local) + if data == nil { resolved = append(resolved, line) continue } - frame, ok := resolver.Lookup(uint32(lineNum-1), uint32(colNum-1)) + frame, ok := sourcemap.LookupTW(data, uint32(lineNum-1), uint32(colNum-1)) if !ok { resolved = append(resolved, line) continue @@ -371,59 +173,58 @@ func ResolveStackTrace(ctx context.Context, projectId uuid.UUID, stackTrace stri return strings.Join(resolved, "\n") } -func frameResolver(ctx context.Context, prefix, fileName, base string, debugIds map[string]string, local map[string]*symbolicator.Resolver) *symbolicator.Resolver { +func frameData(ctx context.Context, prefix, fileName, base string, debugIds map[string]string, local map[string]borrow) []byte { id := NormalizeDebugId(debugIds[fileName]) if id == "" { id = NormalizeDebugId(debugIds[base]) } if id != "" { mapKey := prefix + DebugIdMapName(id) - if !activeSMCache.isNegative(mapKey) { + twKey := twKeyFor(mapKey) + if !sharedCache.IsNegative(twKey) { bundleKey := prefix + DebugIdBundleName(id) - if r, err := getResolver(ctx, mapKey, buildResolver(mapKey, bundleKey), local); err == nil && r != nil { - return r + if data := getBlob(ctx, twKey, loadSourceMapBlob(mapKey, bundleKey), local); data != nil { + return data } } } mapKey := prefix + base + ".map" - if activeSMCache.isNegative(mapKey) { + twKey := twKeyFor(mapKey) + if sharedCache.IsNegative(twKey) { return nil } - r, err := getResolver(ctx, mapKey, buildResolver(mapKey, prefix+base), local) - if err != nil { - return nil - } - return r + return getBlob(ctx, twKey, loadSourceMapBlob(mapKey, prefix+base), local) } -func getResolver(ctx context.Context, cacheKey string, build resolverBuild, local map[string]*symbolicator.Resolver) (*symbolicator.Resolver, error) { - if r, ok := local[cacheKey]; ok { - return r, nil +func getBlob(ctx context.Context, twKey string, load twcache.LoadFunc, local map[string]borrow) []byte { + if br, ok := local[twKey]; ok { + return br.data } - r, err := activeSMCache.getOrBuild(ctx, cacheKey, build) + data, done, err := sharedCache.Get(ctx, twKey, load) if err != nil { - local[cacheKey] = nil - return nil, err + local[twKey] = borrow{} + return nil } - local[cacheKey] = r - return r, nil + local[twKey] = borrow{data: data, done: done} + return data } var smStoreHits, smBuilds atomic.Uint64 -func buildResolver(mapKey, bundleKey string) resolverBuild { - return func(ctx context.Context) (*symbolicator.Resolver, int64, error) { +func loadSourceMapBlob(mapKey, bundleKey string) twcache.LoadFunc { + return func(ctx context.Context) ([]byte, error) { base := context.WithoutCancel(ctx) + twKey := twKeyFor(mapKey) refreshStoreTw := true - twKey := twKeyFor(mapKey) twBytes, err := readWithTimeout(base, twKey) if err == nil { - if r, twErr := symbolicator.OpenTW(twBytes); twErr == nil { + if sourcemap.ValidTW(twBytes) { smStoreHits.Add(1) - return r, r.ApproxSize(), nil + return twBytes, nil } + } else if !errors.Is(err, storage.ErrNotFound) { refreshStoreTw = false traceway.CaptureException(fmt.Errorf("failed to read tw artifact, rebuilding from source map (key=%s): %w", twKey, err)) @@ -431,27 +232,26 @@ func buildResolver(mapKey, bundleKey string) resolverBuild { mapBytes, err := readWithTimeout(base, mapKey) if err != nil { - return nil, 0, err + return nil, err } - var bundleBytes []byte if b, readErr := readWithTimeout(base, bundleKey); readErr == nil { bundleBytes = b } else if !errors.Is(readErr, storage.ErrNotFound) { - return nil, 0, fmt.Errorf("failed to read bundle (key=%s): %w", bundleKey, readErr) + return nil, fmt.Errorf("failed to read bundle (key=%s): %w", bundleKey, readErr) } - resolver, err := symbolicator.NewResolver(mapBytes, bundleBytes) + blob, err := sourcemap.BuildTW(mapBytes, bundleBytes) if err != nil { - return nil, 0, err + return nil, err } smBuilds.Add(1) if refreshStoreTw { - if werr := storage.Store.Write(base, twKey, resolver.MarshalTW()); werr != nil { + if werr := storage.Store.Write(base, twKey, blob); werr != nil { traceway.CaptureException(fmt.Errorf("failed to refresh tw artifact in storage (key=%s): %w", twKey, werr)) } } - return resolver, resolver.ApproxSize(), nil + return blob, nil } } diff --git a/backend/app/services/sourcemap_tw.go b/backend/app/services/sourcemap_tw.go index df2e0495..2b597d87 100644 --- a/backend/app/services/sourcemap_tw.go +++ b/backend/app/services/sourcemap_tw.go @@ -30,8 +30,10 @@ func GenerateTWArtifacts(ctx context.Context, projectId uuid.UUID, fileNames []s } InvalidateSourceMap(projectId, base) - if _, err := activeSMCache.getOrBuild(ctx, mapKey, buildResolver(mapKey, bundleKey)); err != nil && !errors.Is(err, storage.ErrNotFound) { + _, done, err := sharedCache.Get(ctx, twKey, loadSourceMapBlob(mapKey, bundleKey)) + if err != nil && !errors.Is(err, storage.ErrNotFound) { traceway.CaptureException(fmt.Errorf("tw generation: failed to warm resolver (key=%s): %w", mapKey, err)) } + done() } } diff --git a/backend/app/services/symbolicator_cache.go b/backend/app/services/symbolicator_cache.go new file mode 100644 index 00000000..6eaf1f0f --- /dev/null +++ b/backend/app/services/symbolicator_cache.go @@ -0,0 +1,47 @@ +package services + +import ( + "errors" + "fmt" + "strings" + + "github.com/tracewayapp/traceway/backend/app/storage" + "github.com/tracewayapp/traceway/backend/app/symbolicator/dart" + "github.com/tracewayapp/traceway/backend/app/symbolicator/sourcemap" + "github.com/tracewayapp/traceway/backend/app/symbolicator/twcache" + + traceway "go.tracewayapp.com" +) + +func validArtifact(b []byte) bool { + return sourcemap.ValidTW(b) || dart.ValidFlat(b) +} + +var sharedCache = newSymbolicatorCache() + +func isStorageNotFound(err error) bool { return errors.Is(err, storage.ErrNotFound) } + +func newSymbolicatorCache() *twcache.Cache { + c := twcache.NewMem(200, 500<<20) + c.NotFound = isStorageNotFound + c.Validate = validArtifact + c.SetWarn(func(err error) { traceway.CaptureException(err) }) + return c +} + +func EnableSymbolicatorDiskCache(dir string, maxBytes int64) error { + c, err := twcache.NewDisk(dir, maxBytes, func(err error) { traceway.CaptureException(err) }) + if err != nil { + return fmt.Errorf("symbolicator disk cache: %w", err) + } + c.NotFound = isStorageNotFound + c.Validate = validArtifact + sharedCache = c + return nil +} + +func twKeyFor(mapKey string) string { + return strings.TrimSuffix(mapKey, ".map") + ".tw" +} + +func noop() {} diff --git a/backend/app/services/turnstile.service.go b/backend/app/services/turnstile.service.go index 61b524db..af1511ea 100644 --- a/backend/app/services/turnstile.service.go +++ b/backend/app/services/turnstile.service.go @@ -1,9 +1,9 @@ package services import ( - "github.com/tracewayapp/traceway/backend/app/config" "encoding/json" "errors" + "github.com/tracewayapp/traceway/backend/app/config" "net/http" "net/url" "time" diff --git a/backend/app/storage/local.go b/backend/app/storage/local.go index 574502f9..1d22bea4 100644 --- a/backend/app/storage/local.go +++ b/backend/app/storage/local.go @@ -2,6 +2,7 @@ package storage import ( "context" + "errors" "fmt" "os" "path/filepath" @@ -11,6 +12,8 @@ type localStorage struct { basePath string } +var ErrInvalidKey = errors.New("storage: key escapes base path") + func NewLocalStorage(basePath string) (*localStorage, error) { abs, err := filepath.Abs(basePath) if err != nil { @@ -22,8 +25,19 @@ func NewLocalStorage(basePath string) (*localStorage, error) { return &localStorage{basePath: abs}, nil } +func (l *localStorage) resolve(key string) (string, error) { + rel := filepath.FromSlash(key) + if !filepath.IsLocal(rel) { + return "", fmt.Errorf("%w: %q", ErrInvalidKey, key) + } + return filepath.Join(l.basePath, rel), nil +} + func (l *localStorage) Write(_ context.Context, key string, data []byte) error { - fullPath := filepath.Join(l.basePath, key) + fullPath, err := l.resolve(key) + if err != nil { + return err + } dir := filepath.Dir(fullPath) if err := os.MkdirAll(dir, 0755); err != nil { return fmt.Errorf("failed to create directory %s: %w", dir, err) @@ -35,7 +49,10 @@ func (l *localStorage) Write(_ context.Context, key string, data []byte) error { } func (l *localStorage) Delete(_ context.Context, key string) error { - fullPath := filepath.Join(l.basePath, key) + fullPath, err := l.resolve(key) + if err != nil { + return err + } if err := os.Remove(fullPath); err != nil && !os.IsNotExist(err) { return fmt.Errorf("failed to delete file %s: %w", fullPath, err) } @@ -43,7 +60,10 @@ func (l *localStorage) Delete(_ context.Context, key string) error { } func (l *localStorage) Read(_ context.Context, key string) ([]byte, error) { - fullPath := filepath.Join(l.basePath, key) + fullPath, err := l.resolve(key) + if err != nil { + return nil, err + } data, err := os.ReadFile(fullPath) if err != nil { if os.IsNotExist(err) { diff --git a/backend/app/storage/local_test.go b/backend/app/storage/local_test.go new file mode 100644 index 00000000..e5e2d65e --- /dev/null +++ b/backend/app/storage/local_test.go @@ -0,0 +1,42 @@ +package storage + +import ( + "context" + "os" + "path/filepath" + "testing" +) + +func TestLocalStorageRejectsTraversalKeys(t *testing.T) { + base := t.TempDir() + s, err := NewLocalStorage(base) + if err != nil { + t.Fatal(err) + } + ctx := context.Background() + + for _, key := range []string{"../escape.txt", "../../etc/x", "a/../../escape", "/abs/path", ""} { + if err := s.Write(ctx, key, []byte("x")); err == nil { + t.Errorf("Write(%q) succeeded, want rejection", key) + } + if _, err := s.Read(ctx, key); err == nil { + t.Errorf("Read(%q) succeeded, want rejection", key) + } + if err := s.Delete(ctx, key); err == nil { + t.Errorf("Delete(%q) succeeded, want rejection", key) + } + } + + escaped := filepath.Join(filepath.Dir(base), "escape.txt") + if _, err := os.Stat(escaped); err == nil { + t.Errorf("traversal write escaped to %s", escaped) + } + + if err := s.Write(ctx, "ok/nested/file.txt", []byte("data")); err != nil { + t.Fatalf("legit nested key rejected: %v", err) + } + got, err := s.Read(ctx, "ok/nested/file.txt") + if err != nil || string(got) != "data" { + t.Fatalf("legit read failed: got %q err %v", got, err) + } +} diff --git a/backend/app/symbolicator/dart/bench_test.go b/backend/app/symbolicator/dart/bench_test.go new file mode 100644 index 00000000..9ea36fca --- /dev/null +++ b/backend/app/symbolicator/dart/bench_test.go @@ -0,0 +1,51 @@ +package dart + +import ( + "os" + "path/filepath" + "testing" +) + +const benchDir = "fixtures/flutter-macos-arm64-dart3.10.1" + +func BenchmarkBuildFlat(b *testing.B) { + elfBytes, err := readFileBytes(filepath.Join(benchDir, "app.darwin-arm64.symbols")) + if err != nil { + b.Fatal(err) + } + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + if _, err := BuildFlat(elfBytes); err != nil { + b.Fatal(err) + } + } +} + +func BenchmarkLookupFlatTrace(b *testing.B) { + elfBytes, err := readFileBytes(filepath.Join(benchDir, "app.darwin-arm64.symbols")) + if err != nil { + b.Fatal(err) + } + flat, err := BuildFlat(elfBytes) + if err != nil { + b.Fatal(err) + } + traceText, err := readFileBytes(filepath.Join(benchDir, "trace.txt")) + if err != nil { + b.Fatal(err) + } + frames := ParseTrace(string(traceText)).Frames + + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + for _, f := range frames { + _ = LookupFlat(flat, f) + } + } +} + +func readFileBytes(path string) ([]byte, error) { + return os.ReadFile(path) +} diff --git a/backend/app/symbolicator/dart/fixtures/dartaot-macos-arm64-dart2.19.6/app.debug.elf b/backend/app/symbolicator/dart/fixtures/dartaot-macos-arm64-dart2.19.6/app.debug.elf new file mode 100644 index 00000000..23a9357f Binary files /dev/null and b/backend/app/symbolicator/dart/fixtures/dartaot-macos-arm64-dart2.19.6/app.debug.elf differ diff --git a/backend/app/symbolicator/dart/fixtures/dartaot-macos-arm64-dart2.19.6/expected.txt b/backend/app/symbolicator/dart/fixtures/dartaot-macos-arm64-dart2.19.6/expected.txt new file mode 100644 index 00000000..12964f25 --- /dev/null +++ b/backend/app/symbolicator/dart/fixtures/dartaot-macos-arm64-dart2.19.6/expected.txt @@ -0,0 +1,7 @@ +level3 (file:///private/tmp/dartsdks/work-2.19.6/crash.dart:3:3) +level2 (file:///private/tmp/dartsdks/work-2.19.6/crash.dart:6:18) +level1 (file:///private/tmp/dartsdks/work-2.19.6/crash.dart:7:18) +main (file:///private/tmp/dartsdks/work-2.19.6/crash.dart:8:16) +main (file:///private/tmp/dartsdks/work-2.19.6/crash.dart) +_delayEntrypointInvocation. (dart:isolate-patch/isolate_patch.dart:297:19) +_RawReceivePort._handleMessage (dart:isolate-patch/isolate_patch.dart:192:26) diff --git a/backend/app/symbolicator/dart/fixtures/dartaot-macos-arm64-dart2.19.6/trace.txt b/backend/app/symbolicator/dart/fixtures/dartaot-macos-arm64-dart2.19.6/trace.txt new file mode 100644 index 00000000..82077057 --- /dev/null +++ b/backend/app/symbolicator/dart/fixtures/dartaot-macos-arm64-dart2.19.6/trace.txt @@ -0,0 +1,12 @@ +Unhandled exception: +Bad state: boom from level3 +*** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** +pid: 25805, tid: 8806539456, name Dart_Initialize +os: macos arch: arm64 comp: no sim: no +build_id: '8228d9e1d6fa5932606d8ed329fefc6c' +isolate_dso_base: 100ab0000, vm_dso_base: 100ab0000 +isolate_instructions: 100aed780, vm_instructions: 100ae8000 + #00 abs 0000000100b2cb3f virt 000000000007cb3f _kDartIsolateSnapshotInstructions+0x3f3bf + #01 abs 0000000100b2cb0b virt 000000000007cb0b _kDartIsolateSnapshotInstructions+0x3f38b + #02 abs 0000000100b2d0cb virt 000000000007d0cb _kDartIsolateSnapshotInstructions+0x3f94b + #03 abs 0000000100af301b virt 000000000004301b _kDartIsolateSnapshotInstructions+0x589b diff --git a/backend/app/symbolicator/dart/fixtures/dartaot-macos-arm64-dart3.0.0/app.debug.elf b/backend/app/symbolicator/dart/fixtures/dartaot-macos-arm64-dart3.0.0/app.debug.elf new file mode 100644 index 00000000..5b288e10 Binary files /dev/null and b/backend/app/symbolicator/dart/fixtures/dartaot-macos-arm64-dart3.0.0/app.debug.elf differ diff --git a/backend/app/symbolicator/dart/fixtures/dartaot-macos-arm64-dart3.0.0/expected.txt b/backend/app/symbolicator/dart/fixtures/dartaot-macos-arm64-dart3.0.0/expected.txt new file mode 100644 index 00000000..5f42ebb8 --- /dev/null +++ b/backend/app/symbolicator/dart/fixtures/dartaot-macos-arm64-dart3.0.0/expected.txt @@ -0,0 +1,7 @@ +level3 (file:///private/tmp/dartsdks/work-3.0.0/crash.dart:3:3) +level2 (file:///private/tmp/dartsdks/work-3.0.0/crash.dart:6:18) +level1 (file:///private/tmp/dartsdks/work-3.0.0/crash.dart:7:18) +main (file:///private/tmp/dartsdks/work-3.0.0/crash.dart:8:16) +main (file:///private/tmp/dartsdks/work-3.0.0/crash.dart) +_delayEntrypointInvocation. (dart:isolate-patch/isolate_patch.dart:296:19) +_RawReceivePort._handleMessage (dart:isolate-patch/isolate_patch.dart:189:12) diff --git a/backend/app/symbolicator/dart/fixtures/dartaot-macos-arm64-dart3.0.0/trace.txt b/backend/app/symbolicator/dart/fixtures/dartaot-macos-arm64-dart3.0.0/trace.txt new file mode 100644 index 00000000..5c5a8d37 --- /dev/null +++ b/backend/app/symbolicator/dart/fixtures/dartaot-macos-arm64-dart3.0.0/trace.txt @@ -0,0 +1,12 @@ +Unhandled exception: +Bad state: boom from level3 +*** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** +pid: 25794, tid: 8806539456, name Dart_Initialize +os: macos arch: arm64 comp: no sim: no +build_id: '606747fb012550a2c0545e36c9a36d41' +isolate_dso_base: 104c5c000, vm_dso_base: 104c5c000 +isolate_instructions: 104c95ac0, vm_instructions: 104c90000 + #00 abs 0000000104cd20cb virt 00000000000760cb _kDartIsolateSnapshotInstructions+0x3c60b + #01 abs 0000000104cd2097 virt 0000000000076097 _kDartIsolateSnapshotInstructions+0x3c5d7 + #02 abs 0000000104cd2693 virt 0000000000076693 _kDartIsolateSnapshotInstructions+0x3cbd3 + #03 abs 0000000104c9ae9f virt 000000000003ee9f _kDartIsolateSnapshotInstructions+0x53df diff --git a/backend/app/symbolicator/dart/fixtures/dartaot-macos-arm64-dart3.10.1/app.debug.elf b/backend/app/symbolicator/dart/fixtures/dartaot-macos-arm64-dart3.10.1/app.debug.elf new file mode 100644 index 00000000..0600b715 Binary files /dev/null and b/backend/app/symbolicator/dart/fixtures/dartaot-macos-arm64-dart3.10.1/app.debug.elf differ diff --git a/backend/app/symbolicator/dart/fixtures/dartaot-macos-arm64-dart3.10.1/expected.txt b/backend/app/symbolicator/dart/fixtures/dartaot-macos-arm64-dart3.10.1/expected.txt new file mode 100644 index 00000000..7bdfbb25 --- /dev/null +++ b/backend/app/symbolicator/dart/fixtures/dartaot-macos-arm64-dart3.10.1/expected.txt @@ -0,0 +1,9 @@ +level3 (file:///private/tmp/dartfix/crash.dart:3:3) +level2 (file:///private/tmp/dartfix/crash.dart:7:3) +level1 (file:///private/tmp/dartfix/crash.dart:11:3) +main (file:///private/tmp/dartfix/crash.dart:15:3) +main (file:///private/tmp/dartfix/crash.dart:14:1) +_Closure.call (dart:core-patch/function.dart) +_delayEntrypointInvocation. (dart:isolate-patch/isolate_patch.dart:314:19) +_Closure.call (dart:core-patch/function.dart) +_RawReceivePort._handleMessage (dart:isolate-patch/isolate_patch.dart:193:12) diff --git a/backend/app/symbolicator/dart/fixtures/dartaot-macos-arm64-dart3.10.1/trace.txt b/backend/app/symbolicator/dart/fixtures/dartaot-macos-arm64-dart3.10.1/trace.txt new file mode 100644 index 00000000..74c994dc --- /dev/null +++ b/backend/app/symbolicator/dart/fixtures/dartaot-macos-arm64-dart3.10.1/trace.txt @@ -0,0 +1,14 @@ +Unhandled exception: +Bad state: boom from level3 +*** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** +pid: 49838, tid: 8806539456, name Dart_Initialize +os: macos arch: arm64 comp: no sim: no +build_id: '20fb76bfc9d292a85d82b8712754455e' +isolate_dso_base: 104914000, vm_dso_base: 104914000 +isolate_instructions: 1049527c0, vm_instructions: 104948000 + #00 abs 00000001049832cb virt 000000000006f2cb _kDartIsolateSnapshotInstructions+0x30b0b + #01 abs 0000000104983293 virt 000000000006f293 _kDartIsolateSnapshotInstructions+0x30ad3 + #02 abs 000000010498ebd7 virt 000000000007abd7 _kDartIsolateSnapshotInstructions+0x3c417 + #03 abs 00000001049836c3 virt 000000000006f6c3 _kDartIsolateSnapshotInstructions+0x30f03 + #04 abs 000000010498eeb3 virt 000000000007aeb3 _kDartIsolateSnapshotInstructions+0x3c6f3 + #05 abs 0000000104961b07 virt 000000000004db07 _kDartIsolateSnapshotInstructions+0xf347 diff --git a/backend/app/symbolicator/dart/fixtures/flutter-macos-arm64-dart3.10.1/app.darwin-arm64.symbols b/backend/app/symbolicator/dart/fixtures/flutter-macos-arm64-dart3.10.1/app.darwin-arm64.symbols new file mode 100644 index 00000000..06aaa060 Binary files /dev/null and b/backend/app/symbolicator/dart/fixtures/flutter-macos-arm64-dart3.10.1/app.darwin-arm64.symbols differ diff --git a/backend/app/symbolicator/dart/fixtures/flutter-macos-arm64-dart3.10.1/expected.txt b/backend/app/symbolicator/dart/fixtures/flutter-macos-arm64-dart3.10.1/expected.txt new file mode 100644 index 00000000..4a1b158d --- /dev/null +++ b/backend/app/symbolicator/dart/fixtures/flutter-macos-arm64-dart3.10.1/expected.txt @@ -0,0 +1,17 @@ +chargeCard (/Users/dusanstanojevic/Documents/flutter-demo/app/lib/main.dart:20:3) +applyTax (/Users/dusanstanojevic/Documents/flutter-demo/app/lib/main.dart:30:10) +checkout (/Users/dusanstanojevic/Documents/flutter-demo/app/lib/main.dart:35:10) +main. (/Users/dusanstanojevic/Documents/flutter-demo/app/lib/main.dart:47:7) +_rootRun (third_party/dart/sdk/lib/async/zone.dart:1525:13) +_rootRun (third_party/dart/sdk/lib/async/zone.dart:1516:1) +_CustomZone.run (third_party/dart/sdk/lib/async/zone.dart:1422:19) +_runZoned (third_party/dart/sdk/lib/async/zone.dart:2034:6) +runZonedGuarded (third_party/dart/sdk/lib/async/zone.dart:2020:12) +main (/Users/dusanstanojevic/Documents/flutter-demo/app/lib/main.dart:42:3) +main (/Users/dusanstanojevic/Documents/flutter-demo/app/lib/main.dart:38:1) +_Closure.call (third_party/dart/sdk/lib/_internal/vm/lib/function.dart) +_runMain. (lib/ui/hooks.dart:345:23) +_Closure.call (third_party/dart/sdk/lib/_internal/vm/lib/function.dart) +_delayEntrypointInvocation. (third_party/dart/sdk/lib/_internal/vm/lib/isolate_patch.dart:314:19) +_Closure.call (third_party/dart/sdk/lib/_internal/vm/lib/function.dart) +_RawReceivePort._handleMessage (third_party/dart/sdk/lib/_internal/vm/lib/isolate_patch.dart:193:12) diff --git a/backend/app/symbolicator/dart/fixtures/flutter-macos-arm64-dart3.10.1/trace.txt b/backend/app/symbolicator/dart/fixtures/flutter-macos-arm64-dart3.10.1/trace.txt new file mode 100644 index 00000000..ac4a5087 --- /dev/null +++ b/backend/app/symbolicator/dart/fixtures/flutter-macos-arm64-dart3.10.1/trace.txt @@ -0,0 +1,24 @@ +PaymentDeclinedException: card declined for $30.59 +*** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** +pid: 58347, tid: 8806539456, name io.flutter.platform +os: macos arch: arm64 comp: no sim: no +build_id: 'fe664295997135e7b67b648ba66ca9eb' +isolate_dso_base: 1130a0000, vm_dso_base: 1130a0000 +isolate_instructions: 1130aac00, vm_instructions: 1130a0440 + #00 abs 00000001131eca6b _kDartIsolateSnapshotInstructions+0x141e6b + #01 abs 00000001131ec99b _kDartIsolateSnapshotInstructions+0x141d9b + #02 abs 00000001131ec963 _kDartIsolateSnapshotInstructions+0x141d63 + #03 abs 00000001131ebb23 _kDartIsolateSnapshotInstructions+0x140f23 + #04 abs 000000011320abdf _kDartIsolateSnapshotInstructions+0x15ffdf + #05 abs 00000001130c524b _kDartIsolateSnapshotInstructions+0x1a64b + #06 abs 00000001131c50c3 _kDartIsolateSnapshotInstructions+0x11a4c3 + #07 abs 00000001131eaf6f _kDartIsolateSnapshotInstructions+0x14036f + #08 abs 0000000113225087 _kDartIsolateSnapshotInstructions+0x17a487 + #09 abs 00000001131eaeab _kDartIsolateSnapshotInstructions+0x1402ab + #10 abs 0000000113205137 _kDartIsolateSnapshotInstructions+0x15a537 + #11 abs 000000011322be8b _kDartIsolateSnapshotInstructions+0x18128b + #12 abs 00000001130ab25b _kDartIsolateSnapshotInstructions+0x65b + #13 abs 000000011322be8b _kDartIsolateSnapshotInstructions+0x18128b + #14 abs 0000000113205563 _kDartIsolateSnapshotInstructions+0x15a963 + #15 abs 000000011322bbe7 _kDartIsolateSnapshotInstructions+0x180fe7 + #16 abs 00000001130c256b _kDartIsolateSnapshotInstructions+0x1796b diff --git a/backend/app/symbolicator/dart/fixtures_test.go b/backend/app/symbolicator/dart/fixtures_test.go new file mode 100644 index 00000000..d6a257f4 --- /dev/null +++ b/backend/app/symbolicator/dart/fixtures_test.go @@ -0,0 +1,73 @@ +package dart + +import ( + "os" + "path/filepath" + "strings" + "testing" +) + +func TestTraceShapeVaries(t *testing.T) { + for _, dir := range fixtureDirs(t) { + name := filepath.Base(dir) + raw := readFile(t, filepath.Join(dir, "trace.txt")) + trace := ParseTrace(raw) + if !IsNonSymbolic(raw) { + t.Errorf("%s: not detected as non-symbolic", name) + } + elfBuildID, err := ReadBuildID(readBytes(t, findSymbols(t, dir))) + if err != nil { + t.Fatal(err) + } + t.Logf("%-36s frames=%2d header_virt=%-5v header_build_id=%-6v elf_build_id=%q", + name, len(trace.Frames), strings.Contains(raw, " virt "), + trace.BuildID != "", elfBuildID) + } +} + +func fixtureDirs(t *testing.T) []string { + t.Helper() + all, _ := filepath.Glob("fixtures/*") + var dirs []string + for _, d := range all { + if info, err := os.Stat(d); err == nil && info.IsDir() { + dirs = append(dirs, d) + } + } + if len(dirs) == 0 { + t.Fatal("no fixtures under fixtures/") + } + return dirs +} + +func findSymbols(t *testing.T, dir string) string { + t.Helper() + for _, pat := range []string{"*.symbols", "*.elf"} { + if m, _ := filepath.Glob(filepath.Join(dir, pat)); len(m) > 0 { + return m[0] + } + } + t.Fatalf("no .symbols or .elf file in %s", dir) + return "" +} + +func readFile(t *testing.T, path string) string { return string(readBytes(t, path)) } + +func readBytes(t *testing.T, path string) []byte { + t.Helper() + b, err := os.ReadFile(path) + if err != nil { + t.Fatal(err) + } + return b +} + +func nonEmptyLines(s string) []string { + var out []string + for _, line := range strings.Split(s, "\n") { + if strings.TrimSpace(line) != "" { + out = append(out, strings.TrimSpace(line)) + } + } + return out +} diff --git a/backend/app/symbolicator/dart/flat.go b/backend/app/symbolicator/dart/flat.go new file mode 100644 index 00000000..46747357 --- /dev/null +++ b/backend/app/symbolicator/dart/flat.go @@ -0,0 +1,606 @@ +package dart + +import ( + "bytes" + "debug/dwarf" + "debug/elf" + "encoding/binary" + "errors" + "fmt" + "sort" +) + +const maxInlineFrames = 64 + +type SymFrame struct { + Function string + File string + Line int + Col int +} + +func (s SymFrame) Location() string { + if s.Line <= 0 { + return s.File + } + if s.Col <= 0 { + return fmt.Sprintf("%s:%d", s.File, s.Line) + } + return fmt.Sprintf("%s:%d:%d", s.File, s.Line, s.Col) +} + +func dwarfFromELF(elfBytes []byte) (*dwarf.Data, map[string]uint64, string, error) { + f, err := elf.NewFile(bytes.NewReader(elfBytes)) + if err != nil { + return nil, nil, "", fmt.Errorf("not an ELF file: %w", err) + } + data, err := f.DWARF() + if err != nil { + return nil, nil, "", fmt.Errorf("reading DWARF: %w", err) + } + syms, err := f.Symbols() + if err != nil { + return nil, nil, "", fmt.Errorf("reading symbols: %w", err) + } + base := map[string]uint64{} + for _, s := range syms { + switch s.Name { + case isolateInstructionsSymbol: + base["isolate"] = s.Value + case vmInstructionsSymbol: + base["vm"] = s.Value + } + } + return data, base, readBuildID(f), nil +} + +func ReadBuildID(elfBytes []byte) (string, error) { + _, _, buildID, err := dwarfFromELF(elfBytes) + return buildID, err +} + +const ( + isolateInstructionsSymbol = "_kDartIsolateSnapshotInstructions" + vmInstructionsSymbol = "_kDartVmSnapshotInstructions" +) + +func InstructionSymbol(section string) string { + if section == "vm" { + return vmInstructionsSymbol + } + return isolateInstructionsSymbol +} + +type flatBuilder struct { + isolateBase uint64 + vmBase uint64 + buildID string + + entries []flatEntry + frames []flatFrame + files []string + fns []string +} + +type flatEntry struct { + pc, hi uint64 + frameStart uint32 + frameCount uint32 +} + +type flatFrame struct { + fileIdx int32 + line int32 + col int32 + fnIdx int32 +} + +const ( + flatEntrySize = 24 + flatFrameSize = 16 + dwHeaderSize = 48 +) + +func BuildFlat(elfBytes []byte) ([]byte, error) { + data, base, buildID, err := dwarfFromELF(elfBytes) + if err != nil { + return nil, err + } + return flatten(data, base, buildID).marshal(), nil +} + +type funcRange struct { + low, high uint64 + off dwarf.Offset +} + +type lineRow struct { + addr, end uint64 + file string + line, col int +} + +func flatten(data *dwarf.Data, base map[string]uint64, buildID string) *flatBuilder { + f := &flatBuilder{ + isolateBase: base["isolate"], + vmBase: base["vm"], + buildID: buildID, + } + files := newInterner() + fns := newInterner() + + rdr := data.Reader() + for { + cu, err := rdr.Next() + if err != nil || cu == nil { + break + } + if cu.Tag != dwarf.TagCompileUnit { + rdr.SkipChildren() + continue + } + + lr, lrErr := data.LineReader(cu) + + topFuncs, bounds := collectFuncsAndBounds(data, rdr) + if lrErr != nil || lr == nil || len(topFuncs) == 0 { + continue + } + sort.Slice(topFuncs, func(i, j int) bool { return topFuncs[i].low < topFuncs[j].low }) + lineFiles := lr.Files() + + var raw []dwarf.LineEntry + for { + var le dwarf.LineEntry + if err := lr.Next(&le); err != nil { + break + } + raw = append(raw, le) + } + var rows []lineRow + for i := 0; i+1 < len(raw); i++ { + le := raw[i] + if le.EndSequence { + continue + } + end := raw[i+1].Address + if end <= le.Address { + continue + } + file := "" + if le.File != nil { + file = le.File.Name + } + rows = append(rows, lineRow{addr: le.Address, end: end, file: file, line: le.Line, col: le.Column}) + bounds = append(bounds, le.Address, end) + } + sort.Slice(rows, func(i, j int) bool { return rows[i].addr < rows[j].addr }) + bounds = sortUnique(bounds) + + var lastFrames []SymFrame + for bi := 0; bi+1 < len(bounds); bi++ { + a, b := bounds[bi], bounds[bi+1] + fn := findFunc(topFuncs, a) + if fn == nil { + lastFrames = nil + continue + } + var chain []*dwarf.Entry + cr := data.Reader() + cr.Seek(fn.off) + descend(data, cr, a, &chain) + if len(chain) == 0 { + lastFrames = nil + continue + } + lf, ll, lc := rowAt(rows, a) + sfs := framesFromChain(data, chain, lineFiles, lf, ll, lc) + + if lastFrames != nil && len(f.entries) > 0 && f.entries[len(f.entries)-1].hi == a && sameFrames(sfs, lastFrames) { + f.entries[len(f.entries)-1].hi = b + continue + } + start := uint32(len(f.frames)) + for _, sf := range sfs { + f.frames = append(f.frames, flatFrame{ + fileIdx: files.intern(sf.File), + line: int32(sf.Line), + col: int32(sf.Col), + fnIdx: fns.intern(sf.Function), + }) + } + f.entries = append(f.entries, flatEntry{pc: a, hi: b, frameStart: start, frameCount: uint32(len(sfs))}) + lastFrames = sfs + } + } + + f.files = files.list + f.fns = fns.list + sort.Slice(f.entries, func(i, j int) bool { return f.entries[i].pc < f.entries[j].pc }) + return f +} + +func collectFuncsAndBounds(d *dwarf.Data, rdr *dwarf.Reader) ([]funcRange, []uint64) { + var funcs []funcRange + var bounds []uint64 + depth := 0 + for { + e, err := rdr.Next() + if err != nil || e == nil { + return funcs, bounds + } + if e.Tag == 0 { + depth-- + if depth < 0 { + return funcs, bounds + } + continue + } + if e.Tag == dwarf.TagSubprogram || e.Tag == dwarf.TagInlinedSubroutine { + if rs, rerr := d.Ranges(e); rerr == nil { + for _, rg := range rs { + if rg[1] > rg[0] { + bounds = append(bounds, rg[0], rg[1]) + if e.Tag == dwarf.TagSubprogram && depth == 0 { + funcs = append(funcs, funcRange{low: rg[0], high: rg[1], off: e.Offset}) + } + } + } + } + } + if e.Children { + depth++ + } + } +} + +func findFunc(funcs []funcRange, addr uint64) *funcRange { + i := sort.Search(len(funcs), func(i int) bool { return funcs[i].low > addr }) - 1 + if i >= 0 && addr >= funcs[i].low && addr < funcs[i].high { + return &funcs[i] + } + return nil +} + +func rowAt(rows []lineRow, addr uint64) (string, int, int) { + i := sort.Search(len(rows), func(i int) bool { return rows[i].addr > addr }) - 1 + if i >= 0 && addr >= rows[i].addr && addr < rows[i].end { + return rows[i].file, rows[i].line, rows[i].col + } + return "", 0, 0 +} + +func sortUnique(xs []uint64) []uint64 { + if len(xs) < 2 { + return xs + } + sort.Slice(xs, func(i, j int) bool { return xs[i] < xs[j] }) + out := xs[:1] + for _, x := range xs[1:] { + if x != out[len(out)-1] { + out = append(out, x) + } + } + return out +} + +func sameFrames(a, b []SymFrame) bool { + if len(a) != len(b) { + return false + } + for i := range a { + if a[i] != b[i] { + return false + } + } + return true +} + +func framesFromChain(d *dwarf.Data, chain []*dwarf.Entry, files []*dwarf.LineFile, leafFile string, leafLine, leafCol int) []SymFrame { + if len(chain) > maxInlineFrames { + chain = chain[len(chain)-maxInlineFrames:] + } + out := make([]SymFrame, len(chain)) + for i := len(chain) - 1; i >= 0; i-- { + pos := len(chain) - 1 - i + sf := SymFrame{Function: nameOf(d, chain[i])} + if i == len(chain)-1 { + sf.File, sf.Line, sf.Col = leafFile, leafLine, leafCol + } else { + inner := chain[i+1] + if v, ok := inner.Val(dwarf.AttrCallFile).(int64); ok && int(v) < len(files) && files[v] != nil { + sf.File = files[v].Name + } + if v, ok := inner.Val(dwarf.AttrCallLine).(int64); ok { + sf.Line = int(v) + } + if v, ok := inner.Val(dwarf.AttrCallColumn).(int64); ok { + sf.Col = int(v) + } + } + out[pos] = sf + } + return out +} + +func descend(d *dwarf.Data, r *dwarf.Reader, pc uint64, chain *[]*dwarf.Entry) { + for { + e, err := r.Next() + if err != nil || e == nil || e.Tag == 0 { + return + } + scope := e.Tag == dwarf.TagSubprogram || e.Tag == dwarf.TagInlinedSubroutine + block := e.Tag == dwarf.TagLexDwarfBlock + if (scope || block) && rangesContain(d, e, pc) { + if scope { + *chain = append(*chain, e) + } + if e.Children { + descend(d, r, pc, chain) + } + return + } + if e.Children { + r.SkipChildren() + } + } +} + +func rangesContain(d *dwarf.Data, e *dwarf.Entry, pc uint64) bool { + rs, err := d.Ranges(e) + if err != nil { + return false + } + for _, r := range rs { + if pc >= r[0] && pc < r[1] { + return true + } + } + return false +} + +func nameOf(d *dwarf.Data, e *dwarf.Entry) string { + if n, ok := e.Val(dwarf.AttrName).(string); ok && n != "" { + return n + } + for _, attr := range []dwarf.Attr{dwarf.AttrAbstractOrigin, dwarf.AttrSpecification} { + if off, ok := e.Val(attr).(dwarf.Offset); ok { + r := d.Reader() + r.Seek(off) + if ref, err := r.Next(); err == nil && ref != nil { + if n := nameOf(d, ref); n != "" { + return n + } + } + } + } + return "" +} + +func readBuildID(f *elf.File) string { + sec := f.Section(".note.gnu.build-id") + if sec == nil { + return "" + } + data, err := sec.Data() + if err != nil || len(data) < 12 { + return "" + } + nameSize := f.ByteOrder.Uint32(data[0:4]) + descSize := f.ByteOrder.Uint32(data[4:8]) + off := 12 + ((nameSize + 3) &^ 3) + if int(off+descSize) > len(data) { + return "" + } + return fmt.Sprintf("%x", data[off:off+descSize]) +} + +type interner struct { + list []string + idx map[string]int32 +} + +func newInterner() *interner { return &interner{idx: make(map[string]int32)} } + +func (in *interner) intern(s string) int32 { + if i, ok := in.idx[s]; ok { + return i + } + i := int32(len(in.list)) + in.list = append(in.list, s) + in.idx[s] = i + return i +} + +var dwMagic = [4]byte{'T', 'W', 'D', 'F'} + +const dwVersion = 1 + +var ErrInvalidFlat = errors.New("dart: invalid flat data") + +func (f *flatBuilder) marshal() []byte { + le := binary.LittleEndian + fileOffsets, fileBlob := encodeStringTable(f.files) + fnOffsets, fnBlob := encodeStringTable(f.fns) + + size := dwHeaderSize + + len(f.entries)*flatEntrySize + + len(f.frames)*flatFrameSize + + len(fileOffsets)*4 + len(fileBlob) + + len(fnOffsets)*4 + len(fnBlob) + + len(f.buildID) + + out := make([]byte, 0, size) + out = append(out, dwMagic[:]...) + out = le.AppendUint32(out, dwVersion) + out = le.AppendUint64(out, f.isolateBase) + out = le.AppendUint64(out, f.vmBase) + out = le.AppendUint32(out, uint32(len(f.entries))) + out = le.AppendUint32(out, uint32(len(f.frames))) + out = le.AppendUint32(out, uint32(len(f.files))) + out = le.AppendUint32(out, uint32(len(f.fns))) + out = le.AppendUint32(out, uint32(len(f.buildID))) + out = le.AppendUint32(out, 0) + for _, e := range f.entries { + out = le.AppendUint64(out, e.pc) + out = le.AppendUint64(out, e.hi) + out = le.AppendUint32(out, e.frameStart) + out = le.AppendUint32(out, e.frameCount) + } + for _, fr := range f.frames { + out = le.AppendUint32(out, uint32(fr.fileIdx)) + out = le.AppendUint32(out, uint32(fr.line)) + out = le.AppendUint32(out, uint32(fr.col)) + out = le.AppendUint32(out, uint32(fr.fnIdx)) + } + for _, off := range fileOffsets { + out = le.AppendUint32(out, off) + } + out = append(out, fileBlob...) + for _, off := range fnOffsets { + out = le.AppendUint32(out, off) + } + out = append(out, fnBlob...) + out = append(out, f.buildID...) + return out +} + +func encodeStringTable(strs []string) ([]uint32, []byte) { + offsets := make([]uint32, len(strs)+1) + var blob []byte + for i, s := range strs { + offsets[i] = uint32(len(blob)) + blob = append(blob, s...) + } + offsets[len(strs)] = uint32(len(blob)) + return offsets, blob +} + +type flatLayout struct { + isolateBase, vmBase uint64 + entryCount int + framesOff int + frameCount int + fileTableOff int + filesCount int + fnTableOff int + fnsCount int +} + +func readFlatLayout(data []byte) (flatLayout, bool) { + var l flatLayout + if len(data) < dwHeaderSize { + return l, false + } + le := binary.LittleEndian + if data[0] != dwMagic[0] || data[1] != dwMagic[1] || data[2] != dwMagic[2] || data[3] != dwMagic[3] { + return l, false + } + if le.Uint32(data[4:]) != dwVersion { + return l, false + } + l.isolateBase = le.Uint64(data[8:]) + l.vmBase = le.Uint64(data[16:]) + entryCount := int64(le.Uint32(data[24:])) + frameCount := int64(le.Uint32(data[28:])) + l.filesCount = int(le.Uint32(data[32:])) + l.fnsCount = int(le.Uint32(data[36:])) + n := int64(len(data)) + + entriesOff := int64(dwHeaderSize) + framesOff := entriesOff + entryCount*flatEntrySize + fileTableOff := framesOff + frameCount*flatFrameSize + if entryCount < 0 || frameCount < 0 || l.filesCount < 0 || l.fnsCount < 0 || + framesOff < entriesOff || fileTableOff < framesOff { + return l, false + } + fileBlobOff := fileTableOff + int64(l.filesCount+1)*4 + if fileBlobOff > n { + return l, false + } + fileBlobLen := int64(le.Uint32(data[fileTableOff+int64(l.filesCount)*4:])) + fnTableOff := fileBlobOff + fileBlobLen + fnBlobOff := fnTableOff + int64(l.fnsCount+1)*4 + if fnTableOff < fileBlobOff || fnBlobOff > n { + return l, false + } + fnBlobLen := int64(le.Uint32(data[fnTableOff+int64(l.fnsCount)*4:])) + if fnBlobOff+fnBlobLen > n { + return l, false + } + + l.entryCount = int(entryCount) + l.frameCount = int(frameCount) + l.framesOff = int(framesOff) + l.fileTableOff = int(fileTableOff) + l.fnTableOff = int(fnTableOff) + return l, true +} + +func ValidFlat(data []byte) bool { + _, ok := readFlatLayout(data) + return ok +} + +func LookupFlat(data []byte, frame StackFrame) []SymFrame { + l, ok := readFlatLayout(data) + if !ok { + return nil + } + base := l.isolateBase + if frame.Section == "vm" { + base = l.vmBase + } + if base == 0 { + return nil + } + pc := base + frame.Offset + + le := binary.LittleEndian + entryAt := func(i int) (pcLo, hi uint64, frameStart, frameCount uint32) { + rec := data[dwHeaderSize+i*flatEntrySize:] + return le.Uint64(rec), le.Uint64(rec[8:]), le.Uint32(rec[16:]), le.Uint32(rec[20:]) + } + + i := sort.Search(l.entryCount, func(i int) bool { + pcLo, _, _, _ := entryAt(i) + return pcLo > pc + }) - 1 + if i < 0 { + return nil + } + ePc, eHi, frameStart, frameCount := entryAt(i) + if pc < ePc || pc >= eHi { + return nil + } + + if uint64(frameStart)+uint64(frameCount) > uint64(l.frameCount) { + return nil + } + out := make([]SymFrame, 0, frameCount) + for k := uint32(0); k < frameCount; k++ { + rec := data[l.framesOff+int(frameStart+k)*flatFrameSize:] + fileIdx := int32(le.Uint32(rec)) + line := int32(le.Uint32(rec[4:])) + col := int32(le.Uint32(rec[8:])) + fnIdx := int32(le.Uint32(rec[12:])) + file, _ := stringAt(data, l.fileTableOff, l.filesCount, fileIdx) + fn, _ := stringAt(data, l.fnTableOff, l.fnsCount, fnIdx) + out = append(out, SymFrame{Function: fn, File: file, Line: int(line), Col: int(col)}) + } + return out +} + +func stringAt(data []byte, tableOff, count int, idx int32) (string, bool) { + if idx < 0 || int(idx) >= count { + return "", false + } + le := binary.LittleEndian + blobStart := tableOff + (count+1)*4 + o0 := int(le.Uint32(data[tableOff+int(idx)*4:])) + o1 := int(le.Uint32(data[tableOff+(int(idx)+1)*4:])) + if o0 > o1 || blobStart+o1 > len(data) { + return "", false + } + return string(data[blobStart+o0 : blobStart+o1]), true +} diff --git a/backend/app/symbolicator/dart/flat_test.go b/backend/app/symbolicator/dart/flat_test.go new file mode 100644 index 00000000..0d6eea0d --- /dev/null +++ b/backend/app/symbolicator/dart/flat_test.go @@ -0,0 +1,59 @@ +package dart + +import ( + "path/filepath" + "testing" +) + +func TestFlatMatchesGolden(t *testing.T) { + for _, dir := range fixtureDirs(t) { + t.Run(filepath.Base(dir), func(t *testing.T) { + elf := readBytes(t, findSymbols(t, dir)) + + flat, err := BuildFlat(elf) + if err != nil { + t.Fatal(err) + } + if !ValidFlat(flat) { + t.Fatal("BuildFlat produced bytes ValidFlat rejects") + } + + trace := ParseTrace(readFile(t, filepath.Join(dir, "trace.txt"))) + if len(trace.Frames) == 0 { + t.Fatal("parsed 0 frames from trace.txt") + } + var lines []string + for _, fr := range trace.Frames { + got := LookupFlat(flat, fr) + if len(got) == 0 { + t.Errorf("frame did not resolve: %s", fr.Raw) + continue + } + for _, sf := range got { + lines = append(lines, sf.Function+" ("+sf.Location()+")") + } + } + + golden := nonEmptyLines(readFile(t, filepath.Join(dir, "expected.txt"))) + if len(lines) != len(golden) { + t.Fatalf("flat produced %d frames, golden has %d", len(lines), len(golden)) + } + for i := range golden { + if lines[i] != golden[i] { + t.Errorf("flat frame %d: got %q want %q", i, lines[i], golden[i]) + } + } + }) + } +} + +func TestValidFlatRejectsBadData(t *testing.T) { + for _, b := range [][]byte{nil, []byte("no"), []byte("TWDFxxxx"), make([]byte, 64)} { + if ValidFlat(b) { + t.Errorf("expected ValidFlat=false for %d-length input", len(b)) + } + if got := LookupFlat(b, StackFrame{Section: "isolate", Offset: 1}); got != nil { + t.Errorf("expected nil lookup for %d-length bad data, got %v", len(b), got) + } + } +} diff --git a/backend/app/symbolicator/dart/normalize.go b/backend/app/symbolicator/dart/normalize.go new file mode 100644 index 00000000..d75e9622 --- /dev/null +++ b/backend/app/symbolicator/dart/normalize.go @@ -0,0 +1,49 @@ +package dart + +import "strings" + +func NormalizeDebugID(debugID string) string { + var b strings.Builder + for _, r := range strings.ToLower(strings.TrimSpace(debugID)) { + if (r >= '0' && r <= '9') || (r >= 'a' && r <= 'f') { + b.WriteRune(r) + } + } + return b.String() +} + +func NormalizeArch(arch string) string { + a := strings.ToLower(strings.TrimSpace(arch)) + switch a { + case "x86_64", "x64", "amd64": + return "x64" + case "aarch64", "arm64": + return "arm64" + case "armv7", "arm": + return "arm" + case "ia32", "x86", "i386": + return "ia32" + } + var b strings.Builder + for _, r := range a { + if (r >= 'a' && r <= 'z') || (r >= '0' && r <= '9') || r == '_' { + b.WriteRune(r) + } + } + return b.String() +} + +func IsValidArch(arch string) bool { + a := strings.TrimSpace(arch) + if a == "" { + return false + } + for _, r := range a { + switch { + case r >= 'a' && r <= 'z', r >= 'A' && r <= 'Z', r >= '0' && r <= '9', r == '_': + default: + return false + } + } + return true +} diff --git a/backend/app/symbolicator/dart/normalize_test.go b/backend/app/symbolicator/dart/normalize_test.go new file mode 100644 index 00000000..bdd35691 --- /dev/null +++ b/backend/app/symbolicator/dart/normalize_test.go @@ -0,0 +1,46 @@ +package dart + +import ( + "strings" + "testing" +) + +func TestNormalizeArchSanitizesUnsafeInput(t *testing.T) { + cases := map[string]string{ + "arm64": "arm64", + "aarch64": "arm64", + "x86_64": "x64", + "amd64": "x64", + "armv7": "arm", + "i386": "ia32", + " ARM64 ": "arm64", + "riscv64": "riscv64", + "../../../../tmp/x": "tmpx", + "arm64/../../etc": "arm64etc", + `a/b\c`: "abc", + "../../arm64": "arm64", + "..": "", + } + for in, want := range cases { + got := NormalizeArch(in) + if got != want { + t.Errorf("NormalizeArch(%q) = %q, want %q", in, got, want) + } + if strings.ContainsAny(got, `/\.`) { + t.Errorf("NormalizeArch(%q) = %q still contains a path separator", in, got) + } + } +} + +func TestIsValidArch(t *testing.T) { + for _, a := range []string{"arm64", "x64", "x86_64", "ia32", "arm", "riscv64"} { + if !IsValidArch(a) { + t.Errorf("IsValidArch(%q) = false, want true", a) + } + } + for _, a := range []string{"", " ", "../../etc", "arm64/x", "arm-64", "a.b", "arm64\x00", "x64;rm"} { + if IsValidArch(a) { + t.Errorf("IsValidArch(%q) = true, want false", a) + } + } +} diff --git a/backend/app/symbolicator/dart/stack_frame.go b/backend/app/symbolicator/dart/stack_frame.go new file mode 100644 index 00000000..2a46097e --- /dev/null +++ b/backend/app/symbolicator/dart/stack_frame.go @@ -0,0 +1,70 @@ +package dart + +import ( + "regexp" + "strconv" + "strings" +) + +type StackFrame struct { + Index int + Section string + Offset uint64 + Raw string +} + +type StackTrace struct { + BuildID string + OS string + Arch string + Frames []StackFrame +} + +var ( + buildIDRe = regexp.MustCompile(`build_id:\s*'([0-9a-fA-F]+)'`) + osArchRe = regexp.MustCompile(`os:\s*(\S+)\s+arch:\s*(\S+)`) + + frameRe = regexp.MustCompile( + `^\s*#(\d+)\s+abs\s+[0-9a-fA-F]+(?:\s+virt\s+[0-9a-fA-F]+)?\s+(_kDart(?:Isolate|Vm)SnapshotInstructions)\+0x([0-9a-fA-F]+)`, + ) +) + +func ParseTrace(text string) StackTrace { + var t StackTrace + for _, line := range strings.Split(text, "\n") { + if t.BuildID == "" { + if m := buildIDRe.FindStringSubmatch(line); m != nil { + t.BuildID = strings.ToLower(m[1]) + } + } + if t.OS == "" { + if m := osArchRe.FindStringSubmatch(line); m != nil { + t.OS, t.Arch = m[1], m[2] + } + } + if m := frameRe.FindStringSubmatch(line); m != nil { + idx, _ := strconv.Atoi(m[1]) + off, _ := strconv.ParseUint(m[3], 16, 64) + section := "isolate" + if strings.Contains(m[2], "Vm") { + section = "vm" + } + t.Frames = append(t.Frames, StackFrame{ + Index: idx, + Section: section, + Offset: off, + Raw: strings.TrimSpace(line), + }) + } + } + return t +} + +func IsNonSymbolic(text string) bool { + for _, line := range strings.Split(text, "\n") { + if frameRe.MatchString(line) { + return true + } + } + return false +} diff --git a/backend/app/symbolicator/otelprocessor/cache.go b/backend/app/symbolicator/otelprocessor/cache.go index 342e9a66..100278c5 100644 --- a/backend/app/symbolicator/otelprocessor/cache.go +++ b/backend/app/symbolicator/otelprocessor/cache.go @@ -1,155 +1,40 @@ package otelprocessor import ( - "container/list" - "context" "crypto/sha256" "encoding/hex" + "errors" "fmt" "os" - "sync" - "time" - "github.com/tracewayapp/traceway/backend/app/symbolicator" + "github.com/tracewayapp/traceway/backend/app/symbolicator/dart" + "github.com/tracewayapp/traceway/backend/app/symbolicator/sourcemap" "github.com/tracewayapp/traceway/backend/app/symbolicator/twcache" ) -const negativeCacheTTL = time.Minute -const negativeCacheMaxKeys = 10000 - -type buildFunc func(ctx context.Context) (*symbolicator.Resolver, error) - -type resolverCache struct { - mu sync.Mutex - entries map[string]*list.Element - order *list.List - maxEntries int - loading map[string]*resolverLoad - negative map[string]negativeEntry - disk *twcache.Cache -} - -type cacheEntry struct { - key string - resolver *symbolicator.Resolver -} - -type resolverLoad struct { - done chan struct{} - resolver *symbolicator.Resolver - err error -} - -type negativeEntry struct { - expiresAt time.Time - err error -} - -func newResolverCache(cfg *Config) (*resolverCache, error) { - c := &resolverCache{ - entries: make(map[string]*list.Element), - order: list.New(), - maxEntries: cfg.SourceMapCacheSize, - loading: make(map[string]*resolverLoad), - negative: make(map[string]negativeEntry), - } - if cfg.CacheDir != "" { - disk, err := newDiskCache(cfg.CacheDir, int64(cfg.CacheMaxMB)<<20, cfg.CacheMaxDiskPct) - if err != nil { - return nil, err - } - c.disk = disk - } - return c, nil +func validArtifact(b []byte) bool { + return sourcemap.ValidTW(b) || dart.ValidFlat(b) } -func (c *resolverCache) get(ctx context.Context, key string, build buildFunc) (*symbolicator.Resolver, error) { - c.mu.Lock() - if el, ok := c.entries[key]; ok { - c.order.MoveToFront(el) - r := el.Value.(*cacheEntry).resolver - c.mu.Unlock() - return r, nil - } - if n, ok := c.negative[key]; ok { - if time.Now().Before(n.expiresAt) { - c.mu.Unlock() - return nil, n.err - } - delete(c.negative, key) - } - if l, ok := c.loading[key]; ok { - c.mu.Unlock() - <-l.done - return l.resolver, l.err - } - l := &resolverLoad{done: make(chan struct{})} - c.loading[key] = l - c.mu.Unlock() - - l.resolver, l.err = c.load(ctx, key, build) - - c.mu.Lock() - delete(c.loading, key) - if l.err == nil && l.resolver != nil { - el := c.order.PushFront(&cacheEntry{key: key, resolver: l.resolver}) - c.entries[key] = el - for c.order.Len() > c.maxEntries { - back := c.order.Back() - evicted := c.order.Remove(back).(*cacheEntry) - delete(c.entries, evicted.key) - } - } else if l.err != nil { - if len(c.negative) >= negativeCacheMaxKeys { - now := time.Now() - for k, n := range c.negative { - if now.After(n.expiresAt) { - delete(c.negative, k) - } - } - } - if len(c.negative) < negativeCacheMaxKeys { - c.negative[key] = negativeEntry{expiresAt: time.Now().Add(negativeCacheTTL), err: l.err} - } - } - c.mu.Unlock() - close(l.done) - return l.resolver, l.err -} +func isObjectNotFound(err error) bool { return errors.Is(err, errObjectNotFound) } -func (c *resolverCache) load(ctx context.Context, key string, build buildFunc) (*symbolicator.Resolver, error) { - if c.disk == nil { - return build(ctx) - } - name := diskNameFor(key) - if r, err := c.disk.Open(name); err == nil { - return r, nil +func newCache(cfg *Config) (*twcache.Cache, error) { + if cfg.CacheDir == "" { + c := twcache.NewMem(cfg.SourceMapCacheSize, 1<<62) + c.Validate = validArtifact + c.NotFound = isObjectNotFound + return c, nil } - r, err := build(ctx) - if err != nil { - return nil, err - } - if cached, werr := c.disk.Write(name, r.MarshalTW()); werr == nil { - return cached, nil - } - return r, nil -} - -func diskNameFor(key string) string { - sum := sha256.Sum256([]byte(key)) - return hex.EncodeToString(sum[:]) + ".tw" -} - -func newDiskCache(dir string, maxBytes int64, maxPct int) (*twcache.Cache, error) { - if err := os.MkdirAll(dir, 0o755); err != nil { + if err := os.MkdirAll(cfg.CacheDir, 0o755); err != nil { return nil, fmt.Errorf("failed to create cache_dir: %w", err) } - if maxPct > 0 { - total, err := twcache.DiskCapacityBytes(dir) + maxBytes := int64(cfg.CacheMaxMB) << 20 + if cfg.CacheMaxDiskPct > 0 { + total, err := twcache.DiskCapacityBytes(cfg.CacheDir) if err != nil { return nil, fmt.Errorf("cache_max_disk_pct requires disk capacity detection: %w", err) } - pctBytes := total / 100 * int64(maxPct) + pctBytes := total / 100 * int64(cfg.CacheMaxDiskPct) if maxBytes <= 0 || pctBytes < maxBytes { maxBytes = pctBytes } @@ -157,9 +42,21 @@ func newDiskCache(dir string, maxBytes int64, maxPct int) (*twcache.Cache, error if maxBytes <= 0 { return nil, fmt.Errorf("the source map cache requires a positive byte cap (cache_max_mb or cache_max_disk_pct)") } - cache, err := twcache.New(dir, maxBytes, nil) + c, err := twcache.NewDisk(cfg.CacheDir, maxBytes, nil) if err != nil { - return nil, fmt.Errorf("failed to initialize cache_dir: %w", err) + return nil, err } - return cache, nil + c.Validate = validArtifact + c.NotFound = isObjectNotFound + return c, nil +} + +func cacheKey(url, buildUUID string) string { + sum := sha256.Sum256([]byte(url + "|" + buildUUID)) + return hex.EncodeToString(sum[:]) + ".tw" +} + +func dartCacheKey(symbolsKey string) string { + sum := sha256.Sum256([]byte(symbolsKey)) + return hex.EncodeToString(sum[:]) + ".tw" } diff --git a/backend/app/symbolicator/otelprocessor/config.go b/backend/app/symbolicator/otelprocessor/config.go index b586083a..b70082c7 100644 --- a/backend/app/symbolicator/otelprocessor/config.go +++ b/backend/app/symbolicator/otelprocessor/config.go @@ -56,6 +56,8 @@ type Config struct { CacheMaxMB int `mapstructure:"cache_max_mb"` CacheMaxDiskPct int `mapstructure:"cache_max_disk_pct"` + DartDefaultArch string `mapstructure:"dart_default_arch"` + LanguageAttributeKey string `mapstructure:"language_attribute_key"` AllowedLanguages []string `mapstructure:"allowed_languages"` diff --git a/backend/app/symbolicator/otelprocessor/dart_test.go b/backend/app/symbolicator/otelprocessor/dart_test.go new file mode 100644 index 00000000..c85c8296 --- /dev/null +++ b/backend/app/symbolicator/otelprocessor/dart_test.go @@ -0,0 +1,97 @@ +package otelprocessor + +import ( + "context" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/tracewayapp/traceway/backend/app/symbolicator/dart" + "go.opentelemetry.io/collector/pdata/pcommon" +) + +const dartSeedDir = "../../../../benchmarks/processor/seeds/dart" + +func TestSymbolicateDartTrace(t *testing.T) { + elf, err := os.ReadFile(filepath.Join(dartSeedDir, "app.debug.elf")) + if err != nil { + t.Skipf("dart seed not available (%v)", err) + } + rawBytes, err := os.ReadFile(filepath.Join(dartSeedDir, "trace.txt")) + if err != nil { + t.Skipf("dart seed trace not available (%v)", err) + } + raw := string(rawBytes) + + tr := dart.ParseTrace(raw) + if tr.BuildID == "" || tr.Arch == "" { + t.Fatalf("seed trace missing build_id/arch: %+v", tr) + } + store := t.TempDir() + symbolsName := dart.NormalizeDebugID(tr.BuildID) + "-" + dart.NormalizeArch(tr.Arch) + ".symbols" + if err := os.WriteFile(filepath.Join(store, symbolsName), elf, 0o644); err != nil { + t.Fatal(err) + } + + p := newTestProcessor(t, func(c *Config) { + c.SourceMapStoreKey = fileStoreKey + c.LocalSourceMaps = LocalSourceMapsConfig{Path: store} + }) + + attrs := pcommon.NewMap() + attrs.PutStr(p.cfg.ExceptionTypeAttributeKey, "Bad state") + attrs.PutStr(p.cfg.ExceptionMessageAttributeKey, "boom from level3") + attrs.PutStr(p.cfg.StackTraceAttributeKey, raw) + + p.processRecord(context.Background(), attrs, pcommon.NewMap()) + + got := strAttr(attrs, p.cfg.StackTraceAttributeKey) + if strings.Contains(got, "_kDartIsolateSnapshotInstructions") { + t.Errorf("expected all frames resolved, output still has the instruction symbol:\n%s", got) + } + if !strings.Contains(got, "crash.dart") { + t.Errorf("expected resolved dart frames (crash.dart), got:\n%s", got) + } + if !strings.Contains(got, "level3 (file:///private/tmp/dartfix/crash.dart:3:3)") { + t.Errorf("expected golden frame 0 'level3 (...crash.dart:3:3)', got:\n%s", got) + } + if v, ok := attrs.Get(p.cfg.SymbolicatorFailureAttributeKey); !ok || v.Bool() { + t.Errorf("expected symbolicator.failed=false, got %v", v) + } + + attrs2 := pcommon.NewMap() + attrs2.PutStr(p.cfg.StackTraceAttributeKey, raw) + p.processRecord(context.Background(), attrs2, pcommon.NewMap()) + if got2 := strAttr(attrs2, p.cfg.StackTraceAttributeKey); strings.Contains(got2, "_kDartIsolateSnapshotInstructions") { + t.Errorf("warm-cache resolve regressed:\n%s", got2) + } +} + +func TestSymbolicateDartTraceMissingSymbols(t *testing.T) { + rawBytes, err := os.ReadFile(filepath.Join(dartSeedDir, "trace.txt")) + if err != nil { + t.Skipf("dart seed trace not available (%v)", err) + } + raw := string(rawBytes) + + p := newTestProcessor(t, func(c *Config) { + c.SourceMapStoreKey = fileStoreKey + c.LocalSourceMaps = LocalSourceMapsConfig{Path: t.TempDir()} + }) + + attrs := pcommon.NewMap() + attrs.PutStr(p.cfg.StackTraceAttributeKey, raw) + p.processRecord(context.Background(), attrs, pcommon.NewMap()) + + got := strAttr(attrs, p.cfg.StackTraceAttributeKey) + if !strings.Contains(got, "_kDartIsolateSnapshotInstructions+") { + t.Errorf("expected stable unresolved offset frames, got:\n%s", got) + } + if strings.Contains(got, "build_id:") || strings.Contains(got, "abs ") { + t.Errorf("volatile header leaked into output:\n%s", got) + } + if v, ok := attrs.Get(p.cfg.SymbolicatorFailureAttributeKey); !ok || !v.Bool() { + t.Errorf("expected symbolicator.failed=true for missing symbols") + } +} diff --git a/backend/app/symbolicator/otelprocessor/factory.go b/backend/app/symbolicator/otelprocessor/factory.go index 100ec040..6be13b8e 100644 --- a/backend/app/symbolicator/otelprocessor/factory.go +++ b/backend/app/symbolicator/otelprocessor/factory.go @@ -9,7 +9,7 @@ import ( "go.opentelemetry.io/collector/processor" "go.opentelemetry.io/collector/processor/processorhelper" - "github.com/tracewayapp/traceway/backend/app/symbolicator/scopes" + "github.com/tracewayapp/traceway/backend/app/symbolicator/sourcemap/scopes" ) const processorVersion = "0.1.0" @@ -72,7 +72,7 @@ func newSymbolicator(cfg *Config, set processor.Settings) (*symbolicatorProcesso if err != nil { return nil, err } - cache, err := newResolverCache(cfg) + cache, err := newCache(cfg) if err != nil { return nil, err } diff --git a/backend/app/symbolicator/otelprocessor/negative_cache_test.go b/backend/app/symbolicator/otelprocessor/negative_cache_test.go new file mode 100644 index 00000000..d73667f3 --- /dev/null +++ b/backend/app/symbolicator/otelprocessor/negative_cache_test.go @@ -0,0 +1,35 @@ +package otelprocessor + +import ( + "context" + "testing" + + "go.opentelemetry.io/collector/pdata/pcommon" +) + +func TestJSNegativeCacheShortCircuit(t *testing.T) { + p := newTestProcessor(t, func(cfg *Config) { + cfg.LocalSourceMaps.Path = t.TempDir() + }) + + resolveOnce := func() { + attrs := pcommon.NewMap() + putStructuredFrame(attrs) + p.processRecord(context.Background(), attrs, pcommon.NewMap()) + } + + resolveOnce() + first := p.cache.Stats() + if first.Misses == 0 { + t.Fatalf("expected the first resolve to miss the cache and fetch the bundle: %+v", first) + } + + resolveOnce() + second := p.cache.Stats() + if second.Misses != first.Misses { + t.Errorf("missing bundle re-fetched on the second resolve (misses %d -> %d): negative-cache short-circuit not applied", first.Misses, second.Misses) + } + if second.NegativeHits == 0 { + t.Errorf("expected a negative-cache hit on the second resolve: %+v", second) + } +} diff --git a/backend/app/symbolicator/otelprocessor/processor_test.go b/backend/app/symbolicator/otelprocessor/processor_test.go index 135fc5fa..0301fd38 100644 --- a/backend/app/symbolicator/otelprocessor/processor_test.go +++ b/backend/app/symbolicator/otelprocessor/processor_test.go @@ -53,7 +53,7 @@ func newTestProcessor(t *testing.T, mutate func(*Config)) *symbolicatorProcessor if err != nil { t.Fatal(err) } - cache, err := newResolverCache(cfg) + cache, err := newCache(cfg) if err != nil { t.Fatal(err) } diff --git a/backend/app/symbolicator/otelprocessor/store.go b/backend/app/symbolicator/otelprocessor/store.go index e5de7937..b9710045 100644 --- a/backend/app/symbolicator/otelprocessor/store.go +++ b/backend/app/symbolicator/otelprocessor/store.go @@ -17,6 +17,8 @@ import ( awsconfig "github.com/aws/aws-sdk-go-v2/config" "github.com/aws/aws-sdk-go-v2/service/s3" "github.com/aws/aws-sdk-go-v2/service/s3/types" + + "github.com/tracewayapp/traceway/backend/app/symbolicator/dart" ) const ( @@ -104,6 +106,20 @@ func (a *artifactStore) getSourceAndMap(ctx context.Context, frameURL, buildUUID return source, sourceMap, nil } +func (a *artifactStore) dartSymbolsKey(buildID, arch string) string { + base := dart.NormalizeDebugID(buildID) + "-" + dart.NormalizeArch(arch) + ".symbols" + return a.key("", base) +} + +func (a *artifactStore) getDartSymbols(ctx context.Context, buildID, arch string) ([]byte, error) { + key := a.dartSymbolsKey(buildID, arch) + data, err := a.store.fetch(ctx, key) + if err != nil { + return nil, fmt.Errorf("failed to find dart symbols %q: %w", key, err) + } + return data, nil +} + func (a *artifactStore) key(buildUUID, base string) string { parts := make([]string, 0, 3) if a.prefix != "" { diff --git a/backend/app/symbolicator/otelprocessor/symbolicate.go b/backend/app/symbolicator/otelprocessor/symbolicate.go index 63a2b301..1f3513de 100644 --- a/backend/app/symbolicator/otelprocessor/symbolicate.go +++ b/backend/app/symbolicator/otelprocessor/symbolicate.go @@ -7,8 +7,10 @@ import ( "math" "strings" - "github.com/tracewayapp/traceway/backend/app/symbolicator" - "github.com/tracewayapp/traceway/backend/app/symbolicator/jsstack" + "github.com/tracewayapp/traceway/backend/app/symbolicator/dart" + "github.com/tracewayapp/traceway/backend/app/symbolicator/sourcemap" + "github.com/tracewayapp/traceway/backend/app/symbolicator/sourcemap/jsstack" + "github.com/tracewayapp/traceway/backend/app/symbolicator/twcache" "go.opentelemetry.io/collector/pdata/pcommon" "go.uber.org/zap" ) @@ -16,15 +18,17 @@ import ( const ( parsingMethodStructured = "structured_stacktrace_attributes" parsingMethodParsed = "processor_parsed" + maxDartFrames = 50 ) var errMismatchedLength = errors.New("mismatched stacktrace attribute lengths") var errUnparseableStackTrace = errors.New("unable to parse stack trace") +var errMissingDartBuild = errors.New("dart trace missing build_id or arch") type symbolicatorProcessor struct { cfg *Config store *artifactStore - cache *resolverCache + cache *twcache.Cache logger *zap.Logger } @@ -54,6 +58,11 @@ func (p *symbolicatorProcessor) processRecord(ctx context.Context, attrs, resour } originalStack := stackVal.Str() + if dart.IsNonSymbolic(originalStack) { + p.symbolicateDartTrace(ctx, attrs, originalStack) + return + } + buildUUID := "" if v, ok := resource.Get(p.cfg.BuildUUIDAttributeKey); ok { buildUUID = v.Str() @@ -131,10 +140,100 @@ func (p *symbolicatorProcessor) processRecord(ctx context.Context, attrs, resour } attrs.PutStr(p.cfg.SymbolicatorErrorAttributeKey, err.Error()) } + p.putProcessorMeta(attrs) +} + +func (p *symbolicatorProcessor) putProcessorMeta(attrs pcommon.Map) { attrs.PutStr("traceway.processor_type", componentType.String()) attrs.PutStr("traceway.processor_version", processorVersion) } +func (p *symbolicatorProcessor) symbolicateDartTrace(ctx context.Context, attrs pcommon.Map, rawStack string) { + attrs.PutStr(p.cfg.SymbolicatorParsingMethodAttributeKey, parsingMethodParsed) + if p.cfg.PreserveStackTrace { + attrs.PutStr(p.cfg.OriginalStackTraceAttributeKey, rawStack) + } + + trace := dart.ParseTrace(rawStack) + arch := trace.Arch + if arch == "" { + arch = p.cfg.DartDefaultArch + } + + fail := func(err error) { + attrs.PutStr(p.cfg.StackTraceAttributeKey, p.renderDartTrace(attrs, trace, nil)) + attrs.PutBool(p.cfg.SymbolicatorFailureAttributeKey, true) + attrs.PutStr(p.cfg.SymbolicatorErrorAttributeKey, err.Error()) + p.putProcessorMeta(attrs) + } + + if len(trace.Frames) == 0 { + fail(errUnparseableStackTrace) + return + } + if trace.BuildID == "" || arch == "" { + fail(errMissingDartBuild) + return + } + + key := dartCacheKey(p.store.dartSymbolsKey(trace.BuildID, arch)) + if p.cache.IsNegative(key) { + fail(fmt.Errorf("failed to find dart symbols for build %s/%s", trace.BuildID, arch)) + return + } + + data, done, err := p.cache.Get(ctx, key, func(ctx context.Context) ([]byte, error) { + fetchCtx, cancel := context.WithTimeout(ctx, p.cfg.Timeout) + defer cancel() + elf, err := p.store.getDartSymbols(fetchCtx, trace.BuildID, arch) + if err != nil { + return nil, err + } + return dart.BuildFlat(elf) + }) + if err != nil { + fail(err) + return + } + defer done() + + attrs.PutStr(p.cfg.StackTraceAttributeKey, p.renderDartTrace(attrs, trace, data)) + attrs.PutBool(p.cfg.SymbolicatorFailureAttributeKey, false) + p.putProcessorMeta(attrs) +} + +func (p *symbolicatorProcessor) renderDartTrace(attrs pcommon.Map, trace dart.StackTrace, data []byte) string { + var b strings.Builder + excType := strAttr(attrs, p.cfg.ExceptionTypeAttributeKey) + excMessage := strAttr(attrs, p.cfg.ExceptionMessageAttributeKey) + if excType != "" && excMessage != "" { + fmt.Fprintf(&b, "%s: %s\n", excType, excMessage) + } + n := 0 + for _, f := range trace.Frames { + if n >= maxDartFrames { + break + } + var resolved []dart.SymFrame + if data != nil { + resolved = dart.LookupFlat(data, f) + } + if len(resolved) == 0 { + fmt.Fprintf(&b, "#%d %s+%x\n", n, dart.InstructionSymbol(f.Section), f.Offset) + n++ + continue + } + for _, sf := range resolved { + if n >= maxDartFrames { + break + } + fmt.Fprintf(&b, "#%d %s (%s)\n", n, sf.Function, sf.Location()) + n++ + } + } + return strings.TrimRight(b.String(), "\n") +} + func (p *symbolicatorProcessor) languageAllowed(attrs, resource pcommon.Map) bool { if len(p.cfg.AllowedLanguages) == 0 { return true @@ -205,20 +304,25 @@ func (p *symbolicatorProcessor) symbolicateFrame(ctx context.Context, f stackFra return frameResult{err: fmt.Errorf("line/column out of range: %d:%d", f.line, f.col)} } - resolver, err := p.cache.get(ctx, f.url+"|"+buildUUID, func(ctx context.Context) (*symbolicator.Resolver, error) { + key := cacheKey(f.url, buildUUID) + if p.cache.IsNegative(key) { + return frameResult{err: fmt.Errorf("no source map for %s", f.url)} + } + data, done, err := p.cache.Get(ctx, key, func(ctx context.Context) ([]byte, error) { fetchCtx, cancel := context.WithTimeout(ctx, p.cfg.Timeout) defer cancel() source, sourceMap, err := p.store.getSourceAndMap(fetchCtx, f.url, buildUUID) if err != nil { return nil, err } - return symbolicator.NewResolver(sourceMap, source) + return sourcemap.BuildTW(sourceMap, source) }) if err != nil { return frameResult{err: err} } + defer done() - frame, ok := resolver.Lookup(uint32(f.line-1), uint32(f.col-1)) + frame, ok := sourcemap.LookupTW(data, uint32(f.line-1), uint32(f.col-1)) if !ok { return frameResult{err: fmt.Errorf("no mapping at %d:%d", f.line, f.col)} } diff --git a/backend/app/symbolicator/jsstack/canonicalize.go b/backend/app/symbolicator/sourcemap/jsstack/canonicalize.go similarity index 100% rename from backend/app/symbolicator/jsstack/canonicalize.go rename to backend/app/symbolicator/sourcemap/jsstack/canonicalize.go diff --git a/backend/app/symbolicator/jsstack/canonicalize_test.go b/backend/app/symbolicator/sourcemap/jsstack/canonicalize_test.go similarity index 100% rename from backend/app/symbolicator/jsstack/canonicalize_test.go rename to backend/app/symbolicator/sourcemap/jsstack/canonicalize_test.go diff --git a/backend/app/symbolicator/jsstack/frames.go b/backend/app/symbolicator/sourcemap/jsstack/frames.go similarity index 100% rename from backend/app/symbolicator/jsstack/frames.go rename to backend/app/symbolicator/sourcemap/jsstack/frames.go diff --git a/backend/app/symbolicator/oxc_parity_test.go b/backend/app/symbolicator/sourcemap/oxc_parity_test.go similarity index 70% rename from backend/app/symbolicator/oxc_parity_test.go rename to backend/app/symbolicator/sourcemap/oxc_parity_test.go index da35e4d0..6029dc6f 100644 --- a/backend/app/symbolicator/oxc_parity_test.go +++ b/backend/app/symbolicator/sourcemap/oxc_parity_test.go @@ -1,12 +1,11 @@ //go:build oxc && cgo -package symbolicator +package sourcemap import ( "testing" - "github.com/tracewayapp/traceway/backend/app/symbolicator/scopes" - "github.com/tracewayapp/traceway/backend/app/symbolicator/sourcemap" + "github.com/tracewayapp/traceway/backend/app/symbolicator/sourcemap/scopes" ) func TestOxcGojaLookupEquivalence(t *testing.T) { @@ -24,28 +23,28 @@ func TestOxcGojaLookupEquivalence(t *testing.T) { if err := scopes.SetParser("goja"); err != nil { t.Fatal(err) } - gojaResolver, err := NewResolver(mapBytes, bundle) + gojaTW, err := BuildTW(mapBytes, bundle) if err != nil { - t.Fatalf("NewResolver(goja): %v", err) + t.Fatalf("BuildTW(goja): %v", err) } if err := scopes.SetParser("oxc"); err != nil { t.Fatal(err) } - oxcResolver, err := NewResolver(mapBytes, bundle) + oxcTW, err := BuildTW(mapBytes, bundle) if err != nil { - t.Fatalf("NewResolver(oxc): %v", err) + t.Fatalf("BuildTW(oxc): %v", err) } - parsed, err := sourcemap.Parse(mapBytes) + parsed, err := Parse(mapBytes) if err != nil { t.Fatalf("parsing source map: %v", err) } mismatches := 0 for _, token := range parsed.Tokens { - gFrame, gOk := gojaResolver.Lookup(token.GenLine, token.GenCol) - oFrame, oOk := oxcResolver.Lookup(token.GenLine, token.GenCol) + gFrame, gOk := LookupTW(gojaTW, token.GenLine, token.GenCol) + oFrame, oOk := LookupTW(oxcTW, token.GenLine, token.GenCol) if gOk != oOk || gFrame != oFrame { mismatches++ if mismatches <= 10 { diff --git a/backend/app/symbolicator/parity_test.go b/backend/app/symbolicator/sourcemap/parity_test.go similarity index 94% rename from backend/app/symbolicator/parity_test.go rename to backend/app/symbolicator/sourcemap/parity_test.go index 10e1af9a..4130dfda 100644 --- a/backend/app/symbolicator/parity_test.go +++ b/backend/app/symbolicator/sourcemap/parity_test.go @@ -1,22 +1,22 @@ -package symbolicator +package sourcemap import ( "os" "path/filepath" "testing" - "github.com/tracewayapp/traceway/backend/app/symbolicator/scopes" + "github.com/tracewayapp/traceway/backend/app/symbolicator/sourcemap/scopes" ) func fixture(t testing.TB, parts ...string) string { t.Helper() - root, err := filepath.Abs(filepath.Join("..", "..", "..", "..", "symbolic")) + root, err := filepath.Abs(filepath.Join("..", "..", "..", "..", "..", "symbolic")) if err == nil { if _, statErr := os.Stat(root); statErr == nil { return filepath.Join(append([]string{root, "symbolic-testutils", "fixtures"}, parts...)...) } } - return filepath.Join(append([]string{"..", "services", "testdata"}, parts...)...) + return filepath.Join(append([]string{"..", "..", "services", "testdata"}, parts...)...) } func mustRead(t testing.TB, path string) []byte { @@ -140,13 +140,13 @@ func runParityCases(t *testing.T) { mapBytes := mustRead(t, fixture(t, tc.mapPath...)) bundle := readIfSet(t, tc.minifiedPath) - r, err := NewResolver(mapBytes, bundle) + tw, err := BuildTW(mapBytes, bundle) if err != nil { - t.Fatalf("NewResolver: %v", err) + t.Fatalf("BuildTW: %v", err) } for _, exp := range tc.expectations { - frame, ok := r.Lookup(exp.line, exp.col) + frame, ok := LookupTW(tw, exp.line, exp.col) if exp.wantNone { if ok { diff --git a/backend/app/symbolicator/resolver_bench_test.go b/backend/app/symbolicator/sourcemap/resolver_bench_test.go similarity index 71% rename from backend/app/symbolicator/resolver_bench_test.go rename to backend/app/symbolicator/sourcemap/resolver_bench_test.go index aa4b2b6d..690d9de0 100644 --- a/backend/app/symbolicator/resolver_bench_test.go +++ b/backend/app/symbolicator/sourcemap/resolver_bench_test.go @@ -1,12 +1,12 @@ -package symbolicator +package sourcemap import ( "testing" - "github.com/tracewayapp/traceway/backend/app/symbolicator/scopes" + "github.com/tracewayapp/traceway/backend/app/symbolicator/sourcemap/scopes" ) -func BenchmarkNewResolver(b *testing.B) { +func BenchmarkBuildTW(b *testing.B) { mapBytes := mustRead(b, fixture(b, "sourcemapcache", "preact.module.js.map")) bundle := mustRead(b, fixture(b, "sourcemapcache", "preact.module.js")) @@ -21,7 +21,7 @@ func BenchmarkNewResolver(b *testing.B) { b.ReportAllocs() b.ResetTimer() for b.Loop() { - if _, err := NewResolver(mapBytes, bundle); err != nil { + if _, err := BuildTW(mapBytes, bundle); err != nil { b.Fatal(err) } } @@ -29,21 +29,20 @@ func BenchmarkNewResolver(b *testing.B) { } } -func BenchmarkOpenTW(b *testing.B) { +func BenchmarkLookupTW(b *testing.B) { mapBytes := mustRead(b, fixture(b, "sourcemapcache", "preact.module.js.map")) bundle := mustRead(b, fixture(b, "sourcemapcache", "preact.module.js")) - resolver, err := NewResolver(mapBytes, bundle) + tw, err := BuildTW(mapBytes, bundle) if err != nil { b.Fatal(err) } - tw := resolver.MarshalTW() b.SetBytes(int64(len(tw))) b.ReportAllocs() b.ResetTimer() for b.Loop() { - if _, err := OpenTW(tw); err != nil { - b.Fatal(err) + if _, ok := LookupTW(tw, 0, 132); !ok { + b.Fatal("expected a mapping") } } } diff --git a/backend/app/symbolicator/scopes/bench_test.go b/backend/app/symbolicator/sourcemap/scopes/bench_test.go similarity index 100% rename from backend/app/symbolicator/scopes/bench_test.go rename to backend/app/symbolicator/sourcemap/scopes/bench_test.go diff --git a/backend/app/symbolicator/scopes/goja.go b/backend/app/symbolicator/sourcemap/scopes/goja.go similarity index 100% rename from backend/app/symbolicator/scopes/goja.go rename to backend/app/symbolicator/sourcemap/scopes/goja.go diff --git a/backend/app/symbolicator/scopes/oxc-shim/Cargo.lock b/backend/app/symbolicator/sourcemap/scopes/oxc-shim/Cargo.lock similarity index 100% rename from backend/app/symbolicator/scopes/oxc-shim/Cargo.lock rename to backend/app/symbolicator/sourcemap/scopes/oxc-shim/Cargo.lock diff --git a/backend/app/symbolicator/scopes/oxc-shim/Cargo.toml b/backend/app/symbolicator/sourcemap/scopes/oxc-shim/Cargo.toml similarity index 100% rename from backend/app/symbolicator/scopes/oxc-shim/Cargo.toml rename to backend/app/symbolicator/sourcemap/scopes/oxc-shim/Cargo.toml diff --git a/backend/app/symbolicator/scopes/oxc-shim/include/oxc_shim.h b/backend/app/symbolicator/sourcemap/scopes/oxc-shim/include/oxc_shim.h similarity index 100% rename from backend/app/symbolicator/scopes/oxc-shim/include/oxc_shim.h rename to backend/app/symbolicator/sourcemap/scopes/oxc-shim/include/oxc_shim.h diff --git a/backend/app/symbolicator/scopes/oxc-shim/src/lib.rs b/backend/app/symbolicator/sourcemap/scopes/oxc-shim/src/lib.rs similarity index 100% rename from backend/app/symbolicator/scopes/oxc-shim/src/lib.rs rename to backend/app/symbolicator/sourcemap/scopes/oxc-shim/src/lib.rs diff --git a/backend/app/symbolicator/scopes/oxc.go b/backend/app/symbolicator/sourcemap/scopes/oxc.go similarity index 100% rename from backend/app/symbolicator/scopes/oxc.go rename to backend/app/symbolicator/sourcemap/scopes/oxc.go diff --git a/backend/app/symbolicator/scopes/registry.go b/backend/app/symbolicator/sourcemap/scopes/registry.go similarity index 100% rename from backend/app/symbolicator/scopes/registry.go rename to backend/app/symbolicator/sourcemap/scopes/registry.go diff --git a/backend/app/symbolicator/scopes/scopes.go b/backend/app/symbolicator/sourcemap/scopes/scopes.go similarity index 100% rename from backend/app/symbolicator/scopes/scopes.go rename to backend/app/symbolicator/sourcemap/scopes/scopes.go diff --git a/backend/app/symbolicator/sourcemap/tw.go b/backend/app/symbolicator/sourcemap/tw.go new file mode 100644 index 00000000..0261664e --- /dev/null +++ b/backend/app/symbolicator/sourcemap/tw.go @@ -0,0 +1,180 @@ +package sourcemap + +import ( + "encoding/binary" + "sort" +) + +const twVersion = 1 +const twHeaderSize = 24 +const twTokenSize = 24 + +var twMagic = [4]byte{'T', 'W', 'S', 'M'} + +func (b *builder) marshal() []byte { + fileOffsets, fileBlob := encodeStringTable(b.files) + fnOffsets, fnBlob := encodeStringTable(b.fns) + + size := twHeaderSize + + len(b.tokens)*twTokenSize + + len(fileOffsets)*4 + len(fileBlob) + + len(fnOffsets)*4 + len(fnBlob) + + out := make([]byte, 0, size) + out = append(out, twMagic[:]...) + out = binary.LittleEndian.AppendUint32(out, twVersion) + out = binary.LittleEndian.AppendUint32(out, uint32(len(b.tokens))) + out = binary.LittleEndian.AppendUint32(out, uint32(len(b.files))) + out = binary.LittleEndian.AppendUint32(out, uint32(len(b.fns))) + out = binary.LittleEndian.AppendUint32(out, 0) + + for i := range b.tokens { + t := &b.tokens[i] + out = binary.LittleEndian.AppendUint32(out, t.genLine) + out = binary.LittleEndian.AppendUint32(out, t.genCol) + out = binary.LittleEndian.AppendUint32(out, t.srcLine) + out = binary.LittleEndian.AppendUint32(out, t.srcCol) + out = binary.LittleEndian.AppendUint32(out, uint32(t.fileIdx)) + out = binary.LittleEndian.AppendUint32(out, uint32(t.fnIdx)) + } + + for _, off := range fileOffsets { + out = binary.LittleEndian.AppendUint32(out, off) + } + out = append(out, fileBlob...) + for _, off := range fnOffsets { + out = binary.LittleEndian.AppendUint32(out, off) + } + out = append(out, fnBlob...) + return out +} + +func encodeStringTable(strs []string) ([]uint32, []byte) { + offsets := make([]uint32, len(strs)+1) + var blob []byte + for i, s := range strs { + offsets[i] = uint32(len(blob)) + blob = append(blob, s...) + } + offsets[len(strs)] = uint32(len(blob)) + return offsets, blob +} + +type twLayout struct { + tokenCount int + fileTableOff int + filesCount int + fnTableOff int + fnsCount int +} + +func readTWLayout(data []byte) (twLayout, bool) { + var l twLayout + if len(data) < twHeaderSize { + return l, false + } + if [4]byte(data[:4]) != twMagic { + return l, false + } + le := binary.LittleEndian + if le.Uint32(data[4:]) != twVersion { + return l, false + } + tokenCount := int64(le.Uint32(data[8:])) + l.filesCount = int(le.Uint32(data[12:])) + l.fnsCount = int(le.Uint32(data[16:])) + n := int64(len(data)) + + fileTableOff := int64(twHeaderSize) + tokenCount*twTokenSize + if tokenCount < 0 || l.filesCount < 0 || l.fnsCount < 0 || fileTableOff < int64(twHeaderSize) { + return l, false + } + fileBlobOff := fileTableOff + int64(l.filesCount+1)*4 + if fileBlobOff > n { + return l, false + } + fileBlobLen := int64(le.Uint32(data[fileTableOff+int64(l.filesCount)*4:])) + fnTableOff := fileBlobOff + fileBlobLen + fnBlobOff := fnTableOff + int64(l.fnsCount+1)*4 + if fnTableOff < fileBlobOff || fnBlobOff > n { + return l, false + } + fnBlobLen := int64(le.Uint32(data[fnTableOff+int64(l.fnsCount)*4:])) + if fnBlobOff+fnBlobLen > n { + return l, false + } + + l.tokenCount = int(tokenCount) + l.fileTableOff = int(fileTableOff) + l.fnTableOff = int(fnTableOff) + return l, true +} + +func ValidTW(data []byte) bool { + _, ok := readTWLayout(data) + return ok +} + +func LookupTW(data []byte, genLine, genCol uint32) (StackTraceFrame, bool) { + l, ok := readTWLayout(data) + if !ok { + return StackTraceFrame{}, false + } + le := binary.LittleEndian + tokAt := func(i int) (gl, gc uint32) { + rec := data[twHeaderSize+i*twTokenSize:] + return le.Uint32(rec), le.Uint32(rec[4:]) + } + + idx := sort.Search(l.tokenCount, func(i int) bool { + gl, gc := tokAt(i) + return gl > genLine || (gl == genLine && gc > genCol) + }) + if idx == 0 { + return StackTraceFrame{}, false + } + idx-- + for idx > 0 { + gl, gc := tokAt(idx) + pl, pc := tokAt(idx - 1) + if pl != gl || pc != gc { + break + } + idx-- + } + + rec := data[twHeaderSize+idx*twTokenSize:] + tGenLine := le.Uint32(rec) + srcLine := le.Uint32(rec[8:]) + srcCol := le.Uint32(rec[12:]) + fileIdx := int32(le.Uint32(rec[16:])) + fnIdx := int32(le.Uint32(rec[20:])) + if tGenLine < genLine || fileIdx < 0 { + return StackTraceFrame{}, false + } + file, fok := stringAt(data, l.fileTableOff, l.filesCount, fileIdx) + if !fok { + return StackTraceFrame{}, false + } + frame := StackTraceFrame{File: file, Line: srcLine, Col: srcCol} + if fnIdx >= 0 { + if fn, ok := stringAt(data, l.fnTableOff, l.fnsCount, fnIdx); ok { + frame.Fn = fn + } + } + return frame, true +} + +func stringAt(data []byte, tableOff, count int, idx int32) (string, bool) { + if idx < 0 || int(idx) >= count { + return "", false + } + le := binary.LittleEndian + blobStart := tableOff + (count+1)*4 + o0 := int(le.Uint32(data[tableOff+int(idx)*4:])) + o1 := int(le.Uint32(data[tableOff+(int(idx)+1)*4:])) + if o0 > o1 || blobStart+o1 > len(data) { + return "", false + } + return string(data[blobStart+o0 : blobStart+o1]), true +} diff --git a/backend/app/symbolicator/symbolicator.go b/backend/app/symbolicator/sourcemap/tw_build.go similarity index 50% rename from backend/app/symbolicator/symbolicator.go rename to backend/app/symbolicator/sourcemap/tw_build.go index bcf4fe33..da76be6b 100644 --- a/backend/app/symbolicator/symbolicator.go +++ b/backend/app/symbolicator/sourcemap/tw_build.go @@ -1,11 +1,7 @@ -package symbolicator +package sourcemap import ( - "runtime" - "sort" - - "github.com/tracewayapp/traceway/backend/app/symbolicator/scopes" - "github.com/tracewayapp/traceway/backend/app/symbolicator/sourcemap" + "github.com/tracewayapp/traceway/backend/app/symbolicator/sourcemap/scopes" ) type StackTraceFrame struct { @@ -22,14 +18,22 @@ type resolvedToken struct { fnIdx int32 } -type Resolver struct { +type builder struct { tokens []resolvedToken files []string fns []string } -func NewResolver(sourceMap, bundle []byte) (*Resolver, error) { - m, err := sourcemap.Parse(sourceMap) +func BuildTW(sourceMap, bundle []byte) ([]byte, error) { + b, err := newBuilder(sourceMap, bundle) + if err != nil { + return nil, err + } + return b.marshal(), nil +} + +func newBuilder(sourceMap, bundle []byte) (*builder, error) { + m, err := Parse(sourceMap) if err != nil { return nil, err } @@ -52,8 +56,6 @@ func NewResolver(sourceMap, bundle []byte) (*Resolver, error) { for i := range m.Tokens { t := m.Tokens[i] - // Tokens are sorted by generated position, so the floor transition - // pointer only moves forward across the whole map (linear merge). for ti+1 < len(transitions) { next := &transitions[ti+1] if t.GenLine < next.Line || (t.GenLine == next.Line && t.GenCol < next.Col) { @@ -64,17 +66,16 @@ func NewResolver(sourceMap, bundle []byte) (*Resolver, error) { } rt := resolvedToken{genLine: t.GenLine, genCol: t.GenCol, fileIdx: -1, fnIdx: -1} - if t.SrcID != sourcemap.NoID && int(t.SrcID) < len(m.Sources) { + if t.SrcID != NoID && int(t.SrcID) < len(m.Sources) { rt.fileIdx = files.intern(m.Sources[t.SrcID]) rt.srcLine = t.SrcLine + 1 rt.srcCol = t.SrcCol + 1 if ti >= 0 && transitions[ti].Named { - // Resolve the enclosing function's name once per transition, - // not once per token. + if !curResolved { curFnIdx = -1 tr := transitions[ti] - if nt := m.FloorToken(tr.NameLine, tr.NameCol); nt != nil && nt.NameID != sourcemap.NoID { + if nt := m.FloorToken(tr.NameLine, tr.NameCol); nt != nil && nt.NameID != NoID { curFnIdx = fns.intern(m.Names[nt.NameID]) } curResolved = true @@ -85,46 +86,7 @@ func NewResolver(sourceMap, bundle []byte) (*Resolver, error) { tokens[i] = rt } - return &Resolver{tokens: tokens, files: files.list, fns: fns.list}, nil -} - -func (r *Resolver) Lookup(genLine, genCol uint32) (StackTraceFrame, bool) { - defer runtime.KeepAlive(r) - toks := r.tokens - idx := sort.Search(len(toks), func(i int) bool { - return toks[i].genLine > genLine || (toks[i].genLine == genLine && toks[i].genCol > genCol) - }) - if idx == 0 { - return StackTraceFrame{}, false - } - idx-- - for idx > 0 && toks[idx-1].genLine == toks[idx].genLine && toks[idx-1].genCol == toks[idx].genCol { - idx-- - } - - t := toks[idx] - if t.genLine < genLine || t.fileIdx < 0 { - return StackTraceFrame{}, false - } - frame := StackTraceFrame{File: r.files[t.fileIdx], Line: t.srcLine, Col: t.srcCol} - if t.fnIdx >= 0 { - frame.Fn = r.fns[t.fnIdx] - } - return frame, true -} - -// ApproxSize estimates the resolver's retained heap footprint, for cache -// accounting. The flat token table dominates; the bundle and source map are -// not retained. -func (r *Resolver) ApproxSize() int64 { - n := int64(len(r.tokens)) * 24 - for _, s := range r.files { - n += int64(len(s)) + 16 - } - for _, s := range r.fns { - n += int64(len(s)) + 16 - } - return n + return &builder{tokens: tokens, files: files.list, fns: fns.list}, nil } type interner struct { diff --git a/backend/app/symbolicator/sourcemap/tw_test.go b/backend/app/symbolicator/sourcemap/tw_test.go new file mode 100644 index 00000000..d80cc59e --- /dev/null +++ b/backend/app/symbolicator/sourcemap/tw_test.go @@ -0,0 +1,117 @@ +package sourcemap + +import ( + "encoding/binary" + "testing" +) + +const minimalMap = `{"version":3,"sources":["a.js"],"names":["foo"],"mappings":"AAAAA"}` + +func TestLookupTWRoundTrip(t *testing.T) { + tw, err := BuildTW([]byte(minimalMap), nil) + if err != nil { + t.Fatalf("BuildTW: %v", err) + } + if !ValidTW(tw) { + t.Fatal("BuildTW produced bytes ValidTW rejects") + } + frame, ok := LookupTW(tw, 0, 0) + if !ok { + t.Fatal("expected a mapping at (0,0)") + } + if frame.File != "a.js" || frame.Line != 1 || frame.Col != 1 { + t.Errorf("got %+v, want a.js:1:1", frame) + } +} + +func TestLookupTWEmpty(t *testing.T) { + tw, err := BuildTW([]byte(`{"version":3,"sources":[],"names":[],"mappings":""}`), nil) + if err != nil { + t.Fatalf("BuildTW: %v", err) + } + if !ValidTW(tw) { + t.Fatal("empty map should still be a valid artifact") + } + if _, ok := LookupTW(tw, 0, 0); ok { + t.Error("empty resolver should not resolve anything") + } +} + +func TestValidTWRejectsInvalid(t *testing.T) { + cases := map[string][]byte{ + "empty": nil, + "short": []byte("TWSM"), + "bad_magic": append([]byte("XXXX"), make([]byte, 20)...), + "bad_version": append([]byte("TWSM\xff\x00\x00\x00"), make([]byte, 16)...), + "token_overrun": append([]byte("TWSM\x01\x00\x00\x00\xff\xff\xff\x00"), make([]byte, 12)...), + } + for name, data := range cases { + if ValidTW(data) { + t.Errorf("%s: ValidTW should be false", name) + } + if _, ok := LookupTW(data, 0, 0); ok { + t.Errorf("%s: LookupTW should not resolve invalid data", name) + } + } +} + +func buildTW(tokens [][6]uint32, files, fns []string) []byte { + le := binary.LittleEndian + out := append([]byte{}, twMagic[:]...) + out = le.AppendUint32(out, twVersion) + out = le.AppendUint32(out, uint32(len(tokens))) + out = le.AppendUint32(out, uint32(len(files))) + out = le.AppendUint32(out, uint32(len(fns))) + out = le.AppendUint32(out, 0) + for _, tok := range tokens { + for _, v := range tok { + out = le.AppendUint32(out, v) + } + } + out = appendStringTableBytes(out, files) + out = appendStringTableBytes(out, fns) + return out +} + +func appendStringTableBytes(out []byte, ss []string) []byte { + offsets, blob := encodeStringTable(ss) + for _, o := range offsets { + out = binary.LittleEndian.AppendUint32(out, o) + } + return append(out, blob...) +} + +func TestLookupTWOutOfRangeIndexSafe(t *testing.T) { + const noID = uint32(0xFFFFFFFF) + cases := map[string][6]uint32{ + "fileIdx_too_big": {0, 0, 1, 1, 5, 0}, + "fnIdx_too_big": {0, 0, 1, 1, 0, 7}, + } + for name, tok := range cases { + t.Run(name, func(t *testing.T) { + data := buildTW([][6]uint32{tok}, []string{"a.js"}, []string{"fn"}) + frame, ok := LookupTW(data, 0, 0) + if name == "fileIdx_too_big" && ok { + t.Errorf("out-of-range fileIdx should not resolve, got %+v", frame) + } + + if name == "fnIdx_too_big" { + if !ok || frame.File != "a.js" { + t.Errorf("expected file to resolve, got %+v ok=%v", frame, ok) + } + if frame.Fn != "" { + t.Errorf("out-of-range fnIdx should drop the name, got %q", frame.Fn) + } + } + _ = noID + }) + } +} + +func TestLookupTWNegativeIndex(t *testing.T) { + const noID = uint32(0xFFFFFFFF) + data := buildTW([][6]uint32{{0, 0, 1, 1, noID, noID}}, []string{"a.js"}, []string{"fn"}) + if _, ok := LookupTW(data, 0, 0); ok { + t.Fatal("token with fileIdx=-1 should not resolve") + } +} diff --git a/backend/app/symbolicator/tw.go b/backend/app/symbolicator/tw.go deleted file mode 100644 index 4ccf372b..00000000 --- a/backend/app/symbolicator/tw.go +++ /dev/null @@ -1,161 +0,0 @@ -package symbolicator - -import ( - "encoding/binary" - "errors" - "runtime" - "unsafe" -) - -const twVersion = 1 -const twHeaderSize = 24 -const twTokenSize = 24 - -var twMagic = [4]byte{'T', 'W', 'S', 'M'} - -var ErrInvalidTW = errors.New("symbolicator: invalid tw data") - -var hostLittleEndian = func() bool { - var x uint16 = 1 - return *(*byte)(unsafe.Pointer(&x)) == 1 -}() - -func (r *Resolver) MarshalTW() []byte { - defer runtime.KeepAlive(r) - fileOffsets, fileBlob := encodeStringTable(r.files) - fnOffsets, fnBlob := encodeStringTable(r.fns) - - size := twHeaderSize + - len(r.tokens)*twTokenSize + - len(fileOffsets)*4 + len(fileBlob) + - len(fnOffsets)*4 + len(fnBlob) - - out := make([]byte, 0, size) - out = append(out, twMagic[:]...) - out = binary.LittleEndian.AppendUint32(out, twVersion) - out = binary.LittleEndian.AppendUint32(out, uint32(len(r.tokens))) - out = binary.LittleEndian.AppendUint32(out, uint32(len(r.files))) - out = binary.LittleEndian.AppendUint32(out, uint32(len(r.fns))) - out = binary.LittleEndian.AppendUint32(out, 0) - - for i := range r.tokens { - t := &r.tokens[i] - out = binary.LittleEndian.AppendUint32(out, t.genLine) - out = binary.LittleEndian.AppendUint32(out, t.genCol) - out = binary.LittleEndian.AppendUint32(out, t.srcLine) - out = binary.LittleEndian.AppendUint32(out, t.srcCol) - out = binary.LittleEndian.AppendUint32(out, uint32(t.fileIdx)) - out = binary.LittleEndian.AppendUint32(out, uint32(t.fnIdx)) - } - - for _, off := range fileOffsets { - out = binary.LittleEndian.AppendUint32(out, off) - } - out = append(out, fileBlob...) - for _, off := range fnOffsets { - out = binary.LittleEndian.AppendUint32(out, off) - } - out = append(out, fnBlob...) - return out -} - -func encodeStringTable(strs []string) ([]uint32, []byte) { - offsets := make([]uint32, len(strs)+1) - var blob []byte - for i, s := range strs { - offsets[i] = uint32(len(blob)) - blob = append(blob, s...) - } - offsets[len(strs)] = uint32(len(blob)) - return offsets, blob -} - -func OpenTW(data []byte) (*Resolver, error) { - if len(data) < twHeaderSize || [4]byte(data[:4]) != twMagic { - return nil, ErrInvalidTW - } - if binary.LittleEndian.Uint32(data[4:]) != twVersion { - return nil, ErrInvalidTW - } - tokenCount := int(binary.LittleEndian.Uint32(data[8:])) - filesCount := int(binary.LittleEndian.Uint32(data[12:])) - fnsCount := int(binary.LittleEndian.Uint32(data[16:])) - - pos := uint64(twHeaderSize) - tokensOff := pos - pos += uint64(tokenCount) * twTokenSize - if pos > uint64(len(data)) { - return nil, ErrInvalidTW - } - - files, pos, err := decodeStringTable(data, pos, filesCount) - if err != nil { - return nil, err - } - fns, pos, err := decodeStringTable(data, pos, fnsCount) - if err != nil { - return nil, err - } - if pos != uint64(len(data)) { - return nil, ErrInvalidTW - } - - r := &Resolver{files: files, fns: fns} - if tokenCount > 0 { - tokenBytes := data[tokensOff : tokensOff+uint64(tokenCount)*twTokenSize] - if hostLittleEndian && - unsafe.Sizeof(resolvedToken{}) == twTokenSize && - uintptr(unsafe.Pointer(&tokenBytes[0]))%unsafe.Alignof(resolvedToken{}) == 0 { - r.tokens = unsafe.Slice((*resolvedToken)(unsafe.Pointer(&tokenBytes[0])), tokenCount) - } else { - r.tokens = decodeTokens(tokenBytes, tokenCount) - } - } - for i := range r.tokens { - t := &r.tokens[i] - if int(t.fileIdx) >= len(files) || int(t.fnIdx) >= len(fns) { - return nil, ErrInvalidTW - } - } - return r, nil -} - -func decodeTokens(b []byte, count int) []resolvedToken { - tokens := make([]resolvedToken, count) - for i := range tokens { - rec := b[i*twTokenSize:] - tokens[i] = resolvedToken{ - genLine: binary.LittleEndian.Uint32(rec), - genCol: binary.LittleEndian.Uint32(rec[4:]), - srcLine: binary.LittleEndian.Uint32(rec[8:]), - srcCol: binary.LittleEndian.Uint32(rec[12:]), - fileIdx: int32(binary.LittleEndian.Uint32(rec[16:])), - fnIdx: int32(binary.LittleEndian.Uint32(rec[20:])), - } - } - return tokens -} - -func decodeStringTable(data []byte, pos uint64, count int) ([]string, uint64, error) { - offsetsEnd := pos + uint64(count+1)*4 - if count < 0 || offsetsEnd > uint64(len(data)) { - return nil, 0, ErrInvalidTW - } - offsets := make([]uint32, count+1) - for i := range offsets { - offsets[i] = binary.LittleEndian.Uint32(data[pos+uint64(i)*4:]) - if i > 0 && offsets[i-1] > offsets[i] { - return nil, 0, ErrInvalidTW - } - } - blobStart := offsetsEnd - blobEnd := blobStart + uint64(offsets[count]) - if blobEnd > uint64(len(data)) { - return nil, 0, ErrInvalidTW - } - strs := make([]string, count) - for i := range count { - strs[i] = string(data[blobStart+uint64(offsets[i]) : blobStart+uint64(offsets[i+1])]) - } - return strs, blobEnd, nil -} diff --git a/backend/app/symbolicator/tw_test.go b/backend/app/symbolicator/tw_test.go deleted file mode 100644 index cec85c61..00000000 --- a/backend/app/symbolicator/tw_test.go +++ /dev/null @@ -1,155 +0,0 @@ -package symbolicator - -import ( - "encoding/binary" - "testing" - - "github.com/tracewayapp/traceway/backend/app/symbolicator/sourcemap" -) - -func TestTWRoundTrip(t *testing.T) { - for _, tc := range parityCases { - t.Run(tc.name, func(t *testing.T) { - mapBytes := mustRead(t, fixture(t, tc.mapPath...)) - bundle := readIfSet(t, tc.minifiedPath) - - original, err := NewResolver(mapBytes, bundle) - if err != nil { - t.Fatalf("NewResolver: %v", err) - } - - reopened, err := OpenTW(original.MarshalTW()) - if err != nil { - t.Fatalf("OpenTW: %v", err) - } - - parsed, err := sourcemap.Parse(mapBytes) - if err != nil { - t.Fatalf("parsing source map: %v", err) - } - - mismatches := 0 - for _, token := range parsed.Tokens { - oFrame, oOk := original.Lookup(token.GenLine, token.GenCol) - rFrame, rOk := reopened.Lookup(token.GenLine, token.GenCol) - if oOk != rOk || oFrame != rFrame { - mismatches++ - if mismatches <= 10 { - t.Errorf("lookup(%d,%d): original=(%+v,%v) reopened=(%+v,%v)", token.GenLine, token.GenCol, oFrame, oOk, rFrame, rOk) - } - } - } - if mismatches > 10 { - t.Errorf("%d total mismatches", mismatches) - } - - for _, exp := range tc.expectations { - frame, ok := reopened.Lookup(exp.line, exp.col) - if exp.wantNone { - if ok { - t.Errorf("lookup(%d,%d): got %+v, want no mapping", exp.line, exp.col, frame) - } - continue - } - if !ok { - t.Errorf("lookup(%d,%d): got no mapping, want a result", exp.line, exp.col) - continue - } - if exp.file != "" && frame.File != exp.file { - t.Errorf("lookup(%d,%d) file: got %q, want %q", exp.line, exp.col, frame.File, exp.file) - } - if frame.Line != exp.srcLine+1 || frame.Col != exp.srcCol+1 { - t.Errorf("lookup(%d,%d) pos: got %d:%d, want %d:%d", exp.line, exp.col, frame.Line, frame.Col, exp.srcLine+1, exp.srcCol+1) - } - if frame.Fn != exp.fn { - t.Errorf("lookup(%d,%d) fn: got %q, want %q", exp.line, exp.col, frame.Fn, exp.fn) - } - } - }) - } -} - -func TestOpenTWRejectsInvalid(t *testing.T) { - cases := map[string][]byte{ - "empty": nil, - "short": []byte("TWSM"), - "bad_magic": append([]byte("XXXX"), make([]byte, 20)...), - "bad_version": append([]byte("TWSM\xff\x00\x00\x00"), make([]byte, 16)...), - "truncated": func() []byte { r := &Resolver{}; return r.MarshalTW()[:twHeaderSize-1] }(), - "token_overrun": append([]byte("TWSM\x01\x00\x00\x00\xff\xff\xff\x00"), make([]byte, 12)...), - } - for name, data := range cases { - if _, err := OpenTW(data); err == nil { - t.Errorf("%s: expected error, got nil", name) - } - } -} - -func buildTW(t *testing.T, tokens [][6]uint32, fileOffsets []uint32, fileBlob []byte, fnOffsets []uint32, fnBlob []byte) []byte { - t.Helper() - out := []byte("TWSM") - out = binary.LittleEndian.AppendUint32(out, 1) - out = binary.LittleEndian.AppendUint32(out, uint32(len(tokens))) - out = binary.LittleEndian.AppendUint32(out, uint32(len(fileOffsets)-1)) - out = binary.LittleEndian.AppendUint32(out, uint32(len(fnOffsets)-1)) - out = binary.LittleEndian.AppendUint32(out, 0) - for _, tok := range tokens { - for _, v := range tok { - out = binary.LittleEndian.AppendUint32(out, v) - } - } - for _, off := range fileOffsets { - out = binary.LittleEndian.AppendUint32(out, off) - } - out = append(out, fileBlob...) - for _, off := range fnOffsets { - out = binary.LittleEndian.AppendUint32(out, off) - } - out = append(out, fnBlob...) - return out -} - -func TestOpenTWRejectsCorruptStringOffsets(t *testing.T) { - data := buildTW(t, nil, []uint32{0, 1000, 5}, []byte("abcde"), []uint32{0}, nil) - if _, err := OpenTW(data); err == nil { - t.Fatal("expected error for non-monotonic string offsets, got nil") - } -} - -func TestOpenTWRejectsOutOfRangeTokenIndexes(t *testing.T) { - cases := map[string][6]uint32{ - "fileIdx_too_big": {0, 0, 1, 1, 5, 0xFFFFFFFF}, - "fnIdx_too_big": {0, 0, 1, 1, 0, 7}, - } - for name, tok := range cases { - t.Run(name, func(t *testing.T) { - data := buildTW(t, [][6]uint32{tok}, []uint32{0, 4}, []byte("a.js"), []uint32{0, 2}, []byte("fn")) - if _, err := OpenTW(data); err == nil { - t.Fatal("expected error for out-of-range token index, got nil") - } - }) - } -} - -func TestOpenTWAcceptsNegativeTokenIndexes(t *testing.T) { - noID := uint32(0xFFFFFFFF) - data := buildTW(t, [][6]uint32{{0, 0, 1, 1, noID, noID}}, []uint32{0, 4}, []byte("a.js"), []uint32{0, 2}, []byte("fn")) - r, err := OpenTW(data) - if err != nil { - t.Fatalf("OpenTW: %v", err) - } - if _, ok := r.Lookup(0, 0); ok { - t.Fatal("token with fileIdx=-1 should not resolve") - } -} - -func TestMarshalTWEmpty(t *testing.T) { - r := &Resolver{} - reopened, err := OpenTW(r.MarshalTW()) - if err != nil { - t.Fatalf("OpenTW: %v", err) - } - if _, ok := reopened.Lookup(0, 0); ok { - t.Error("empty resolver should not resolve anything") - } -} diff --git a/backend/app/symbolicator/twcache/disk.go b/backend/app/symbolicator/twcache/disk.go new file mode 100644 index 00000000..e3fed459 --- /dev/null +++ b/backend/app/symbolicator/twcache/disk.go @@ -0,0 +1,238 @@ +package twcache + +import ( + "container/list" + "errors" + "fmt" + "io/fs" + "os" + "path/filepath" + "sort" + "strings" + "sync" + "time" +) + +const mtimeRefreshInterval = 10 * time.Minute + +type twcachedisk struct { + dirPath string + warn func(error) + + mu sync.Mutex + files map[string]*list.Element + order *list.List + maxBytes int64 + curBytes int64 + evictions uint64 +} + +type diskEntry struct { + name string + size int64 + touchedAt time.Time +} + +func newDiskStore(dir string, maxBytes int64, warn func(error)) (*twcachedisk, error) { + if maxBytes <= 0 { + return nil, errors.New("twcache: maxBytes must be positive") + } + if err := os.MkdirAll(dir, 0o755); err != nil { + return nil, fmt.Errorf("failed to create tw cache dir: %w", err) + } + s := &twcachedisk{ + dirPath: dir, + warn: warn, + files: make(map[string]*list.Element), + order: list.New(), + maxBytes: maxBytes, + } + if err := s.scan(); err != nil { + return nil, fmt.Errorf("failed to scan tw cache dir: %w", err) + } + return s, nil +} + +func (s *twcachedisk) dir() string { return s.dirPath } + +func (s *twcachedisk) path(name string) (string, error) { + rel := filepath.FromSlash(name) + if !filepath.IsLocal(rel) { + return "", ErrInvalidName + } + return filepath.Join(s.dirPath, rel), nil +} + +func (s *twcachedisk) scan() error { + type scanned struct { + name string + size int64 + mtime time.Time + } + var found []scanned + err := filepath.WalkDir(s.dirPath, func(path string, dirEntry fs.DirEntry, err error) error { + if err != nil { + if s.warn != nil { + s.warn(fmt.Errorf("skipping unreadable tw cache entry (path=%s): %w", path, err)) + } + return nil + } + if dirEntry.IsDir() || !strings.HasSuffix(path, ".tw") { + return nil + } + info, err := dirEntry.Info() + if err != nil { + return nil + } + rel, err := filepath.Rel(s.dirPath, path) + if err != nil { + return nil + } + found = append(found, scanned{name: filepath.ToSlash(rel), size: info.Size(), mtime: info.ModTime()}) + return nil + }) + if err != nil { + return err + } + sort.Slice(found, func(i, j int) bool { return found[i].mtime.Before(found[j].mtime) }) + s.mu.Lock() + defer s.mu.Unlock() + for _, f := range found { + el := s.order.PushFront(&diskEntry{name: f.name, size: f.size}) + s.files[f.name] = el + s.curBytes += f.size + } + s.evictLocked() + return nil +} + +func (s *twcachedisk) get(name string) ([]byte, func(), bool) { + path, err := s.path(name) + if err != nil { + return nil, noop, false + } + data, unmap, err := mmapFile(path) + if err != nil { + return nil, noop, false + } + s.noteUse(name, int64(len(data))) + return data, unmap, true +} + +func (s *twcachedisk) contains(name string) bool { + s.mu.Lock() + defer s.mu.Unlock() + _, ok := s.files[name] + return ok +} + +func (s *twcachedisk) put(name string, data []byte) error { + if err := s.persist(name, data); err != nil { + return err + } + s.noteUse(name, int64(len(data))) + return nil +} + +func (s *twcachedisk) persist(name string, data []byte) error { + path, err := s.path(name) + if err != nil { + return err + } + if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil { + return err + } + tmp, err := os.CreateTemp(filepath.Dir(path), ".tw-*") + if err != nil { + return err + } + if _, err := tmp.Write(data); err != nil { + tmp.Close() + os.Remove(tmp.Name()) + return err + } + if err := tmp.Close(); err != nil { + os.Remove(tmp.Name()) + return err + } + if err := os.Rename(tmp.Name(), path); err != nil { + os.Remove(tmp.Name()) + return err + } + return nil +} + +func (s *twcachedisk) noteUse(name string, size int64) { + now := time.Now() + updateMtime := false + s.mu.Lock() + if el, ok := s.files[name]; ok { + e := el.Value.(*diskEntry) + s.curBytes += size - e.size + e.size = size + s.order.MoveToFront(el) + if now.Sub(e.touchedAt) > mtimeRefreshInterval { + e.touchedAt = now + updateMtime = true + } + } else { + s.files[name] = s.order.PushFront(&diskEntry{name: name, size: size, touchedAt: now}) + s.curBytes += size + } + s.evictLocked() + s.mu.Unlock() + if updateMtime { + if path, err := s.path(name); err == nil { + _ = os.Chtimes(path, now, now) + } + } +} + +func (s *twcachedisk) remove(name string) { + s.mu.Lock() + defer s.mu.Unlock() + if el, ok := s.files[name]; ok { + s.dropLocked(el) + } else if path, err := s.path(name); err == nil { + os.Remove(path) + } +} + +func (s *twcachedisk) setLimits(_ int, maxBytes int64) { + s.mu.Lock() + defer s.mu.Unlock() + if maxBytes > 0 { + s.maxBytes = maxBytes + } + s.evictLocked() +} + +func (s *twcachedisk) dropLocked(el *list.Element) { + e := s.order.Remove(el).(*diskEntry) + delete(s.files, e.name) + s.curBytes -= e.size + os.Remove(filepath.Join(s.dirPath, filepath.FromSlash(e.name))) +} + +func (s *twcachedisk) evictLocked() { + for s.curBytes > s.maxBytes { + back := s.order.Back() + if back == nil { + break + } + s.dropLocked(back) + s.evictions++ + } +} + +func (s *twcachedisk) stats() storeStats { + s.mu.Lock() + defer s.mu.Unlock() + return storeStats{ + Mode: "disk", + Entries: s.order.Len(), + Bytes: s.curBytes, + MaxBytes: s.maxBytes, + Evictions: s.evictions, + } +} diff --git a/backend/app/symbolicator/twcache/mem.go b/backend/app/symbolicator/twcache/mem.go new file mode 100644 index 00000000..0ad7443c --- /dev/null +++ b/backend/app/symbolicator/twcache/mem.go @@ -0,0 +1,119 @@ +package twcache + +import ( + "container/list" + "sync" +) + +type twcachemem struct { + mu sync.Mutex + items map[string]*list.Element + order *list.List + maxEntries int + maxBytes int64 + curBytes int64 + evictions uint64 +} + +type memEntry struct { + name string + data []byte + size int64 +} + +func newMemStore(maxEntries int, maxBytes int64) *twcachemem { + return &twcachemem{ + items: make(map[string]*list.Element), + order: list.New(), + maxEntries: maxEntries, + maxBytes: maxBytes, + } +} + +func (s *twcachemem) get(name string) ([]byte, func(), bool) { + s.mu.Lock() + defer s.mu.Unlock() + el, ok := s.items[name] + if !ok { + return nil, noop, false + } + s.order.MoveToFront(el) + return el.Value.(*memEntry).data, noop, true +} + +func (s *twcachemem) contains(name string) bool { + s.mu.Lock() + defer s.mu.Unlock() + _, ok := s.items[name] + return ok +} + +func (s *twcachemem) put(name string, data []byte) error { + s.mu.Lock() + defer s.mu.Unlock() + size := int64(len(data)) + if el, ok := s.items[name]; ok { + e := el.Value.(*memEntry) + s.curBytes += size - e.size + e.data = data + e.size = size + s.order.MoveToFront(el) + } else { + s.items[name] = s.order.PushFront(&memEntry{name: name, data: data, size: size}) + s.curBytes += size + } + s.evictLocked() + return nil +} + +func (s *twcachemem) remove(name string) { + s.mu.Lock() + defer s.mu.Unlock() + if el, ok := s.items[name]; ok { + s.dropLocked(el) + } +} + +func (s *twcachemem) setLimits(maxEntries int, maxBytes int64) { + s.mu.Lock() + defer s.mu.Unlock() + if maxEntries > 0 { + s.maxEntries = maxEntries + } + if maxBytes > 0 { + s.maxBytes = maxBytes + } + s.evictLocked() +} + +func (s *twcachemem) dropLocked(el *list.Element) { + e := s.order.Remove(el).(*memEntry) + delete(s.items, e.name) + s.curBytes -= e.size +} + +func (s *twcachemem) evictLocked() { + for s.order.Len() > s.maxEntries || s.curBytes > s.maxBytes { + back := s.order.Back() + if back == nil { + break + } + s.dropLocked(back) + s.evictions++ + } +} + +func (s *twcachemem) stats() storeStats { + s.mu.Lock() + defer s.mu.Unlock() + return storeStats{ + Mode: "memory", + Entries: s.order.Len(), + Bytes: s.curBytes, + MaxBytes: s.maxBytes, + MaxEntries: s.maxEntries, + Evictions: s.evictions, + } +} + +func (s *twcachemem) dir() string { return "" } diff --git a/backend/app/symbolicator/twcache/twcache.go b/backend/app/symbolicator/twcache/twcache.go index 7bb7b1ef..0b395996 100644 --- a/backend/app/symbolicator/twcache/twcache.go +++ b/backend/app/symbolicator/twcache/twcache.go @@ -1,249 +1,294 @@ package twcache import ( - "container/list" + "context" "errors" "fmt" - "io/fs" - "os" - "path/filepath" - "runtime" - "sort" - "strings" "sync" + "sync/atomic" "time" - - "github.com/tracewayapp/traceway/backend/app/symbolicator" ) -const mtimeRefreshInterval = 10 * time.Minute +const ( + negativeBaseTTL = time.Minute + transientNegativeTTL = 15 * time.Second + negativeMaxTTL = 15 * time.Minute + negativeMaxKeys = 10000 + failReportInterval = time.Minute +) var ErrInvalidName = errors.New("twcache: cache name escapes the cache directory") -type Cache struct { - dir string - warn func(error) +type LoadFunc func(ctx context.Context) ([]byte, error) - mu sync.Mutex - files map[string]*list.Element - order *list.List - maxBytes int64 - curBytes int64 - hits uint64 - evictions uint64 -} +type store interface { + get(name string) (data []byte, done func(), ok bool) -type entry struct { - name string - size int64 - touchedAt time.Time + contains(name string) bool + put(name string, data []byte) error + remove(name string) + setLimits(maxEntries int, maxBytes int64) + stats() storeStats + dir() string } -type Stats struct { - Entries int - Bytes int64 - MaxBytes int64 - Hits uint64 - Evictions uint64 +type storeStats struct { + Mode string + Entries int + Bytes int64 + MaxBytes int64 + MaxEntries int + Evictions uint64 } -func New(dir string, maxBytes int64, warn func(error)) (*Cache, error) { - if maxBytes <= 0 { - return nil, errors.New("twcache: maxBytes must be positive") - } - if err := os.MkdirAll(dir, 0o755); err != nil { - return nil, fmt.Errorf("failed to create tw cache dir: %w", err) - } - c := &Cache{ - dir: dir, - warn: warn, - files: make(map[string]*list.Element), - order: list.New(), - maxBytes: maxBytes, - } - if err := c.scan(); err != nil { - return nil, fmt.Errorf("failed to scan tw cache dir: %w", err) - } - return c, nil +func noop() {} + +type Cache struct { + name string + store store + warn func(error) + + NotFound func(error) bool + + Validate func([]byte) bool + + mu sync.Mutex + loading map[string]*cacheLoad + negative map[string]*negEntry + negativeHits uint64 + lastParseMs float64 + failuresSinceReport uint64 + lastFailAt time.Time + + hits atomic.Uint64 + misses atomic.Uint64 + failures atomic.Uint64 + notFound atomic.Uint64 } -func (c *Cache) Dir() string { - return c.dir +type negEntry struct { + expiresAt time.Time + failures uint32 } -func (c *Cache) path(name string) (string, error) { - rel := filepath.FromSlash(name) - if !filepath.IsLocal(rel) { - return "", ErrInvalidName - } - return filepath.Join(c.dir, rel), nil +type cacheLoad struct { + done chan struct{} + err error } -func (c *Cache) scan() error { - type scanned struct { - name string - size int64 - mtime time.Time - } - var found []scanned - err := filepath.WalkDir(c.dir, func(path string, dirEntry fs.DirEntry, err error) error { - if err != nil { - if c.warn != nil { - c.warn(fmt.Errorf("skipping unreadable tw cache entry (path=%s): %w", path, err)) - } - return nil - } - if dirEntry.IsDir() || !strings.HasSuffix(path, ".tw") { - return nil - } - info, err := dirEntry.Info() - if err != nil { - return nil - } - rel, err := filepath.Rel(c.dir, path) - if err != nil { - return nil - } - found = append(found, scanned{name: filepath.ToSlash(rel), size: info.Size(), mtime: info.ModTime()}) - return nil - }) - if err != nil { - return err - } - sort.Slice(found, func(i, j int) bool { return found[i].mtime.Before(found[j].mtime) }) - c.mu.Lock() - defer c.mu.Unlock() - for _, f := range found { - el := c.order.PushFront(&entry{name: f.name, size: f.size}) - c.files[f.name] = el - c.curBytes += f.size +func newCache(s store, warn func(error)) *Cache { + return &Cache{ + name: "symbolication artifact", + store: s, + warn: warn, + loading: make(map[string]*cacheLoad), + negative: make(map[string]*negEntry), } - c.evictLocked() - return nil } -// Open mmaps the named .tw file and returns a resolver backed by it. The -// mapping is released by a runtime cleanup once the resolver is collected. -// Corrupt files are removed so the caller's rebuild can replace them. -func (c *Cache) Open(name string) (*symbolicator.Resolver, error) { - return c.open(name, true) +func NewMem(maxEntries int, maxBytes int64) *Cache { + return newCache(newMemStore(maxEntries, maxBytes), nil) } -func (c *Cache) open(name string, countHit bool) (*symbolicator.Resolver, error) { - path, err := c.path(name) - if err != nil { - return nil, err - } - data, unmap, err := mmapFile(path) - if err != nil { - return nil, err - } - resolver, err := symbolicator.OpenTW(data) +func NewDisk(dir string, maxBytes int64, warn func(error)) (*Cache, error) { + s, err := newDiskStore(dir, maxBytes, warn) if err != nil { - unmap() - c.Remove(name) return nil, err } - runtime.AddCleanup(resolver, func(u func()) { u() }, unmap) - c.noteUse(name, int64(len(data)), countHit) - return resolver, nil + return newCache(s, warn), nil } -// Write atomically persists data as the named .tw file and returns a -// resolver mmapped from it. Write does not count as a cache hit. -func (c *Cache) Write(name string, data []byte) (*symbolicator.Resolver, error) { - path, err := c.path(name) - if err != nil { - return nil, err - } - if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil { - return nil, err - } - tmp, err := os.CreateTemp(filepath.Dir(path), ".tw-*") - if err != nil { - return nil, err - } - if _, err := tmp.Write(data); err != nil { - tmp.Close() - os.Remove(tmp.Name()) - return nil, err +func (c *Cache) SetWarn(warn func(error)) { c.warn = warn } + +func (c *Cache) SetLimits(maxEntries int, maxBytes int64) { + c.store.setLimits(maxEntries, maxBytes) +} + +func (c *Cache) Dir() string { return c.store.dir() } + +func (c *Cache) Get(ctx context.Context, key string, load LoadFunc) (data []byte, done func(), err error) { + if data, done, ok := c.store.get(key); ok { + if c.Validate == nil || c.Validate(data) { + c.hits.Add(1) + return data, done, nil + } + + done() + c.store.remove(key) } - if err := tmp.Close(); err != nil { - os.Remove(tmp.Name()) - return nil, err + if err := c.ensureBuilt(ctx, key, load); err != nil { + return nil, noop, err } - if err := os.Rename(tmp.Name(), path); err != nil { - os.Remove(tmp.Name()) - return nil, err + if data, done, ok := c.store.get(key); ok { + return data, done, nil } - return c.open(name, false) + return nil, noop, fmt.Errorf("%s: %q evicted before use", c.name, key) } -func (c *Cache) noteUse(name string, size int64, countHit bool) { - now := time.Now() - updateMtime := false +func (c *Cache) ensureBuilt(ctx context.Context, key string, load LoadFunc) error { c.mu.Lock() - if countHit { - c.hits++ - } - if el, ok := c.files[name]; ok { - e := el.Value.(*entry) - c.curBytes += size - e.size - e.size = size - c.order.MoveToFront(el) - if now.Sub(e.touchedAt) > mtimeRefreshInterval { - e.touchedAt = now - updateMtime = true + if l, ok := c.loading[key]; ok { + c.mu.Unlock() + <-l.done + if l.err == nil { + c.hits.Add(1) } - } else { - c.files[name] = c.order.PushFront(&entry{name: name, size: size, touchedAt: now}) - c.curBytes += size + return l.err + } + + if c.store.contains(key) { + c.mu.Unlock() + c.hits.Add(1) + return nil } - c.evictLocked() + c.misses.Add(1) + l := &cacheLoad{done: make(chan struct{})} + c.loading[key] = l c.mu.Unlock() - if updateMtime { - if path, err := c.path(name); err == nil { - _ = os.Chtimes(path, now, now) + + var ms float64 + func() { + defer func() { + if r := recover(); r != nil { + l.err = fmt.Errorf("%s load panicked (key=%s): %v", c.name, key, r) + } + }() + start := time.Now() + blob, lerr := load(ctx) + ms = float64(time.Since(start).Microseconds()) / 1000.0 + if lerr != nil { + l.err = lerr + return } + l.err = c.store.put(key, blob) + }() + + c.mu.Lock() + delete(c.loading, key) + if l.err == nil { + c.lastParseMs = ms + delete(c.negative, key) + } else { + c.markNegativeLocked(key, l.err) } + c.mu.Unlock() + + close(l.done) + if l.err != nil && !c.isNotFound(l.err) { + c.reportFailure(l.err) + } + return l.err +} + +func (c *Cache) isNotFound(err error) bool { + return c.NotFound == nil || c.NotFound(err) } -func (c *Cache) Remove(name string) { +func (c *Cache) IsNegative(key string) bool { c.mu.Lock() defer c.mu.Unlock() - if el, ok := c.files[name]; ok { - c.dropLocked(el) - } else if path, err := c.path(name); err == nil { - os.Remove(path) + e, ok := c.negative[key] + if !ok || time.Now().After(e.expiresAt) { + return false } + c.negativeHits++ + return true } -func (c *Cache) dropLocked(el *list.Element) { - e := c.order.Remove(el).(*entry) - delete(c.files, e.name) - c.curBytes -= e.size - os.Remove(filepath.Join(c.dir, filepath.FromSlash(e.name))) +func (c *Cache) Invalidate(key string) { + c.mu.Lock() + delete(c.negative, key) + c.mu.Unlock() + c.store.remove(key) } -func (c *Cache) evictLocked() { - for c.curBytes > c.maxBytes { - back := c.order.Back() - if back == nil { +func (c *Cache) markNegativeLocked(key string, loadErr error) { + base := transientNegativeTTL + if c.isNotFound(loadErr) { + base = negativeBaseTTL + c.notFound.Add(1) + } else { + c.failures.Add(1) + } + e := c.negative[key] + if e == nil { + if len(c.negative) >= negativeMaxKeys { + c.pruneNegativeLocked() + } + e = &negEntry{} + c.negative[key] = e + } + ttl := min(base<= failReportInterval { + report = c.failuresSinceReport + c.failuresSinceReport = 0 + c.lastFailAt = time.Now() + } + c.mu.Unlock() + if report > 0 && c.warn != nil { + c.warn(fmt.Errorf("%s loads failed %d time(s) since last report: %w", c.name, report, err)) + } +} + +type Stats struct { + Mode string + Entries int + Bytes int64 + MaxBytes int64 + MaxEntries int + Hits uint64 + Misses uint64 + Evictions uint64 + Failures uint64 + NotFound uint64 + NegativeHits uint64 + NegativeEntries int + LastParseMs float64 +} + func (c *Cache) Stats() Stats { + ss := c.store.stats() c.mu.Lock() - defer c.mu.Unlock() + negEntries := len(c.negative) + negHits := c.negativeHits + lastMs := c.lastParseMs + c.mu.Unlock() return Stats{ - Entries: c.order.Len(), - Bytes: c.curBytes, - MaxBytes: c.maxBytes, - Hits: c.hits, - Evictions: c.evictions, + Mode: ss.Mode, + Entries: ss.Entries, + Bytes: ss.Bytes, + MaxBytes: ss.MaxBytes, + MaxEntries: ss.MaxEntries, + Evictions: ss.Evictions, + Hits: c.hits.Load(), + Misses: c.misses.Load(), + Failures: c.failures.Load(), + NotFound: c.notFound.Load(), + NegativeHits: negHits, + NegativeEntries: negEntries, + LastParseMs: lastMs, } } diff --git a/backend/app/symbolicator/twcache/twcache_test.go b/backend/app/symbolicator/twcache/twcache_test.go new file mode 100644 index 00000000..9f90f61e --- /dev/null +++ b/backend/app/symbolicator/twcache/twcache_test.go @@ -0,0 +1,160 @@ +package twcache + +import ( + "context" + "errors" + "sync" + "sync/atomic" + "testing" +) + +func blobLoad(data []byte, calls *atomic.Int64) LoadFunc { + return func(ctx context.Context) ([]byte, error) { + if calls != nil { + calls.Add(1) + } + return data, nil + } +} + +func caches(t *testing.T) map[string]*Cache { + t.Helper() + disk, err := NewDisk(t.TempDir(), 64<<20, nil) + if err != nil { + t.Fatal(err) + } + return map[string]*Cache{ + "mem": NewMem(100, 64<<20), + "disk": disk, + } +} + +func TestGetBuildsThenHits(t *testing.T) { + ctx := context.Background() + for mode, c := range caches(t) { + t.Run(mode, func(t *testing.T) { + var calls atomic.Int64 + data, done, err := c.Get(ctx, "k.tw", blobLoad([]byte("hello"), &calls)) + if err != nil { + t.Fatalf("Get: %v", err) + } + if string(data) != "hello" { + t.Errorf("got %q, want hello", data) + } + done() + + data2, done2, err := c.Get(ctx, "k.tw", blobLoad([]byte("DIFFERENT"), &calls)) + if err != nil { + t.Fatalf("Get (warm): %v", err) + } + if string(data2) != "hello" { + t.Errorf("warm get: got %q, want the cached hello", data2) + } + done2() + if calls.Load() != 1 { + t.Errorf("expected load called once, got %d", calls.Load()) + } + }) + } +} + +func TestDiskPersistsAcrossRestart(t *testing.T) { + ctx := context.Background() + dir := t.TempDir() + c, err := NewDisk(dir, 64<<20, nil) + if err != nil { + t.Fatal(err) + } + data, done, err := c.Get(ctx, "k.tw", blobLoad([]byte("persisted"), nil)) + if err != nil { + t.Fatal(err) + } + if string(data) != "persisted" { + t.Fatalf("got %q", data) + } + done() + + c2, err := NewDisk(dir, 64<<20, nil) + if err != nil { + t.Fatal(err) + } + failLoad := func(ctx context.Context) ([]byte, error) { return nil, errors.New("should not build") } + data2, done2, err := c2.Get(ctx, "k.tw", failLoad) + if err != nil { + t.Fatalf("restart get: %v", err) + } + if string(data2) != "persisted" { + t.Errorf("restart got %q, want persisted", data2) + } + done2() +} + +func TestSingleflight(t *testing.T) { + ctx := context.Background() + for mode, c := range caches(t) { + t.Run(mode, func(t *testing.T) { + var calls atomic.Int64 + release := make(chan struct{}) + load := func(ctx context.Context) ([]byte, error) { + calls.Add(1) + <-release + return []byte("v"), nil + } + const n = 16 + var wg sync.WaitGroup + start := make(chan struct{}) + for range n { + wg.Add(1) + go func() { + defer wg.Done() + <-start + _, done, err := c.Get(ctx, "k.tw", load) + if err == nil { + done() + } + }() + } + close(start) + close(release) + wg.Wait() + if calls.Load() != 1 { + t.Errorf("expected 1 build for concurrent gets, got %d", calls.Load()) + } + }) + } +} + +func TestNegativeAndInvalidate(t *testing.T) { + ctx := context.Background() + for mode, c := range caches(t) { + t.Run(mode, func(t *testing.T) { + boom := errors.New("boom") + if _, _, err := c.Get(ctx, "k.tw", func(ctx context.Context) ([]byte, error) { return nil, boom }); err == nil { + t.Fatal("expected error") + } + if !c.IsNegative("k.tw") { + t.Error("expected negative entry after a failed load") + } + + var calls atomic.Int64 + _, done, err := c.Get(ctx, "k.tw", blobLoad([]byte("v"), &calls)) + if err != nil { + t.Fatal(err) + } + done() + if c.IsNegative("k.tw") { + t.Error("successful load should clear the negative entry") + } + + c.Invalidate("k.tw") + _, done2, err := c.Get(ctx, "k.tw", blobLoad([]byte("v"), &calls)) + if err != nil { + t.Fatal(err) + } + done2() + if calls.Load() != 2 { + t.Errorf("invalidate should force a rebuild: got %d builds, want 2", calls.Load()) + } + }) + } +} diff --git a/backend/cmd/options.go b/backend/cmd/options.go index db872cbe..5f0a763e 100644 --- a/backend/cmd/options.go +++ b/backend/cmd/options.go @@ -1,12 +1,12 @@ package cmd type options struct { - sqlitePath string - port int - serverURL string - disableLogging bool - defaultUser *defaultUserOpts - defaultProjects []defaultProjectOpts + sqlitePath string + port int + serverURL string + disableLogging bool + defaultUser *defaultUserOpts + defaultProjects []defaultProjectOpts monitoringTracewayURL string } @@ -16,9 +16,10 @@ type defaultUserOpts struct { } type defaultProjectOpts struct { - name string - framework string - token string + name string + framework string + token string + sourceMapToken string } type Option func(*options) @@ -61,6 +62,16 @@ func WithDefaultProject(name, framework, token string) Option { } } +func WithDefaultProjectSourceMapToken(name, token string) Option { + return func(o *options) { + for i := range o.defaultProjects { + if o.defaultProjects[i].name == name { + o.defaultProjects[i].sourceMapToken = token + } + } + } +} + func WithMonitoringURL(url string) Option { return func(o *options) { o.monitoringTracewayURL = url diff --git a/backend/cmd/run.go b/backend/cmd/run.go index df462670..09db49cc 100644 --- a/backend/cmd/run.go +++ b/backend/cmd/run.go @@ -25,7 +25,7 @@ import ( "github.com/tracewayapp/traceway/backend/app/services" "github.com/tracewayapp/traceway/backend/app/sourcemapbackfill" "github.com/tracewayapp/traceway/backend/app/storage" - "github.com/tracewayapp/traceway/backend/app/symbolicator/scopes" + "github.com/tracewayapp/traceway/backend/app/symbolicator/sourcemap/scopes" "github.com/tracewayapp/traceway/backend/static" "github.com/coreos/go-systemd/v22/daemon" @@ -123,7 +123,7 @@ func Run(opts ...Option) { dir = "./twcache" } maxBytes := int64(parsePositiveInt(cfg.SourceMapDiskCacheMaxMB, 2048)) * 1024 * 1024 - if err := services.EnableSourceMapDiskCache(dir, maxBytes); err != nil { + if err := services.EnableSymbolicatorDiskCache(dir, maxBytes); err != nil { panic(fmt.Errorf("source map disk cache init failed: %w", err)) } default: @@ -238,9 +238,6 @@ func Run(opts ...Option) { } } -// applyEnvOverrides lets the dev-options path pick up OAuth/OIDC and -// symbolicator env vars so embedded examples (e.g. devtesting-embedded) can be -// configured without exposing every knob as a WithXxx option. func applyEnvOverrides(cfg *config.Cfg) { for _, m := range []struct { envVar string diff --git a/backend/cmd/seed.go b/backend/cmd/seed.go index 606154c6..903aa792 100644 --- a/backend/cmd/seed.go +++ b/backend/cmd/seed.go @@ -58,6 +58,10 @@ func seed(opts *options) error { OrganizationId: &org.Id, CreatedAt: time.Now().UTC(), } + if p.sourceMapToken != "" { + token := p.sourceMapToken + project.SourceMapToken = &token + } if err := lit.InsertExistingUuid(tx, project); err != nil { return struct{}{}, err } diff --git a/backend/tools/cachebench/corpus.go b/backend/tools/cachebench/corpus.go index d304543a..17200143 100644 --- a/backend/tools/cachebench/corpus.go +++ b/backend/tools/cachebench/corpus.go @@ -12,7 +12,6 @@ import ( "github.com/tracewayapp/traceway/backend/app/services" "github.com/tracewayapp/traceway/backend/app/storage" - "github.com/tracewayapp/traceway/backend/app/symbolicator" "github.com/tracewayapp/traceway/backend/app/symbolicator/sourcemap" "github.com/google/uuid" @@ -96,14 +95,13 @@ func buildCanonical(tokens int) (bundle, mapJSON []byte) { func generateCorpus(ctx context.Context, corpusDir string, projectId uuid.UUID, entries, tokens, workers int) error { bundle, mapJSON := buildCanonical(tokens) - resolver, err := symbolicator.NewResolver(mapJSON, bundle) + tw, err := sourcemap.BuildTW(mapJSON, bundle) if err != nil { - return fmt.Errorf("building canonical resolver: %w", err) + return fmt.Errorf("building canonical artifact: %w", err) } - tw := resolver.MarshalTW() - if frame, ok := resolver.Lookup(0, 10); !ok || !strings.HasPrefix(frame.File, "src/module-") { - return fmt.Errorf("canonical resolver sanity check failed: %+v ok=%v", frame, ok) + if frame, ok := sourcemap.LookupTW(tw, 0, 10); !ok || !strings.HasPrefix(frame.File, "src/module-") { + return fmt.Errorf("canonical artifact sanity check failed: %+v ok=%v", frame, ok) } var idx atomic.Int64 @@ -140,7 +138,7 @@ func generateCorpus(ctx context.Context, corpusDir string, projectId uuid.UUID, BundleBytes: len(bundle), MapBytes: len(mapJSON), TwBytes: len(tw), - ResolverBytes: resolver.ApproxSize(), + ResolverBytes: int64(len(tw)), } data, err := json.MarshalIndent(manifest, "", " ") if err != nil { diff --git a/backend/tools/cachebench/run.go b/backend/tools/cachebench/run.go index 2369ffa3..a09ae60d 100644 --- a/backend/tools/cachebench/run.go +++ b/backend/tools/cachebench/run.go @@ -115,7 +115,7 @@ func runBench(cfg runConfig) error { if err := os.RemoveAll(cfg.diskDir); err != nil { return err } - if err := services.EnableSourceMapDiskCache(cfg.diskDir, int64(cfg.diskMB)<<20); err != nil { + if err := services.EnableSymbolicatorDiskCache(cfg.diskDir, int64(cfg.diskMB)<<20); err != nil { return err } default: diff --git a/backend/traceway.go b/backend/traceway.go index b5f1ba4c..981f5048 100644 --- a/backend/traceway.go +++ b/backend/traceway.go @@ -1,7 +1,3 @@ -// Package tracewaybackend provides an embeddable Traceway backend that can be -// run inside your own Go application. Uses SQLite for all storage — no external -// databases required. -// package tracewaybackend import "github.com/tracewayapp/traceway/backend/cmd" @@ -17,4 +13,6 @@ var ( WithDefaultProject = cmd.WithDefaultProject WithMonitoringURL = cmd.WithMonitoringURL DisableLogging = cmd.DisableLogging + + WithDefaultProjectSourceMapToken = cmd.WithDefaultProjectSourceMapToken ) diff --git a/benchmarks/processor/README.md b/benchmarks/processor/README.md index 65491251..0ca8d786 100644 --- a/benchmarks/processor/README.md +++ b/benchmarks/processor/README.md @@ -34,9 +34,26 @@ stacktraces/sec; the drain's symbolicated percentage is the correctness check. | `traceway-oxc-disk` | otelcol-bench-traceway | oxc | mmap'd `.tw` disk tier | | `traceway-goja-mem` | otelcol-bench-traceway | goja | in-memory resolvers only | | `traceway-goja-disk` | otelcol-bench-traceway | goja | mmap'd `.tw` disk tier | - -One traceway binary (built with `-tags oxc`) serves all four variants; parser -and cache mode are runtime config (`parser`, `cache_dir`). +| `traceway-dart-mem` | otelcol-bench-traceway | DWARF flatten | in-memory `.tw` only | +| `traceway-dart-disk` | otelcol-bench-traceway | DWARF flatten | mmap'd `.tw` disk tier | + +One traceway binary (built with `-tags oxc`) serves all variants; language +(JS source maps vs. Dart symbols), parser, and cache mode are runtime config. +The processor auto-routes a non-symbolic Dart AOT trace to its Dart path; on a +cache miss it flattens the build's `.symbols` DWARF to a `.tw` (the analog of the +JS map compile). There is no Honeycomb Dart comparison — Honeycomb's processor is +JS-only. + +## Language + +`benchmark-processor` takes a `language` input: `js` (the 5 JS impls incl. +Honeycomb), `dart` (the two `traceway-dart-*` impls), or `both`. Locally, just +list the impls in `IMPLS`. The Dart corpus replicates a committed seed +(`seeds/dart/app.debug.elf`, a real pure-Dart AOT `.symbols` ELF + its trace) +under N synthetic build-ids — hardlinked, so N builds cost ~one inode — so the +churn/oom scenarios exercise the cache the same way the JS corpus does. The +drain's symbolicated check uses Dart markers (`crash.dart` resolved vs. +`_kDart…SnapshotInstructions` unresolved) selected automatically per impl. ## Scenarios @@ -72,7 +89,8 @@ timeline, not a single number. ``` ./run-local.sh -IMPLS="traceway-oxc traceway-goja honeycomb" SCENARIOS=churn CONNECTIONS=4,16,64 ./run-local.sh +IMPLS="traceway-oxc-mem traceway-goja-mem honeycomb" SCENARIOS=churn CONNECTIONS=4,16,64 ./run-local.sh +IMPLS="traceway-dart-mem traceway-dart-disk" SCENARIOS="hot churn" ./run-local.sh # Dart ``` Needs go, cargo, node, jq. Builds both collectors (the Traceway one with @@ -98,9 +116,10 @@ but the collector. ## GitHub Action `benchmark-processor` (workflow_dispatch) builds all artifacts on the runner, -then runs the two implementations as parallel matrix entries, each on its own -Hetzner server pair. Needs the `HCLOUD_TOKEN` secret. Results are uploaded as -`results-traceway-oxc` and `results-honeycomb` artifacts. +then runs the impls as parallel matrix entries, each on its own Hetzner server +pair. Needs the `HCLOUD_TOKEN` secret. The `language` input selects the matrix: +`js` (the 5 JS impls), `dart` (`traceway-dart-mem`/`-disk`), or `both`. Results +are uploaded as `results-` artifacts. ## Knobs diff --git a/benchmarks/processor/corpusgen/main.go b/benchmarks/processor/corpusgen/main.go index bef38264..8e928745 100644 --- a/benchmarks/processor/corpusgen/main.go +++ b/benchmarks/processor/corpusgen/main.go @@ -6,11 +6,23 @@ import ( "fmt" "os" "path/filepath" + "regexp" "strings" ) type corpus struct { - Urls []string `json:"urls"` + Language string `json:"language"` + Urls []string `json:"urls"` +} + +type dartBuild struct { + BuildID string `json:"buildId"` + Trace string `json:"trace"` +} + +type dartCorpus struct { + Language string `json:"language"` + Builds []dartBuild `json:"builds"` } const vlqChars = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/" @@ -150,8 +162,11 @@ func padMap(mapBytes []byte, kb, mappingsKB, seed int) []byte { } func main() { + language := flag.String("language", "js", "corpus language: js or dart") bundle := flag.String("bundle", "../../testing/symbolication/node-app/dist/app.mjs", "") mapFile := flag.String("map", "../../testing/symbolication/node-app/dist/app.mjs.map", "") + symbols := flag.String("symbols", "seeds/dart/app.debug.elf", "dart: seed .symbols/.elf") + traceFile := flag.String("trace", "seeds/dart/trace.txt", "dart: seed non-symbolic trace") entries := flag.Int("entries", 1, "") padKB := flag.Int("pad-kb", 256, "") mapPadKB := flag.Int("map-pad-kb", 0, "") @@ -159,6 +174,14 @@ func main() { out := flag.String("out", "./corpus", "") flag.Parse() + if err := os.MkdirAll(*out, 0o755); err != nil { + panic(err) + } + if *language == "dart" { + generateDart(*out, *symbols, *traceFile, *entries) + return + } + bundleBytes, err := os.ReadFile(*bundle) if err != nil { panic(err) @@ -169,10 +192,7 @@ func main() { } firstLine := strings.SplitN(string(bundleBytes), "\n", 2)[0] - if err := os.MkdirAll(*out, 0o755); err != nil { - panic(err) - } - c := corpus{} + c := corpus{Language: "js"} for n := 0; n < *entries; n++ { var pad strings.Builder chunk := "function __benchPad%d_%d(a,b){var c=a*b+%d;for(var i=0;i<3;i++){c+=i*a-b}return c}\n" @@ -201,3 +221,51 @@ func main() { } fmt.Printf("wrote %d entries (%d KB padding each) to %s\n", *entries, *padKB, *out) } + +var archRe = regexp.MustCompile(`os:\s*\S+\s+arch:\s*(\S+)`) +var buildIDLineRe = regexp.MustCompile(`build_id:\s*'[0-9a-fA-F]+'`) + +func generateDart(out, symbolsPath, tracePath string, entries int) { + elf, err := os.ReadFile(symbolsPath) + if err != nil { + panic(fmt.Errorf("reading dart seed symbols %q: %w", symbolsPath, err)) + } + template, err := os.ReadFile(tracePath) + if err != nil { + panic(fmt.Errorf("reading dart seed trace %q: %w", tracePath, err)) + } + absSeed, err := filepath.Abs(symbolsPath) + if err != nil { + panic(err) + } + arch := "arm64" + if m := archRe.FindStringSubmatch(string(template)); m != nil { + arch = m[1] + } + + c := dartCorpus{Language: "dart"} + for n := 0; n < entries; n++ { + buildID := fmt.Sprintf("%032x", n) + dest := filepath.Join(out, buildID+"-"+arch+".symbols") + _ = os.Remove(dest) + if err := os.Link(absSeed, dest); err != nil { + + if werr := os.WriteFile(dest, elf, 0o644); werr != nil { + panic(werr) + } + } + c.Builds = append(c.Builds, dartBuild{BuildID: buildID, Trace: substituteBuildID(string(template), buildID)}) + } + data, _ := json.MarshalIndent(c, "", " ") + if err := os.WriteFile(filepath.Join(out, "corpus.json"), data, 0o644); err != nil { + panic(err) + } + fmt.Printf("wrote %d dart builds (arch %s, seed %s) to %s\n", entries, arch, symbolsPath, out) +} + +func substituteBuildID(trace, buildID string) string { + if buildIDLineRe.MatchString(trace) { + return buildIDLineRe.ReplaceAllString(trace, "build_id: '"+buildID+"'") + } + return "build_id: '" + buildID + "'\n" + trace +} diff --git a/benchmarks/processor/drain/src/main.rs b/benchmarks/processor/drain/src/main.rs index 8bc34721..dc315971 100644 --- a/benchmarks/processor/drain/src/main.rs +++ b/benchmarks/processor/drain/src/main.rs @@ -8,8 +8,8 @@ use std::io::Read; use std::sync::atomic::{AtomicU64, Ordering}; use std::sync::Arc; -const OK_MARKER: &[u8] = b"../src/inventory.js"; -const FAIL_MARKER: &[u8] = b".mjs:1:"; +const DEFAULT_OK_MARKER: &str = "../src/inventory.js"; +const DEFAULT_FAIL_MARKER: &str = ".mjs:1:"; #[derive(Default)] struct Counters { @@ -19,11 +19,17 @@ struct Counters { bytes: AtomicU64, } +struct App { + counters: Counters, + ok: Vec, + fail: Vec, +} + fn contains(haystack: &[u8], needle: &[u8]) -> bool { haystack.windows(needle.len()).any(|w| w == needle) } -async fn ingest(State(c): State>, headers: HeaderMap, body: Bytes) -> &'static str { +async fn ingest(State(c): State>, headers: HeaderMap, body: Bytes) -> &'static str { let decoded: Vec; let data: &[u8] = if headers .get("content-encoding") @@ -33,7 +39,7 @@ async fn ingest(State(c): State>, headers: HeaderMap, body: Bytes) let mut d = GzDecoder::new(body.as_ref()); let mut out = Vec::with_capacity(body.len() * 4); if d.read_to_end(&mut out).is_err() { - c.unsymbolicated.fetch_add(1, Ordering::Relaxed); + c.counters.unsymbolicated.fetch_add(1, Ordering::Relaxed); return ""; } decoded = out; @@ -41,43 +47,50 @@ async fn ingest(State(c): State>, headers: HeaderMap, body: Bytes) } else { body.as_ref() }; - c.requests.fetch_add(1, Ordering::Relaxed); - c.bytes.fetch_add(data.len() as u64, Ordering::Relaxed); - if contains(data, OK_MARKER) && !contains(data, FAIL_MARKER) { - c.symbolicated.fetch_add(1, Ordering::Relaxed); + c.counters.requests.fetch_add(1, Ordering::Relaxed); + c.counters.bytes.fetch_add(data.len() as u64, Ordering::Relaxed); + if contains(data, &c.ok) && !contains(data, &c.fail) { + c.counters.symbolicated.fetch_add(1, Ordering::Relaxed); } else { - c.unsymbolicated.fetch_add(1, Ordering::Relaxed); + c.counters.unsymbolicated.fetch_add(1, Ordering::Relaxed); } "" } -async fn stats(State(c): State>) -> String { +async fn stats(State(c): State>) -> String { serde_json::json!({ - "requests": c.requests.load(Ordering::Relaxed), - "symbolicated": c.symbolicated.load(Ordering::Relaxed), - "unsymbolicated": c.unsymbolicated.load(Ordering::Relaxed), - "bytes": c.bytes.load(Ordering::Relaxed), + "requests": c.counters.requests.load(Ordering::Relaxed), + "symbolicated": c.counters.symbolicated.load(Ordering::Relaxed), + "unsymbolicated": c.counters.unsymbolicated.load(Ordering::Relaxed), + "bytes": c.counters.bytes.load(Ordering::Relaxed), }) .to_string() } -async fn reset(State(c): State>) -> &'static str { - c.requests.store(0, Ordering::Relaxed); - c.symbolicated.store(0, Ordering::Relaxed); - c.unsymbolicated.store(0, Ordering::Relaxed); - c.bytes.store(0, Ordering::Relaxed); +async fn reset(State(c): State>) -> &'static str { + c.counters.requests.store(0, Ordering::Relaxed); + c.counters.symbolicated.store(0, Ordering::Relaxed); + c.counters.unsymbolicated.store(0, Ordering::Relaxed); + c.counters.bytes.store(0, Ordering::Relaxed); "reset" } #[tokio::main] async fn main() { - let counters = Arc::new(Counters::default()); + let ok = std::env::var("OK_MARKER").unwrap_or_else(|_| DEFAULT_OK_MARKER.to_string()); + let fail = std::env::var("FAIL_MARKER").unwrap_or_else(|_| DEFAULT_FAIL_MARKER.to_string()); + println!("drain markers: ok={:?} fail={:?}", ok, fail); + let state = Arc::new(App { + counters: Counters::default(), + ok: ok.into_bytes(), + fail: fail.into_bytes(), + }); let app = Router::new() .route("/v1/traces", post(ingest)) .route("/v1/logs", post(ingest)) .route("/stats", get(stats)) .route("/reset", post(reset)) - .with_state(counters); + .with_state(state); let addr = std::env::var("DRAIN_ADDR").unwrap_or_else(|_| "0.0.0.0:9319".to_string()); let listener = tokio::net::TcpListener::bind(&addr).await.unwrap(); println!("drain listening on {}", addr); diff --git a/benchmarks/processor/loadgen/main.go b/benchmarks/processor/loadgen/main.go index 99fb45d2..b2f666de 100644 --- a/benchmarks/processor/loadgen/main.go +++ b/benchmarks/processor/loadgen/main.go @@ -21,17 +21,32 @@ import ( "go.opentelemetry.io/collector/pdata/ptrace/ptraceotlp" ) -func buildBody(url string, spans int) []byte { - td := ptrace.NewTraces() - rs := td.ResourceSpans().AppendEmpty() - rs.Resource().Attributes().PutStr("service.name", "processor-bench") - rs.Resource().Attributes().PutStr("telemetry.sdk.language", "nodejs") - ss := rs.ScopeSpans().AppendEmpty() +func orDefault(s, def string) string { + if s == "" { + return def + } + return s +} + +func buildBodyJS(url string, spans int) []byte { stack := fmt.Sprintf(`RangeError: cannot reserve 3 units for order ord_1042, only 2 in stock at file:///bench/%s:1:435 at p (file:///bench/%s:1:554) at file:///bench/%s:1:1180 at file:///bench/%s:1:1717`, url, url, url, url) + return buildBody("nodejs", "RangeError", "bench", stack, spans) +} + +func buildBodyDart(trace string, spans int) []byte { + return buildBody("dart", "DartError", "bench", trace, spans) +} + +func buildBody(lang, excType, excMsg, stack string, spans int) []byte { + td := ptrace.NewTraces() + rs := td.ResourceSpans().AppendEmpty() + rs.Resource().Attributes().PutStr("service.name", "processor-bench") + rs.Resource().Attributes().PutStr("telemetry.sdk.language", lang) + ss := rs.ScopeSpans().AppendEmpty() for i := 0; i < spans; i++ { sp := ss.Spans().AppendEmpty() sp.SetName("POST /orders/fulfill") @@ -44,8 +59,8 @@ func buildBody(url string, spans int) []byte { sp.SetSpanID(sid) ev := sp.Events().AppendEmpty() ev.SetName("exception") - ev.Attributes().PutStr("exception.type", "RangeError") - ev.Attributes().PutStr("exception.message", "bench") + ev.Attributes().PutStr("exception.type", excType) + ev.Attributes().PutStr("exception.message", excMsg) ev.Attributes().PutStr("exception.stacktrace", stack) } req := ptraceotlp.NewExportRequestFromTraces(td) @@ -57,16 +72,16 @@ func buildBody(url string, spans int) []byte { } type stepResult struct { - Connections int `json:"connections"` - DurationSec float64 `json:"duration_sec"` - Sent int64 `json:"sent"` - Ok int64 `json:"ok"` - Rejected int64 `json:"rejected"` - Errors int64 `json:"errors"` - OkReqPerSec float64 `json:"ok_req_per_sec"` + Connections int `json:"connections"` + DurationSec float64 `json:"duration_sec"` + Sent int64 `json:"sent"` + Ok int64 `json:"ok"` + Rejected int64 `json:"rejected"` + Errors int64 `json:"errors"` + OkReqPerSec float64 `json:"ok_req_per_sec"` StacksPerSec float64 `json:"stacks_per_sec"` - P50Ms float64 `json:"p50_ms"` - P99Ms float64 `json:"p99_ms"` + P50Ms float64 `json:"p50_ms"` + P99Ms float64 `json:"p99_ms"` } func main() { @@ -83,16 +98,32 @@ func main() { panic(err) } var c struct { - Urls []string `json:"urls"` + Language string `json:"language"` + Urls []string `json:"urls"` + Builds []struct { + BuildID string `json:"buildId"` + Trace string `json:"trace"` + } `json:"builds"` } if err := json.Unmarshal(raw, &c); err != nil { panic(err) } - bodies := make([][]byte, len(c.Urls)) - for i, u := range c.Urls { - bodies[i] = buildBody(u, *spansPerReq) + var bodies [][]byte + if c.Language == "dart" { + bodies = make([][]byte, len(c.Builds)) + for i, b := range c.Builds { + bodies[i] = buildBodyDart(b.Trace, *spansPerReq) + } + } else { + bodies = make([][]byte, len(c.Urls)) + for i, u := range c.Urls { + bodies[i] = buildBodyJS(u, *spansPerReq) + } + } + if len(bodies) == 0 { + panic("corpus has no entries") } - fmt.Printf("prepared %d bodies, %d spans each, %d bytes first\n", len(bodies), *spansPerReq, len(bodies[0])) + fmt.Printf("prepared %d %s bodies, %d spans each, %d bytes first\n", len(bodies), orDefault(c.Language, "js"), *spansPerReq, len(bodies[0])) var results []stepResult for _, s := range strings.Split(*connSteps, ",") { @@ -150,16 +181,16 @@ func main() { return lats[int(float64(len(lats)-1)*p)] } r := stepResult{ - Connections: conns, - DurationSec: stepDur.Seconds(), - Sent: sent, - Ok: ok, - Rejected: rejected, - Errors: errs, - OkReqPerSec: float64(ok) / stepDur.Seconds(), + Connections: conns, + DurationSec: stepDur.Seconds(), + Sent: sent, + Ok: ok, + Rejected: rejected, + Errors: errs, + OkReqPerSec: float64(ok) / stepDur.Seconds(), StacksPerSec: float64(ok*int64(*spansPerReq)) / stepDur.Seconds(), - P50Ms: pct(0.50), - P99Ms: pct(0.99), + P50Ms: pct(0.50), + P99Ms: pct(0.99), } results = append(results, r) line, _ := json.Marshal(r) diff --git a/benchmarks/processor/run-hetzner.sh b/benchmarks/processor/run-hetzner.sh index e0cf8af0..9f44a25a 100755 --- a/benchmarks/processor/run-hetzner.sh +++ b/benchmarks/processor/run-hetzner.sh @@ -27,14 +27,22 @@ command -v hcloud >/dev/null || { echo "hcloud CLI required" >&2; exit 1; } [ -d artifacts ] || { echo "artifacts/ missing, build first (see workflow or run-local.sh build_all)" >&2; exit 1; } case "$IMPL" in - honeycomb) COL_BIN=otelcol-bench-honeycomb; COL_CFG=config-honeycomb.yaml; PARSER=goja; DISK= ;; - traceway-oxc-mem) COL_BIN=otelcol-bench-traceway; COL_CFG=config-traceway.yaml; PARSER=oxc; DISK= ;; - traceway-oxc-disk) COL_BIN=otelcol-bench-traceway; COL_CFG=config-traceway.yaml; PARSER=oxc; DISK=1 ;; - traceway-goja-mem) COL_BIN=otelcol-bench-traceway; COL_CFG=config-traceway.yaml; PARSER=goja; DISK= ;; - traceway-goja-disk) COL_BIN=otelcol-bench-traceway; COL_CFG=config-traceway.yaml; PARSER=goja; DISK=1 ;; + honeycomb) COL_BIN=otelcol-bench-honeycomb; COL_CFG=config-honeycomb.yaml; PARSER=goja; DISK=; LANG=js ;; + traceway-oxc-mem) COL_BIN=otelcol-bench-traceway; COL_CFG=config-traceway.yaml; PARSER=oxc; DISK=; LANG=js ;; + traceway-oxc-disk) COL_BIN=otelcol-bench-traceway; COL_CFG=config-traceway.yaml; PARSER=oxc; DISK=1; LANG=js ;; + traceway-goja-mem) COL_BIN=otelcol-bench-traceway; COL_CFG=config-traceway.yaml; PARSER=goja; DISK=; LANG=js ;; + traceway-goja-disk) COL_BIN=otelcol-bench-traceway; COL_CFG=config-traceway.yaml; PARSER=goja; DISK=1; LANG=js ;; + traceway-dart-mem) COL_BIN=otelcol-bench-traceway; COL_CFG=config-traceway.yaml; PARSER=goja; DISK=; LANG=dart ;; + traceway-dart-disk) COL_BIN=otelcol-bench-traceway; COL_CFG=config-traceway.yaml; PARSER=goja; DISK=1; LANG=dart ;; *) echo "unknown impl $IMPL" >&2; exit 1 ;; esac +if [ "$LANG" = dart ]; then + OK_MARKER=crash.dart; FAIL_MARKER=_kDartIsolateSnapshotInstructions +else + OK_MARKER=../src/inventory.js; FAIL_MARKER=.mjs:1: +fi + scenario_params() { case "$1" in hot) echo "1 $PAD_KB 0:0 128 $CONNECTIONS $STEP_DURATION $SUT_TYPE" ;; @@ -75,7 +83,7 @@ provision() { hcloud ssh-key create --name "bench-key-$RUN_ID" --public-key-from-file "$KEY_FILE.pub" > /dev/null LDG_IP=$(provision "bench-ldg-$RUN_ID" "$LDG_TYPE") -$SSH "root@$LDG_IP" "nohup env DRAIN_ADDR=0.0.0.0:9319 /opt/bench/drain > /opt/bench/drain.log 2>&1 & sleep 1; curl -sf http://127.0.0.1:9319/stats" +$SSH "root@$LDG_IP" "nohup env DRAIN_ADDR=0.0.0.0:9319 OK_MARKER='$OK_MARKER' FAIL_MARKER='$FAIL_MARKER' /opt/bench/drain > /opt/bench/drain.log 2>&1 & sleep 1; curl -sf http://127.0.0.1:9319/stats" PREV_SUT_TYPE="" SUT_IP="" @@ -90,8 +98,13 @@ for scenario in $SCENARIOS; do outdir="$RESULTS/$tag" mkdir -p "$outdir" + if [ "$LANG" = dart ]; then + corpusgen_cmd="./corpusgen --language dart --symbols seeds/dart/app.debug.elf --trace seeds/dart/trace.txt --entries $entries --out corpus-$scenario" + else + corpusgen_cmd="./corpusgen --bundle app.mjs --map app.mjs.map --entries $entries --pad-kb $pad --map-pad-kb ${mappad%%:*} --mappings-pad-kb ${mappad##*:} --out corpus-$scenario" + fi for ip in "$SUT_IP" "$LDG_IP"; do - $SSH "root@$ip" "cd /opt/bench && [ -f corpus-$scenario/corpus.json ] || ./corpusgen --bundle app.mjs --map app.mjs.map --entries $entries --pad-kb $pad --map-pad-kb ${mappad%%:*} --mappings-pad-kb ${mappad##*:} --out corpus-$scenario" + $SSH "root@$ip" "cd /opt/bench && [ -f corpus-$scenario/corpus.json ] || $corpusgen_cmd" done CACHE_DIR_REMOTE="" diff --git a/benchmarks/processor/run-local.sh b/benchmarks/processor/run-local.sh index f9e57723..d4b42183 100755 --- a/benchmarks/processor/run-local.sh +++ b/benchmarks/processor/run-local.sh @@ -40,7 +40,7 @@ build_all() { (cd loadgen && go mod tidy >/dev/null 2>&1; go build -o loadgen .) (cd corpusgen && go build -o corpusgen .) - if [ ! -f ../../testing/symbolication/node-app/dist/app.mjs ]; then + if echo "$IMPLS" | grep -qE 'honeycomb|oxc|goja' && [ ! -f ../../testing/symbolication/node-app/dist/app.mjs ]; then (cd ../../testing/symbolication/node-app && npm install && npm run build) fi } @@ -54,32 +54,47 @@ scenario_params() { } gen_corpus() { - local scenario="$1" entries="$2" pad="$3" mappad="$4" - local dir="./corpus-$scenario" + local lang="$1" scenario="$2" entries="$3" pad="$4" mappad="$5" + local dir="./corpus-$lang-$scenario" if [ ! -f "$dir/corpus.json" ]; then - ./corpusgen/corpusgen --entries "$entries" --pad-kb "$pad" --map-pad-kb "${mappad%%:*}" --mappings-pad-kb "${mappad##*:}" --out "$dir" >&2 + if [ "$lang" = dart ]; then + ./corpusgen/corpusgen --language dart --entries "$entries" --out "$dir" >&2 + else + ./corpusgen/corpusgen --entries "$entries" --pad-kb "$pad" --map-pad-kb "${mappad%%:*}" --mappings-pad-kb "${mappad##*:}" --out "$dir" >&2 + fi fi echo "$dir" } impl_env() { case "$1" in - honeycomb) echo "BIN=./build-honeycomb/otelcol-bench-honeycomb CFG=config-honeycomb.yaml PARSER= DISK=" ;; - traceway-oxc-mem) echo "BIN=./build-traceway/otelcol-bench-traceway CFG=config-traceway.yaml PARSER=oxc DISK=" ;; - traceway-oxc-disk) echo "BIN=./build-traceway/otelcol-bench-traceway CFG=config-traceway.yaml PARSER=oxc DISK=1" ;; - traceway-goja-mem) echo "BIN=./build-traceway/otelcol-bench-traceway CFG=config-traceway.yaml PARSER=goja DISK=" ;; - traceway-goja-disk) echo "BIN=./build-traceway/otelcol-bench-traceway CFG=config-traceway.yaml PARSER=goja DISK=1" ;; + honeycomb) echo "BIN=./build-honeycomb/otelcol-bench-honeycomb CFG=config-honeycomb.yaml PARSER= DISK= LANG=js" ;; + traceway-oxc-mem) echo "BIN=./build-traceway/otelcol-bench-traceway CFG=config-traceway.yaml PARSER=oxc DISK= LANG=js" ;; + traceway-oxc-disk) echo "BIN=./build-traceway/otelcol-bench-traceway CFG=config-traceway.yaml PARSER=oxc DISK=1 LANG=js" ;; + traceway-goja-mem) echo "BIN=./build-traceway/otelcol-bench-traceway CFG=config-traceway.yaml PARSER=goja DISK= LANG=js" ;; + traceway-goja-disk) echo "BIN=./build-traceway/otelcol-bench-traceway CFG=config-traceway.yaml PARSER=goja DISK=1 LANG=js" ;; + traceway-dart-mem) echo "BIN=./build-traceway/otelcol-bench-traceway CFG=config-traceway.yaml PARSER= DISK= LANG=dart" ;; + traceway-dart-disk) echo "BIN=./build-traceway/otelcol-bench-traceway CFG=config-traceway.yaml PARSER= DISK=1 LANG=dart" ;; *) echo "unknown impl $1" >&2; return 1 ;; esac } +drain_markers() { + case "$1" in + dart) echo "OK_MARKER=crash.dart FAIL_MARKER=_kDartIsolateSnapshotInstructions" ;; + *) echo "OK_MARKER=../src/inventory.js FAIL_MARKER=.mjs:1:" ;; + esac +} + run_one() { local impl="$1" scenario="$2" read -r entries pad mappad cachesize conns dur <<< "$(scenario_params "$scenario")" - local store - store=$(gen_corpus "$scenario" "$entries" "$pad" "$mappad") - local BIN CFG PARSER DISK + local BIN CFG PARSER DISK LANG eval "$(impl_env "$impl")" + local store + store=$(gen_corpus "$LANG" "$scenario" "$entries" "$pad" "$mappad") + local OK_MARKER FAIL_MARKER + eval "$(drain_markers "$LANG")" local tag="$impl-$scenario" local outdir="$RESULTS/$tag" mkdir -p "$outdir" @@ -91,7 +106,7 @@ run_one() { sleep 1 done - DRAIN_ADDR=127.0.0.1:9319 ./drain/target/release/drain & + DRAIN_ADDR=127.0.0.1:9319 OK_MARKER="$OK_MARKER" FAIL_MARKER="$FAIL_MARKER" ./drain/target/release/drain & local drain_pid=$! sleep 1 curl -sf -X POST http://127.0.0.1:9319/reset > /dev/null diff --git a/benchmarks/processor/seeds/dart/app.debug.elf b/benchmarks/processor/seeds/dart/app.debug.elf new file mode 100644 index 00000000..0600b715 Binary files /dev/null and b/benchmarks/processor/seeds/dart/app.debug.elf differ diff --git a/benchmarks/processor/seeds/dart/trace.txt b/benchmarks/processor/seeds/dart/trace.txt new file mode 100644 index 00000000..74c994dc --- /dev/null +++ b/benchmarks/processor/seeds/dart/trace.txt @@ -0,0 +1,14 @@ +Unhandled exception: +Bad state: boom from level3 +*** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** +pid: 49838, tid: 8806539456, name Dart_Initialize +os: macos arch: arm64 comp: no sim: no +build_id: '20fb76bfc9d292a85d82b8712754455e' +isolate_dso_base: 104914000, vm_dso_base: 104914000 +isolate_instructions: 1049527c0, vm_instructions: 104948000 + #00 abs 00000001049832cb virt 000000000006f2cb _kDartIsolateSnapshotInstructions+0x30b0b + #01 abs 0000000104983293 virt 000000000006f293 _kDartIsolateSnapshotInstructions+0x30ad3 + #02 abs 000000010498ebd7 virt 000000000007abd7 _kDartIsolateSnapshotInstructions+0x3c417 + #03 abs 00000001049836c3 virt 000000000006f6c3 _kDartIsolateSnapshotInstructions+0x30f03 + #04 abs 000000010498eeb3 virt 000000000007aeb3 _kDartIsolateSnapshotInstructions+0x3c6f3 + #05 abs 0000000104961b07 virt 000000000004db07 _kDartIsolateSnapshotInstructions+0xf347 diff --git a/examples/devtesting-embedded/main.go b/examples/devtesting-embedded/main.go index 29e1ac7f..84998479 100644 --- a/examples/devtesting-embedded/main.go +++ b/examples/devtesting-embedded/main.go @@ -38,10 +38,12 @@ var cdnHTML []byte var staticFS embed.FS const ( - appPort = 8080 - backendToken = "backend-dev-token" - frontendToken = "frontend-dev-token" - monitoringToken = "monitoring-dev-token" + appPort = 8080 + backendToken = "backend-dev-token" + frontendToken = "frontend-dev-token" + monitoringToken = "monitoring-dev-token" + flutterToken = "flutter-dev-token" + flutterUploadToken = "flutter-upload-token" backendServiceName = "backend-service" workerServiceName = "worker-service" @@ -49,8 +51,6 @@ const ( otlpHost = "localhost:8082" ) -// otelService bundles a TracerProvider + LoggerProvider for a single logical service -// so we can run two services in-process to exercise the distributed-logs flow. type otelService struct { name string tp *sdktrace.TracerProvider @@ -113,8 +113,6 @@ func initOtelService(ctx context.Context, serviceName, token string, extraResour }, nil } -// log emits an OTel log record. The SDK auto-attaches trace_id / span_id from ctx -// so logs emitted inside a span get chip-linked on the trace detail page. func (s *otelService) log(ctx context.Context, sev otellog.Severity, sevText, body string, attrs ...otellog.KeyValue) { rec := otellog.Record{} now := time.Now() @@ -137,11 +135,11 @@ func main() { tracewaybackend.WithDefaultProject("Backend API", "opentelemetry", backendToken), tracewaybackend.WithDefaultProject("jQuery Frontend", "jquery", frontendToken), tracewaybackend.WithDefaultProject("Traceway Monitoring", "gin", monitoringToken), + tracewaybackend.WithDefaultProject("Flutter App", "flutter", flutterToken), + tracewaybackend.WithDefaultProjectSourceMapToken("Flutter App", flutterUploadToken), tracewaybackend.WithMonitoringURL(monitoringToken+"@http://localhost:8082/api/report"), ) - // Give the backend a moment to start listening and register project tokens - // before we start sending OTel data at it. time.Sleep(2 * time.Second) ctx := context.Background() @@ -152,9 +150,6 @@ func main() { } defer backendSvc.shutdown(ctx) - // Second OTel provider with a different service.name. Reports to the same - // Backend API project so both sides of the "distributed trace" are visible - // in one project. Used by /api/test-distributed-logs. workerSvc, err := initOtelService(ctx, workerServiceName, backendToken) if err != nil { panic(err) @@ -189,8 +184,6 @@ func main() { c.Next() }) - // otelgin creates a SERVER span per request and puts it in c.Request.Context(), - // which is what the OTel logger reads to attach trace_id + span_id to each log. router.Use(otelgin.Middleware(backendServiceName, otelgin.WithTracerProvider(backendSvc.tp))) router.GET("/", func(c *gin.Context) { @@ -204,7 +197,6 @@ func main() { staticSub, _ := fs.Sub(staticFS, "static") router.StaticFS("/static", http.FS(staticSub)) - // Emits an error log + records an exception on the root span. router.GET("/api/test-error", func(c *gin.Context) { ctx := c.Request.Context() backendSvc.log(ctx, otellog.SeverityInfo, "INFO", "received test-error request") @@ -217,7 +209,6 @@ func main() { c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()}) }) - // Emits a DEBUG + INFO log on a successful request. router.GET("/api/test-success", func(c *gin.Context) { ctx := c.Request.Context() backendSvc.log(ctx, otellog.SeverityDebug, "DEBUG", "test-success entered") @@ -226,8 +217,6 @@ func main() { c.JSON(http.StatusOK, gin.H{"message": "success"}) }) - // Emits one log at each severity level — useful for inspecting the /logs page - // and confirming the SeverityBadge renders every variant. router.GET("/api/test-log-levels", func(c *gin.Context) { ctx := c.Request.Context() backendSvc.log(ctx, otellog.SeverityTrace1, "TRACE", "trace-level log for visual testing") @@ -239,11 +228,6 @@ func main() { c.JSON(http.StatusOK, gin.H{"emitted": 6}) }) - // Nested child spans + logs from each level. On the endpoint detail page: - // - root-span logs chip as the endpoint name - // - child-span logs chip as that child span's name (db.query / cache.lookup / auth.verify) - // Also populates parent_span_id on each non-root span, and attributes on each - // child span so the span tree + attribute popover can be exercised. router.GET("/api/test-spans-with-logs", func(c *gin.Context) { ctx := c.Request.Context() backendSvc.log(ctx, otellog.SeverityInfo, "INFO", "handler: entering /test-spans-with-logs") @@ -280,10 +264,6 @@ func main() { c.JSON(http.StatusOK, gin.H{"status": "ok"}) }) - // Distributed logs: emits logs from two different services (backend-service - // and worker-service) with a shared traceway.distributed_trace_id so the - // "Load logs from other traces" button on the trace detail page has - // something to pull in. router.GET("/api/test-distributed-logs", func(c *gin.Context) { ctx := c.Request.Context() dtid := uuid.New().String() @@ -294,9 +274,6 @@ func main() { backendSvc.log(ctx, otellog.SeverityInfo, "INFO", "backend: received request, about to call worker", otellog.String("distributed_trace_id", dtid)) - // Fresh root context (no parent) so the worker registers as a separate - // trace in Traceway. WithSpanKind(Consumer) makes the converter treat - // this as a task rather than another endpoint. workerCtx, workerSpan := workerSvc.tr.Start(context.Background(), "worker.process-job", trace.WithSpanKind(trace.SpanKindConsumer)) workerSpan.SetAttributes(attribute.String("traceway.distributed_trace_id", dtid)) @@ -347,15 +324,12 @@ func main() { writeSSEStream(c, 30*time.Second, time.Second) }) - // Short SSE for quicker iteration. router.GET("/api/test-sse-short", func(c *gin.Context) { span := trace.SpanFromContext(c.Request.Context()) span.SetAttributes(attribute.Bool("traceway.is_stream", true)) writeSSEStream(c, 5*time.Second, 500*time.Millisecond) }) - // Long-poll style — no SSE Content-Type, just a long-held connection that - // returns JSON at the end. Flagged via the same span attribute. router.GET("/api/test-long-poll", func(c *gin.Context) { span := trace.SpanFromContext(c.Request.Context()) span.SetAttributes(attribute.Bool("traceway.is_stream", true)) diff --git a/scripts/build-oxc-shim.sh b/scripts/build-oxc-shim.sh index 5e177448..95dea7e4 100755 --- a/scripts/build-oxc-shim.sh +++ b/scripts/build-oxc-shim.sh @@ -9,7 +9,7 @@ if ! command -v cargo &> /dev/null; then exit 1 fi -cd "$ROOT_DIR/backend/app/symbolicator/scopes/oxc-shim" +cd "$ROOT_DIR/backend/app/symbolicator/sourcemap/scopes/oxc-shim" cargo build --release echo "Built liboxc_shim.a" diff --git a/testing/flutter-symbolication/run-local.sh b/testing/flutter-symbolication/run-local.sh new file mode 100755 index 00000000..c1ed188e --- /dev/null +++ b/testing/flutter-symbolication/run-local.sh @@ -0,0 +1,112 @@ +#!/usr/bin/env bash + +set -euo pipefail + +ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" +FIXTURE="$ROOT/backend/app/symbolicator/dart/fixtures/flutter-macos-arm64-dart3.10.1" +SYMBOLS="$FIXTURE/app.darwin-arm64.symbols" +TRACE="$FIXTURE/trace.txt" + +INGEST_TOKEN="flutter-dev-token" +UPLOAD_TOKEN="flutter-upload-token" +BASE="http://localhost:8082" + +for f in "$SYMBOLS" "$TRACE"; do + [ -f "$f" ] || { echo "missing fixture: $f"; exit 1; } +done + +WORK="$(mktemp -d)" +mkdir -p "$WORK/storage" +echo "work dir: $WORK" + +echo "==> building devtesting-embedded" +(cd "$ROOT/examples/devtesting-embedded" && go build -o "$WORK/devtesting-embedded" .) + +echo "==> starting backend (port 8082)" +( cd "$WORK" && "$WORK/devtesting-embedded" ) > "$WORK/server.log" 2>&1 & +SERVER_PID=$! +cleanup() { kill "$SERVER_PID" 2>/dev/null || true; } +trap cleanup EXIT + +ready= +for _ in $(seq 1 60); do + if curl -fsS "$BASE/api/has-organizations" >/dev/null 2>&1; then ready=1; break; fi + sleep 1 +done +[ -n "$ready" ] || { echo "FAIL: backend did not come up"; tail -40 "$WORK/server.log"; exit 1; } +echo " backend is up" + +DEBUG_ID="fe664295997135e7b67b648ba66ca9eb" +echo "==> uploading symbols (arch=arm64 debug_id=$DEBUG_ID)" +UP="$(curl -fsS -X POST "$BASE/api/symbols/upload" \ + -H "Authorization: Bearer $UPLOAD_TOKEN" \ + -F "files=@$SYMBOLS" -F "arch=arm64" -F "debug_id=$DEBUG_ID")" +echo " response: $UP" +echo "$UP" | grep -q '"uploaded":1' || { echo "FAIL: symbols upload"; exit 1; } + +post_report() { + python3 - "$1" "$BASE/api/report" "$INGEST_TOKEN" <<'PY' +import sys, json, gzip, urllib.request +trace = open(sys.argv[1]).read() +url, token = sys.argv[2], sys.argv[3] +body = { + "collectionFrames": [{ + "stackTraces": [{ + "stackTrace": trace, + "recordedAt": "2026-06-13T00:00:00Z", + "isMessage": False, + "attributes": {}, + }], + "metrics": [], "traces": [], "sessions": [], + }], + "appVersion": "1.0.0", "serverName": "", +} +data = gzip.compress(json.dumps(body).encode()) +req = urllib.request.Request(url, data=data, method="POST", headers={ + "Content-Type": "application/json", + "Content-Encoding": "gzip", + "Authorization": "Bearer " + token, +}) +with urllib.request.urlopen(req) as r: + print(" report HTTP", r.status) +PY +} + +echo "==> posting report #1 (raw non-symbolic trace)" +post_report "$TRACE" + +echo "==> posting report #2 (same crash, different load address)" +sed -E 's/abs [0-9a-fA-F]+/abs 0000000700000000/' "$TRACE" > "$WORK/trace2.txt" +post_report "$WORK/trace2.txt" + +TELDB="$WORK/storage/traceway_telemetry.db" +echo "==> reading back from $TELDB" + +got= +for _ in $(seq 1 25); do + if [ -f "$TELDB" ]; then + got="$(sqlite3 "$TELDB" "SELECT stack_trace FROM exception_stack_traces WHERE stack_trace LIKE '%chargeCard%' LIMIT 1;" 2>/dev/null || true)" + [ -n "$got" ] && break + fi + sleep 1 +done + +if [ -z "$got" ]; then + echo "FAIL: no symbolicated exception stored" + echo "--- tables ---"; sqlite3 "$TELDB" ".tables" 2>/dev/null || true + echo "--- server log (tail) ---"; tail -40 "$WORK/server.log" + exit 1 +fi + +echo "--- stored symbolicated stack trace ---" +echo "$got" | head -6 +echo "$got" | grep -q "main.dart:20:3" || { echo "FAIL: chargeCard not resolved to main.dart:20:3"; exit 1; } + +counts="$(sqlite3 "$TELDB" "SELECT COUNT(*), COUNT(DISTINCT exception_hash) FROM exception_stack_traces WHERE stack_trace LIKE '%chargeCard%';")" +total="${counts%%|*}" +distinct="${counts##*|}" +echo "==> grouping: $total row(s), $distinct distinct hash(es)" +[ "$distinct" = "1" ] || { echo "FAIL: expected 1 distinct exception hash, got $distinct"; exit 1; } + +echo +echo "PASS: Flutter symbolication works end-to-end (upload -> report -> symbolicated + grouped)"