perf(core): Prototype breadcrumb persistence as a JSONL append log

sentry-android-core/src/test/java/io/sentry/android/core/ApplicationExitInfoEventProcessorTest.kt

@@ -41,7 +41,6 @@ import io.sentry.cache.PersistingScopeObserver.TAGS_FILENAME
 import io.sentry.cache.PersistingScopeObserver.TRACE_FILENAME
 import io.sentry.cache.PersistingScopeObserver.TRANSACTION_FILENAME
 import io.sentry.cache.PersistingScopeObserver.USER_FILENAME
-import io.sentry.cache.tape.QueueFile
 import io.sentry.hints.AbnormalExit
 import io.sentry.hints.Backfillable
 import io.sentry.protocol.Browser
@@ -60,8 +59,8 @@ import io.sentry.protocol.SentryStackTrace
 import io.sentry.protocol.SentryThread
 import io.sentry.protocol.User
 import io.sentry.util.HintUtils
-import java.io.ByteArrayOutputStream
 import java.io.File
+import java.io.StringWriter
 import kotlin.test.BeforeTest
 import kotlin.test.Test
 import kotlin.test.assertEquals
@@ -166,12 +165,12 @@ class ApplicationExitInfoEventProcessorTest {
       val dir = File(options.cacheDirPath, SCOPE_CACHE).also { it.mkdirs() }
       val file = File(dir, filename)
       if (filename == BREADCRUMBS_FILENAME) {
-        val queueFile = QueueFile.Builder(file).build()
-        (entity as List<Breadcrumb>).forEach { crumb ->
-          val baos = ByteArrayOutputStream()
-          options.serializer.serialize(crumb, baos.writer())
-          queueFile.add(baos.toByteArray())
-        }
+        // breadcrumbs are stored as newline-delimited JSON, one breadcrumb per line
+        file.writeText(
+          (entity as List<Breadcrumb>).joinToString(separator = "") { crumb ->
+            StringWriter().also { options.serializer.serialize(crumb, it) }.toString() + "\n"
+          }
+        )
       } else {
         options.serializer.serialize(entity, file.writer())
       }

sentry-android-core/src/test/java/io/sentry/android/core/SentryAndroidTest.kt

@@ -44,15 +44,14 @@ import io.sentry.cache.PersistingScopeObserver.BREADCRUMBS_FILENAME
 import io.sentry.cache.PersistingScopeObserver.REPLAY_FILENAME
 import io.sentry.cache.PersistingScopeObserver.SCOPE_CACHE
 import io.sentry.cache.PersistingScopeObserver.TRANSACTION_FILENAME
-import io.sentry.cache.tape.QueueFile
 import io.sentry.protocol.Contexts
 import io.sentry.protocol.SentryId
 import io.sentry.spotlight.SpotlightIntegration
 import io.sentry.test.applyTestOptions
 import io.sentry.transport.NoOpEnvelopeCache
 import io.sentry.util.StringUtils
-import java.io.ByteArrayOutputStream
 import java.io.File
+import java.io.StringWriter
 import java.nio.file.Files
 import java.util.concurrent.TimeUnit
 import java.util.concurrent.atomic.AtomicBoolean
@@ -564,17 +563,14 @@ class SentryAndroidTest {
 
   private fun prefillScopeCache(options: SentryOptions, cacheDir: String) {
     val scopeDir = File(cacheDir, SCOPE_CACHE).also { it.mkdirs() }
-    val queueFile = QueueFile.Builder(File(scopeDir, BREADCRUMBS_FILENAME)).build()
-    val baos = ByteArrayOutputStream()
-    options.serializer.serialize(
+    val breadcrumb =
       Breadcrumb(DateUtils.getDateTime("2009-11-16T01:08:47.000Z")).apply {
         message = "Debug!"
         type = "debug"
         level = DEBUG
-      },
-      baos.writer(),
-    )
-    queueFile.add(baos.toByteArray())
+      }
+    val serialized = StringWriter().also { options.serializer.serialize(breadcrumb, it) }.toString()
+    File(scopeDir, BREADCRUMBS_FILENAME).writeText("$serialized\n")
     File(scopeDir, TRANSACTION_FILENAME).writeText("\"MainActivity\"")
     File(scopeDir, REPLAY_FILENAME).writeText("\"afcb46b1140ade5187c4bbb5daa804df\"")
     File(options.getCacheDirPath(), "replay_afcb46b1140ade5187c4bbb5daa804df").mkdirs()

sentry-android-replay/src/test/java/io/sentry/android/replay/ReplayIntegrationTest.kt

@@ -36,7 +36,6 @@ import io.sentry.android.replay.capture.SessionCaptureStrategyTest.Fixture.Compa
 import io.sentry.android.replay.gestures.GestureRecorder
 import io.sentry.android.replay.util.ReplayShadowMediaCodec
 import io.sentry.cache.PersistingScopeObserver
-import io.sentry.cache.tape.QueueFile
 import io.sentry.protocol.SentryException
 import io.sentry.protocol.SentryId
 import io.sentry.rrweb.RRWebBreadcrumbEvent
@@ -48,8 +47,8 @@ import io.sentry.transport.CurrentDateProvider
 import io.sentry.transport.ICurrentDateProvider
 import io.sentry.transport.RateLimiter
 import io.sentry.util.Random
-import java.io.ByteArrayOutputStream
 import java.io.File
+import java.io.StringWriter
 import kotlin.test.BeforeTest
 import kotlin.test.Test
 import kotlin.test.assertEquals
@@ -501,18 +500,16 @@ class ReplayIntegrationTest {
       it.writeText("\"$oldReplayId\"")
     }
     val breadcrumbsFile = File(scopeCache, PersistingScopeObserver.BREADCRUMBS_FILENAME)
-    val queueFile = QueueFile.Builder(breadcrumbsFile).build()
-    val baos = ByteArrayOutputStream()
-    fixture.options.serializer.serialize(
+    val breadcrumb =
       Breadcrumb(DateUtils.getDateTime("2024-07-11T10:25:23.454Z")).apply {
         category = "navigation"
         type = "navigation"
         setData("from", "from")
         setData("to", "to")
-      },
-      baos.writer(),
-    )
-    queueFile.add(baos.toByteArray())
+      }
+    val serialized =
+      StringWriter().also { fixture.options.serializer.serialize(breadcrumb, it) }.toString()
+    breadcrumbsFile.writeText("$serialized\n")
     File(oldReplay, ONGOING_SEGMENT).also {
       it.writeText(
         """

sentry/api/sentry.api

@@ -4821,6 +4821,13 @@ public final class io/sentry/backpressure/NoOpBackpressureMonitor : io/sentry/ba
 	public fun start ()V
 }
 
+public final class io/sentry/cache/BreadcrumbAppendLog {
+	public fun <init> (Lio/sentry/SentryOptions;Ljava/io/File;)V
+	public fun append (Ljava/util/List;)V
+	public fun clear ()V
+	public fun read ()Ljava/util/List;
+}
+
 public final class io/sentry/cache/CacheUtils {
 	public static fun read (Lio/sentry/SentryOptions;Ljava/lang/String;Ljava/lang/String;Ljava/lang/Class;Lio/sentry/JsonDeserializer;)Ljava/lang/Object;
 	public static fun store (Lio/sentry/SentryOptions;Ljava/lang/Object;Ljava/lang/String;Ljava/lang/String;)V

sentry/src/main/java/io/sentry/cache/BreadcrumbAppendLog.java

@@ -0,0 +1,250 @@
+package io.sentry.cache;
+
+import static io.sentry.SentryLevel.DEBUG;
+import static io.sentry.SentryLevel.ERROR;
+
+import io.sentry.Breadcrumb;
+import io.sentry.SentryOptions;
+import java.io.BufferedReader;
+import java.io.BufferedWriter;
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileOutputStream;
+import java.io.InputStreamReader;
+import java.io.OutputStreamWriter;
+import java.io.StringReader;
+import java.io.Writer;
+import java.nio.charset.Charset;
+import java.util.ArrayList;
+import java.util.List;
+import org.jetbrains.annotations.ApiStatus;
+import org.jetbrains.annotations.NotNull;
+import org.jetbrains.annotations.Nullable;
+
+/**
+ * A bounded, newline-delimited JSON append log for breadcrumbs.
+ *
+ * <p>Breadcrumbs are only ever appended in bulk and read back as a whole list on the next launch,
+ * so this stores one serialized breadcrumb per line rather than using a random-access queue. A torn
+ * trailing line — the realistic failure when the process dies mid-write — fails to deserialize and
+ * is skipped, leaving every earlier breadcrumb readable.
+ *
+ * <p>The log is bounded by rewriting it down to the newest {@code maxBreadcrumbs} entries once it
+ * grows past {@code compactionThresholdFactor} times that limit, so appends stay O(1) amortized.
+ *
+ * <p>Not thread-safe; callers must serialize access. {@link PersistingScopeObserver} only touches
+ * it from the flush task on its single-threaded executor.
+ *
+ * <p>A null file means there is nowhere to persist to (no cache dir configured); every operation
+ * then becomes a no-op.
+ */
+@ApiStatus.Internal
+public final class BreadcrumbAppendLog {
+
+  @SuppressWarnings("CharsetObjectCanBeUsed")
+  private static final Charset UTF_8 = Charset.forName("UTF-8");
+
+  /** Compact once the log holds this many times {@code maxBreadcrumbs} lines. */
+  private static final int COMPACTION_THRESHOLD_FACTOR = 2;
+
+  private final @NotNull SentryOptions options;
+  private final @Nullable File file;
+  private final int maxBreadcrumbs;
+
+  /**
+   * Lines currently in the file. Tracked in memory so appends don't have to count them on disk;
+   * seeded from the file on first use so a log inherited from the previous run is still bounded.
+   */
+  private int lineCount;
+
+  private boolean lineCountKnown;
+
+  public BreadcrumbAppendLog(final @NotNull SentryOptions options, final @Nullable File file) {
+    this.options = options;
+    this.file = file;
+    // a non-positive limit would make the compaction threshold zero and compact on every append
+    this.maxBreadcrumbs = Math.max(1, options.getMaxBreadcrumbs());
+  }
+
+  /** Appends breadcrumbs, compacting afterwards if the log has outgrown its bound. */
+  public void append(final @NotNull List<Breadcrumb> breadcrumbs) {
+    if (file == null || breadcrumbs.isEmpty()) {
+      return;
+    }
+    // otherwise we'd append newline-delimited JSON onto a file in some older format
+    discardForeignFormat();
+    ensureLineCount();
+
+    int appended = 0;
+    try (final Writer writer =
+        new BufferedWriter(new OutputStreamWriter(new FileOutputStream(file, true), UTF_8))) {
+      for (final @NotNull Breadcrumb crumb : breadcrumbs) {
+        try {
+          options.getSerializer().serialize(crumb, writer);
+        } catch (Throwable e) {
+          // the partially written line would corrupt the log, so stop rather than write more
+          options.getLogger().log(ERROR, e, "Error serializing breadcrumb, dropping the rest");
+          break;
+        }
+        writer.write('\n');
+        appended++;
+      }
+    } catch (Throwable e) {
+      // the line count is unreliable once a write fails part-way, so re-read it next time
+      options.getLogger().log(ERROR, e, "Error appending breadcrumbs to %s", file.getName());
+      lineCountKnown = false;
+      return;
+    }
+
+    lineCount += appended;
+    if (lineCount > maxBreadcrumbs * COMPACTION_THRESHOLD_FACTOR) {
+      compact();
+    }
+  }
+
+  /** Reads the persisted breadcrumbs, oldest first, skipping any line that fails to parse. */
+  public @NotNull List<Breadcrumb> read() {
+    final @NotNull List<Breadcrumb> breadcrumbs = readAll();
+    return breadcrumbs.size() <= maxBreadcrumbs
+        ? breadcrumbs
+        : new ArrayList<>(
+            breadcrumbs.subList(breadcrumbs.size() - maxBreadcrumbs, breadcrumbs.size()));
+  }
+
+  /** Empties the log. */
+  public void clear() {
+    if (file == null) {
+      return;
+    }
+    if (file.exists() && !file.delete()) {
+      options.getLogger().log(DEBUG, "Failed to delete %s", file.getAbsolutePath());
+    }
+    lineCount = 0;
+    lineCountKnown = true;
+  }
+
+  private @NotNull List<Breadcrumb> readAll() {
+    discardForeignFormat();
+    if (file == null || !file.exists()) {
+      return new ArrayList<>();
+    }
+
+    final @NotNull List<Breadcrumb> breadcrumbs = new ArrayList<>();
+    try (final BufferedReader reader =
+        new BufferedReader(new InputStreamReader(new FileInputStream(file), UTF_8))) {
+      String line;
+      while ((line = reader.readLine()) != null) {
+        if (line.isEmpty()) {
+          continue;
+        }
+        final @Nullable Breadcrumb crumb = deserialize(line);
+        if (crumb != null) {
+          breadcrumbs.add(crumb);
+        }
+      }
+    } catch (Throwable e) {
+      options.getLogger().log(ERROR, e, "Error reading breadcrumbs from %s", file.getName());
+    }
+    return breadcrumbs;
+  }
+
+  /**
+   * Deletes a file left behind by an older SDK version, which stored breadcrumbs either as a
+   * QueueFile ring buffer or as a single JSON array. Neither is newline-delimited, so reading them
+   * line-by-line yields nothing useful; delete instead of leaving stale bytes to be appended to.
+   */
+  private void discardForeignFormat() {
+    if (file == null || !file.exists()) {
+      return;
+    }
+    final int firstByte = readFirstByte();
+    // every line this class writes is a JSON object, so anything else is a foreign format
+    if (firstByte == -1 || firstByte == '{') {
+      return;
+    }
+    options.getLogger().log(DEBUG, "Discarding breadcrumbs in an unrecognized format");
+    clear();
+  }
+
+  private int readFirstByte() {
+    if (file == null) {
+      return -1;
+    }
+    try (final FileInputStream stream = new FileInputStream(file)) {
+      return stream.read();
+    } catch (Throwable e) {
+      options.getLogger().log(ERROR, e, "Error reading breadcrumbs from %s", file.getName());
+      return -1;
+    }
+  }
+
+  private @Nullable Breadcrumb deserialize(final @NotNull String line) {
+    try (final StringReader reader = new StringReader(line)) {
+      return options.getSerializer().deserialize(reader, Breadcrumb.class);
+    } catch (Throwable e) {
+      // a torn trailing line is expected after the process died mid-write, so this isn't an error
+      options.getLogger().log(DEBUG, "Skipping unreadable breadcrumb line");
+      return null;
+    }
+  }
+
+  /** Rewrites the log with only the newest {@code maxBreadcrumbs} entries. */
+  private void compact() {
+    if (file == null) {
+      return;
+    }
+    final @NotNull List<Breadcrumb> retained = read();
+    final @NotNull File tempFile = new File(file.getPath() + ".tmp");
+
+    int written = 0;
+    try (final Writer writer =
+        new BufferedWriter(new OutputStreamWriter(new FileOutputStream(tempFile), UTF_8))) {
+      for (final @NotNull Breadcrumb crumb : retained) {
+        options.getSerializer().serialize(crumb, writer);
+        writer.write('\n');
+        written++;
+      }
+    } catch (Throwable e) {
+      options.getLogger().log(ERROR, e, "Error compacting breadcrumbs, keeping the existing log");
+      if (tempFile.exists() && !tempFile.delete()) {
+        options.getLogger().log(DEBUG, "Failed to delete %s", tempFile.getAbsolutePath());
+      }
+      return;
+    }
+
+    // renameTo is atomic, so a crash here leaves either the old log or the compacted one
+    if (!tempFile.renameTo(file)) {
+      options.getLogger().log(ERROR, "Failed to replace %s with the compacted log", file.getName());
+      if (!tempFile.delete()) {
+        options.getLogger().log(DEBUG, "Failed to delete %s", tempFile.getAbsolutePath());
+      }
+      return;
+    }
+    lineCount = written;
+    lineCountKnown = true;
+  }
+
+  private void ensureLineCount() {
+    if (lineCountKnown) {
+      return;
+    }
+    lineCount = countLines();
+    lineCountKnown = true;
+  }
+
+  private int countLines() {
+    if (file == null || !file.exists()) {
+      return 0;
+    }
+    int lines = 0;
+    try (final BufferedReader reader =
+        new BufferedReader(new InputStreamReader(new FileInputStream(file), UTF_8))) {
+      while (reader.readLine() != null) {
+        lines++;
+      }
+    } catch (Throwable e) {
+      options.getLogger().log(ERROR, e, "Error counting breadcrumbs in %s", file.getName());
+    }
+    return lines;
+  }
+}