upload screenshots along with traces

Author: rcholic

src/agent.ts

@@ -88,6 +88,7 @@ export class SentienceAgent {
   private stepCount: number;
   private history: HistoryEntry[];
   private tokenUsage: TokenStats;
+  private screenshotSequence: number;
 
   /**
    * Initialize Sentience Agent
@@ -117,6 +118,9 @@ export class SentienceAgent {
       totalTokens: 0,
       byAction: []
     };
+    
+    // Screenshot sequence counter
+    this.screenshotSequence = 0;
   }
 
   /**
@@ -171,6 +175,23 @@ export class SentienceAgent {
           throw new Error(`Snapshot failed: ${snap.error}`);
         }
 
+        // Store screenshot if captured
+        if (snap.screenshot && this.tracer) {
+          this.screenshotSequence += 1;
+          const seq = this.screenshotSequence;
+          
+          // Store screenshot in CloudTraceSink if available
+          const sink = (this.tracer as any).sink;
+          if (sink && typeof sink.storeScreenshot === 'function') {
+            sink.storeScreenshot(
+              seq,
+              snap.screenshot,
+              snap.screenshot_format || 'jpeg',
+              stepId
+            );
+          }
+        }
+
         // Apply element filtering based on goal
         const filteredElements = this.filterElements(snap, goal);
 

src/tracing/cloud-sink.ts

@@ -18,6 +18,7 @@ import * as https from 'https';
 import * as http from 'http';
 import { URL } from 'url';
 import { TraceSink } from './sink';
+import { ScreenshotMetadata } from '../types';
 
 /**
  * Optional logger interface for SDK users
@@ -75,10 +76,19 @@ export class CloudTraceSink extends TraceSink {
   private apiUrl: string;
   private logger?: SentienceLogger;
 
-  // File size tracking (NEW)
+  // File size tracking
   private traceFileSizeBytes: number = 0;
   private screenshotTotalSizeBytes: number = 0;
 
+  // Screenshot storage directory
+  private screenshotDir: string;
+  
+  // Screenshot metadata tracking (sequence -> ScreenshotMetadata)
+  private screenshotMetadata: Map<number, ScreenshotMetadata> = new Map();
+  
+  // Upload success flag
+  private uploadSuccessful: boolean = false;
+
   /**
    * Create a new CloudTraceSink
    *
@@ -107,6 +117,12 @@ export class CloudTraceSink extends TraceSink {
     const cacheDir = getPersistentCacheDir();
     this.tempFilePath = path.join(cacheDir, `${this.runId}.jsonl`);
 
+    // Screenshot storage directory
+    this.screenshotDir = path.join(cacheDir, `${this.runId}_screenshots`);
+    if (!fs.existsSync(this.screenshotDir)) {
+      fs.mkdirSync(this.screenshotDir, { recursive: true });
+    }
+
     try {
       // Open file in append mode
       this.writeStream = fs.createWriteStream(this.tempFilePath, {
@@ -282,17 +298,25 @@ export class CloudTraceSink extends TraceSink {
       const statusCode = await this._uploadToCloud(compressedData);
 
       if (statusCode === 200) {
+        this.uploadSuccessful = true;
         console.log('✅ [Sentience] Trace uploaded successfully');
 
+        // Upload screenshots after trace upload succeeds
+        if (this.screenshotMetadata.size > 0) {
+          console.log(`📸 [Sentience] Uploading ${this.screenshotMetadata.size} screenshots...`);
+          await this._uploadScreenshots();
+        }
+
         // Upload trace index file
         await this._uploadIndex();
 
         // Call /v1/traces/complete to report file sizes
         await this._completeTrace();
 
-        // 4. Delete temp file on success
-        await fsPromises.unlink(this.tempFilePath);
+        // 4. Delete files on success
+        await this._cleanupFiles();
       } else {
+        this.uploadSuccessful = false;
         console.error(`❌ [Sentience] Upload failed: HTTP ${statusCode}`);
         console.error(`   Local trace preserved at: ${this.tempFilePath}`);
       }
@@ -323,6 +347,7 @@ export class CloudTraceSink extends TraceSink {
         stats: {
           trace_file_size_bytes: this.traceFileSizeBytes,
           screenshot_total_size_bytes: this.screenshotTotalSizeBytes,
+          screenshot_count: this.screenshotMetadata.size,
         },
       });
 

src/types.ts

@@ -50,6 +50,18 @@ export interface Snapshot {
   requires_license?: boolean;
 }
 
+/**
+ * Metadata for a stored screenshot.
+ * Used by CloudTraceSink to track screenshots before upload.
+ */
+export interface ScreenshotMetadata {
+  sequence: number;
+  format: "png" | "jpeg";
+  sizeBytes: number;
+  stepId: string | null;
+  filepath: string;
+}
+
 export interface ActionResult {
   success: boolean;
   duration_ms: number;