Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
225 changes: 210 additions & 15 deletions src/OpenClaw.Shared/Capabilities/ScreenCapability.cs
Original file line number Diff line number Diff line change
Expand Up @@ -10,31 +10,42 @@ namespace OpenClaw.Shared.Capabilities;
public class ScreenCapability : NodeCapabilityBase
{
public override string Category => "screen";

private static readonly string[] _commands = new[]
{
"screen.snapshot"
// Future: "screen.record"
"screen.snapshot",
"screen.list",
"screen.record",
"screen.record.start",
"screen.record.stop",
};

public override IReadOnlyList<string> Commands => _commands;

// Events for UI/platform-specific implementation
public event Func<ScreenCaptureArgs, Task<ScreenCaptureResult>>? CaptureRequested;

public event Func<Task<ScreenInfo[]>>? ListRequested;
public event Func<ScreenRecordArgs, Task<ScreenRecordResult>>? RecordRequested;
public event Func<ScreenRecordStartArgs, Task<string>>? StartRequested;
public event Func<string, Task<ScreenRecordResult>>? StopRequested;

public ScreenCapability(IOpenClawLogger logger) : base(logger)
{
}

public override async Task<NodeInvokeResponse> ExecuteAsync(NodeInvokeRequest request)
{
return request.Command switch
{
"screen.snapshot" => await HandleCaptureAsync(request),
"screen.snapshot" => await HandleCaptureAsync(request),
"screen.list" => await HandleListAsync(request),
"screen.record" => await HandleRecordAsync(request),
"screen.record.start" => await HandleStartAsync(request),
"screen.record.stop" => await HandleStopAsync(request),
_ => Error($"Unknown command: {request.Command}")
};
}

private async Task<NodeInvokeResponse> HandleCaptureAsync(NodeInvokeRequest request)
{
var format = GetStringArg(request.Args, "format", "png");
Expand All @@ -43,14 +54,14 @@ private async Task<NodeInvokeResponse> HandleCaptureAsync(NodeInvokeRequest requ
var monitor = GetIntArg(request.Args, "monitor", 0);
var screenIndex = GetIntArg(request.Args, "screenIndex", monitor);
var includePointer = GetBoolArg(request.Args, "includePointer", true);

Logger.Info($"screen.snapshot: format={format}, maxWidth={maxWidth}, monitor={screenIndex}");

if (CaptureRequested == null)
{
return Error("Screen capture not available");
}

try
{
var result = await CaptureRequested(new ScreenCaptureArgs
Expand All @@ -61,10 +72,10 @@ private async Task<NodeInvokeResponse> HandleCaptureAsync(NodeInvokeRequest requ
MonitorIndex = screenIndex,
IncludePointer = includePointer
});

var image = $"data:image/{result.Format.ToLowerInvariant()};base64,{result.Base64}";
return Success(new
{
return Success(new
{
format = result.Format,
width = result.Width,
height = result.Height,
Expand All @@ -78,6 +89,176 @@ private async Task<NodeInvokeResponse> HandleCaptureAsync(NodeInvokeRequest requ
return Error($"Capture failed: {ex.Message}");
}
}

private async Task<NodeInvokeResponse> HandleListAsync(NodeInvokeRequest request)
{
Logger.Info("screen.list");

if (ListRequested == null)
{
return Error("Screen list not available");
}

try
{
var screens = await ListRequested();
var formatted = new List<object>();
foreach (var screen in screens)
{
formatted.Add(new
{
index = screen.Index,
name = screen.Name,
primary = screen.IsPrimary,
bounds = new { x = screen.X, y = screen.Y, width = screen.Width, height = screen.Height },
workingArea = new { x = screen.WorkingX, y = screen.WorkingY, width = screen.WorkingWidth, height = screen.WorkingHeight }
});
}
return Success(new { screens = formatted });
}
catch (Exception ex)
{
Logger.Error("Screen list failed", ex);
return Error($"List failed: {ex.Message}");
}
}

private async Task<NodeInvokeResponse> HandleRecordAsync(NodeInvokeRequest request)
{
var durationMs = GetIntArg(request.Args, "durationMs", 5000);
var fps = GetIntArg(request.Args, "fps", 10);
var screenIndex = GetIntArg(request.Args, "screenIndex", GetIntArg(request.Args, "monitor", 0));

Logger.Info($"screen.record: durationMs={durationMs} fps={fps} screenIndex={screenIndex}");

if (RecordRequested == null)
return Error("Screen recording not available");

try
{
var result = await RecordRequested(new ScreenRecordArgs
{
DurationMs = durationMs,
Fps = fps,
ScreenIndex = screenIndex,
});

return Success(new
{
format = result.Format,
base64 = result.Base64,
filePath = result.FilePath,
durationMs = result.DurationMs,
fps = result.Fps,
screenIndex = result.ScreenIndex,
width = result.Width,
height = result.Height,
hasAudio = result.HasAudio,
});
}
catch (Exception ex)
{
Logger.Error("screen.record failed", ex);
return Error($"Record failed: {ex.GetType().Name}: {ex.Message} | {ex.StackTrace?.Split('\n').FirstOrDefault()?.Trim()}");
}
}

private async Task<NodeInvokeResponse> HandleStartAsync(NodeInvokeRequest request)
{
var fps = GetIntArg(request.Args, "fps", 10);
var screenIndex = GetIntArg(request.Args, "screenIndex", GetIntArg(request.Args, "monitor", 0));

Logger.Info($"screen.record.start: fps={fps} screenIndex={screenIndex}");

if (StartRequested == null)
return Error("Screen recording not available");

try
{
var recordingId = await StartRequested(new ScreenRecordStartArgs
{
Fps = fps,
ScreenIndex = screenIndex,
});
return Success(new { recordingId });
}
catch (Exception ex)
{
Logger.Error("screen.record.start failed", ex);
return Error($"Start failed: {ex.Message}");
}
}

private async Task<NodeInvokeResponse> HandleStopAsync(NodeInvokeRequest request)
{
var recordingId = GetStringArg(request.Args, "recordingId", "");

Logger.Info($"screen.record.stop: recordingId={recordingId}");

if (string.IsNullOrEmpty(recordingId))
return Error("recordingId is required");

if (StopRequested == null)
return Error("Screen recording not available");

try
{
var result = await StopRequested(recordingId);
return Success(new
{
format = result.Format,
base64 = result.Base64,
filePath = result.FilePath,
durationMs = result.DurationMs,
fps = result.Fps,
screenIndex = result.ScreenIndex,
width = result.Width,
height = result.Height,
hasAudio = result.HasAudio,
});
}
catch (Exception ex)
{
Logger.Error("screen.record.stop failed", ex);
return Error($"Stop failed: {ex.Message}");
}
}
}

/// <summary>
/// Parameters for a fixed-duration screen recording.
/// Memory usage: width × height × 4 bytes × (durationMs/1000 × fps) frames.
/// Recommended limits: durationMs ≤ 10 000, fps ≤ 10 for 1080p to stay under 500 MB.
/// The service enforces a hard 500 MB frame-buffer cap and stops capture early if exceeded.
/// </summary>
public class ScreenRecordArgs
{
public int DurationMs { get; set; } = 5000;
public int Fps { get; set; } = 10;
public int ScreenIndex { get; set; }
}

/// <summary>
/// Parameters for an open-ended screen recording session (screen.record.start / screen.record.stop).
/// The same 500 MB frame-buffer cap applies; capture stops automatically if the limit is hit.
/// </summary>
public class ScreenRecordStartArgs
{
public int Fps { get; set; } = 10;
public int ScreenIndex { get; set; }
}

public class ScreenRecordResult
{
public string Base64 { get; set; } = "";
public string Format { get; set; } = "mp4";
public string? FilePath { get; set; }
public int DurationMs { get; set; }
public int Fps { get; set; }
public int ScreenIndex { get; set; }
public int Width { get; set; }
public int Height { get; set; }
public bool HasAudio { get; set; }
}

public class ScreenCaptureArgs
Expand All @@ -97,3 +278,17 @@ public class ScreenCaptureResult
public string Base64 { get; set; } = "";
}

public class ScreenInfo
{
public int Index { get; set; }
public string Name { get; set; } = "";
public int Width { get; set; }
public int Height { get; set; }
public int X { get; set; }
public int Y { get; set; }
public int WorkingX { get; set; }
public int WorkingY { get; set; }
public int WorkingWidth { get; set; }
public int WorkingHeight { get; set; }
public bool IsPrimary { get; set; }
}
49 changes: 44 additions & 5 deletions src/OpenClaw.Tray.WinUI/Services/NodeService.cs
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@ public class NodeService : IDisposable
private WindowsNodeClient? _nodeClient;
private CanvasWindow? _canvasWindow;
private ScreenCaptureService? _screenCaptureService;
private ScreenRecordingService? _screenRecordingService;
private CameraCaptureService? _cameraCaptureService;
private DateTime _lastScreenCaptureNotification = DateTime.MinValue;
private string? _a2uiHostUrl;
Expand Down Expand Up @@ -52,8 +53,9 @@ public NodeService(IOpenClawLogger logger, DispatcherQueue dispatcherQueue, stri
_logger = logger;
_dispatcherQueue = dispatcherQueue;
_dataPath = dataPath;
_screenCaptureService = new ScreenCaptureService(logger);
_cameraCaptureService = new CameraCaptureService(logger);
_screenCaptureService = new ScreenCaptureService(logger);
_screenRecordingService = new ScreenRecordingService(logger);
_cameraCaptureService = new CameraCaptureService(logger);
}

/// <summary>
Expand Down Expand Up @@ -96,7 +98,9 @@ public async Task DisconnectAsync()
_nodeClient.Dispose();
_nodeClient = null;
}


_screenRecordingService?.StopAllSessions();

// Close canvas window
if (_canvasWindow != null && !_canvasWindow.IsClosed)
{
Expand Down Expand Up @@ -129,7 +133,11 @@ private void RegisterCapabilities()

// Screen capability
_screenCapability = new ScreenCapability(_logger);
_screenCapability.ListRequested += OnScreenList;
_screenCapability.CaptureRequested += OnScreenCapture;
_screenCapability.RecordRequested += OnScreenRecord;
_screenCapability.StartRequested += OnScreenRecordStart;
_screenCapability.StopRequested += OnScreenRecordStop;
_nodeClient.RegisterCapability(_screenCapability);

// Camera capability
Expand Down Expand Up @@ -412,7 +420,13 @@ private void OnCanvasA2UIReset(object? sender, EventArgs args)
#endregion

#region Screen Capability Handlers


private Task<ScreenInfo[]> OnScreenList()
{
return _screenCaptureService?.ListScreensAsync()
?? Task.FromResult(Array.Empty<ScreenInfo>());
}

private async Task<ScreenCaptureResult> OnScreenCapture(ScreenCaptureArgs args)
{
if (_screenCaptureService == null)
Expand All @@ -437,7 +451,31 @@ private async Task<ScreenCaptureResult> OnScreenCapture(ScreenCaptureArgs args)

return await _screenCaptureService.CaptureAsync(args);
}


private Task<ScreenRecordResult> OnScreenRecord(ScreenRecordArgs args)
{
if (_screenRecordingService == null)
throw new InvalidOperationException("Screen recording service not available");

return _screenRecordingService.RecordAsync(args);
}

private Task<string> OnScreenRecordStart(ScreenRecordStartArgs args)
{
if (_screenRecordingService == null)
throw new InvalidOperationException("Screen recording service not available");

return _screenRecordingService.StartAsync(args);
}

private Task<ScreenRecordResult> OnScreenRecordStop(string recordingId)
{
if (_screenRecordingService == null)
throw new InvalidOperationException("Screen recording service not available");

return _screenRecordingService.StopAsync(recordingId);
}

#endregion

#region Camera Capability Handlers
Expand Down Expand Up @@ -537,6 +575,7 @@ public void Dispose()
_nodeClient = null;
try { client?.Dispose(); } catch { /* ignore */ }

try { _screenRecordingService?.Dispose(); } catch { /* ignore */ }
try { _cameraCaptureService?.Dispose(); } catch { /* ignore */ }

if (_canvasWindow != null && !_canvasWindow.IsClosed)
Expand Down
Loading
Loading