From adb8044c9ff751138523bbb3846a9f7547089580 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Norbert=20Musia=C5=82?= <54953461+normusF7@users.noreply.github.com> Date: Fri, 14 Nov 2025 15:30:19 +0100 Subject: [PATCH 1/2] Add configurable client-side video filter support --- app/deps/ffmpeg.sh | 9 +- app/meson.build | 3 + app/scrcpy.1 | 6 ++ app/src/cli.c | 13 +++ app/src/options.c | 1 + app/src/options.h | 1 + app/src/scrcpy.c | 28 ++++- app/src/video_filter.c | 240 +++++++++++++++++++++++++++++++++++++++++ app/src/video_filter.h | 32 ++++++ doc/build.md | 4 +- doc/video.md | 14 +++ 11 files changed, 345 insertions(+), 6 deletions(-) create mode 100644 app/src/video_filter.c create mode 100644 app/src/video_filter.h diff --git a/app/deps/ffmpeg.sh b/app/deps/ffmpeg.sh index fb8b9a2516..08c6f10242 100755 --- a/app/deps/ffmpeg.sh +++ b/app/deps/ffmpeg.sh @@ -51,15 +51,14 @@ else --extra-cflags="-O2 -fPIC" --disable-programs --disable-doc - --disable-swscale --disable-postproc - --disable-avfilter --disable-network --disable-everything --disable-vulkan --disable-vaapi --disable-vdpau --enable-swresample + --enable-swscale --enable-libdav1d --enable-decoder=h264 --enable-decoder=hevc @@ -79,6 +78,12 @@ else --enable-muxer=opus --enable-muxer=flac --enable-muxer=wav + --enable-filter=buffer + --enable-filter=buffersink + --enable-filter=format + --enable-filter=scale + --enable-filter=lenscorrection + --enable-filter=v360 ) if [[ "$HOST" == linux ]] diff --git a/app/meson.build b/app/meson.build index f7df69eb22..99a3fc222b 100644 --- a/app/meson.build +++ b/app/meson.build @@ -32,6 +32,7 @@ src = [ 'src/recorder.c', 'src/scrcpy.c', 'src/screen.c', + 'src/video_filter.c', 'src/server.c', 'src/version.c', 'src/hid/hid_gamepad.c', @@ -116,6 +117,8 @@ dependencies = [ dependency('libavformat', version: '>= 57.33', static: static), dependency('libavcodec', version: '>= 57.37', static: static), dependency('libavutil', static: static), + dependency('libavfilter', static: static), + dependency('libswscale', static: static), dependency('libswresample', static: static), dependency('sdl2', version: '>= 2.0.5', static: static), ] diff --git a/app/scrcpy.1 b/app/scrcpy.1 index d481ddd16c..162574470d 100644 --- a/app/scrcpy.1 +++ b/app/scrcpy.1 @@ -614,6 +614,12 @@ The list of possible codec options is available in the Android documentation: +.TP +.BI "\-\-video\-filter " filter +Apply an FFmpeg filter graph to the decoded video before it is displayed or forwarded to V4L2. + +Example: --video-filter="lenscorrection=cx=0.5:cy=0.5:k1=-0.2:k2=0" + .TP .BI "\-\-video\-encoder " name Use a specific MediaCodec video encoder (depending on the codec provided by \fB\-\-video\-codec\fR). diff --git a/app/src/cli.c b/app/src/cli.c index b2e3e30a53..a9a25940e1 100644 --- a/app/src/cli.c +++ b/app/src/cli.c @@ -41,6 +41,7 @@ enum { OPT_NO_MIPMAPS, OPT_CODEC_OPTIONS, OPT_VIDEO_CODEC_OPTIONS, + OPT_VIDEO_FILTER, OPT_FORCE_ADB_FORWARD, OPT_DISABLE_SCREENSAVER, OPT_SHORTCUT_MOD, @@ -1002,6 +1003,15 @@ static const struct sc_option options[] = { "Android documentation: " "", }, + { + .longopt_id = OPT_VIDEO_FILTER, + .longopt = "video-filter", + .argdesc = "filter", + .text = "Apply an FFmpeg filter graph on the decoded video before it " + "is rendered or forwarded to V4L2.\n" + "Example: --video-filter=\"lenscorrection=cx=0.5:cy=0.5:" + "k1=-0.2:k2=0\"", + }, { .longopt_id = OPT_VIDEO_ENCODER, .longopt = "video-encoder", @@ -2596,6 +2606,9 @@ parse_args_with_getopt(struct scrcpy_cli_args *args, int argc, char *argv[], case OPT_VIDEO_CODEC_OPTIONS: opts->video_codec_options = optarg; break; + case OPT_VIDEO_FILTER: + opts->video_filter = optarg; + break; case OPT_AUDIO_CODEC_OPTIONS: opts->audio_codec_options = optarg; break; diff --git a/app/src/options.c b/app/src/options.c index 0fe82d291b..9ea031349b 100644 --- a/app/src/options.c +++ b/app/src/options.c @@ -10,6 +10,7 @@ const struct scrcpy_options scrcpy_options_default = { .push_target = NULL, .render_driver = NULL, .video_codec_options = NULL, + .video_filter = NULL, .audio_codec_options = NULL, .video_encoder = NULL, .audio_encoder = NULL, diff --git a/app/src/options.h b/app/src/options.h index 03b4291344..9785f28b0b 100644 --- a/app/src/options.h +++ b/app/src/options.h @@ -235,6 +235,7 @@ struct scrcpy_options { const char *push_target; const char *render_driver; const char *video_codec_options; + const char *video_filter; const char *audio_codec_options; const char *video_encoder; const char *audio_encoder; diff --git a/app/src/scrcpy.c b/app/src/scrcpy.c index b3ff9b368d..e5e1e5fcfa 100644 --- a/app/src/scrcpy.c +++ b/app/src/scrcpy.c @@ -26,6 +26,7 @@ #include "recorder.h" #include "screen.h" #include "server.h" +#include "video_filter.h" #include "uhid/gamepad_uhid.h" #include "uhid/keyboard_uhid.h" #include "uhid/mouse_uhid.h" @@ -55,6 +56,7 @@ struct scrcpy { struct sc_decoder audio_decoder; struct sc_recorder recorder; struct sc_delay_buffer video_buffer; + struct sc_video_filter video_filter; #ifdef HAVE_V4L2 struct sc_v4l2_sink v4l2_sink; struct sc_delay_buffer v4l2_buffer; @@ -392,6 +394,7 @@ scrcpy(struct scrcpy_options *options) { #ifdef HAVE_V4L2 bool v4l2_sink_initialized = false; #endif + bool video_filter_initialized = false; bool video_demuxer_started = false; bool audio_demuxer_started = false; #ifdef HAVE_USB @@ -583,10 +586,12 @@ scrcpy(struct scrcpy_options *options) { #ifdef HAVE_V4L2 needs_video_decoder |= !!options->v4l2_device; #endif + bool video_decoder_initialized = false; if (needs_video_decoder) { sc_decoder_init(&s->video_decoder, "video"); sc_packet_source_add_sink(&s->video_demuxer.packet_source, &s->video_decoder.packet_sink); + video_decoder_initialized = true; } if (needs_audio_decoder) { sc_decoder_init(&s->audio_decoder, "audio"); @@ -787,6 +792,21 @@ scrcpy(struct scrcpy_options *options) { // There is a controller if and only if control is enabled assert(options->control == !!controller); + if (options->video_filter && !video_decoder_initialized) { + LOGW("Ignoring --video-filter: video playback is disabled"); + } + + struct sc_frame_source *video_source = &s->video_decoder.frame_source; + if (video_decoder_initialized && options->video_filter) { + if (!sc_video_filter_init(&s->video_filter, options->video_filter)) { + goto end; + } + + video_filter_initialized = true; + sc_frame_source_add_sink(video_source, &s->video_filter.frame_sink); + video_source = &s->video_filter.frame_source; + } + if (options->window) { const char *window_title = options->window_title ? options->window_title : info->device_name; @@ -821,7 +841,7 @@ scrcpy(struct scrcpy_options *options) { screen_initialized = true; if (options->video_playback) { - struct sc_frame_source *src = &s->video_decoder.frame_source; + struct sc_frame_source *src = video_source; if (options->video_buffer) { sc_delay_buffer_init(&s->video_buffer, options->video_buffer, true); @@ -846,7 +866,7 @@ scrcpy(struct scrcpy_options *options) { goto end; } - struct sc_frame_source *src = &s->video_decoder.frame_source; + struct sc_frame_source *src = video_source; if (options->v4l2_buffer) { sc_delay_buffer_init(&s->v4l2_buffer, options->v4l2_buffer, true); sc_frame_source_add_sink(src, &s->v4l2_buffer.frame_sink); @@ -1047,6 +1067,10 @@ scrcpy(struct scrcpy_options *options) { sc_file_pusher_destroy(&s->file_pusher); } + if (video_filter_initialized) { + sc_video_filter_destroy(&s->video_filter); + } + if (server_started) { sc_server_join(&s->server); } diff --git a/app/src/video_filter.c b/app/src/video_filter.c new file mode 100644 index 0000000000..15eea736fc --- /dev/null +++ b/app/src/video_filter.c @@ -0,0 +1,240 @@ +#include "video_filter.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "trait/frame_source.h" +#include "util/log.h" + +/** Downcast frame_sink to sc_video_filter */ +#define DOWNCAST(SINK) container_of(SINK, struct sc_video_filter, frame_sink) + +static void +sc_video_filter_reset(struct sc_video_filter *vf) { + avfilter_graph_free(&vf->graph); + vf->buffersrc_ctx = NULL; + vf->buffersink_ctx = NULL; + + if (vf->output_ctx) { + avcodec_free_context(&vf->output_ctx); + } + + if (vf->frame) { + av_frame_free(&vf->frame); + } +} + +static bool +sc_video_filter_open(struct sc_frame_sink *sink, const AVCodecContext *ctx) { + struct sc_video_filter *vf = DOWNCAST(sink); + + assert(vf->filter_desc); + + vf->frame = av_frame_alloc(); + if (!vf->frame) { + LOG_OOM(); + return false; + } + + vf->graph = avfilter_graph_alloc(); + if (!vf->graph) { + LOG_OOM(); + goto error; + } + + const AVFilter *buffersrc = avfilter_get_by_name("buffer"); + const AVFilter *buffersink = avfilter_get_by_name("buffersink"); + if (!buffersrc || !buffersink) { + LOGE("Required ffmpeg filters 'buffer' or 'buffersink' are not available"); + goto error; + } + + AVRational time_base = ctx->time_base; + if (!time_base.num || !time_base.den) { + time_base = (AVRational) {1, 1000000}; + } + + AVRational sar = ctx->sample_aspect_ratio; + if (!sar.num || !sar.den) { + sar = (AVRational) {1, 1}; + } + + char args[256]; + int r = snprintf(args, sizeof(args), + "video_size=%dx%d:pix_fmt=%d:time_base=%d/%d:pixel_aspect=%d/%d", + ctx->width, ctx->height, ctx->pix_fmt, + time_base.num, time_base.den, + sar.num, sar.den); + if (r < 0 || (size_t) r >= sizeof(args)) { + LOGE("Could not build buffer source arguments"); + goto error; + } + + r = avfilter_graph_create_filter(&vf->buffersrc_ctx, buffersrc, "in", + args, NULL, vf->graph); + if (r < 0) { + LOGE("Could not create buffer source: %d", r); + goto error; + } + + r = avfilter_graph_create_filter(&vf->buffersink_ctx, buffersink, "out", + NULL, NULL, vf->graph); + if (r < 0) { + LOGE("Could not create buffer sink: %d", r); + goto error; + } + + enum AVPixelFormat pix_fmts[] = { ctx->pix_fmt, AV_PIX_FMT_NONE }; + r = av_opt_set_int_list(vf->buffersink_ctx, "pix_fmts", pix_fmts, + AV_PIX_FMT_NONE, AV_OPT_SEARCH_CHILDREN); + if (r < 0) { + LOGE("Could not configure buffer sink pixel format: %d", r); + goto error; + } + + AVFilterInOut *outputs = avfilter_inout_alloc(); + AVFilterInOut *inputs = avfilter_inout_alloc(); + if (!outputs || !inputs) { + LOG_OOM(); + avfilter_inout_free(&inputs); + avfilter_inout_free(&outputs); + goto error; + } + + outputs->name = av_strdup("in"); + inputs->name = av_strdup("out"); + if (!outputs->name || !inputs->name) { + LOG_OOM(); + avfilter_inout_free(&inputs); + avfilter_inout_free(&outputs); + goto error; + } + + outputs->filter_ctx = vf->buffersrc_ctx; + outputs->pad_idx = 0; + outputs->next = NULL; + + inputs->filter_ctx = vf->buffersink_ctx; + inputs->pad_idx = 0; + inputs->next = NULL; + + r = avfilter_graph_parse_ptr(vf->graph, vf->filter_desc, &inputs, &outputs, + NULL); + avfilter_inout_free(&inputs); + avfilter_inout_free(&outputs); + if (r < 0) { + LOGE("Could not parse video filter graph: %d", r); + goto error; + } + + r = avfilter_graph_config(vf->graph, NULL); + if (r < 0) { + LOGE("Could not configure video filter graph: %d", r); + goto error; + } + + vf->output_ctx = avcodec_alloc_context3(NULL); + if (!vf->output_ctx) { + LOG_OOM(); + goto error; + } + + vf->output_ctx->pix_fmt = av_buffersink_get_pix_fmt(vf->buffersink_ctx); + vf->output_ctx->width = av_buffersink_get_w(vf->buffersink_ctx); + vf->output_ctx->height = av_buffersink_get_h(vf->buffersink_ctx); + vf->output_ctx->sample_aspect_ratio = + av_buffersink_get_sample_aspect_ratio(vf->buffersink_ctx); + vf->output_ctx->time_base = + av_buffersink_get_time_base(vf->buffersink_ctx); + + if (!sc_frame_source_sinks_open(&vf->frame_source, vf->output_ctx)) { + goto error; + } + + return true; + +error: + sc_video_filter_reset(vf); + return false; +} + +static void +sc_video_filter_close(struct sc_frame_sink *sink) { + struct sc_video_filter *vf = DOWNCAST(sink); + + sc_frame_source_sinks_close(&vf->frame_source); + sc_video_filter_reset(vf); +} + +static bool +sc_video_filter_push(struct sc_frame_sink *sink, const AVFrame *frame) { + struct sc_video_filter *vf = DOWNCAST(sink); + + int r = av_buffersrc_add_frame_flags(vf->buffersrc_ctx, frame, + AV_BUFFERSRC_FLAG_KEEP_REF); + if (r < 0) { + LOGE("Could not push frame into video filter: %d", r); + return false; + } + + for (;;) { + r = av_buffersink_get_frame(vf->buffersink_ctx, vf->frame); + if (r == AVERROR(EAGAIN) || r == AVERROR_EOF) { + break; + } + if (r < 0) { + LOGE("Could not retrieve filtered frame: %d", r); + return false; + } + + bool ok = sc_frame_source_sinks_push(&vf->frame_source, vf->frame); + av_frame_unref(vf->frame); + if (!ok) { + return false; + } + } + + return true; +} + +bool +sc_video_filter_init(struct sc_video_filter *vf, const char *filter_desc) { + assert(filter_desc); + + char *dup = strdup(filter_desc); + if (!dup) { + LOG_OOM(); + return false; + } + + sc_frame_source_init(&vf->frame_source); + + static const struct sc_frame_sink_ops ops = { + .open = sc_video_filter_open, + .close = sc_video_filter_close, + .push = sc_video_filter_push, + }; + + vf->frame_sink.ops = &ops; + vf->filter_desc = dup; + vf->graph = NULL; + vf->buffersrc_ctx = NULL; + vf->buffersink_ctx = NULL; + vf->output_ctx = NULL; + vf->frame = NULL; + + return true; +} + +void +sc_video_filter_destroy(struct sc_video_filter *vf) { + free(vf->filter_desc); + vf->filter_desc = NULL; +} diff --git a/app/src/video_filter.h b/app/src/video_filter.h new file mode 100644 index 0000000000..c574fa45ec --- /dev/null +++ b/app/src/video_filter.h @@ -0,0 +1,32 @@ +#ifndef SC_VIDEO_FILTER_H +#define SC_VIDEO_FILTER_H + +#include "common.h" + +#include +#include +#include + +#include "trait/frame_sink.h" +#include "trait/frame_source.h" + +struct sc_video_filter { + struct sc_frame_source frame_source; // frame source trait + struct sc_frame_sink frame_sink; // frame sink trait + + char *filter_desc; // owned copy of the filter description + + AVFilterGraph *graph; + AVFilterContext *buffersrc_ctx; + AVFilterContext *buffersink_ctx; + AVCodecContext *output_ctx; + AVFrame *frame; +}; + +bool +sc_video_filter_init(struct sc_video_filter *vf, const char *filter_desc); + +void +sc_video_filter_destroy(struct sc_video_filter *vf); + +#endif diff --git a/doc/build.md b/doc/build.md index afe8b21bb8..f00b7464ee 100644 --- a/doc/build.md +++ b/doc/build.md @@ -54,8 +54,8 @@ sudo apt install ffmpeg libsdl2-2.0-0 adb libusb-1.0-0 # client build dependencies sudo apt install gcc git pkg-config meson ninja-build libsdl2-dev \ - libavcodec-dev libavdevice-dev libavformat-dev libavutil-dev \ - libswresample-dev libusb-1.0-0-dev + libavcodec-dev libavdevice-dev libavfilter-dev libavformat-dev \ + libavutil-dev libswresample-dev libswscale-dev libusb-1.0-0-dev # server build dependencies sudo apt install openjdk-17-jdk diff --git a/doc/video.md b/doc/video.md index 4de6814a76..740d0eb68a 100644 --- a/doc/video.md +++ b/doc/video.md @@ -198,6 +198,20 @@ For display mirroring, `--max-size` is applied after cropping. For camera, resizing the content). +## Client-side filters + +FFmpeg filters may be applied on the client after decoding and before the +frames are rendered or forwarded to V4L2. + +For example, to compensate for a fisheye lens distortion: + +```bash +scrcpy --video-filter="lenscorrection=cx=0.5:cy=0.5:k1=-0.2:k2=0" +``` + +Any filter graph supported by the packaged FFmpeg build can be used. + + ## Display If several displays are available on the Android device, it is possible to From 8f5078f5ac89ac47ae11443758dad16a0c62815a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Norbert=20Musia=C5=82?= <54953461+normusF7@users.noreply.github.com> Date: Fri, 14 Nov 2025 15:42:38 +0100 Subject: [PATCH 2/2] Fix video filter FFmpeg compatibility --- app/src/video_filter.c | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/app/src/video_filter.c b/app/src/video_filter.c index 15eea736fc..50577d7c8a 100644 --- a/app/src/video_filter.c +++ b/app/src/video_filter.c @@ -4,6 +4,7 @@ #include #include #include +#include #include #include #include @@ -146,7 +147,9 @@ sc_video_filter_open(struct sc_frame_sink *sink, const AVCodecContext *ctx) { goto error; } - vf->output_ctx->pix_fmt = av_buffersink_get_pix_fmt(vf->buffersink_ctx); + // Limit the sink to the decoder pixel format, so the output matches the + // input format even if the filter graph does not specify one explicitly. + vf->output_ctx->pix_fmt = ctx->pix_fmt; vf->output_ctx->width = av_buffersink_get_w(vf->buffersink_ctx); vf->output_ctx->height = av_buffersink_get_h(vf->buffersink_ctx); vf->output_ctx->sample_aspect_ratio = @@ -177,8 +180,15 @@ static bool sc_video_filter_push(struct sc_frame_sink *sink, const AVFrame *frame) { struct sc_video_filter *vf = DOWNCAST(sink); - int r = av_buffersrc_add_frame_flags(vf->buffersrc_ctx, frame, + AVFrame *tmp = av_frame_clone(frame); + if (!tmp) { + LOG_OOM(); + return false; + } + + int r = av_buffersrc_add_frame_flags(vf->buffersrc_ctx, tmp, AV_BUFFERSRC_FLAG_KEEP_REF); + av_frame_free(&tmp); if (r < 0) { LOGE("Could not push frame into video filter: %d", r); return false;