Media/embedded timecode sync

CMakeLists.txt

@@ -18,6 +18,7 @@ include(helpers)
 option(ENABLE_UI "Enable building with UI (requires Qt)" ON)
 option(ENABLE_SCRIPTING "Enable scripting support" ON)
 option(ENABLE_HEVC "Enable HEVC encoders" ON)
+option(ENABLE_UNIT_TESTS "Enable unit tests" OFF)
 
 add_subdirectory(libobs)
 if(OS_WINDOWS)
@@ -31,6 +32,10 @@ endif()
 add_subdirectory(plugins)
 
 add_subdirectory(test/test-input)
+if(ENABLE_UNIT_TESTS)
+  enable_testing()
+  add_subdirectory(test/media-playback)
+endif()
 
 add_subdirectory(frontend)
 

plugins/obs-ffmpeg/data/locale/en-US.ini

@@ -77,6 +77,7 @@ Looping="Loop"
 Input="Input"
 InputFormat="Input Format"
 BufferingMB="Network Buffering"
+SyncToTimecodes="Use embedded timecode for sync"
 HardwareDecode="Use hardware decoding when available"
 ClearOnMediaEnd="Show nothing when playback ends"
 RestartWhenActivated="Restart playback when source becomes active"

plugins/obs-ffmpeg/obs-ffmpeg-source.c

@@ -46,6 +46,7 @@ struct ffmpeg_source {
 	bool is_local_file;
 	bool is_hw_decoding;
 	bool full_decode;
+	bool sync_to_timecodes;
 	bool is_clear_on_media_end;
 	bool restart_on_activate;
 	bool close_when_inactive;
@@ -99,12 +100,14 @@ static bool is_local_file_modified(obs_properties_t *props, obs_property_t *prop
 	obs_property_t *local_file = obs_properties_get(props, "local_file");
 	obs_property_t *looping = obs_properties_get(props, "looping");
 	obs_property_t *buffering = obs_properties_get(props, "buffering_mb");
+	obs_property_t *sync_to_timecodes = obs_properties_get(props, "sync_to_timecodes");
 	obs_property_t *seekable = obs_properties_get(props, "seekable");
 	obs_property_t *speed = obs_properties_get(props, "speed_percent");
 	obs_property_t *reconnect_delay_sec = obs_properties_get(props, "reconnect_delay_sec");
 	obs_property_set_visible(input, !enabled);
 	obs_property_set_visible(input_format, !enabled);
 	obs_property_set_visible(buffering, !enabled);
+	obs_property_set_visible(sync_to_timecodes, !enabled);
 	obs_property_set_visible(local_file, enabled);
 	obs_property_set_visible(looping, enabled);
 	obs_property_set_visible(speed, enabled);
@@ -121,6 +124,7 @@ static void ffmpeg_source_defaults(obs_data_t *settings)
 	obs_data_set_default_bool(settings, "clear_on_media_end", true);
 	obs_data_set_default_bool(settings, "restart_on_activate", true);
 	obs_data_set_default_bool(settings, "linear_alpha", false);
+	obs_data_set_default_bool(settings, "sync_to_timecodes", false);
 	obs_data_set_default_int(settings, "reconnect_delay_sec", 10);
 	obs_data_set_default_int(settings, "buffering_mb", 2);
 	obs_data_set_default_int(settings, "speed_percent", 100);
@@ -189,6 +193,8 @@ static obs_properties_t *ffmpeg_source_getproperties(void *data)
 
 	obs_properties_add_bool(props, "hw_decode", obs_module_text("HardwareDecode"));
 
+	obs_properties_add_bool(props, "sync_to_timecodes", obs_module_text("SyncToTimecodes"));
+
 	obs_properties_add_bool(props, "clear_on_media_end", obs_module_text("ClearOnMediaEnd"));
 
 	prop = obs_properties_add_bool(props, "close_when_inactive", obs_module_text("CloseFileWhenInactive"));
@@ -229,12 +235,14 @@ static void dump_source_info(struct ffmpeg_source *s, const char *input, const c
 		"\tis_clear_on_media_end:   %s\n"
 		"\trestart_on_activate:     %s\n"
 		"\tclose_when_inactive:     %s\n"
+		"\tsync_to_timecodes:       %s\n"
 		"\tfull_decode:             %s\n"
 		"\tffmpeg_options:          %s",
 		input ? input : "(null)", input_format ? input_format : "(null)", s->speed_percent,
 		s->is_looping ? "yes" : "no", s->is_linear_alpha ? "yes" : "no", s->is_hw_decoding ? "yes" : "no",
 		s->is_clear_on_media_end ? "yes" : "no", s->restart_on_activate ? "yes" : "no",
-		s->close_when_inactive ? "yes" : "no", s->full_decode ? "yes" : "no", s->ffmpeg_options);
+		s->close_when_inactive ? "yes" : "no", s->sync_to_timecodes ? "yes" : "no",
+		s->full_decode ? "yes" : "no", s->ffmpeg_options);
 }
 
 static void get_frame(void *opaque, struct obs_source_frame *f)
@@ -309,6 +317,7 @@ static void ffmpeg_source_open(struct ffmpeg_source *s)
 			.reconnecting = s->reconnecting,
 			.request_preload = s->is_stinger,
 			.full_decode = s->full_decode,
+			.sync_to_timecodes = s->sync_to_timecodes,
 		};
 
 		s->media = media_playback_create(&info);
@@ -407,13 +416,15 @@ static void ffmpeg_source_update(void *data, obs_data_t *settings)
 	bool is_linear_alpha;
 	int speed_percent;
 	bool is_looping;
+	bool sync_to_timecodes = false;
 
 	bfree(s->input_format);
 
 	if (is_local_file) {
 		input = obs_data_get_string(settings, "local_file");
 		input_format = NULL;
 		is_looping = obs_data_get_bool(settings, "looping");
+		sync_to_timecodes = false;
 
 		if (s->input && !should_restart_media)
 			should_restart_media |= strcmp(s->input, input) != 0;
@@ -424,6 +435,7 @@ static void ffmpeg_source_update(void *data, obs_data_t *settings)
 		s->reconnect_delay_sec = (int)obs_data_get_int(settings, "reconnect_delay_sec");
 		s->reconnect_delay_sec = s->reconnect_delay_sec == 0 ? 10 : s->reconnect_delay_sec;
 		is_looping = false;
+		sync_to_timecodes = obs_data_get_bool(settings, "sync_to_timecodes");
 	}
 
 	stop_reconnect_thread(s);
@@ -437,6 +449,7 @@ static void ffmpeg_source_update(void *data, obs_data_t *settings)
 
 	/* Restart media source if these properties are changed */
 	if (s->is_hw_decoding != is_hw_decoding || s->range != range || s->speed_percent != speed_percent ||
+	    s->sync_to_timecodes != sync_to_timecodes ||
 	    (s->ffmpeg_options && strcmp(s->ffmpeg_options, ffmpeg_options) != 0))
 		should_restart_media = true;
 
@@ -456,6 +469,7 @@ static void ffmpeg_source_update(void *data, obs_data_t *settings)
 	s->input_format = input_format ? bstrdup(input_format) : NULL;
 	s->is_hw_decoding = is_hw_decoding;
 	s->full_decode = obs_data_get_bool(settings, "full_decode");
+	s->sync_to_timecodes = sync_to_timecodes;
 	s->is_clear_on_media_end = obs_data_get_bool(settings, "clear_on_media_end");
 	s->restart_on_activate = !astrcmpi_n(input, RIST_PROTO, sizeof(RIST_PROTO) - 1)
 					 ? false

shared/media-playback/CMakeLists.txt

@@ -17,6 +17,8 @@ target_sources(
     media-playback/media-playback.h
     media-playback/media.c
     media-playback/media.h
+    media-playback/timecode.c
+    media-playback/timecode.h
 )
 
 target_include_directories(media-playback INTERFACE "${CMAKE_CURRENT_SOURCE_DIR}")

shared/media-playback/media-playback/decode.c

@@ -18,7 +18,11 @@
 
 #include "media-playback.h"
 #include "media.h"
+#include "timecode.h"
 #include <libavutil/mastering_display_metadata.h>
+#include <libavutil/frame.h>
+
+#define NSEC_PER_SEC 1000000000LL
 
 enum AVHWDeviceType hw_priority[] = {
 	AV_HWDEVICE_TYPE_CUDA,  AV_HWDEVICE_TYPE_D3D11VA, AV_HWDEVICE_TYPE_DXVA2,        AV_HWDEVICE_TYPE_VAAPI,
@@ -131,6 +135,22 @@ static uint16_t get_max_luminance(const AVStream *stream)
 	return max_luminance;
 }
 
+static AVRational get_stream_frame_rate(const AVStream *stream)
+{
+	AVRational frame_rate = stream->avg_frame_rate.num && stream->avg_frame_rate.den ? stream->avg_frame_rate
+											 : stream->r_frame_rate;
+
+	if (frame_rate.num > 0 && frame_rate.den > 0)
+		return frame_rate;
+
+	return (AVRational){30, 1};
+}
+
+static int64_t get_frame_duration(AVRational frame_rate)
+{
+	return av_rescale_q(frame_rate.den, (AVRational){1, frame_rate.num}, (AVRational){1, NSEC_PER_SEC});
+}
+
 bool mp_decode_init(mp_media_t *m, enum AVMediaType type, bool hw)
 {
 	struct mp_decode *d = type == AVMEDIA_TYPE_VIDEO ? &m->v : &m->a;
@@ -150,6 +170,8 @@ bool mp_decode_init(mp_media_t *m, enum AVMediaType type, bool hw)
 
 	if (type == AVMEDIA_TYPE_VIDEO)
 		d->max_luminance = get_max_luminance(stream);
+	d->timecode_frame_rate = get_stream_frame_rate(stream);
+	d->timecode_frame_duration = get_frame_duration(d->timecode_frame_rate);
 
 	if (id == AV_CODEC_ID_VP8 || id == AV_CODEC_ID_VP9) {
 		AVDictionaryEntry *tag = NULL;
@@ -213,6 +235,7 @@ void mp_decode_clear_packets(struct mp_decode *d)
 		deque_pop_front(&d->packets, &pkt, sizeof(pkt));
 		mp_media_free_packet(d->m, pkt);
 	}
+	d->frame_timecode_valid = false;
 }
 
 void mp_decode_free(struct mp_decode *d)
@@ -320,13 +343,26 @@ static int decode_packet(struct mp_decode *d, int *got_frame)
 	return ret;
 }
 
+static bool frame_timecode(struct mp_decode *d, int64_t *timestamp)
+{
+	const AVFrameSideData *side_data = NULL;
+
+	if (d->audio || !d->m->sync_to_timecodes)
+		return false;
+
+	side_data = av_frame_get_side_data(d->frame, AV_FRAME_DATA_S12M_TIMECODE);
+	return side_data && mp_s12m_timecode_parse(side_data->data, side_data->size, d->timecode_frame_rate,
+						   d->timecode_frame_duration, timestamp);
+}
+
 bool mp_decode_next(struct mp_decode *d)
 {
 	bool eof = d->m->eof;
 	int got_frame;
 	int ret;
 
 	d->frame_ready = false;
+	d->frame_timecode_valid = false;
 
 	if (!eof && !d->packets.size)
 		return true;
@@ -392,6 +428,8 @@ bool mp_decode_next(struct mp_decode *d)
 			d->frame_pts = av_rescale_q(d->in_frame->best_effort_timestamp, d->stream->time_base,
 						    (AVRational){1, 1000000000});
 
+		d->frame_timecode_valid = frame_timecode(d, &d->frame_timecode);
+
 		int64_t duration = d->in_frame->duration;
 		if (!duration)
 			duration = get_estimated_duration(d, last_pts);
@@ -417,5 +455,6 @@ void mp_decode_flush(struct mp_decode *d)
 	d->eof = false;
 	d->frame_pts = 0;
 	d->frame_ready = false;
+	d->frame_timecode_valid = false;
 	d->next_pts = 0;
 }

shared/media-playback/media-playback/decode.h

@@ -50,13 +50,17 @@ struct mp_decode {
 	int64_t last_duration;
 	int64_t frame_pts;
 	int64_t next_pts;
+	AVRational timecode_frame_rate;
+	int64_t timecode_frame_duration;
+	int64_t frame_timecode;
 	AVFrame *in_frame;
 	AVFrame *sw_frame;
 	AVFrame *hw_frame;
 	AVFrame *frame;
 	enum AVPixelFormat hw_format;
 	bool got_first_keyframe;
 	bool frame_ready;
+	bool frame_timecode_valid;
 	bool eof;
 	bool hw;
 	uint16_t max_luminance;

shared/media-playback/media-playback/media-playback.h

@@ -46,6 +46,7 @@ struct mp_media_info {
 	bool reconnecting;
 	bool request_preload;
 	bool full_decode;
+	bool sync_to_timecodes;
 };
 
 extern media_playback_t *media_playback_create(const struct mp_media_info *info);

shared/media-playback/media-playback/media.c

@@ -27,6 +27,66 @@
 
 static int64_t base_sys_ts = 0;
 
+#define NSEC_PER_SEC 1000000000LL
+#define TIMECODE_DAY_NS (24LL * 60 * 60 * NSEC_PER_SEC)
+#define TIMECODE_SYNC_DELAY_NS (2LL * NSEC_PER_SEC)
+
+static pthread_mutex_t timecode_sync_mutex = PTHREAD_MUTEX_INITIALIZER;
+static bool timecode_sync_anchor_valid = false;
+static int64_t timecode_sync_anchor_code_ns = 0;
+static int64_t timecode_sync_anchor_obs_ns = 0;
+
+static inline int64_t timecode_wrap_delta(int64_t delta)
+{
+	while (delta > TIMECODE_DAY_NS / 2)
+		delta -= TIMECODE_DAY_NS;
+	while (delta < -TIMECODE_DAY_NS / 2)
+		delta += TIMECODE_DAY_NS;
+
+	return delta;
+}
+
+static inline int64_t timecode_mod_day(int64_t timestamp)
+{
+	timestamp %= TIMECODE_DAY_NS;
+	if (timestamp < 0)
+		timestamp += TIMECODE_DAY_NS;
+
+	return timestamp;
+}
+
+static int64_t timecode_sync_timestamp(int64_t timecode_ns)
+{
+	const int64_t now = (int64_t)os_gettime_ns() - base_sys_ts;
+	int64_t timestamp = 0;
+
+	pthread_mutex_lock(&timecode_sync_mutex);
+
+	if (!timecode_sync_anchor_valid) {
+		timecode_sync_anchor_code_ns = timecode_ns;
+		timecode_sync_anchor_obs_ns = now + TIMECODE_SYNC_DELAY_NS;
+		timecode_sync_anchor_valid = true;
+		timestamp = timecode_sync_anchor_obs_ns;
+	} else {
+		const int64_t expected_code_ns = timecode_sync_anchor_code_ns + (now - timecode_sync_anchor_obs_ns);
+		const int64_t expected_day_ns = timecode_mod_day(expected_code_ns);
+		const int64_t delta = timecode_wrap_delta(timecode_ns - expected_day_ns);
+		const int64_t extended_code_ns = expected_code_ns + delta;
+
+		timestamp = timecode_sync_anchor_obs_ns + (extended_code_ns - timecode_sync_anchor_code_ns);
+	}
+
+	pthread_mutex_unlock(&timecode_sync_mutex);
+
+	return timestamp;
+}
+
+static inline void reset_timecode_sync(mp_media_t *m)
+{
+	m->timecode_sync_active = false;
+	m->timecode_sync_offset = 0;
+}
+
 static inline enum video_format convert_pixel_format(int f)
 {
 	switch (f) {
@@ -366,6 +426,9 @@ void mp_media_next_audio(mp_media_t *m)
 	audio.timestamp = m->full_decode ? d->frame_pts
 					 : m->base_ts + d->frame_pts - m->start_ts + m->play_sys_ts - base_sys_ts;
 
+	if (!m->full_decode && m->timecode_sync_active)
+		audio.timestamp += m->timecode_sync_offset;
+
 	if (audio.format == AUDIO_FORMAT_UNKNOWN)
 		return;
 
@@ -453,6 +516,18 @@ void mp_media_next_video(mp_media_t *m, bool preload)
 	frame->timestamp = m->full_decode ? d->frame_pts
 					  : (m->base_ts + d->frame_pts - m->start_ts + m->play_sys_ts - base_sys_ts);
 
+	if (!m->full_decode) {
+		if (m->sync_to_timecodes && d->frame_timecode_valid) {
+			const int64_t timestamp = timecode_sync_timestamp(d->frame_timecode);
+
+			m->timecode_sync_offset = timestamp - frame->timestamp;
+			m->timecode_sync_active = true;
+		}
+
+		if (m->timecode_sync_active)
+			frame->timestamp += m->timecode_sync_offset;
+	}
+
 	frame->width = f->width;
 	frame->height = f->height;
 	frame->max_luminance = d->max_luminance;
@@ -562,6 +637,7 @@ bool mp_media_reset(mp_media_t *m)
 	m->eof = false;
 	m->base_ts += next_ts;
 	m->seek_next_ts = false;
+	reset_timecode_sync(m);
 
 	seek_to(m, start_time);
 
@@ -723,6 +799,7 @@ static bool init_avformat(mp_media_t *m)
 
 static void reset_ts(mp_media_t *m)
 {
+	reset_timecode_sync(m);
 	m->base_ts += mp_media_get_base_pts(m);
 	m->play_sys_ts = (int64_t)os_gettime_ns();
 	m->start_ts = m->next_pts_ns = mp_media_get_next_min_pts(m);
@@ -889,6 +966,7 @@ bool mp_media_init(mp_media_t *media, const struct mp_media_info *info)
 	media->speed = info->speed;
 	media->request_preload = info->request_preload;
 	media->is_local_file = info->is_local_file;
+	media->sync_to_timecodes = info->sync_to_timecodes && !info->is_local_file && !info->full_decode;
 	da_init(media->packet_pool);
 
 	if (!info->is_local_file || media->speed < 1 || media->speed > 200)

shared/media-playback/media-playback/media.h

@@ -83,6 +83,9 @@ struct mp_media {
 	int64_t start_ts;
 	int64_t base_ts;
 	bool full_decode;
+	bool sync_to_timecodes;
+	bool timecode_sync_active;
+	int64_t timecode_sync_offset;
 
 	uint64_t interrupt_poll_ts;