Refactor video buffer and scaler, better A/V sync

Signed-off-by: Andrew Opalach <andrew@akon.city>

3 files changed, 84 insertions, 57 deletions

diff --git a/src/buffer/audio.c b/src/buffer/audio.c
index f3e0b74..113ea1d 100644
--- a/src/buffer/audio.c
+++ b/src/buffer/audio.c
@@ -9,9 +9,9 @@
 #include "common_internal.h"
 #include "volume.h"
 
-#define BUFFER_USEC (6 * 1000000L)
+#define BUFFER_SIZE (6 * 1000000L)
 #define BUFFER_MARK_MIN (2.6 * 1000000L) // Must be a most half of the buffer size.
-#define BUFFER_MARK_BUFFERED (1.5 * 1000000L)
+#define BUFFER_MARK_BUFFERED (2.5 * 1000000L)
 
 #ifdef CAMU_AUDIO_BUFFER_FADE
 #define FADE_STEP(fmt) (1.1f / (fmt)->sample_rate)
@@ -75,7 +75,7 @@ bool camu_audio_buffer_configure(struct camu_audio_buffer *buf, struct camu_code
         in_format_name, buf->fmt.in.channel_count, buf->fmt.in.sample_rate,
         req_format_name, buf->fmt.req.channel_count, buf->fmt.req.sample_rate);
 
-    buf->size = camu_audio_format_usec_to_bytes(&buf->fmt.req, BUFFER_USEC);
+    buf->size = camu_audio_format_usec_to_bytes(&buf->fmt.req, BUFFER_SIZE);
     buf->data = (u8 *)al_malloc(buf->size);
     al_ring_buffer_init(&buf->rb, buf->data, buf->size);
 
diff --git a/src/buffer/video.c b/src/buffer/video.c
index 04a9dbf..0ffc00d 100644
--- a/src/buffer/video.c
+++ b/src/buffer/video.c
@@ -4,10 +4,16 @@
 #include "common.h"
 #include "common_internal.h"
 
-#define BUFFER_WATERMARK_LOW 6 // frames.
-#define BUFFER_WATERMARK_BUFFERED 4
-#define BUFFER_WATERMARK_HIGH 10
-#define BUFFER_WATERMARK_RESET BUFFER_WATERMARK_HIGH + 10
+#ifdef CAMU_VIDEO_BUFFER_FORCE_SCALER
+#ifdef CAMU_HAVE_FFMPEG
+#include "../codec/ffmpeg/scaler.h"
+#endif
+#endif
+
+#define BUFFER_MARK_LOW 0.199998
+#define BUFFER_MARK_BUFFERED 0.133332
+#define BUFFER_MARK_HIGH 0.33333
+#define BUFFER_MARK_RESET (BUFFER_MARK_HIGH * 2.0)
 
 bool camu_video_buffer_init(struct camu_video_buffer *buf, struct camu_clock *clock, struct camu_renderer *renderer)
 {
@@ -29,53 +35,73 @@ bool camu_video_buffer_init(struct camu_video_buffer *buf, struct camu_clock *cl
 bool camu_video_buffer_configure(struct camu_video_buffer *buf, struct camu_codec_stream *stream)
 {
     buf->stream = stream;
+    struct camu_video_format *fmt = &buf->stream->video.fmt;
     switch (stream->mode) {
     case CAMU_NORMAL: {
-        s32 width = buf->stream->video.width;
-        s32 height = buf->stream->video.height;
+        camu_video_format_copy(&buf->fmt.in, fmt);
+
         buf->single_frame = true;
         buf->avg_frame_duration = 0.0;
-        const char *format_name = camu_pixel_format_name(buf->stream->video.format);
-        al_log_info("video_buffer", "Stream: %s (%dx%d) %s.", format_name, width, height, "IMAGE");
+
+        const char *format_name = camu_pixel_format_name(fmt->format);
+        al_log_info("video_buffer", "Stream: %s (%dx%d) %s.",
+            format_name, fmt->width, fmt->height, "IMAGE");
+
         break;
     }
 #ifdef CAMU_HAVE_FFMPEG
     case CAMU_FFMPEG_COMPAT: {
-        s32 width = stream->av.stream->codecpar->width;
-        s32 height = stream->av.stream->codecpar->height;
-        buf->fmt.in_width = buf->stream->video.width = width;
-        buf->fmt.in_height = buf->stream->video.height = height;
-        buf->fmt.in_format = stream->av.stream->codecpar->format;
-        buf->single_frame = stream->av.stream->duration == 0 ||
-            stream->av.stream->avg_frame_rate.den == 0;
-        AVRational frame_rate = buf->single_frame ? ((AVRational){ 0, 1 })
-            : stream->av.stream->avg_frame_rate;
-        buf->avg_frame_duration = av_q2d(av_inv_q(frame_rate));
-        const char *format_name = av_get_pix_fmt_name(buf->fmt.in_format);
-        al_log_info("video_buffer", "Stream: %s (%dx%d) %s %.2ffps.", format_name,
-            width, height, buf->single_frame ? "IMAGE" : "VIDEO", av_q2d(frame_rate));
+        AVCodecParameters *codecpar = stream->av.stream->codecpar;
+        s64 duration = stream->av.stream->duration;
+        AVRational frame_rate = stream->av.stream->avg_frame_rate;
+
+        fmt->width = codecpar->width;
+        fmt->height = codecpar->height;
+        fmt->format = codecpar->format;
+        camu_video_format_copy(&buf->fmt.in, fmt);
+
+        buf->single_frame = duration == 0 || frame_rate.den == 0;
+        buf->avg_frame_duration = buf->single_frame ? 0.0 : av_q2d(av_inv_q(frame_rate));
+
+        const char *format_name = av_get_pix_fmt_name(buf->fmt.in.format);
+        al_log_info("video_buffer", "Stream: %s (%dx%d) %s %.2ffps.",
+            format_name, fmt->width, fmt->height, buf->single_frame ? "IMAGE" : "VIDEO", av_q2d(frame_rate));
+
+        break;
+    }
+#endif
+    }
 #ifdef CAMU_VIDEO_BUFFER_FORCE_SCALER
-        buf->fmt.req_width = width;
-        buf->fmt.req_height = height;
-        buf->fmt.req_format = AV_PIX_FMT_RGBA;
-        if (camu_ff_scaler_init(&buf->scale, &buf->fmt) && buf->fmt.scaler_needed) {
-            format_name = av_get_pix_fmt_name(buf->fmt.req_format);
-            al_log_info("video_buffer", "Scaling to: %s (%dx%d)", format_name,
-                buf->fmt.req_width, buf->fmt.req_height);
+#ifdef CAMU_HAVE_FFMPEG
+    if (fmt->format != CAMU_PIXEL_FORMAT_RGBA && fmt->format != CAMU_PIXEL_FORMAT_RGB) {
+        buf->fmt.scaler_needed = true;
+        buf->fmt.req.width = fmt->width;
+        buf->fmt.req.height = fmt->height;
+        buf->fmt.req.format = CAMU_PIXEL_FORMAT_RGBA;
+        buf->scale = camu_ff_scaler_create();
+        if (buf->fmt.scaler_needed && buf->scale->init(buf->scale, &buf->fmt)) {
+            const char *format_name = av_get_pix_fmt_name(buf->fmt.req.format);
+            al_log_info("video_buffer", "Scaling to: %s (%dx%d).",
+                format_name, buf->fmt.req.width, buf->fmt.req.height);
         } else {
             return false;
         }
-#endif
-        break;
+    } else {
+        buf->fmt.scaler_needed = false;
+        camu_video_format_copy(&buf->fmt.req, &buf->fmt.in);
     }
+#else
+    return false;
+#endif
+#else
+    camu_video_format_copy(&buf->fmt.req, &buf->fmt.in);
 #endif
-    }
     return true;
 }
 
-void camu_video_buffer_set_latency(struct camu_video_buffer *buf, f64 latency)
+void camu_video_buffer_set_latency(struct camu_video_buffer *buf, s32 frames)
 {
-    buf->latency = latency;
+    buf->latency = frames * buf->avg_frame_duration;
 }
 
 bool camu_video_buffer_is_single_frame(struct camu_video_buffer *buf)
@@ -85,17 +111,17 @@ bool camu_video_buffer_is_single_frame(struct camu_video_buffer *buf)
 
 static void after_push_internal(struct camu_video_buffer *buf)
 {
-    s32 count = buf->queue->count(buf->queue);
-    if (!buf->buffered && (buf->single_frame || count >= BUFFER_WATERMARK_BUFFERED)) {
-        al_log_debug("video_buffer", "Buffered (watermark: %d frames).", count);
+    f64 have = buf->queue->count(buf->queue) * buf->avg_frame_duration;
+    if (!buf->buffered && (buf->single_frame || have >= BUFFER_MARK_BUFFERED)) {
+        al_log_debug("video_buffer", "Buffered (mark: %.2fs).", have);
         // Preserve order of: flush -> callback -> set flow, for single frames.
         if (buf->single_frame) buf->queue->flush(buf->queue);
         buf->callback(buf->userdata, CAMU_BUFFER_BUFFERED);
         buf->buffered = true;
         if (buf->single_frame) al_atomic_store(u8)(&buf->flow, FLUSHED, AL_ATOMIC_RELAXED);
     }
-    if (count >= BUFFER_WATERMARK_RESET) buf->queue->reset(buf->queue);
-    else if (count >= BUFFER_WATERMARK_HIGH) buf->callback(buf->userdata, CAMU_BUFFER_CORK);
+    if (have >= BUFFER_MARK_RESET) buf->queue->reset(buf->queue);
+    else if (have >= BUFFER_MARK_HIGH) buf->callback(buf->userdata, CAMU_BUFFER_CORK);
 }
 
 #ifdef CAMU_HAVE_FFMPEG
@@ -113,11 +139,11 @@ static bool push_av_frame_internal(struct camu_video_buffer *buf, AVFrame *frame
     if (buf->pts == -1.0) buf->pts = pts;
 #ifdef CAMU_VIDEO_BUFFER_FORCE_SCALER
     if (buf->fmt.scaler_needed) {
-        if (!camu_ff_scaler_scale(&buf->scale, (const u8 **)frame->data, frame->linesize)) {
-            return;
+        if (!buf->scale->scale(buf->scale, (const u8 **)frame->data, frame->linesize)) {
+            return false;
         }
         av_frame_free(&frame);
-        frame = av_frame_clone(buf->scale.frame);
+        frame = av_frame_clone(buf->scale->get_frame(buf->scale));
     }
 #endif
     if (frame) frame->opaque = buf;
@@ -163,7 +189,8 @@ void camu_video_buffer_flush(struct camu_video_buffer *buf)
 {
     buf->queue->flush(buf->queue);
     if (!buf->buffered) {
-        al_log_debug("video_buffer", "Buffered (watermark: %d frames).", buf->queue->count(buf->queue));
+        f64 have = buf->queue->count(buf->queue) * buf->avg_frame_duration;
+        al_log_debug("video_buffer", "Buffered (mark: %.2fs).", have);
         buf->callback(buf->userdata, CAMU_BUFFER_BUFFERED);
         buf->buffered = true;
     }
@@ -186,8 +213,11 @@ bool camu_video_buffer_read(struct camu_video_buffer *buf, void *out)
         buf->callback(buf->userdata, CAMU_BUFFER_EOF);
         al_atomic_store(u8)(&buf->flow, SIGNALED, AL_ATOMIC_RELEASE);
         al_log_debug("video_buffer", "Flushed.");
-    } else if (flow == FLOWING && buf->queue->count(buf->queue) <= BUFFER_WATERMARK_LOW) {
-        buf->callback(buf->userdata, CAMU_BUFFER_UNCORK);
+    } else if (flow == FLOWING) {
+        f64 have = buf->queue->count(buf->queue) * buf->avg_frame_duration;
+        if (have <= BUFFER_MARK_LOW) {
+            buf->callback(buf->userdata, CAMU_BUFFER_UNCORK);
+        }
     }
     return ret == CAMU_QUEUE_OK || ret == CAMU_QUEUE_MORE;
 }
@@ -195,7 +225,7 @@ bool camu_video_buffer_read(struct camu_video_buffer *buf, void *out)
 void camu_video_buffer_free(struct camu_video_buffer *buf)
 {
 #ifdef CAMU_VIDEO_BUFFER_FORCE_SCALER
-    if (buf->fmt.scaler_needed) camu_ff_scaler_close(&buf->scale);
+    if (buf->fmt.scaler_needed) buf->scale->free(&buf->scale);
 #endif
     buf->queue->free(&buf->queue);
 }
diff --git a/src/buffer/video.h b/src/buffer/video.h
index 0e8bfff..0b48361 100644
--- a/src/buffer/video.h
+++ b/src/buffer/video.h
@@ -8,9 +8,6 @@
 #include "../render/renderer.h"
 #include "../screen/screen.h"
 #include "../screen/view.h"
-#ifdef CAMU_VIDEO_BUFFER_FORCE_SCALER
-#endif
-#include "../codec/ffmpeg/scaler.h"
 
 #include "clock.h"
 #include "frame_queue.h"
@@ -23,23 +20,23 @@ struct camu_video_buffer {
     f64 pts;
 
     bool single_frame;
-    f32 avg_frame_duration;
+    f64 avg_frame_duration;
 
+    struct camu_scaler_format fmt;
 #ifdef CAMU_VIDEO_BUFFER_FORCE_SCALER
-    struct camu_ff_scaler scale;
+    struct camu_scaler *scale;
 #endif
-    // TODO: Terrible
-    struct camu_ff_scale_fmt fmt;
 
     struct camu_frame_queue *queue;
     bool buffered;
 
     atomic(u8) flow;
 
+    struct camu_view view;
+
 #ifdef CAMU_SCREEN_THREADED
     atomic(u8) ref;
 #endif
-    struct camu_view view;
 
     void (*callback)(void *, u8);
     void *userdata;
@@ -47,7 +44,7 @@ struct camu_video_buffer {
 
 bool camu_video_buffer_init(struct camu_video_buffer *buf, struct camu_clock *clock, struct camu_renderer *renderer);
 bool camu_video_buffer_configure(struct camu_video_buffer *buf, struct camu_codec_stream *stream);
-void camu_video_buffer_set_latency(struct camu_video_buffer *buf, f64 latency);
+void camu_video_buffer_set_latency(struct camu_video_buffer *buf, s32 frames);
 bool camu_video_buffer_is_single_frame(struct camu_video_buffer *buf);
 void camu_video_buffer_push(struct camu_video_buffer *buf, struct camu_codec_frame *frame);
 void camu_video_buffer_reset(struct camu_video_buffer *buf);