Improve audio/video_buffer_push() logic

Signed-off-by: Andrew Opalach <andrew@akon.city>

4 files changed, 84 insertions, 85 deletions

diff --git a/src/buffer/audio.c b/src/buffer/audio.c
index 89f1b70..0d78382 100644
--- a/src/buffer/audio.c
+++ b/src/buffer/audio.c
@@ -14,7 +14,7 @@
 #define BUFFER_MARK_BUFFERED 4.0
 
 #ifdef CAMU_AUDIO_BUFFER_FADE
-#define FADE_STEP(fmt) (1.f / (fmt)->sample_rate)
+#define FADE_STEP(fmt) (1.75f / (fmt)->sample_rate)
 #endif
 
 enum {
@@ -60,8 +60,9 @@ bool camu_audio_buffer_configure(struct camu_audio_buffer *buf, struct camu_code
     buf->fmt.resampler_needed = !camu_resampler_format_matches(&buf->fmt);
     if (buf->fmt.resampler_needed) {
 #ifdef CAMU_HAVE_FFMPEG
-        buf->resamp = camu_ff_resampler_create();
-        if (!buf->resamp->init(buf->resamp, &buf->fmt)) {
+        buf->resampler = camu_ff_resampler_create();
+        if (!buf->resampler->init(buf->resampler, &buf->fmt)) {
+            // Resampler will be freed in audio_buffer_free().
             return false;
         }
 #else
@@ -106,8 +107,34 @@ void camu_audio_buffer_set_latency(struct camu_audio_buffer *buf, f64 latency)
 #define OCCUPIED_SECONDS_DEBUG(buf) 0
 #endif
 
-static bool push_internal(struct camu_audio_buffer *buf, u8 *data, s32 sample_count)
+static inline bool frame_is_late(struct camu_clock *clock, f64 base, f64 pts, f64 duration)
 {
+    if (base == -1.0) base = camu_clock_get_base_pts(clock);
+    return pts + duration < base;
+}
+
+// return value of false indicates we pushed to the peak buffer.
+static bool push_internal(struct camu_audio_buffer *buf, f64 pts, u8 **data, s32 sample_count, bool flush)
+{
+    if (!flush) {
+        f64 duration = camu_audio_format_samples_to_sec(&buf->fmt.in, sample_count);
+        f64 base = al_atomic_load(f64)(&buf->pts, AL_ATOMIC_ACQUIRE);
+        if (frame_is_late(buf->clock, base, pts, duration)) return true;
+        if (buf->fmt.resampler_needed) {
+            sample_count = buf->resampler->convert(buf->resampler, (const u8 **)data, sample_count);
+            data = buf->resampler->get_data(buf->resampler);
+        }
+        if (sample_count == 0) return true;
+        if (base == -1.0) al_atomic_store(f64)(&buf->pts, pts, AL_ATOMIC_RELEASE);
+    } else {
+        al_assert(!data && sample_count == 0);
+        if (buf->fmt.resampler_needed) {
+            sample_count = buf->resampler->flush(buf->resampler);
+            data = buf->resampler->get_data(buf->resampler);
+        }
+        if (sample_count == 0) return true;
+    }
+
     size_t space = al_ring_buffer_space(&buf->rb);
     if (!buf->buffered && buf->size - space > buf->mark.buffered) {
         al_log_debug("audio_buffer", "Buffered (mark: %.2fs).", OCCUPIED_SECONDS_DEBUG(buf));
@@ -119,7 +146,7 @@ static bool push_internal(struct camu_audio_buffer *buf, u8 *data, s32 sample_co
     size_t peak = camu_peak_buffer_size(&buf->peak);
     if (space < have + peak) {
         al_assert(have < buf->mark.min);
-        camu_peak_buffer_push(&buf->peak, data, have);
+        camu_peak_buffer_push(&buf->peak, data[0], have);
         peak += have;
         if (peak >= buf->mark.min) {
             // If this happens the writer of this buffer is taking way too long to stop.
@@ -138,20 +165,11 @@ static bool push_internal(struct camu_audio_buffer *buf, u8 *data, s32 sample_co
         al_ring_buffer_write(&buf->rb, camu_peak_buffer_flush(&buf->peak), peak);
     }
 
-    al_ring_buffer_write(&buf->rb, data, have);
+    al_ring_buffer_write(&buf->rb, data[0], have);
 
     return true;
 }
 
-static bool frame_is_late(struct camu_audio_buffer *buf, f64 pts, f64 duration)
-{
-    f64 base = al_atomic_load(f64)(&buf->pts, AL_ATOMIC_ACQUIRE);
-    if (base == -1.0) {
-        base = camu_clock_get_base_pts(buf->clock);
-        al_atomic_store(f64)(&buf->pts, base, AL_ATOMIC_RELEASE);
-    }
-    return pts + duration < base;
-}
 
 #ifdef CAMU_HAVE_FFMPEG
 static void push_av_frame_internal(struct camu_audio_buffer *buf, AVFrame *frame)
@@ -159,15 +177,7 @@ static void push_av_frame_internal(struct camu_audio_buffer *buf, AVFrame *frame
     s32 sample_count = frame->nb_samples;
     AVStream *stream = buf->stream->av.stream;
     f64 pts = frame->best_effort_timestamp * av_q2d(stream->time_base);
-    f64 duration = camu_audio_format_samples_to_sec(&buf->fmt.in, sample_count);
-    if (!frame_is_late(buf, pts, duration)) {
-        u8 **data = frame->data;
-        if (buf->fmt.resampler_needed) {
-            sample_count = buf->resamp->convert(buf->resamp, (const u8 **)data, sample_count);
-            data = buf->resamp->get_data(buf->resamp);
-        }
-        if (sample_count > 0) push_internal(buf, data[0], sample_count);
-    }
+    push_internal(buf, pts, frame->data, sample_count, false);
     av_frame_free(&frame);
 }
 #endif
@@ -180,17 +190,9 @@ void camu_audio_buffer_push(struct camu_audio_buffer *buf, struct camu_codec_fra
     case CAMU_NORMAL: {
         s32 sample_count = frame->audio.sample_count;
         f64 pts = frame->pts;
-        f64 duration = camu_audio_format_samples_to_sec(&buf->fmt.in, sample_count);
-        if (!frame_is_late(buf, pts, duration)) {
-            u8 *store[CAMU_NUM_DATA_POINTERS] = { 0 };
-            store[0] = frame->data;
-            u8 **data = store;
-            if (buf->fmt.resampler_needed) {
-                sample_count = buf->resamp->convert(buf->resamp, (const u8 **)store, sample_count);
-                data = buf->resamp->get_data(buf->resamp);
-            }
-            push_internal(buf, data[0], sample_count);
-        }
+        u8 *planes[CAMU_PLANAR_DATA_POINTERS] = { 0 };
+        planes[0] = frame->data;
+        push_internal(buf, pts, planes, sample_count, false);
         break;
     }
 #ifdef CAMU_HAVE_FFMPEG
@@ -203,23 +205,12 @@ void camu_audio_buffer_push(struct camu_audio_buffer *buf, struct camu_codec_fra
     al_free(frame);
 }
 
-void camu_audio_buffer_unpause(struct camu_audio_buffer *buf)
-{
-    al_atomic_add(s32)(&buf->unpause, 1, AL_ATOMIC_RELAXED);
-}
-
 // flush() always comes from the same thread as push().
 void camu_audio_buffer_flush(struct camu_audio_buffer *buf)
 {
     al_log_debug("audio_buffer", "Flush requested.");
-    if (buf->fmt.resampler_needed) {
-        s32 sample_count = buf->resamp->flush(buf->resamp);
-        if (sample_count > 0) {
-            u8 **data = buf->resamp->get_data(buf->resamp);
-            if (!push_internal(buf, data[0], sample_count)) {
-                al_log_debug("audio_buffer", "Buffer filled by resampler flush.");
-            }
-        }
+    if (!push_internal(buf, 0.0, NULL, 0, true)) {
+        al_log_debug("audio_buffer", "Buffer filled by flush.");
     }
     if (!buf->buffered) {
         al_log_debug("audio_buffer", "Buffered (mark: %.2fs).", OCCUPIED_SECONDS_DEBUG(buf));
@@ -236,10 +227,15 @@ void camu_audio_buffer_reset(struct camu_audio_buffer *buf)
     al_ring_buffer_reset(&buf->rb);
     camu_peak_buffer_flush(&buf->peak);
     if (buf->fmt.resampler_needed) {
-        buf->resamp->flush(buf->resamp);
+        buf->resampler->flush(buf->resampler);
     }
 }
 
+void camu_audio_buffer_unpause(struct camu_audio_buffer *buf)
+{
+    al_atomic_add(s32)(&buf->unpause, 1, AL_ATOMIC_RELAXED);
+}
+
 static void increment_pts(struct camu_audio_buffer *buf, f64 amount, f64 *base)
 {
     *base += amount;
@@ -426,7 +422,7 @@ size_t camu_audio_buffer_read(struct camu_audio_buffer *buf, u8 *data, size_t re
 void camu_audio_buffer_free(struct camu_audio_buffer *buf)
 {
     if (buf->fmt.resampler_needed) {
-        buf->resamp->free(&buf->resamp);
+        buf->resampler->free(&buf->resampler);
     }
     if (buf->data) al_free(buf->data);
     camu_peak_buffer_free(&buf->peak);
diff --git a/src/buffer/audio.h b/src/buffer/audio.h
index b4d7594..6fc3cca 100644
--- a/src/buffer/audio.h
+++ b/src/buffer/audio.h
@@ -22,7 +22,7 @@ struct camu_audio_buffer {
     f64 latency;
 
     struct camu_resampler_format fmt;
-    struct camu_resampler *resamp;
+    struct camu_resampler *resampler;
 
     u8 *data;
     size_t size;
@@ -62,8 +62,8 @@ bool camu_audio_buffer_configure(struct camu_audio_buffer *buf, struct camu_code
 void camu_audio_buffer_set_volume(struct camu_audio_buffer *buf, f32 volume);
 void camu_audio_buffer_set_latency(struct camu_audio_buffer *buf, f64 latency);
 void camu_audio_buffer_push(struct camu_audio_buffer *buf, struct camu_codec_frame *frame);
-void camu_audio_buffer_unpause(struct camu_audio_buffer *buf);
 void camu_audio_buffer_flush(struct camu_audio_buffer *buf);
 void camu_audio_buffer_reset(struct camu_audio_buffer *buf);
+void camu_audio_buffer_unpause(struct camu_audio_buffer *buf);
 size_t camu_audio_buffer_read(struct camu_audio_buffer *buf, u8 *data, size_t req);
 void camu_audio_buffer_free(struct camu_audio_buffer *buf);
diff --git a/src/buffer/video.c b/src/buffer/video.c
index 92bcb2d..ba43685 100644
--- a/src/buffer/video.c
+++ b/src/buffer/video.c
@@ -44,7 +44,9 @@ bool camu_video_buffer_configure(struct camu_video_buffer *buf, struct camu_code
     case CAMU_NORMAL: {
         camu_video_format_copy(&buf->fmt.in, fmt);
         const char *format_name = camu_pixel_format_name(fmt->format);
+        if (!format_name) format_name = "unknown";
         al_log_info("video_buffer", "Stream: %s (%dx%d) %s.", format_name, fmt->width, fmt->height, "IMAGE");
+
         break;
     }
 #ifdef CAMU_HAVE_FFMPEG
@@ -71,17 +73,19 @@ bool camu_video_buffer_configure(struct camu_video_buffer *buf, struct camu_code
     }
 #ifdef CAMU_VIDEO_BUFFER_FORCE_SCALER
 #ifdef CAMU_HAVE_FFMPEG
-    if (fmt->format != CAMU_PIXEL_FORMAT_RGBA && fmt->format != CAMU_PIXEL_FORMAT_RGB) {
+    if (fmt->width > 0 && fmt->height > 0 && fmt->format != CAMU_PIXEL_FORMAT_RGBA &&
+        fmt->format != CAMU_PIXEL_FORMAT_RGB) {
         buf->fmt.scaler_needed = true;
         buf->fmt.req.width = fmt->width;
         buf->fmt.req.height = fmt->height;
         buf->fmt.req.format = CAMU_PIXEL_FORMAT_RGBA;
-        buf->scale = camu_ff_scaler_create();
-        if (buf->fmt.scaler_needed && buf->scale->init(buf->scale, &buf->fmt)) {
+        buf->scaler = camu_ff_scaler_create();
+        if (buf->scaler->init(buf->scaler, &buf->fmt)) {
             const char *format_name = av_get_pix_fmt_name(buf->fmt.req.format);
             al_log_info("video_buffer", "Scaling to: %s (%dx%d).",
                 format_name, buf->fmt.req.width, buf->fmt.req.height);
         } else {
+            // Scaler will be freed in video_buffer_free().
             return false;
         }
     } else {
@@ -97,6 +101,14 @@ bool camu_video_buffer_configure(struct camu_video_buffer *buf, struct camu_code
     return true;
 }
 
+#ifdef CAMU_HAVE_FFMPEG
+bool camu_video_buffer_configure_subtitles(struct camu_video_buffer *buf, AVCodecParameters *codecpar)
+{
+    struct camu_video_format *fmt = &buf->stream->video.fmt;
+    return buf->queue->configure_subtitles(buf->queue, fmt->width, fmt->height, codecpar);
+}
+#endif
+
 void camu_video_buffer_set_latency(struct camu_video_buffer *buf, s32 frames)
 {
     buf->latency = frames * buf->avg_frame_duration;
@@ -122,35 +134,31 @@ static void after_push_internal(struct camu_video_buffer *buf)
     else if (have >= BUFFER_MARK_HIGH) buf->callback(buf->userdata, CAMU_BUFFER_CORK);
 }
 
+static inline bool frame_is_late(struct camu_clock *clock, f64 base, f64 pts, f64 duration)
+{
+    if (base == -1.0) base = camu_clock_get_base_pts(clock);
+    return pts + duration < base;
+}
+
 #ifdef CAMU_HAVE_FFMPEG
 static bool push_av_frame_internal(struct camu_video_buffer *buf, AVFrame *frame)
 {
-    f64 pts = 0.0;
+    AVStream *stream = buf->stream->av.stream;
+    f64 pts = frame->best_effort_timestamp * av_q2d(stream->time_base);
+    f64 duration = frame->duration * av_q2d(stream->time_base);
     f64 base = al_atomic_load(f64)(&buf->pts, AL_ATOMIC_ACQUIRE);
-    if (!buf->single_frame) {
-        AVStream *stream = buf->stream->av.stream;
-        pts = frame->best_effort_timestamp * av_q2d(stream->time_base);
-        // If buf->pts = -1.0, this is the first frame pushed either on
-        // a new stream or after a reset(). Meaning, we can safely set buf->pts
-        // from this thread.
-        if (base == -1.0) {
-            base = camu_clock_get_base_pts(buf->clock);
-            al_atomic_store(f64)(&buf->pts, base, AL_ATOMIC_RELEASE);
-        }
-        if (pts + buf->avg_frame_duration < base) {
-            av_frame_free(&frame);
-            return false;
-        }
-    } else if (base == -1.0) {
-        al_atomic_store(f64)(&buf->pts, 0.0, AL_ATOMIC_RELEASE);
+    if (!buf->single_frame && frame_is_late(buf->clock, base, pts, duration)) {
+        av_frame_free(&frame);
+        return false;
     }
+    if (base == -1.0) al_atomic_store(f64)(&buf->pts, pts, AL_ATOMIC_RELEASE);
 #ifdef CAMU_VIDEO_BUFFER_FORCE_SCALER
     if (buf->fmt.scaler_needed) {
-        if (!buf->scale->scale(buf->scale, (const u8 **)frame->data, frame->linesize)) {
+        if (!buf->scaler->scale(buf->scaler, (const u8 **)frame->data, frame->linesize)) {
             return false;
         }
         av_frame_free(&frame);
-        frame = av_frame_clone(buf->scale->get_frame(buf->scale)->av.frame);
+        frame = av_frame_clone(buf->scaler->get_frame(buf->scaler)->av.frame);
     }
 #endif
     buf->queue->push_av_frame(buf->queue, frame, pts);
@@ -187,12 +195,6 @@ void camu_video_buffer_push(struct camu_video_buffer *buf, struct camu_codec_fra
 }
 
 #ifdef CAMU_HAVE_FFMPEG
-bool camu_video_buffer_configure_subtitles(struct camu_video_buffer *buf, AVCodecParameters *codecpar)
-{
-    struct camu_video_format *fmt = &buf->stream->video.fmt;
-    return buf->queue->configure_subtitles(buf->queue, fmt->width, fmt->height, codecpar);
-}
-
 void camu_video_buffer_push_subtitle(struct camu_video_buffer *buf, AVPacket *pkt)
 {
     buf->queue->push_subtitle(buf->queue, pkt);
@@ -224,9 +226,8 @@ void camu_video_buffer_reset(struct camu_video_buffer *buf)
 
 bool camu_video_buffer_read(struct camu_video_buffer *buf, void *out)
 {
-    f64 base = 0.0;
+    f64 base = al_atomic_load(f64)(&buf->pts, AL_ATOMIC_ACQUIRE);
     if (!buf->single_frame) {
-        base = al_atomic_load(f64)(&buf->pts, AL_ATOMIC_ACQUIRE);
         f64 pts = camu_clock_get_pts(buf->clock, buf->latency);
         if (pts > base) {
             base = pts;
@@ -251,7 +252,7 @@ bool camu_video_buffer_read(struct camu_video_buffer *buf, void *out)
 void camu_video_buffer_free(struct camu_video_buffer *buf)
 {
 #ifdef CAMU_VIDEO_BUFFER_FORCE_SCALER
-    if (buf->fmt.scaler_needed) buf->scale->free(&buf->scale);
+    if (buf->fmt.scaler_needed) buf->scaler->free(&buf->scaler);
 #endif
     if (buf->queue) buf->queue->free(&buf->queue);
 }
diff --git a/src/buffer/video.h b/src/buffer/video.h
index 3953fc1..f6f4e28 100644
--- a/src/buffer/video.h
+++ b/src/buffer/video.h
@@ -23,7 +23,7 @@ struct camu_video_buffer {
 
     struct camu_scaler_format fmt;
 #ifdef CAMU_VIDEO_BUFFER_FORCE_SCALER
-    struct camu_scaler *scale;
+    struct camu_scaler *scaler;
 #endif
 
     struct camu_frame_queue *queue;
@@ -44,11 +44,13 @@ struct camu_video_buffer {
 bool camu_video_buffer_init(struct camu_video_buffer *buf, struct camu_clock *clock);
 bool camu_video_buffer_configure(struct camu_video_buffer *buf, struct camu_codec_stream *stream,
     struct camu_renderer *renderer);
+#ifdef CAMU_HAVE_FFMPEG
+bool camu_video_buffer_configure_subtitles(struct camu_video_buffer *buf, AVCodecParameters *codecpar);
+#endif
 void camu_video_buffer_set_latency(struct camu_video_buffer *buf, s32 frames);
 bool camu_video_buffer_is_single_frame(struct camu_video_buffer *buf);
 void camu_video_buffer_push(struct camu_video_buffer *buf, struct camu_codec_frame *frame);
 #ifdef CAMU_HAVE_FFMPEG
-bool camu_video_buffer_configure_subtitles(struct camu_video_buffer *buf, AVCodecParameters *codecpar);
 void camu_video_buffer_push_subtitle(struct camu_video_buffer *buf, AVPacket *pkt);
 #endif
 void camu_video_buffer_flush(struct camu_video_buffer *buf);