Optimizations based on video loop performance

 - Support nn_packet_stream direct mode
 - Hook up FFmpeg hardware accelerated decoding
 - Refactor VCR
 - Reduce locking when returning packets to a packet pool

Signed-off-by: Andrew Opalach <andrew@akon.city>

10 files changed, 156 insertions, 62 deletions

diff --git a/src/codec/codec.h b/src/codec/codec.h
index 697f456..6ddcff9 100644
--- a/src/codec/codec.h
+++ b/src/codec/codec.h
@@ -27,8 +27,8 @@ enum {
 };
 enum {
     CAMU_STREAM_UNKNOWN = AVMEDIA_TYPE_UNKNOWN,
-    CAMU_STREAM_AUDIO = AVMEDIA_TYPE_AUDIO,
-    CAMU_STREAM_VIDEO = AVMEDIA_TYPE_VIDEO,
+    CAMU_STREAM_AUDIO = AVMEDIA_TYPE_AUDIO, // 1
+    CAMU_STREAM_VIDEO = AVMEDIA_TYPE_VIDEO, // 0
     CAMU_STREAM_SUBTITLE = AVMEDIA_TYPE_SUBTITLE,
     CAMU_STREAM_ATTACHMENT = AVMEDIA_TYPE_ATTACHMENT
 };
@@ -65,7 +65,7 @@ enum {
     CAMU_SEEK_SIZE = 0x10000
 };
 enum {
-    CAMU_STREAM_UNKNOWN = 0,
+    CAMU_STREAM_UNKNOWN = -1,
     CAMU_STREAM_AUDIO,
     CAMU_STREAM_VIDEO,
     CAMU_STREAM_SUBTITLE,
@@ -177,9 +177,13 @@ struct camu_demuxer {
 };
 
 struct camu_decoder {
+    u8 mode;
     bool (*init)(struct camu_decoder *, struct camu_renderer *, struct camu_codec_stream *,
         void (*callback)(void *, struct camu_codec_frame *), void *);
     s32 (*push)(struct camu_decoder *, struct camu_codec_packet *);
+#ifdef CAMU_HAVE_FFMPEG
+    s32 (*push_av_packet)(struct camu_decoder *, AVPacket *);
+#endif
     s32 (*process)(struct camu_decoder *);
     void (*flush)(struct camu_decoder *);
     void (*free)(struct camu_decoder **);
diff --git a/src/codec/ffmpeg/common.c b/src/codec/ffmpeg/common.c
index 16bbeab..b2e727d 100644
--- a/src/codec/ffmpeg/common.c
+++ b/src/codec/ffmpeg/common.c
@@ -4,7 +4,7 @@
 
 #include "common.h"
 
-void camu_ff_set_log_callback(void (*callback)(void *, int, const char *, va_list))
+void camu_ff_set_log_callback(void (*callback)(void *, s32, const char *, va_list))
 {
     av_log_set_callback(callback);
 }
@@ -15,14 +15,18 @@ static s32 pos = 0;
 // If a line takes more than 2 steps to print, make sure we stay within AL_LOG_MESSAGE_SIZE.
 #define CHUNK_SIZE (AL_LOG_MESSAGE_SIZE / 2)
 
-static void av_log_callback(void *userdata, int level, const char *fmt, va_list args)
+static void av_log_callback(void *userdata, s32 level, const char *fmt, va_list args)
 {
     (void)userdata;
     al_assert(buf);
     if (level < AV_LOG_DEBUG) {
         pos += al_vsnprintf(&buf[pos], CHUNK_SIZE, fmt, args);
         if (buf[pos - 1] == '\n' || pos >= CHUNK_SIZE) {
-            al_log_info("ff", buf);
+            if (level <= AV_LOG_INFO) {
+                al_log_info("ff", buf);
+            } else {
+                al_log_debug("ff", buf);
+            }
             pos = 0;
         }
     }
diff --git a/src/codec/ffmpeg/decoder.c b/src/codec/ffmpeg/decoder.c
index 3fdb534..ce974f3 100644
--- a/src/codec/ffmpeg/decoder.c
+++ b/src/codec/ffmpeg/decoder.c
@@ -4,11 +4,51 @@
 
 #include "decoder.h"
 
+#if !defined CAMU_SINK_NO_VIDEO
+static s32 get_buffer2(AVCodecContext *context, AVFrame *pic, s32 flags)
+{
+    struct camu_ff_decoder *av = (struct camu_ff_decoder *)context->opaque;
+    context->opaque = av->renderer->opaque;
+    s32 ret = av->renderer->get_buffer2(context, pic, flags);
+    context->opaque = av;
+    return ret;
+}
+
+static enum AVPixelFormat get_hw_format(AVCodecContext *context, const enum AVPixelFormat *pix_fmts)
+{
+    struct camu_ff_decoder *av = (struct camu_ff_decoder *)context->opaque;
+
+    const enum AVPixelFormat *fmt = pix_fmts;
+    for (; *fmt != AV_PIX_FMT_NONE; fmt++) {
+        if (*fmt == av->hw_pix_fmt) return *fmt;
+    }
+
+    al_log_error("ff_decoder", "Failed to get HW surface format.");
+
+    return avcodec_default_get_format(context, pix_fmts);
+}
+#endif
+
 static void close_internal(struct camu_ff_decoder *av)
 {
     if (av->codec_context) avcodec_free_context(&av->codec_context);
 }
 
+static s32 init_hw_decoder(struct camu_ff_decoder *av, AVCodecContext *context, const enum AVHWDeviceType device_type)
+{
+    s32 ret = av_hwdevice_ctx_create(&av->hw_context, device_type, NULL, NULL, 0);
+    if (ret < 0) {
+        al_log_error("ff_decoder", "Failed to create specified HW device.");
+        return ret;
+    }
+
+    context->hw_device_ctx = av_buffer_ref(av->hw_context);
+
+    al_log_info("ff_decoder", "Using %s hardware decoding.", av_hwdevice_get_type_name(device_type));
+
+    return ret;
+}
+
 static bool ff_decoder_init(struct camu_decoder *dec, struct camu_renderer *renderer, struct camu_codec_stream *stream,
     void (*callback)(void *, struct camu_codec_frame *), void *userdata)
 {
@@ -18,14 +58,65 @@ static bool ff_decoder_init(struct camu_decoder *dec, struct camu_renderer *rend
 
     AVCodecParameters *codecpar = stream->av.stream->codecpar;
     const AVCodec *codec = avcodec_find_decoder(codecpar->codec_id);
+    if (strcmp(codec->name, "libdav1d") == 0) {
+        codec = avcodec_find_decoder_by_name("av1");
+    }
 
     if (!codec) {
         al_log_error("ff_decoder", "Failed to find decoder.");
         goto err;
     }
 
-    av->codec_context = avcodec_alloc_context3(codec);
+    enum AVHWDeviceType device_type = AV_HWDEVICE_TYPE_NONE;
+#ifdef CAMU_FF_DECODER_HWACCEL
+    if (codecpar->codec_type == AVMEDIA_TYPE_VIDEO) {
+#ifdef _WIN32
+        const char *hw_device = "vulkan";
+#else
+        const char *hw_device = "vaapi";
+#endif
+        device_type = av_hwdevice_find_type_by_name(hw_device);
+        if (device_type == AV_HWDEVICE_TYPE_NONE) {
+            al_log_info("ff_decoder", "Hardware device type \"%s\" not supported.", hw_device);
+            al_log_info("ff_decoder", "Types available on this system:");
+            while ((device_type = av_hwdevice_iterate_types(device_type)) != AV_HWDEVICE_TYPE_NONE) {
+                al_log_info("ff_decoder", "\t%s", av_hwdevice_get_type_name(device_type));
+            }
+        }
+    }
+#endif
+
+    if (device_type != AV_HWDEVICE_TYPE_NONE) {
+        bool supports_frames_context = false;
+        for (s32 i = 0; ; i++) {
+            const AVCodecHWConfig *config = avcodec_get_hw_config(codec, i);
+            if (!config) {
+                if (supports_frames_context) {
+                    al_log_warn("ff_decoder", "Selected hardware decoder (%s) only supports"
+                        "hardware frames for %s, which is unimplemented.",
+                        av_hwdevice_get_type_name(device_type), codec->name);
+                } else {
+                    al_log_warn("ff_decoder", "Selected hardware decoder (%s) does not support %s.",
+                        av_hwdevice_get_type_name(device_type), codec->name);
+                }
+                device_type = AV_HWDEVICE_TYPE_NONE;
+                break;
+            }
+
+            if (config->methods & AV_CODEC_HW_CONFIG_METHOD_HW_FRAMES_CTX) {
+                supports_frames_context = true;
+            }
 
+            if (config->methods & AV_CODEC_HW_CONFIG_METHOD_HW_DEVICE_CTX) {
+                if (config->device_type == device_type) {
+                    av->hw_pix_fmt = config->pix_fmt;
+                    break;
+                }
+            }
+        }
+    }
+
+    av->codec_context = avcodec_alloc_context3(codec);
     if (!av->codec_context) {
         al_log_error("ff_decoder", "Failed to alloc codec context.");
         goto err;
@@ -36,26 +127,36 @@ static bool ff_decoder_init(struct camu_decoder *dec, struct camu_renderer *rend
         goto err;
     }
 
-    s32 cpus = av_cpu_count();
-    if (av->codec_context->codec_type == AVMEDIA_TYPE_VIDEO) {
-        cpus = MIN(4, MAX(1, cpus / 2));
-    } else {
-        cpus = MIN(2, MAX(1, cpus / 4));
-    }
-    av->codec_context->thread_count = cpus;
-    // FF_THREAD_FRAME or FF_THREAD_SLICE.
-    av->codec_context->thread_type = FF_THREAD_FRAME;
-    al_log_debug("ff_decoder", "Using %i threads for decoder.", cpus);
-
-#if !defined CAMU_SINK_NO_VIDEO && defined CAMU_RENDERER_VULKAN
+#if !defined CAMU_SINK_NO_VIDEO
     if (codecpar->codec_type == AVMEDIA_TYPE_VIDEO && renderer && renderer->get_buffer2) {
-        av->codec_context->get_buffer2 = renderer->get_buffer2;
-        av->codec_context->opaque = renderer->opaque;
+        av->renderer = renderer;
+        av->codec_context->opaque = av;
+        av->codec_context->get_buffer2 = get_buffer2;
+        if (device_type != AV_HWDEVICE_TYPE_NONE) {
+            av->codec_context->get_format = get_hw_format;
+        }
     }
 #else
     (void)renderer;
 #endif
 
+    if (stream->duration > 0 && device_type == AV_HWDEVICE_TYPE_NONE) {
+        s32 cpus = 0;
+        if (av->codec_context->codec_type == AVMEDIA_TYPE_VIDEO) {
+            cpus = av_cpu_count();
+            cpus = MIN(3, MAX(1, cpus / 2));
+        }
+        av->codec_context->thread_count = cpus;
+        // FF_THREAD_FRAME or FF_THREAD_SLICE.
+        av->codec_context->thread_type = FF_THREAD_FRAME;
+        al_log_debug("ff_decoder", "Using %i threads for decoder.", cpus);
+    }
+
+    if ((device_type != AV_HWDEVICE_TYPE_NONE) &&
+        (init_hw_decoder(av, av->codec_context, device_type) < 0)) {
+        return false;
+    }
+
     if (avcodec_open2(av->codec_context, codec, NULL) < 0) {
         al_log_error("ff_decoder", "Failed to open codec (%s).", codec->name);
         goto err;
@@ -65,12 +166,6 @@ static bool ff_decoder_init(struct camu_decoder *dec, struct camu_renderer *rend
     s64 kbps = codecpar->bit_rate > 0 ? codecpar->bit_rate / 1000L : 0L;
     al_log_info("ff_decoder", "Codec: %s (%s) %ldkbps.", codec->name, long_name, kbps);
 
-    //av->duration = stream->duration;
-    //av->time_base = stream->av.stream->time_base;
-    //av->last_pts = 0;
-    //av->last_duration = 0;
-    //av->seek_pos = -1;
-
     av->callback = callback;
     av->userdata = userdata;
 
@@ -82,26 +177,17 @@ err:
 
 static s32 send_packet(struct camu_ff_decoder *av, AVPacket *pkt)
 {
-    /*
-    if (pkt && av->seek_pos >= 0 && pkt->pts + pkt->duration < av->seek_pos) {
-        av->codec_context->skip_frame = AVDISCARD_NONKEY;
-    } else {
-        av->codec_context->skip_frame = AVDISCARD_NONE;
-    }
-    */
-
     s32 ret = avcodec_send_packet(av->codec_context, pkt);
     if (ret < 0 && ret != AVERROR(EAGAIN) && ret != AVERROR_EOF) {
         al_log_error("ff_decoder", "Error sending packet to the decoder (%s).", av_err2str(ret));
     }
-
     return ret;
 }
 
-static s32 ff_decoder_push(struct camu_decoder *dec, struct camu_codec_packet *packet)
+static s32 ff_decoder_push_av_packet(struct camu_decoder *dec, AVPacket *pkt)
 {
     struct camu_ff_decoder *av = (struct camu_ff_decoder *)dec;
-    return send_packet(av, packet ? packet->av.pkt : NULL);
+    return send_packet(av, pkt);
 }
 
 static void ff_decoder_flush(struct camu_decoder *dec)
@@ -123,18 +209,9 @@ static s32 receive_frames(struct camu_ff_decoder *av)
             al_free(frame);
             // Checking for EAGAIN should prevent an infinite loop.
             if (ret == AVERROR(EAGAIN) || ret == AVERROR_EOF) break;
-            al_log_error("ff_decoder", "Error receiving packet from the decoder (%s).", av_err2str(ret));
             continue;
         }
-        // Track pts and duration of the previous frame so we can handle multiple frames in a single packet.
-        //if (av->codec_context->codec_type == AVMEDIA_TYPE_AUDIO) {
-        //    if (frame->av.frame->best_effort_timestamp < 0) {
-        //        frame->av.frame->best_effort_timestamp = av->last_pts + av->last_duration;
-        //    }
-        //    av->last_pts = frame->av.frame->best_effort_timestamp;
-        //    av->last_duration = av_get_audio_frame_duration(av->codec_context, frame->av.frame->linesize[0])
-        //      / (av->codec_context->sample_rate * av->time_base.num / (f64)av->time_base.den);
-        //}
+        al_assert(frame->av.frame->best_effort_timestamp != AV_NOPTS_VALUE);
         av->callback(av->userdata, frame);
     }
     return ret;
@@ -157,8 +234,9 @@ static void ff_decoder_free(struct camu_decoder **dec)
 struct camu_decoder *camu_ff_decoder_create(void)
 {
     struct camu_ff_decoder *av = al_alloc_object(struct camu_ff_decoder);
+    av->dec.mode = CAMU_FFMPEG_COMPAT;
     av->dec.init = ff_decoder_init;
-    av->dec.push = ff_decoder_push;
+    av->dec.push_av_packet = ff_decoder_push_av_packet;
     av->dec.process = ff_decoder_process;
     av->dec.flush = ff_decoder_flush;
     av->dec.free = ff_decoder_free;
diff --git a/src/codec/ffmpeg/decoder.h b/src/codec/ffmpeg/decoder.h
index 2e35bb1..8d707f3 100644
--- a/src/codec/ffmpeg/decoder.h
+++ b/src/codec/ffmpeg/decoder.h
@@ -5,14 +5,16 @@
 
 #include "../codec.h"
 
+#if !(defined _WIN32 && defined CAMU_RENDERER_OPENGL)
+//#define CAMU_FF_DECODER_HWACCEL
+#endif
+
 struct camu_ff_decoder {
     struct camu_decoder dec;
     AVCodecContext *codec_context;
-    //AVRational time_base;
-    //u64 duration;
-    //s64 last_pts;
-    //s64 last_duration;
-    //s64 seek_pos;
+    struct camu_renderer *renderer;
+    AVBufferRef *hw_context;
+    enum AVPixelFormat hw_pix_fmt;
     void (*callback)(void *, struct camu_codec_frame *);
     void *userdata;
 };
diff --git a/src/codec/ffmpeg/demuxer.c b/src/codec/ffmpeg/demuxer.c
index 0184d4d..286be73 100644
--- a/src/codec/ffmpeg/demuxer.c
+++ b/src/codec/ffmpeg/demuxer.c
@@ -51,6 +51,9 @@ static bool ff_demuxer_init(struct camu_demuxer *demux, struct cch_handle *handl
     av_dict_set(&opts, "analyzeduration", "10000000", 0);
     av_dict_set(&opts, "probesize", "20M", 0);
 
+    // FFmpeg says the avio protocol configures "buffers and access patterns".
+    // I think we already do enough buffering but, better access patterns for
+    // network streams sounds like something we want.
     if (avformat_open_input(&av->format_context, "file:", NULL, &opts) < 0) {
         al_log_error("ff_demuxer", "Failed to open input.");
         av_dict_free(&opts);
@@ -127,9 +130,10 @@ static s32 ff_demuxer_get_packet(struct camu_demuxer *demux, struct camu_codec_p
 {
     struct camu_ff_demuxer *av = (struct camu_ff_demuxer *)demux;
     packet->mode = CAMU_FFMPEG_COMPAT;
+    AVPacket *pkt = packet->av.pkt;
     s32 ret = 0;
     for (;;) {
-        ret = av_read_frame(av->format_context, packet->av.pkt);
+        ret = av_read_frame(av->format_context, pkt);
         if (ret == AVERROR_EOF) {
             av->eof = true;
             break;
@@ -138,10 +142,11 @@ static s32 ff_demuxer_get_packet(struct camu_demuxer *demux, struct camu_codec_p
             al_log_error("ff_demuxer", "Failed to read frame (%s).", av_err2str(ret));
             continue;
         }
-        if (!subscribed_to_index(demux, packet->av.pkt->stream_index)) {
-            av_packet_unref(packet->av.pkt);
+        if (!subscribed_to_index(demux, pkt->stream_index)) {
+            av_packet_unref(pkt);
             continue;
         }
+        if (pkt->pts == AV_NOPTS_VALUE) pkt->pts = 0L;
         break;
     }
     return ret;
diff --git a/src/codec/ffmpeg/packet_ext.c b/src/codec/ffmpeg/packet_ext.c
index 759719b..2ff425e 100644
--- a/src/codec/ffmpeg/packet_ext.c
+++ b/src/codec/ffmpeg/packet_ext.c
@@ -145,7 +145,7 @@ void nn_packet_read_av_dictionary(struct nn_packet *packet, AVDictionary **dict)
         u32 len;
         NNWT_PACKET_READ_TYPE(packet, u32, len);
         NNWT_PACKET_READ_DATA(packet, len, key);
-        // Key and value MUST be allocated with av_malloc functions
+        // Key and value MUST be allocated with av_malloc functions,
         // just like all FFmpeg structures.
         key = av_strndup(key, len);
         NNWT_PACKET_READ_TYPE(packet, u32, len);
@@ -170,9 +170,8 @@ AVStream *nn_packet_read_av_stream(AVFormatContext *format_context, const AVCode
     return stream;
 }
 
-AVPacket *nn_packet_read_av_packet(struct nn_packet *packet)
+void nn_packet_read_av_packet(struct nn_packet *packet, AVPacket *pkt)
 {
-    AVPacket *pkt = av_packet_alloc();
     NNWT_PACKET_READ_TYPE(packet, s64, pkt->pts);
     NNWT_PACKET_READ_TYPE(packet, s64, pkt->dts);
     NNWT_PACKET_READ_TYPE(packet, s32, pkt->size);
@@ -197,5 +196,4 @@ AVPacket *nn_packet_read_av_packet(struct nn_packet *packet)
     NNWT_PACKET_READ_TYPE(packet, s64, pkt->duration);
     NNWT_PACKET_READ_TYPE(packet, s64, pkt->pos);
     NNWT_PACKET_READ_TYPE(packet, AVRational, pkt->time_base);
-    return pkt;
 }
diff --git a/src/codec/ffmpeg/packet_ext.h b/src/codec/ffmpeg/packet_ext.h
index 91254c2..fab9b36 100644
--- a/src/codec/ffmpeg/packet_ext.h
+++ b/src/codec/ffmpeg/packet_ext.h
@@ -14,4 +14,4 @@ void nn_packet_write_av_packet(struct nn_packet *packet, AVPacket *pkt);
 void nn_packet_read_av_codec_parameters(struct nn_packet *packet, AVCodecParameters *codecpar);
 enum AVCodecID nn_packet_read_av_codec_id(struct nn_packet *packet);
 AVStream *nn_packet_read_av_stream(AVFormatContext *format_context, const AVCodec *codec, struct nn_packet *packet);
-AVPacket *nn_packet_read_av_packet(struct nn_packet *packet);
+void nn_packet_read_av_packet(struct nn_packet *packet, AVPacket *pkt);
diff --git a/src/codec/spng/impl.c b/src/codec/spng/impl.c
index 58b68f6..3fb7e21 100644
--- a/src/codec/spng/impl.c
+++ b/src/codec/spng/impl.c
@@ -154,6 +154,7 @@ struct camu_demuxer *camu_spng_demuxer_create(void)
 struct camu_decoder *camu_spng_decoder_create(void)
 {
     struct camu_spng_decoder *spng = al_alloc_object(struct camu_spng_decoder);
+    spng->dec.mode = CAMU_NORMAL;
     spng->dec.init = spng_decoder_init;
     spng->dec.push = spng_decoder_push;
     spng->dec.process = spng_decoder_process;
diff --git a/src/codec/stb_image/impl.c b/src/codec/stb_image/impl.c
index bb38844..1d3a4a1 100644
--- a/src/codec/stb_image/impl.c
+++ b/src/codec/stb_image/impl.c
@@ -159,6 +159,7 @@ struct camu_demuxer *camu_stbi_demuxer_create(void)
 struct camu_decoder *camu_stbi_decoder_create(void)
 {
     struct camu_stbi_decoder *stb = al_alloc_object(struct camu_stbi_decoder);
+    stb->dec.mode = CAMU_NORMAL;
     stb->dec.init = stbi_decoder_init;
     stb->dec.push = stbi_decoder_push;
     stb->dec.process = stbi_decoder_process;
diff --git a/src/codec/wuffs/impl.c b/src/codec/wuffs/impl.c
index 378ac00..2c8f162 100644
--- a/src/codec/wuffs/impl.c
+++ b/src/codec/wuffs/impl.c
@@ -194,6 +194,7 @@ struct camu_demuxer *camu_wuffs_demuxer_create(void)
 struct camu_decoder *camu_wuffs_decoder_create(void)
 {
     struct camu_wuffs_decoder *wfs = al_alloc_object(struct camu_wuffs_decoder);
+    wfs->dec.mode = CAMU_NORMAL;
     wfs->dec.init = wuffs_decoder_init;
     wfs->dec.push = wuffs_decoder_push;
     wfs->dec.process = wuffs_decoder_process;