Improve FFmpeg hwaccel fallback, VR emulation

 - Optimize sink seeking with single video frame.
 - Small cleanups.

Signed-off-by: Andrew Opalach <andrew@akon.city>

8 files changed, 101 insertions, 70 deletions

diff --git a/src/codec/codec.h b/src/codec/codec.h
index 9515220..3440e9f 100644
--- a/src/codec/codec.h
+++ b/src/codec/codec.h
@@ -60,6 +60,7 @@ enum {
 enum {
     CAMU_PIXEL_FORMAT_RGBA = AV_PIX_FMT_RGBA,
     CAMU_PIXEL_FORMAT_RGB = AV_PIX_FMT_RGB8,
+    CAMU_PIXEL_FORMAT_RGB32 = AV_PIX_FMT_RGB32,
     CAMU_PIXEL_FORMAT_GREYA = AV_PIX_FMT_GRAY8A,
     CAMU_PIXEL_FORMAT_GREY = AV_PIX_FMT_GRAY8,
     CAMU_PIXEL_FORMAT_PAL8 = AV_PIX_FMT_PAL8
@@ -295,7 +296,7 @@ static inline void camu_audio_format_copy(struct camu_audio_format *dest, struct
 static inline const char *camu_audio_format_name(s32 format)
 {
 #ifdef CAMU_HAVE_FFMPEG
-    return av_get_sample_fmt_name(format);
+    return av_get_sample_fmt_name((enum AVSampleFormat)format);
 #else
     (void)format;
     return "(undefined)";
@@ -305,7 +306,7 @@ static inline const char *camu_audio_format_name(s32 format)
 static inline size_t camu_audio_format_bytes_per_sample(struct camu_audio_format *fmt)
 {
 #ifdef CAMU_HAVE_FFMPEG
-    return (size_t)av_get_bytes_per_sample(fmt->format);
+    return (size_t)av_get_bytes_per_sample((enum AVSampleFormat)fmt->format);
 #else
     switch (fmt->format) {
     case CAMU_SAMPLE_FORMAT_U8:
@@ -375,7 +376,7 @@ static inline void camu_video_format_copy(struct camu_video_format *dest, struct
 static inline const char *camu_pixel_format_name(s32 format)
 {
 #ifdef CAMU_HAVE_FFMPEG
-    return av_get_pix_fmt_name(format);
+    return av_get_pix_fmt_name((enum AVPixelFormat)format);
 #else
     switch (format) {
     case CAMU_PIXEL_FORMAT_RGBA: return "rgba";
diff --git a/src/codec/ffmpeg/decoder.c b/src/codec/ffmpeg/decoder.c
index f201ea3..9693dd9 100644
--- a/src/codec/ffmpeg/decoder.c
+++ b/src/codec/ffmpeg/decoder.c
@@ -7,19 +7,47 @@
 
 #if !defined CAMU_SINK_NO_VIDEO && defined CAMU_FF_DECODER_HWACCEL
 #include "../../render/renderer.h"
-#endif
-
-#ifdef CAMU_FF_DECODER_HWACCEL
+static const char *hwdevces[] = {
 #if defined STELA_API_VULKAN
-static const char *hwdevces[] = { "vulkan" };
+    "vulkan",
 #else
 #if defined NAUNET_ON_WINDOWS
-static const char *hwdevces[] = { "d3d11va" };
+    "d3d11va",
 #else
-static const char *hwdevces[] = { "vaapi" };
+    "vaapi",
+#endif
 #endif
+};
 #endif
 
+static bool alloc_codec_context_internal(struct camu_ff_decoder *av, const AVCodec *codec, AVCodecParameters *codecpar)
+{
+    if (!(av->codec_context = avcodec_alloc_context3(codec))) {
+        log_error("Failed to alloc codec context.");
+        return false;
+    }
+
+    if (avcodec_parameters_to_context(av->codec_context, codecpar) < 0) {
+        log_error("Failed to copy codec parameters.");
+        return false;
+    }
+
+    return true;
+}
+
+static bool open_avcodec_internal(struct camu_ff_decoder *av, const AVCodec *codec)
+{
+    AVDictionary *opts = NULL;
+    s32 ret = avcodec_open2(av->codec_context, codec, &opts);
+    if (ret < 0) {
+        log_error("Failed to open codec %s (%s).", codec->name, av_err2str(ret));
+        return false;
+    }
+    av_dict_free(&opts);
+    return true;
+}
+
+#ifdef CAMU_FF_DECODER_HWACCEL
 static s32 get_buffer2(AVCodecContext *context, AVFrame *pic, s32 flags)
 {
     struct camu_ff_decoder *av = (struct camu_ff_decoder *)context->opaque;
@@ -63,19 +91,36 @@ static enum AVPixelFormat get_hw_format(AVCodecContext *context, const enum AVPi
 {
     struct camu_ff_decoder *av = (struct camu_ff_decoder *)context->opaque;
 
+    const AVCodec *codec = av->codec_context->codec;
+    const char *hwdevice_name = av_hwdevice_get_type_name(av->hw_device_type);
+
     const enum AVPixelFormat *fmt = pix_fmts;
     for (; *fmt != AV_PIX_FMT_NONE; fmt++) {
         if (*fmt == av->hw_pix_fmt) {
             if (av->use_frames_context) {
                 init_hwframe_context(av, av->codec_context, av->hw_context);
             }
-            return *fmt;
+            log_info("Using %s hardware decoding.", hwdevice_name);
+            goto out;
         }
     }
 
-    log_error("Failed to get HW surface format.");
+    log_error("Failed to get %s HW surface format.", hwdevice_name);
+
+    // Remake AVCodecContext as a software decoder.
+    AVCodecParameters *codecpar = avcodec_parameters_alloc();
+    avcodec_parameters_from_context(codecpar, av->codec_context);
+    av->errored_hw_context = av->codec_context;
+    alloc_codec_context_internal(av, codec, codecpar);
+    // Note about thread_type choice in ff_decoder_init().
+    av->codec_context->thread_type = FF_THREAD_FRAME;
+    av->codec_context->thread_count = av->thread_count;
+    if (open_avcodec_internal(av, codec)) {
+        log_info("Software fallback, using %i threads for decoder.", av->thread_count);
+    }
 
-    return avcodec_default_get_format(context, pix_fmts);
+out:
+    return *fmt;
 }
 
 static s32 init_hwdevice_context(struct camu_ff_decoder *av, AVCodecContext *context)
@@ -90,9 +135,7 @@ static s32 init_hwdevice_context(struct camu_ff_decoder *av, AVCodecContext *con
 
     context->hw_device_ctx = av_buffer_ref(av->hw_context);
     // Note that context->extra_hw_frames has the ability to cause corruption.
-    context->extra_hw_frames = 18;
-
-    log_info("Using %s hardware decoding.", av_hwdevice_get_type_name(av->hw_device_type));
+    //context->extra_hw_frames = 18;
 
     return ret;
 }
@@ -139,7 +182,7 @@ static bool get_hwdevice_config(struct camu_ff_decoder *av, const AVCodec *codec
     return false;
 }
 
-static bool collect_supported_hwaccels(struct camu_ff_decoder *av)
+static void collect_supported_hwaccels(struct camu_ff_decoder *av)
 {
     const AVCodec *codec;
     void *iter = NULL;
@@ -149,15 +192,10 @@ static bool collect_supported_hwaccels(struct camu_ff_decoder *av)
             al_array_push(av->supported_hw_codecs, codec);
         }
     }
-
     enum AVHWDeviceType hw_device_type = AV_HWDEVICE_TYPE_NONE;
     while ((hw_device_type = av_hwdevice_iterate_types(hw_device_type)) != AV_HWDEVICE_TYPE_NONE) {
         al_array_push(av->supported_hw_devices, hw_device_type);
     }
-
-    av->hw_device_type = AV_HWDEVICE_TYPE_NONE;
-
-    return true;
 }
 #endif
 
@@ -165,6 +203,7 @@ static void close_internal(struct camu_ff_decoder *av)
 {
     if (av->codec_context) avcodec_free_context(&av->codec_context);
 #ifdef CAMU_FF_DECODER_HWACCEL
+    if (av->errored_hw_context) avcodec_free_context(&av->errored_hw_context);
     al_array_free(av->supported_hw_codecs);
     al_array_free(av->supported_hw_devices);
 #endif
@@ -177,6 +216,7 @@ static bool ff_decoder_init(struct camu_decoder *dec, struct camu_renderer *rend
 
     av->codec_context = NULL;
 #ifdef CAMU_FF_DECODER_HWACCEL
+    av->errored_hw_context = NULL;
     al_array_init(av->supported_hw_codecs);
     al_array_init(av->supported_hw_devices);
 #endif
@@ -188,21 +228,21 @@ static bool ff_decoder_init(struct camu_decoder *dec, struct camu_renderer *rend
         goto err;
     }
 
-#ifdef CAMU_FF_DECODER_HWACCEL
-    bool attempt_hwdec = stream->duration > 0 && codecpar->codec_type == AVMEDIA_TYPE_VIDEO;
+    bool is_video = codecpar->codec_type == AVMEDIA_TYPE_VIDEO && stream->duration > 0;
 
-    if (attempt_hwdec) {
+#ifdef CAMU_FF_DECODER_HWACCEL
+    if (is_video) {
         collect_supported_hwaccels(av);
-
+        av->hw_device_type = AV_HWDEVICE_TYPE_NONE;
         av->hw_pix_fmt = AV_PIX_FMT_NONE;
-
         for (u32 i = 0; i < ARRAY_SIZE(hwdevces); i++) {
             av->hw_device_type = hw_device_supported_by_name(av, hwdevces[i]);
-            if (get_hwdevice_config(av, codec, av->hw_device_type)) {
+            if (!get_hwdevice_config(av, codec, av->hw_device_type)) {
+                av->hw_device_type = AV_HWDEVICE_TYPE_NONE;
+            } else {
                 break;
             }
         }
-
         if (av->hw_device_type == AV_HWDEVICE_TYPE_NONE) {
             const AVCodec *hw_codec;
             al_array_foreach(av->supported_hw_codecs, i, hw_codec) {
@@ -216,27 +256,18 @@ static bool ff_decoder_init(struct camu_decoder *dec, struct camu_renderer *rend
                 }
             }
         }
-
         if (av->hw_device_type == AV_HWDEVICE_TYPE_NONE) {
             log_warn("Hardware accelerated video decoding of %s not supported.", codec->name);
         }
     }
 #endif
 
-    av->codec_context = avcodec_alloc_context3(codec);
-    if (!av->codec_context) {
-        log_error("Failed to alloc codec context.");
-        goto err;
-    }
-
-    if (avcodec_parameters_to_context(av->codec_context, codecpar) < 0) {
-        log_error("Failed to copy codec parameters.");
+    if (!alloc_codec_context_internal(av, codec, codecpar)) {
         goto err;
     }
 
 #ifdef CAMU_FF_DECODER_HWACCEL
-    if (attempt_hwdec && av->hw_device_type != AV_HWDEVICE_TYPE_NONE) {
-        //av->codec_context->codec_id = codec->id;
+    if (is_video && av->hw_device_type != AV_HWDEVICE_TYPE_NONE) {
         if (init_hwdevice_context(av, av->codec_context) == 0) {
             av->renderer = renderer;
             av->codec_context->opaque = av;
@@ -250,37 +281,30 @@ static bool ff_decoder_init(struct camu_decoder *dec, struct camu_renderer *rend
     (void)renderer;
 #endif
 
+    if (is_video) {
+        av->thread_count = MIN(6, MAX(1, av_cpu_count() / 2));
 #ifdef CAMU_FF_DECODER_HWACCEL
-    if ((!attempt_hwdec || av->hw_device_type == AV_HWDEVICE_TYPE_NONE) && stream->duration > 0) {
-#else
-    if (codecpar->codec_type == AVMEDIA_TYPE_VIDEO && stream->duration > 0) {
+        if (av->hw_device_type == AV_HWDEVICE_TYPE_NONE) {
 #endif
-        s32 cpus = 0;
-        if (av->codec_context->codec_type == AVMEDIA_TYPE_VIDEO) {
-            cpus = av_cpu_count();
-            cpus = MIN(6, MAX(1, cpus / 2));
-        }
-        if (cpus > 0) {
-            av->codec_context->thread_count = cpus;
             // FF_THREAD_FRAME or FF_THREAD_SLICE.
             // Both modes can get choked on certain videos, sometimes SLICE can squeeze
             // out a little more performance but a higher thread_count with FRAME seems
             // like the safest default.
             av->codec_context->thread_type = FF_THREAD_FRAME;
-            log_info("Using %i threads for decoder.", cpus);
+            av->codec_context->thread_count = av->thread_count;
+            log_info("Using %i threads for decoder.", av->thread_count);
+#ifdef CAMU_FF_DECODER_HWACCEL
         }
+#endif
     }
 
-    AVDictionary *opts = NULL;
-    s32 ret = avcodec_open2(av->codec_context, codec, &opts);
-    if (ret < 0) {
-        log_error("Failed to open codec %s (%s).", codec->name, av_err2str(ret));
+    if (!open_avcodec_internal(av, codec)) {
         goto err;
     }
 
-    const char *long_name = codec->long_name ? codec->long_name : codec->name;
-    s64 kbps = (codecpar->bit_rate > 0) ? codecpar->bit_rate / 1000 : 0;
-    log_info("Codec: %s (%s) %lldkbps.", codec->name, long_name, kbps);
+    log_info("Codec: %s (%s) %lldkbps.", codec->name,
+        codec->long_name ? codec->long_name : codec->name,
+        (av->codec_context->bit_rate > 0) ? av->codec_context->bit_rate / 1000 : 0);
 
     av->callback = callback;
     av->userdata = userdata;
diff --git a/src/codec/ffmpeg/decoder.h b/src/codec/ffmpeg/decoder.h
index 4797060..85b88dc 100644
--- a/src/codec/ffmpeg/decoder.h
+++ b/src/codec/ffmpeg/decoder.h
@@ -11,6 +11,7 @@
 struct camu_ff_decoder {
     struct camu_decoder dec;
     AVCodecContext *codec_context;
+    s32 thread_count;
 #ifdef CAMU_FF_DECODER_HWACCEL
     struct camu_renderer *renderer;
     array(const AVCodec *) supported_hw_codecs;
@@ -19,6 +20,7 @@ struct camu_ff_decoder {
     enum AVHWDeviceType hw_device_type;
     enum AVPixelFormat hw_pix_fmt;
     bool use_frames_context;
+    AVCodecContext *errored_hw_context;
 #endif
     void (*callback)(void *, struct camu_codec_frame *);
     void *userdata;
diff --git a/src/codec/ffmpeg/demuxer.c b/src/codec/ffmpeg/demuxer.c
index a5821a2..6560e0e 100644
--- a/src/codec/ffmpeg/demuxer.c
+++ b/src/codec/ffmpeg/demuxer.c
@@ -80,6 +80,7 @@ static bool ff_demuxer_init(struct camu_demuxer *demux, struct cch_handle *handl
     s64 default_duration = (av->format_context->duration < 0) ? 0 : av->format_context->duration;
     av->duration = 0;
 
+    // @TODO: Handle no-duration video.
 #define GUESS_STREAM_IS_IMAGE(stream) \
     ((stream->duration >= 0 && stream->nb_frames <= 1 && \
       (stream->avg_frame_rate.den == 0 && stream->r_frame_rate.den > 0)) || \
@@ -87,7 +88,6 @@ static bool ff_demuxer_init(struct camu_demuxer *demux, struct cch_handle *handl
 
     for (u32 i = 0; i < av->format_context->nb_streams; i++) {
         AVStream *stream = av->format_context->streams[i];
-        enum AVMediaType type = stream->codecpar->codec_type;
 #ifdef CAMU_OLD_FFMPEG
         // This is probably not correct.
         if (stream->codecpar->channel_layout == 0) {
@@ -95,6 +95,7 @@ static bool ff_demuxer_init(struct camu_demuxer *demux, struct cch_handle *handl
             stream->codecpar->channels = 2;
         }
 #endif
+        enum AVMediaType type = stream->codecpar->codec_type;
         u64 duration = 0;
         if (type == AVMEDIA_TYPE_AUDIO || type == AVMEDIA_TYPE_VIDEO || type == AVMEDIA_TYPE_SUBTITLE) {
             // Try to detect attached images.
diff --git a/src/codec/ffmpeg/encoder.c b/src/codec/ffmpeg/encoder.c
index 5d90785..641acc5 100644
--- a/src/codec/ffmpeg/encoder.c
+++ b/src/codec/ffmpeg/encoder.c
@@ -39,7 +39,7 @@ bool camu_ff_encoder_init(struct camu_ff_encoder *enc, char *encoder_name, char
     enc->codec_context->time_base = (AVRational){ 1, enc->codec_context->sample_rate };
     enc->codec_context->strict_std_compliance = FF_COMPLIANCE_EXPERIMENTAL;
 
-    enc->output_format = av_guess_format(ext, NULL, NULL);
+    enc->output_format = av_guess_format(encoder_name, ext, NULL);
     if (enc->output_format && enc->output_format->flags & AVFMT_GLOBALHEADER) {
         enc->codec_context->flags |= AV_CODEC_FLAG_GLOBAL_HEADER;
     }
@@ -52,21 +52,23 @@ bool camu_ff_encoder_init(struct camu_ff_encoder *enc, char *encoder_name, char
     av_dict_set(&opts, "compression_level", "10", 0);
     av_dict_set(&opts, "frame_duration", "20", 0);
     av_dict_set(&opts, "application", "audio", 0);
-
     if (avcodec_open2(enc->codec_context, codec, &opts) < 0) {
         log_error("Failed to open codec.");
         av_dict_free(&opts);
         goto err;
     }
-
     av_dict_free(&opts);
 
+    log_info("Encoder: %s (%s) %sbps.", codec->name,
+        codec->long_name ? codec->long_name : codec->name, kbps);
+
     AVCodecContext *out_codec = enc->codec_context;
     AVFrame *frame = av_frame_alloc();
     frame->format = out_codec->sample_fmt;
     frame->sample_rate = out_codec->sample_rate;
     camu_copy_channel_layout(&AV_CODECPAR_CHANNEL_LAYOUT(frame), &AV_CODECPAR_CHANNEL_LAYOUT(out_codec));
-    av_samples_alloc(frame->data, NULL, AV_CODECPAR_CHANNELS(out_codec), SAMPLES_PER_FRAME, frame->format, 1);
+    av_samples_alloc(frame->data, NULL, AV_CODECPAR_CHANNELS(out_codec),
+        SAMPLES_PER_FRAME, (enum AVSampleFormat)frame->format, 1);
     frame->nb_samples = SAMPLES_PER_FRAME;
     frame->pts = 0;
 
diff --git a/src/codec/ffmpeg/packet_ext.c b/src/codec/ffmpeg/packet_ext.c
index a1a6a5e..7dcda88 100644
--- a/src/codec/ffmpeg/packet_ext.c
+++ b/src/codec/ffmpeg/packet_ext.c
@@ -114,7 +114,7 @@ void nn_packet_read_av_codec_parameters(struct nn_packet *packet, AVCodecParamet
     NNWT_PACKET_READ_TYPE(packet, enum AVCodecID, codecpar->codec_id);
     NNWT_PACKET_READ_TYPE(packet, u32, codecpar->codec_tag);
     NNWT_PACKET_READ_TYPE(packet, s32, codecpar->extradata_size);
-    codecpar->extradata = av_malloc(codecpar->extradata_size);
+    codecpar->extradata = (u8 *)av_malloc(codecpar->extradata_size);
     u8 *extradata;
     NNWT_PACKET_READ_DATA(packet, codecpar->extradata_size, extradata);
     al_memcpy(codecpar->extradata, extradata, codecpar->extradata_size);
diff --git a/src/codec/ffmpeg/resampler.c b/src/codec/ffmpeg/resampler.c
index 48cca8e..b141681 100644
--- a/src/codec/ffmpeg/resampler.c
+++ b/src/codec/ffmpeg/resampler.c
@@ -23,8 +23,8 @@ static bool ff_resampler_init(struct camu_resampler *resamp, struct camu_resampl
 #endif
     av_opt_set_int(swr, "in_sample_rate", fmt->in.sample_rate, 0);
     av_opt_set_int(swr, "out_sample_rate", fmt->req.sample_rate, 0);
-    av_opt_set_sample_fmt(swr, "in_sample_fmt", fmt->in.format, 0);
-    av_opt_set_sample_fmt(swr, "out_sample_fmt", fmt->req.format, 0);
+    av_opt_set_sample_fmt(swr, "in_sample_fmt", (enum AVSampleFormat)fmt->in.format, 0);
+    av_opt_set_sample_fmt(swr, "out_sample_fmt", (enum AVSampleFormat)fmt->req.format, 0);
 
 #ifdef CAMU_HAVE_SOXR
     // Slower but more accurate resampler.
@@ -54,7 +54,7 @@ static inline void alloc_sample_data(struct camu_ff_resampler *av, s32 sample_co
     if (av->sample_count >= sample_count) return;
     free_sample_data(av);
     s32 nb_channels = av->fmt.channel_count;
-    av_samples_alloc(av->data, NULL, nb_channels, sample_count, av->fmt.format, 32);
+    av_samples_alloc(av->data, NULL, nb_channels, sample_count, (enum AVSampleFormat)av->fmt.format, 32);
     av->sample_count = sample_count;
 }
 
diff --git a/src/codec/ffmpeg/scaler.c b/src/codec/ffmpeg/scaler.c
index 93bbac6..0cd7fcb 100644
--- a/src/codec/ffmpeg/scaler.c
+++ b/src/codec/ffmpeg/scaler.c
@@ -41,16 +41,17 @@ static bool ff_scaler_init(struct camu_scaler *scale, struct camu_scaler_format
 
     av->fmt = *fmt;
 
-    s32 flags = SWS_BITEXACT;
-    av->scaler_context = sws_getContext(fmt->in.width, fmt->in.height, fmt->in.format,
-        fmt->req.width, fmt->req.height, fmt->req.format, flags, NULL, NULL, NULL);
+    av->scaler_context = sws_getContext(
+        fmt->in.width, fmt->in.height, (enum AVPixelFormat)fmt->in.format,
+        fmt->req.width, fmt->req.height, (enum AVPixelFormat)fmt->req.format,
+        SWS_BITEXACT, NULL, NULL, NULL);
     if (!av->scaler_context) {
         log_error("Failed to create scaler context.");
         return false;
     }
 
     struct camu_codec_frame *frame = &av->frame;
-    frame->av.frame = alloc_picture(fmt->req.format, fmt->req.width, fmt->req.height);
+    frame->av.frame = alloc_picture((enum AVPixelFormat)fmt->req.format, fmt->req.width, fmt->req.height);
     if (!frame->av.frame) {
         log_error("Failed to allocate frame.");
         return false;