From 2bee71a7e032c0972418e324bb1d7e6b02330b18 Mon Sep 17 00:00:00 2001 From: Andrew Opalach Date: Wed, 19 Feb 2025 13:40:38 -0500 Subject: Server resource unload, many tweaks and fixes - Initial liana client preferences. - Hook up libplacebo dx11 backend. - Make usage of FFmpeg hardware decoding api make some sense. Signed-off-by: Andrew Opalach --- src/codec/ffmpeg/common.c | 56 +++++++------ src/codec/ffmpeg/common.h | 6 +- src/codec/ffmpeg/decoder.c | 187 ++++++++++++++++++++---------------------- src/codec/ffmpeg/decoder.h | 3 +- src/codec/ffmpeg/meson.build | 18 ++-- src/codec/ffmpeg/packet_ext.c | 8 +- src/codec/ffmpeg/version.sh | 7 ++ 7 files changed, 145 insertions(+), 140 deletions(-) create mode 100755 src/codec/ffmpeg/version.sh (limited to 'src/codec/ffmpeg') diff --git a/src/codec/ffmpeg/common.c b/src/codec/ffmpeg/common.c index c5fe7cd..044908c 100644 --- a/src/codec/ffmpeg/common.c +++ b/src/codec/ffmpeg/common.c @@ -13,44 +13,46 @@ s64 camu_ff_frame_duration(AVFrame *frame) #endif } -void camu_ff_set_log_callback(void (*callback)(void *, s32, const char *, va_list)) -{ - av_log_set_callback(callback); -} +static __thread struct { + char buf[AL_LOG_MESSAGE_SIZE]; + size_t pos; +} log_context; -static char *buf = NULL; -static size_t pos = 0; +// If a line takes more than 1 step to print, make sure we stay within AL_LOG_MESSAGE_SIZE. +#define AV_LOG_MAX_CHUNK (AL_LOG_MESSAGE_SIZE / 2) -// If a line takes more than 2 steps to print, make sure we stay within AL_LOG_MESSAGE_SIZE. -#define CHUNK_SIZE (AL_LOG_MESSAGE_SIZE / 2u) - -// @TODO: thread-saftey. static void av_log_callback(void *userdata, s32 level, const char *fmt, va_list args) { (void)userdata; - al_assert(buf); - if (level < AV_LOG_DEBUG) { - s32 ret = al_vsnprintf(&buf[pos], CHUNK_SIZE, fmt, args); - al_assert(ret > 0); - pos += ret; - if ((pos > 0 && buf[pos - 1] == '\n') || pos >= CHUNK_SIZE) { - if (level <= AV_LOG_INFO) { - al_log_info("ff", buf); - } else { - al_log_debug("ff", buf); - } - pos = 0; + + if (level >= AV_LOG_DEBUG) { + return; + } + + char *offset = log_context.buf + log_context.pos; + s32 ret = al_vsnprintf(offset, AV_LOG_MAX_CHUNK, fmt, args); + al_assert(ret > 0); + + offset += ret; + log_context.pos += ret; + + if ((log_context.pos > 0 && *(offset - 1) == '\n') || log_context.pos >= AV_LOG_MAX_CHUNK) { + if (level <= AV_LOG_INFO) { + al_log_info("ff", log_context.buf); + } else { + al_log_debug("ff", log_context.buf); } + log_context.pos = 0; } } -void camu_ff_set_default_log_callback() +void camu_ff_set_log_callback(void (*callback)(void *, s32, const char *, va_list)) { - buf = (char *)al_malloc(AL_LOG_MESSAGE_SIZE); - camu_ff_set_log_callback(av_log_callback); + av_log_set_callback(callback); } -void camu_ff_free_default_log_callback() +void camu_ff_common_init(void) { - al_free(buf); + al_log_info("ff", "Using FFmpeg version "FFMPEG_VERSION"."); + camu_ff_set_log_callback(av_log_callback); } diff --git a/src/codec/ffmpeg/common.h b/src/codec/ffmpeg/common.h index 885dd4d..944f915 100644 --- a/src/codec/ffmpeg/common.h +++ b/src/codec/ffmpeg/common.h @@ -4,12 +4,8 @@ #include #include -void camu_ff_common_init(void); - s64 camu_ff_frame_duration(AVFrame *frame); // userdata, level, fmt, args void camu_ff_set_log_callback(void (*callback)(void *, int, const char *, va_list)); - -void camu_ff_set_default_log_callback(); -void camu_ff_free_default_log_callback(); +void camu_ff_common_init(void); diff --git a/src/codec/ffmpeg/decoder.c b/src/codec/ffmpeg/decoder.c index dd0cc0b..02f077d 100644 --- a/src/codec/ffmpeg/decoder.c +++ b/src/codec/ffmpeg/decoder.c @@ -8,8 +8,19 @@ #include "decoder.h" -#ifndef CAMU_SINK_NO_VIDEO #ifdef CAMU_FF_DECODER_HWACCEL +#if defined CAMU_RENDERER_VULKAN +static const char *hwdevces[] = { "vulkan" }; +#elif defined CAMU_RENDERER_DX11 +static const char *hwdevces[] = { "d3d11va" }; +#elif defined CAMU_RENDERER_OPENGL +#ifdef NAUNET_ON_WINDOWS +static const char *hwdevces[] = { "d3d11va" }; +#else +static const char *hwdevces[] = { "vaapi" }; +#endif +#endif + static s32 get_buffer2(AVCodecContext *context, AVFrame *pic, s32 flags) { struct camu_ff_decoder *av = (struct camu_ff_decoder *)context->opaque; @@ -29,6 +40,7 @@ static s32 init_hwframe_context(struct camu_ff_decoder *av, AVCodecContext *cont AVHWFramesContext *frames_ctx = (AVHWFramesContext *)(hw_frames_ref->data); frames_ctx->format = av->hw_pix_fmt; + // codec_context->sw_pix_fmt is only set in ffmpeg before calling get_hw_format(). frames_ctx->sw_format = av->codec_context->sw_pix_fmt; frames_ctx->width = av->codec_context->width; frames_ctx->height = av->codec_context->height; @@ -56,12 +68,6 @@ static enum AVPixelFormat get_hw_format(AVCodecContext *context, const enum AVPi for (; *fmt != AV_PIX_FMT_NONE; fmt++) { if (*fmt == av->hw_pix_fmt) { if (av->use_frames_context) { - av->hw_context = NULL; - s32 ret = av_hwdevice_ctx_create(&av->hw_context, av->hw_device_type, NULL, NULL, 0); - if (ret < 0 || !av->hw_context) { - al_log_error("ff_decoder", "Failed to initialize hardware device (%s).", av_err2str(ret)); - break; - } init_hwframe_context(av, av->codec_context, av->hw_context); } return *fmt; @@ -75,6 +81,8 @@ static enum AVPixelFormat get_hw_format(AVCodecContext *context, const enum AVPi static s32 init_hwdevice_context(struct camu_ff_decoder *av, AVCodecContext *context) { + av->hw_context = NULL; + s32 ret = av_hwdevice_ctx_create(&av->hw_context, av->hw_device_type, NULL, NULL, 0); if (ret < 0) { al_log_error("ff_decoder", "Failed to create specified HW device."); @@ -82,27 +90,15 @@ static s32 init_hwdevice_context(struct camu_ff_decoder *av, AVCodecContext *con } context->hw_device_ctx = av_buffer_ref(av->hw_context); + context->extra_hw_frames = 40; al_log_info("ff_decoder", "Using %s hardware decoding.", av_hwdevice_get_type_name(av->hw_device_type)); return ret; } -static s32 hw_device_priority(enum AVHWDeviceType type) -{ - switch (type) { - case AV_HWDEVICE_TYPE_MEDIACODEC: - return 4; - case AV_HWDEVICE_TYPE_VAAPI: - return 3; - case AV_HWDEVICE_TYPE_VULKAN: - return 2; - default: - return -1; - }; -} - -static bool hw_device_supported(struct camu_ff_decoder *av, enum AVHWDeviceType type) +/* +static bool hw_device_supported_by_type(struct camu_ff_decoder *av, enum AVHWDeviceType type) { enum AVHWDeviceType supported_type; al_array_foreach(av->supported_hw_devices, i, supported_type) { @@ -110,6 +106,7 @@ static bool hw_device_supported(struct camu_ff_decoder *av, enum AVHWDeviceType } return false; } +*/ static enum AVHWDeviceType hw_device_supported_by_name(struct camu_ff_decoder *av, const char *name) { @@ -124,14 +121,46 @@ static enum AVHWDeviceType hw_device_supported_by_name(struct camu_ff_decoder *a static bool hw_config_needs_frames_ctx(const AVCodecHWConfig *config) { - if ((config->methods & AV_CODEC_HW_CONFIG_METHOD_HW_FRAMES_CTX) && - !(config->methods & AV_CODEC_HW_CONFIG_METHOD_HW_DEVICE_CTX)) { - return true; - } else { - return false; + return (config->methods & AV_CODEC_HW_CONFIG_METHOD_HW_FRAMES_CTX) && + !(config->methods & AV_CODEC_HW_CONFIG_METHOD_HW_DEVICE_CTX); +} + +static bool get_hwdevice_config(struct camu_ff_decoder *av, const AVCodec *codec, enum AVHWDeviceType hw_device_type) +{ + s32 iter = 0; + const AVCodecHWConfig *config = NULL; + while ((config = avcodec_get_hw_config(codec, iter++))) { + if (config->device_type == hw_device_type) { + av->hw_pix_fmt = config->pix_fmt; + av->use_frames_context = hw_config_needs_frames_ctx(config); + return true; + } } + return false; +} + +static bool collect_supported_hwaccels(struct camu_ff_decoder *av) +{ + al_array_init(av->supported_hw_codecs); + const AVCodec *codec; + void *iter = NULL; + while ((codec = av_codec_iterate(&iter))) { + if (!av_codec_is_decoder(codec)) continue; + if (codec->capabilities & (AV_CODEC_CAP_HARDWARE | AV_CODEC_CAP_HYBRID)) { + al_array_push(av->supported_hw_codecs, codec); + } + } + + al_array_init(av->supported_hw_devices); + enum AVHWDeviceType hw_device_type = AV_HWDEVICE_TYPE_NONE; + while ((hw_device_type = av_hwdevice_iterate_types(hw_device_type)) != AV_HWDEVICE_TYPE_NONE) { + al_array_push(av->supported_hw_devices, hw_device_type); + } + + av->hw_device_type = AV_HWDEVICE_TYPE_NONE; + + return true; } -#endif #endif static void close_internal(struct camu_ff_decoder *av) @@ -147,47 +176,24 @@ static bool ff_decoder_init(struct camu_decoder *dec, struct camu_renderer *rend av->codec_context = NULL; AVCodecParameters *codecpar = stream->av.stream->codecpar; - const AVCodec *codec; -#ifdef CAMU_FF_DECODER_HWACCEL - void *state = NULL; - al_array_init(av->supported_hw_codecs); - while ((codec = av_codec_iterate(&state))) { - if (codec->capabilities & (AV_CODEC_CAP_HARDWARE | AV_CODEC_CAP_HYBRID)) { - al_array_push(av->supported_hw_codecs, codec); - } - } -#endif - - codec = avcodec_find_decoder(codecpar->codec_id); + const AVCodec *codec = avcodec_find_decoder(codecpar->codec_id); if (!codec) { al_log_error("ff_decoder", "Failed to find decoder."); goto err; } - av->hw_device_type = AV_HWDEVICE_TYPE_NONE; -#ifdef CAMU_FF_DECODER_HWACCEL - if (codecpar->codec_type == AVMEDIA_TYPE_VIDEO) { - al_array_init(av->supported_hw_devices); - enum AVHWDeviceType hw_device_type = AV_HWDEVICE_TYPE_NONE; - while ((hw_device_type = av_hwdevice_iterate_types(hw_device_type)) != AV_HWDEVICE_TYPE_NONE) { - al_array_push(av->supported_hw_devices, hw_device_type); - } + bool attempt_hwdec = stream->duration > 0 && codecpar->codec_type == AVMEDIA_TYPE_VIDEO; - const AVCodecHWConfig *config = NULL; - for (s32 i = 0; ; i++) { - config = avcodec_get_hw_config(codec, i); - if (!config) { - break; - } +#ifdef CAMU_FF_DECODER_HWACCEL + if (attempt_hwdec) { + collect_supported_hwaccels(av); - if (!hw_device_supported(av, config->device_type)) { - continue; - } + av->hw_pix_fmt = AV_PIX_FMT_NONE; - if (hw_device_priority(config->device_type) > hw_device_priority(av->hw_device_type)) { - av->hw_device_type = config->device_type; - av->hw_pix_fmt = config->pix_fmt; - av->use_frames_context = hw_config_needs_frames_ctx(config); + for (u32 i = 0; i < ARRAY_SIZE(hwdevces); i++) { + av->hw_device_type = hw_device_supported_by_name(av, hwdevces[i]); + if (get_hwdevice_config(av, codec, av->hw_device_type)) { + break; } } @@ -196,30 +202,17 @@ static bool ff_decoder_init(struct camu_decoder *dec, struct camu_renderer *rend al_array_foreach(av->supported_hw_codecs, i, hw_codec) { if (hw_codec->id == codecpar->codec_id) { av->hw_device_type = hw_device_supported_by_name(av, hw_codec->wrapper_name); - if (av->hw_device_type != AV_HWDEVICE_TYPE_NONE) { - for (u32 j = 0; ; j++) { - config = avcodec_get_hw_config(hw_codec, j); - if (!config) { - al_log_warn("ff_decoder", "Hardware codec has no hw_configs."); - av->hw_device_type = AV_HWDEVICE_TYPE_NONE; - break; - } - codec = hw_codec; - av->hw_pix_fmt = config->pix_fmt; - av->use_frames_context = hw_config_needs_frames_ctx(config); - break; - } - } - if (av->hw_device_type != AV_HWDEVICE_TYPE_NONE) { + if (get_hwdevice_config(av, hw_codec, av->hw_device_type)) { + codec = hw_codec; break; } + av->hw_device_type = AV_HWDEVICE_TYPE_NONE; } } } if (av->hw_device_type == AV_HWDEVICE_TYPE_NONE) { - al_log_warn("ff_decoder", - "Hardware accelerated video decoding of %s not supported.", codec->name); + al_log_warn("ff_decoder", "Hardware accelerated video decoding of %s not supported.", codec->name); } } #endif @@ -236,42 +229,38 @@ static bool ff_decoder_init(struct camu_decoder *dec, struct camu_renderer *rend } #ifdef CAMU_FF_DECODER_HWACCEL - if ((av->hw_device_type != AV_HWDEVICE_TYPE_NONE && !av->use_frames_context) && - (init_hwdevice_context(av, av->codec_context) < 0)) { - av->hw_device_type = AV_HWDEVICE_TYPE_NONE; - } -#endif - -#ifndef CAMU_SINK_NO_VIDEO - if (codecpar->codec_type == AVMEDIA_TYPE_VIDEO && renderer && renderer->get_buffer2) { + if (attempt_hwdec && av->hw_device_type != AV_HWDEVICE_TYPE_NONE) { + //av->codec_context->codec_id = codec->id; + if (init_hwdevice_context(av, av->codec_context) < 0) { + av->hw_device_type = AV_HWDEVICE_TYPE_NONE; + } av->renderer = renderer; - // libplacebo's get_buffer2 take a really long time to discard frames on opengl + no hwaccel. -#ifdef CAMU_FF_DECODER_HWACCEL av->codec_context->opaque = av; av->codec_context->get_buffer2 = get_buffer2; - if (av->hw_device_type != AV_HWDEVICE_TYPE_NONE) { - av->codec_context->get_format = get_hw_format; - } -#endif + av->codec_context->get_format = get_hw_format; } #else (void)renderer; #endif - if (stream->duration > 0 && av->hw_device_type == AV_HWDEVICE_TYPE_NONE) { + if ((!attempt_hwdec || av->hw_device_type == AV_HWDEVICE_TYPE_NONE) && stream->duration > 0) { s32 cpus = 0; if (av->codec_context->codec_type == AVMEDIA_TYPE_VIDEO) { cpus = av_cpu_count(); cpus = MIN(3, MAX(1, cpus / 2)); } - av->codec_context->thread_count = cpus; - // FF_THREAD_FRAME or FF_THREAD_SLICE. - av->codec_context->thread_type = FF_THREAD_FRAME; - al_log_debug("ff_decoder", "Using %i threads for decoder.", cpus); + if (cpus > 0) { + av->codec_context->thread_count = cpus; + // FF_THREAD_FRAME or FF_THREAD_SLICE. + av->codec_context->thread_type = FF_THREAD_FRAME; + al_log_info("ff_decoder", "Using %i threads for decoder.", cpus); + } } - if (avcodec_open2(av->codec_context, codec, NULL) < 0) { - al_log_error("ff_decoder", "Failed to open codec (%s).", codec->name); + AVDictionary *opts = NULL; + s32 ret = avcodec_open2(av->codec_context, codec, &opts); + if (ret < 0) { + al_log_error("ff_decoder", "Failed to open codec %s (%s).", codec->name, av_err2str(ret)); goto err; } diff --git a/src/codec/ffmpeg/decoder.h b/src/codec/ffmpeg/decoder.h index b1cbf19..4aa3d32 100644 --- a/src/codec/ffmpeg/decoder.h +++ b/src/codec/ffmpeg/decoder.h @@ -5,8 +5,7 @@ #include "../codec.h" -// We need EGL for hwaccel on OpenGL. -#if !(defined CAMU_RENDERER_OPENGL && !defined STELA_USE_EGL) +#ifndef CAMU_SINK_NO_VIDEO //#define CAMU_FF_DECODER_HWACCEL #endif diff --git a/src/codec/ffmpeg/meson.build b/src/codec/ffmpeg/meson.build index cba033c..b93e556 100644 --- a/src/codec/ffmpeg/meson.build +++ b/src/codec/ffmpeg/meson.build @@ -27,18 +27,20 @@ ffmpeg_client_deps = [] libavutil = dependency('libavutil', required: false) libavformat = dependency('libavformat', required: false) libavcodec = dependency('libavcodec', required: false) +libavdevice = dependency('libavdevice', required: false) libswresample = dependency('libswresample', required: false) libswscale = dependency('libswscale', required: false) -if not (libavutil.found() and libavformat.found() and libavcodec.found() and libswresample.found() and libswscale.found()) +if not (libavutil.found() and libavformat.found() and libavcodec.found() and libavdevice.found() and libswresample.found() and libswscale.found()) ffmpeg_proj = subproject('ffmpeg', required: false) if ffmpeg_proj.found() libavutil = ffmpeg_proj.get_variable('avutil') libavformat = ffmpeg_proj.get_variable('avformat') libavcodec = ffmpeg_proj.get_variable('avcodec') + libavdevice = ffmpeg_proj.get_variable('avdevice') libswresample = ffmpeg_proj.get_variable('swresample') libswscale = ffmpeg_proj.get_variable('swscale') # Only used to set include dirs for FFmpeg. - ffmpeg_deps += [ffmpeg_proj.get_variable('ffmpeg_inc_dep')] + ffmpeg_deps += [ffmpeg_proj.get_variable('ffmpeg_include')] # We explicitly enable zlib in the FFmpeg build. ffmpeg_deps += [dependency('zlib')] if is_windows @@ -48,18 +50,24 @@ if not (libavutil.found() and libavformat.found() and libavcodec.found() and lib # Android MediaCodec HW decoding. ffmpeg_client_deps += [compiler.find_library('mediandk')] endif + ffmpeg_version_string = ffmpeg_proj.get_variable('ffmpeg_version_string') endif +else + ffmpeg_version_string = run_command('./version.sh', check: true).stdout().strip() endif +# This is probably meaningless compared to libav* versions, it's just something I wanted. +ffmpeg_args += ['-DFFMPEG_VERSION="' + ffmpeg_version_string + '"'] + soxr = compiler.find_library('soxr', required: false) if soxr.found() ffmpeg_client_deps += [soxr] ffmpeg_args += ['-DCAMU_HAVE_SOXR'] endif -if libavutil.found() and libavformat.found() and libavcodec.found() and libswresample.found() and libswscale.found() - ffmpeg_server_deps += [libavutil, libavformat, libavcodec] - ffmpeg_client_deps += [libavutil, libavformat, libavcodec, libswresample, libswscale] +if libavutil.found() and libavformat.found() and libavcodec.found() and libavdevice.found() and libswresample.found() and libswscale.found() + ffmpeg_server_deps += [libavutil, libavformat, libavcodec, libavdevice] + ffmpeg_client_deps += [libavutil, libavformat, libavcodec, libavdevice, libswresample, libswscale] codec_server_deps += [ declare_dependency(sources: ffmpeg_server_src, dependencies: [ffmpeg_deps, ffmpeg_server_deps], compile_args: ffmpeg_args) diff --git a/src/codec/ffmpeg/packet_ext.c b/src/codec/ffmpeg/packet_ext.c index f3704f2..9aa1992 100644 --- a/src/codec/ffmpeg/packet_ext.c +++ b/src/codec/ffmpeg/packet_ext.c @@ -26,7 +26,8 @@ void nn_packet_write_av_codec_parameters(struct nn_packet *packet, AVCodecParame NNWT_PACKET_WRITE_TYPE(packet, enum AVColorSpace, codecpar->color_space); NNWT_PACKET_WRITE_TYPE(packet, enum AVChromaLocation, codecpar->chroma_location); NNWT_PACKET_WRITE_TYPE(packet, s32, codecpar->video_delay); - NNWT_PACKET_WRITE_TYPE(packet, AVChannelLayout, codecpar->ch_layout); + NNWT_PACKET_WRITE_TYPE(packet, u64, codecpar->ch_layout.u.mask); + //NNWT_PACKET_WRITE_TYPE(packet, AVChannelLayout, codecpar->ch_layout); NNWT_PACKET_WRITE_TYPE(packet, s32, codecpar->sample_rate); NNWT_PACKET_WRITE_TYPE(packet, s32, codecpar->block_align); NNWT_PACKET_WRITE_TYPE(packet, s32, codecpar->frame_size); @@ -124,7 +125,10 @@ void nn_packet_read_av_codec_parameters(struct nn_packet *packet, AVCodecParamet NNWT_PACKET_READ_TYPE(packet, enum AVColorSpace, codecpar->color_space); NNWT_PACKET_READ_TYPE(packet, enum AVChromaLocation, codecpar->chroma_location); NNWT_PACKET_READ_TYPE(packet, s32, codecpar->video_delay); - NNWT_PACKET_READ_TYPE(packet, AVChannelLayout, codecpar->ch_layout); + u64 ch_layout_mask; + NNWT_PACKET_READ_TYPE(packet, u64, ch_layout_mask); + av_channel_layout_from_mask(&codecpar->ch_layout, ch_layout_mask); + //NNWT_PACKET_READ_TYPE(packet, AVChannelLayout, codecpar->ch_layout); NNWT_PACKET_READ_TYPE(packet, s32, codecpar->sample_rate); NNWT_PACKET_READ_TYPE(packet, s32, codecpar->block_align); NNWT_PACKET_READ_TYPE(packet, s32, codecpar->frame_size); diff --git a/src/codec/ffmpeg/version.sh b/src/codec/ffmpeg/version.sh new file mode 100755 index 0000000..6afdb2a --- /dev/null +++ b/src/codec/ffmpeg/version.sh @@ -0,0 +1,7 @@ +#! /usr/bin/env sh +if ! command -v ffprobe 2>&1 >/dev/null +then + echo "(unknown)" +else + ffprobe -v 0 -of default=nw=1:nk=1 -show_program_version | head -1 +fi -- cgit v1.2.3-101-g0448