From f3590721df77b7557a80c437acca33b1a4514a9a Mon Sep 17 00:00:00 2001 From: Andrew Opalach Date: Mon, 20 Jan 2025 20:38:33 -0500 Subject: Hardware decoding on Android - This is bad, hardware decoding selection needs to be completely rethought. Signed-off-by: Andrew Opalach --- flake.nix | 12 +- hardware/cmsb-001/configuration.nix | 30 ++-- src/codec/ffmpeg/decoder.c | 155 ++++++++++++++------- src/codec/ffmpeg/decoder.h | 4 +- src/codec/meson.build | 2 + src/render/renderer_libplacebo.c | 2 +- .../packagefiles/glfm/get_egl_internal.diff | 39 ++++++ 7 files changed, 174 insertions(+), 70 deletions(-) create mode 100644 subprojects/packagefiles/glfm/get_egl_internal.diff diff --git a/flake.nix b/flake.nix index fbfde71..10738f4 100644 --- a/flake.nix +++ b/flake.nix @@ -91,7 +91,7 @@ ninja meson cmake - wayland-scanner + wayland-scanner.bin ((python3.override { enableOptimizations = true; }).withPackages (python-pkgs: [ python-pkgs.setuptools python-pkgs.pip @@ -142,14 +142,14 @@ }; }; # https://cryptoisland.blog/posts/2024/12/09/flake-based-recipe-for-offline-dev-shells-in-nixos/ - # lib.strings.makeLibraryPath for PKG_CONFIG_PATH and LD_LIBRARY_PATH? shellWrapper = with pkgs; let shell = devShells.${system}.default; in pkgs.writeScriptBin "camu-devshell" '' - export PATH=${pkgs.lib.makeBinPath shell.nativeBuildInputs}:${pkgs.lib.makeBinPath shell.buildInputs}:$PATH - ${shell.shellHook} - ${pkgs.bashInteractive}/bin/bash $@ - ''; + export PATH=${pkgs.lib.makeBinPath shell.nativeBuildInputs}:$PATH + export PKG_CONFIG_PATH=${pkgs.lib.makeSearchPathOutput "dev" "lib/pkgconfig" shell.buildInputs}:$PKG_CONFIG_PATH + export PKG_CONFIG_PATH=${pkgs.lib.makeSearchPathOutput "dev" "share/pkgconfig" shell.buildInputs}:$PKG_CONFIG_PATH + ${pkgs.bashInteractive}/bin/bash --rcfile <(echo '${shell.shellHook}') $@ + ''; }); devShells = forAllDevSystems (pkgs: rec { diff --git a/hardware/cmsb-001/configuration.nix b/hardware/cmsb-001/configuration.nix index a4fb9bb..3390cfe 100644 --- a/hardware/cmsb-001/configuration.nix +++ b/hardware/cmsb-001/configuration.nix @@ -141,16 +141,16 @@ }; }; - #services.cage = { - # enable = true; - # user = "naunet"; - # environment = { - # WLR_LIBINPUT_NO_DEVICES = "1"; - # }; - # program = "${pkgs.camu}/bin/cmv"; - #}; - #systemd.services."cage-tty1".serviceConfig.StandardOutput = lib.mkForce "journal+console"; - #systemd.services."cage-tty1".wantedBy = []; + services.cage = { + enable = true; + user = "naunet"; + environment = { + WLR_LIBINPUT_NO_DEVICES = "1"; + }; + program = "${pkgs.camu}/bin/cmv"; + }; + systemd.services."cage-tty1".serviceConfig.StandardOutput = lib.mkForce "journal+console"; + systemd.services."cage-tty1".wantedBy = []; # https://pablo.tools/blog/computers/nixos-generate-raspberry-images/ #sdImage = { @@ -170,15 +170,15 @@ # https://retropie.org.uk/docs/Overclocking/ # BOOT/config.txt (changes) - # gpu_mem=64 + # gpu_mem=128 # total_mem=1024 - # arm_freq=1300 + # arm_freq=1250 # gpu_freq=500 - # core_freq=500 + # core_freq=450 # sdram_freq=500 # sdram_schmoo=0x02000020 - # over_voltage=2 - # over_voltage_sdram=2 + # over_voltage=2 + # over_voltage_sdram=2 # https://github.com/NixOS/nixos-hardware/issues/631 hardware.deviceTree = { diff --git a/src/codec/ffmpeg/decoder.c b/src/codec/ffmpeg/decoder.c index b502e88..36e59dd 100644 --- a/src/codec/ffmpeg/decoder.c +++ b/src/codec/ffmpeg/decoder.c @@ -14,6 +14,7 @@ static s32 get_buffer2(AVCodecContext *context, AVFrame *pic, s32 flags) return ret; } +#ifdef CAMU_FF_DECODER_HWACCEL static s32 init_hwframe_context(struct camu_ff_decoder *av, AVCodecContext *context, AVBufferRef *hw_device_context) { AVBufferRef *hw_frames_ref; @@ -52,9 +53,9 @@ static enum AVPixelFormat get_hw_format(AVCodecContext *context, const enum AVPi if (*fmt == av->hw_pix_fmt) { if (av->use_frames_context) { av->hw_context = NULL; - s32 ret = av_hwdevice_ctx_create(&av->hw_context, AV_HWDEVICE_TYPE_VAAPI, NULL, NULL, 0); + s32 ret = av_hwdevice_ctx_create(&av->hw_context, av->hw_device_type, NULL, NULL, 0); if (ret < 0 || !av->hw_context) { - al_log_error("ff_decoder", "Failed to initialize VAAPI device (%s).", av_err2str(ret)); + al_log_error("ff_decoder", "Failed to initialize hardware device (%s).", av_err2str(ret)); break; } init_hwframe_context(av, av->codec_context, av->hw_context); @@ -67,7 +68,6 @@ static enum AVPixelFormat get_hw_format(AVCodecContext *context, const enum AVPi return avcodec_default_get_format(context, pix_fmts); } -#endif static s32 init_hwdevice_context(struct camu_ff_decoder *av, AVCodecContext *context) { @@ -84,6 +84,52 @@ static s32 init_hwdevice_context(struct camu_ff_decoder *av, AVCodecContext *con return ret; } +static s32 hw_device_priority(enum AVHWDeviceType type) +{ + switch (type) { + case AV_HWDEVICE_TYPE_MEDIACODEC: + return 4; + case AV_HWDEVICE_TYPE_VAAPI: + return 3; + case AV_HWDEVICE_TYPE_VULKAN: + return 2; + default: + return -1; + }; +} + +static bool hw_device_supported(struct camu_ff_decoder *av, enum AVHWDeviceType type) +{ + enum AVHWDeviceType supported_type; + al_array_foreach(av->supported_hw_devices, i, supported_type) { + if (supported_type == type) return true; + } + return false; +} + +static enum AVHWDeviceType hw_device_supported_by_name(struct camu_ff_decoder *av, const char *name) +{ + enum AVHWDeviceType supported_type; + al_array_foreach(av->supported_hw_devices, i, supported_type) { + if (strcmp(av_hwdevice_get_type_name(supported_type), name) == 0) { + return supported_type; + } + } + return AV_HWDEVICE_TYPE_NONE; +} + +static bool hw_config_needs_frames_ctx(const AVCodecHWConfig *config) +{ + if ((config->methods & AV_CODEC_HW_CONFIG_METHOD_HW_FRAMES_CTX) && + !(config->methods & AV_CODEC_HW_CONFIG_METHOD_HW_DEVICE_CTX)) { + return true; + } else { + return false; + } +} +#endif +#endif + static void close_internal(struct camu_ff_decoder *av) { if (av->codec_context) avcodec_free_context(&av->codec_context); @@ -97,13 +143,16 @@ static bool ff_decoder_init(struct camu_decoder *dec, struct camu_renderer *rend av->codec_context = NULL; AVCodecParameters *codecpar = stream->av.stream->codecpar; - const AVCodec *codec = avcodec_find_decoder(codecpar->codec_id); -#ifdef CAMU_FF_DECODER_HWACCEL - if (strcmp(codec->name, "libdav1d") == 0) { - codec = avcodec_find_decoder_by_name("av1"); + void *state = NULL; + const AVCodec *codec; + al_array_init(av->supported_hw_codecs); + while ((codec = av_codec_iterate(&state))) { + if (codec->capabilities & (AV_CODEC_CAP_HARDWARE | AV_CODEC_CAP_HYBRID)) { + al_array_push(av->supported_hw_codecs, codec); + } } -#endif + codec = avcodec_find_decoder(codecpar->codec_id); if (!codec) { al_log_error("ff_decoder", "Failed to find decoder."); goto err; @@ -112,53 +161,62 @@ static bool ff_decoder_init(struct camu_decoder *dec, struct camu_renderer *rend av->hw_device_type = AV_HWDEVICE_TYPE_NONE; #ifdef CAMU_FF_DECODER_HWACCEL if (codecpar->codec_type == AVMEDIA_TYPE_VIDEO) { -#ifdef _WIN32 // TEMP - const char *hw_device = "vulkan"; -#else - const char *hw_device = "vaapi"; -#endif - av->hw_device_type = av_hwdevice_find_type_by_name(hw_device); - if (av->hw_device_type == AV_HWDEVICE_TYPE_NONE) { - al_log_info("ff_decoder", "Hardware device type \"%s\" not supported.", hw_device); - al_log_info("ff_decoder", "Types available on this system:"); - while ((av->hw_device_type = av_hwdevice_iterate_types(av->hw_device_type)) != AV_HWDEVICE_TYPE_NONE) { - al_log_info("ff_decoder", "\t%s", av_hwdevice_get_type_name(av->hw_device_type)); - } + al_array_init(av->supported_hw_devices); + enum AVHWDeviceType hw_device_type = AV_HWDEVICE_TYPE_NONE; + while ((hw_device_type = av_hwdevice_iterate_types(hw_device_type)) != AV_HWDEVICE_TYPE_NONE) { + al_array_push(av->supported_hw_devices, hw_device_type); } - } -#endif - if (av->hw_device_type != AV_HWDEVICE_TYPE_NONE) { - av->use_frames_context = false; + const AVCodecHWConfig *config = NULL; for (s32 i = 0; ; i++) { - const AVCodecHWConfig *config = avcodec_get_hw_config(codec, i); + config = avcodec_get_hw_config(codec, i); if (!config) { - if (av->use_frames_context) { - al_log_warn("ff_decoder", "Selected hardware decoder (%s) only supports" - "hardware frames for %s, which is unimplemented.", - av_hwdevice_get_type_name(av->hw_device_type), codec->name); - } else { - al_log_warn("ff_decoder", "Selected hardware decoder (%s) does not support %s.", - av_hwdevice_get_type_name(av->hw_device_type), codec->name); - } - av->hw_device_type = AV_HWDEVICE_TYPE_NONE; break; } - if (config->methods & AV_CODEC_HW_CONFIG_METHOD_HW_FRAMES_CTX) { + if (!hw_device_supported(av, config->device_type)) { + continue; + } + + if (hw_device_priority(config->device_type) > hw_device_priority(av->hw_device_type)) { + av->hw_device_type = config->device_type; av->hw_pix_fmt = config->pix_fmt; - av->use_frames_context = true; + av->use_frames_context = hw_config_needs_frames_ctx(config); } + } - if (config->methods & AV_CODEC_HW_CONFIG_METHOD_HW_DEVICE_CTX) { - if (config->device_type == av->hw_device_type) { - av->hw_pix_fmt = config->pix_fmt; - av->use_frames_context = false; - break; + if (av->hw_device_type == AV_HWDEVICE_TYPE_NONE) { + const AVCodec *hw_codec; + al_array_foreach(av->supported_hw_codecs, i, hw_codec) { + if (hw_codec->id == codecpar->codec_id) { + av->hw_device_type = hw_device_supported_by_name(av, hw_codec->wrapper_name); + if (av->hw_device_type != AV_HWDEVICE_TYPE_NONE) { + for (u32 j = 0; ; j++) { + config = avcodec_get_hw_config(hw_codec, j); + if (!config) { + al_log_warn("ff_decoder", "Hardware codec has no hw_configs."); + av->hw_device_type = AV_HWDEVICE_TYPE_NONE; + break; + } + codec = hw_codec; + av->hw_pix_fmt = config->pix_fmt; + av->use_frames_context = hw_config_needs_frames_ctx(config); + break; + } + } + if (av->hw_device_type != AV_HWDEVICE_TYPE_NONE) { + break; + } } } } + + if (av->hw_device_type == AV_HWDEVICE_TYPE_NONE) { + al_log_warn("ff_decoder", + "Hardware accelerated video decoding of %s not supported.", codec->name); + } } +#endif av->codec_context = avcodec_alloc_context3(codec); if (!av->codec_context) { @@ -171,14 +229,23 @@ static bool ff_decoder_init(struct camu_decoder *dec, struct camu_renderer *rend goto err; } +#ifdef CAMU_FF_DECODER_HWACCEL + if ((av->hw_device_type != AV_HWDEVICE_TYPE_NONE && !av->use_frames_context) && + (init_hwdevice_context(av, av->codec_context) < 0)) { + av->hw_device_type = AV_HWDEVICE_TYPE_NONE; + } +#endif + #if !defined CAMU_SINK_NO_VIDEO if (codecpar->codec_type == AVMEDIA_TYPE_VIDEO && renderer && renderer->get_buffer2) { av->renderer = renderer; av->codec_context->opaque = av; av->codec_context->get_buffer2 = get_buffer2; +#ifdef CAMU_FF_DECODER_HWACCEL if (av->hw_device_type != AV_HWDEVICE_TYPE_NONE) { av->codec_context->get_format = get_hw_format; } +#endif } #else (void)renderer; @@ -196,12 +263,6 @@ static bool ff_decoder_init(struct camu_decoder *dec, struct camu_renderer *rend al_log_debug("ff_decoder", "Using %i threads for decoder.", cpus); } - if ((av->hw_device_type != AV_HWDEVICE_TYPE_NONE) && - (init_hwdevice_context(av, av->codec_context) < 0)) { - av->hw_pix_fmt = AV_PIX_FMT_NONE; - return false; - } - if (avcodec_open2(av->codec_context, codec, NULL) < 0) { al_log_error("ff_decoder", "Failed to open codec (%s).", codec->name); goto err; diff --git a/src/codec/ffmpeg/decoder.h b/src/codec/ffmpeg/decoder.h index 827afd2..b1cbf19 100644 --- a/src/codec/ffmpeg/decoder.h +++ b/src/codec/ffmpeg/decoder.h @@ -14,9 +14,11 @@ struct camu_ff_decoder { struct camu_decoder dec; AVCodecContext *codec_context; struct camu_renderer *renderer; + array(const AVCodec *) supported_hw_codecs; + array(enum AVHWDeviceType) supported_hw_devices; AVBufferRef *hw_context; - enum AVPixelFormat hw_pix_fmt; enum AVHWDeviceType hw_device_type; + enum AVPixelFormat hw_pix_fmt; bool use_frames_context; void (*callback)(void *, struct camu_codec_frame *); void *userdata; diff --git a/src/codec/meson.build b/src/codec/meson.build index 2a72b55..2e89d95 100644 --- a/src/codec/meson.build +++ b/src/codec/meson.build @@ -39,6 +39,8 @@ if get_option('codecs').contains('ffmpeg') if is_windows # Extra static dependencies for Windows. ffmpeg_deps += [compiler.find_library('bcrypt')] + elif is_android + ffmpeg_deps += [compiler.find_library('mediandk')] endif soxr = compiler.find_library('soxr', required: false) if soxr.found() diff --git a/src/render/renderer_libplacebo.c b/src/render/renderer_libplacebo.c index a15adfd..e6cfa0c 100644 --- a/src/render/renderer_libplacebo.c +++ b/src/render/renderer_libplacebo.c @@ -294,7 +294,7 @@ static void renderer_lp_render(struct camu_renderer *renderer, struct camu_scree f64 tick = nn_get_tick(); f64 frame_time = tick - lr->last_render_tick; - if (frame_time > 0.020) { + if (frame_time > 0.050) { al_log_info("render_libplacebo", "FRAME_TIME: %fs", frame_time); } lr->last_render_tick = tick; diff --git a/subprojects/packagefiles/glfm/get_egl_internal.diff b/subprojects/packagefiles/glfm/get_egl_internal.diff new file mode 100644 index 0000000..3c9098c --- /dev/null +++ b/subprojects/packagefiles/glfm/get_egl_internal.diff @@ -0,0 +1,39 @@ +diff --git a/include/glfm.h b/include/glfm.h +index ceace39..d18711c 100644 +--- a/include/glfm.h ++++ b/include/glfm.h +@@ -892,6 +892,9 @@ ANativeActivity *glfmAndroidGetActivity(void) GLFM_DEPRECATED("Use glfmGetAndroi + /// ``GLFMAppFocusFunc``. + void *glfmGetAndroidActivity(const GLFMDisplay *display); + ++void *glfmGetEglDisplay(const GLFMDisplay *display); ++void *glfmGetEglContext(const GLFMDisplay *display); ++ + #endif // GLFM_EXPOSE_NATIVE_ANDROID + + #ifdef __cplusplus +diff --git a/src/glfm_android.c b/src/glfm_android.c +index 67d1927..7942ddf 100644 +--- a/src/glfm_android.c ++++ b/src/glfm_android.c +@@ -2800,4 +2800,20 @@ void *glfmGetAndroidActivity(const GLFMDisplay *display) { + return platformData->activity; + } + ++void *glfmGetEglDisplay(const GLFMDisplay *display) { ++ if (!display || !display->platformData) { ++ return NULL; ++ } ++ GLFMPlatformData *platformData = display->platformData; ++ return platformData->eglDisplay; ++} ++ ++void *glfmGetEglContext(const GLFMDisplay *display) { ++ if (!display || !display->platformData) { ++ return NULL; ++ } ++ GLFMPlatformData *platformData = display->platformData; ++ return platformData->eglContext; ++} ++ + #endif // __ANDROID__ -- cgit v1.2.3-101-g0448