Hardware decoding on Android

 - This is bad, hardware decoding selection needs to be completely
   rethought.

Signed-off-by: Andrew Opalach <andrew@akon.city>

7 files changed, 174 insertions, 70 deletions

diff --git a/flake.nix b/flake.nix
index fbfde71..10738f4 100644
--- a/flake.nix
+++ b/flake.nix
@@ -91,7 +91,7 @@
           ninja
           meson
           cmake
-          wayland-scanner
+          wayland-scanner.bin
           ((python3.override { enableOptimizations = true; }).withPackages (python-pkgs: [
             python-pkgs.setuptools
             python-pkgs.pip
@@ -142,14 +142,14 @@
         };
       };
       # https://cryptoisland.blog/posts/2024/12/09/flake-based-recipe-for-offline-dev-shells-in-nixos/
-      # lib.strings.makeLibraryPath for PKG_CONFIG_PATH and LD_LIBRARY_PATH?
       shellWrapper = with pkgs; let
         shell = devShells.${system}.default;
       in pkgs.writeScriptBin "camu-devshell" ''
-          export PATH=${pkgs.lib.makeBinPath shell.nativeBuildInputs}:${pkgs.lib.makeBinPath shell.buildInputs}:$PATH
-          ${shell.shellHook}
-          ${pkgs.bashInteractive}/bin/bash $@
-        '';
+        export PATH=${pkgs.lib.makeBinPath shell.nativeBuildInputs}:$PATH
+        export PKG_CONFIG_PATH=${pkgs.lib.makeSearchPathOutput "dev" "lib/pkgconfig" shell.buildInputs}:$PKG_CONFIG_PATH
+        export PKG_CONFIG_PATH=${pkgs.lib.makeSearchPathOutput "dev" "share/pkgconfig" shell.buildInputs}:$PKG_CONFIG_PATH
+        ${pkgs.bashInteractive}/bin/bash --rcfile <(echo '${shell.shellHook}') $@
+      '';
     });
 
     devShells = forAllDevSystems (pkgs: rec {
diff --git a/hardware/cmsb-001/configuration.nix b/hardware/cmsb-001/configuration.nix
index a4fb9bb..3390cfe 100644
--- a/hardware/cmsb-001/configuration.nix
+++ b/hardware/cmsb-001/configuration.nix
@@ -141,16 +141,16 @@
     };
   };
 
-  #services.cage = {
-  #  enable = true;
-  #  user = "naunet";
-  #  environment = {
-  #    WLR_LIBINPUT_NO_DEVICES = "1";
-  #  };
-  #  program = "${pkgs.camu}/bin/cmv";
-  #};
-  #systemd.services."cage-tty1".serviceConfig.StandardOutput = lib.mkForce "journal+console";
-  #systemd.services."cage-tty1".wantedBy = [];
+  services.cage = {
+    enable = true;
+    user = "naunet";
+    environment = {
+      WLR_LIBINPUT_NO_DEVICES = "1";
+    };
+    program = "${pkgs.camu}/bin/cmv";
+  };
+  systemd.services."cage-tty1".serviceConfig.StandardOutput = lib.mkForce "journal+console";
+  systemd.services."cage-tty1".wantedBy = [];
 
   # https://pablo.tools/blog/computers/nixos-generate-raspberry-images/
   #sdImage = {
@@ -170,15 +170,15 @@
 
   # https://retropie.org.uk/docs/Overclocking/
   # BOOT/config.txt (changes)
-  #  gpu_mem=64
+  #  gpu_mem=128
   #  total_mem=1024
-  #  arm_freq=1300
+  #  arm_freq=1250
   #  gpu_freq=500
-  #  core_freq=500
+  #  core_freq=450
   #  sdram_freq=500
   #  sdram_schmoo=0x02000020
-  #  over_voltage=2
-  #  over_voltage_sdram=2
+  # over_voltage=2
+  # over_voltage_sdram=2
 
   # https://github.com/NixOS/nixos-hardware/issues/631
   hardware.deviceTree = {
diff --git a/src/codec/ffmpeg/decoder.c b/src/codec/ffmpeg/decoder.c
index b502e88..36e59dd 100644
--- a/src/codec/ffmpeg/decoder.c
+++ b/src/codec/ffmpeg/decoder.c
@@ -14,6 +14,7 @@ static s32 get_buffer2(AVCodecContext *context, AVFrame *pic, s32 flags)
     return ret;
 }
 
+#ifdef CAMU_FF_DECODER_HWACCEL
 static s32 init_hwframe_context(struct camu_ff_decoder *av, AVCodecContext *context, AVBufferRef *hw_device_context)
 {
     AVBufferRef *hw_frames_ref;
@@ -52,9 +53,9 @@ static enum AVPixelFormat get_hw_format(AVCodecContext *context, const enum AVPi
         if (*fmt == av->hw_pix_fmt) {
             if (av->use_frames_context) {
                 av->hw_context = NULL;
-                s32 ret = av_hwdevice_ctx_create(&av->hw_context, AV_HWDEVICE_TYPE_VAAPI, NULL, NULL, 0);
+                s32 ret = av_hwdevice_ctx_create(&av->hw_context, av->hw_device_type, NULL, NULL, 0);
                 if (ret < 0 || !av->hw_context) {
-                    al_log_error("ff_decoder", "Failed to initialize VAAPI device (%s).", av_err2str(ret));
+                    al_log_error("ff_decoder", "Failed to initialize hardware device (%s).", av_err2str(ret));
                     break;
                 }
                 init_hwframe_context(av, av->codec_context, av->hw_context);
@@ -67,7 +68,6 @@ static enum AVPixelFormat get_hw_format(AVCodecContext *context, const enum AVPi
 
     return avcodec_default_get_format(context, pix_fmts);
 }
-#endif
 
 static s32 init_hwdevice_context(struct camu_ff_decoder *av, AVCodecContext *context)
 {
@@ -84,6 +84,52 @@ static s32 init_hwdevice_context(struct camu_ff_decoder *av, AVCodecContext *con
     return ret;
 }
 
+static s32 hw_device_priority(enum AVHWDeviceType type)
+{
+    switch (type) {
+    case AV_HWDEVICE_TYPE_MEDIACODEC:
+        return 4;
+    case AV_HWDEVICE_TYPE_VAAPI:
+        return 3;
+    case AV_HWDEVICE_TYPE_VULKAN:
+        return 2;
+    default:
+        return -1;
+    };
+}
+
+static bool hw_device_supported(struct camu_ff_decoder *av, enum AVHWDeviceType type)
+{
+    enum AVHWDeviceType supported_type;
+    al_array_foreach(av->supported_hw_devices, i, supported_type) {
+        if (supported_type == type) return true;
+    }
+    return false;
+}
+
+static enum AVHWDeviceType hw_device_supported_by_name(struct camu_ff_decoder *av, const char *name)
+{
+    enum AVHWDeviceType supported_type;
+    al_array_foreach(av->supported_hw_devices, i, supported_type) {
+        if (strcmp(av_hwdevice_get_type_name(supported_type), name) == 0) {
+            return supported_type;
+        }
+    }
+    return AV_HWDEVICE_TYPE_NONE;
+}
+
+static bool hw_config_needs_frames_ctx(const AVCodecHWConfig *config)
+{
+    if ((config->methods & AV_CODEC_HW_CONFIG_METHOD_HW_FRAMES_CTX) &&
+        !(config->methods & AV_CODEC_HW_CONFIG_METHOD_HW_DEVICE_CTX)) {
+        return true;
+    } else {
+        return false;
+    }
+}
+#endif
+#endif
+
 static void close_internal(struct camu_ff_decoder *av)
 {
     if (av->codec_context) avcodec_free_context(&av->codec_context);
@@ -97,13 +143,16 @@ static bool ff_decoder_init(struct camu_decoder *dec, struct camu_renderer *rend
     av->codec_context = NULL;
 
     AVCodecParameters *codecpar = stream->av.stream->codecpar;
-    const AVCodec *codec = avcodec_find_decoder(codecpar->codec_id);
-#ifdef CAMU_FF_DECODER_HWACCEL
-    if (strcmp(codec->name, "libdav1d") == 0) {
-        codec = avcodec_find_decoder_by_name("av1");
+    void *state = NULL;
+    const AVCodec *codec;
+    al_array_init(av->supported_hw_codecs);
+    while ((codec = av_codec_iterate(&state))) {
+        if (codec->capabilities & (AV_CODEC_CAP_HARDWARE | AV_CODEC_CAP_HYBRID)) {
+            al_array_push(av->supported_hw_codecs, codec);
+        }
     }
-#endif
 
+    codec = avcodec_find_decoder(codecpar->codec_id);
     if (!codec) {
         al_log_error("ff_decoder", "Failed to find decoder.");
         goto err;
@@ -112,53 +161,62 @@ static bool ff_decoder_init(struct camu_decoder *dec, struct camu_renderer *rend
     av->hw_device_type = AV_HWDEVICE_TYPE_NONE;
 #ifdef CAMU_FF_DECODER_HWACCEL
     if (codecpar->codec_type == AVMEDIA_TYPE_VIDEO) {
-#ifdef _WIN32 // TEMP
-        const char *hw_device = "vulkan";
-#else
-        const char *hw_device = "vaapi";
-#endif
-        av->hw_device_type = av_hwdevice_find_type_by_name(hw_device);
-        if (av->hw_device_type == AV_HWDEVICE_TYPE_NONE) {
-            al_log_info("ff_decoder", "Hardware device type \"%s\" not supported.", hw_device);
-            al_log_info("ff_decoder", "Types available on this system:");
-            while ((av->hw_device_type = av_hwdevice_iterate_types(av->hw_device_type)) != AV_HWDEVICE_TYPE_NONE) {
-                al_log_info("ff_decoder", "\t%s", av_hwdevice_get_type_name(av->hw_device_type));
-            }
+        al_array_init(av->supported_hw_devices);
+        enum AVHWDeviceType hw_device_type = AV_HWDEVICE_TYPE_NONE;
+        while ((hw_device_type = av_hwdevice_iterate_types(hw_device_type)) != AV_HWDEVICE_TYPE_NONE) {
+            al_array_push(av->supported_hw_devices, hw_device_type);
         }
-    }
-#endif
 
-    if (av->hw_device_type != AV_HWDEVICE_TYPE_NONE) {
-        av->use_frames_context = false;
+        const AVCodecHWConfig *config = NULL;
         for (s32 i = 0; ; i++) {
-            const AVCodecHWConfig *config = avcodec_get_hw_config(codec, i);
+            config = avcodec_get_hw_config(codec, i);
             if (!config) {
-                if (av->use_frames_context) {
-                    al_log_warn("ff_decoder", "Selected hardware decoder (%s) only supports"
-                        "hardware frames for %s, which is unimplemented.",
-                        av_hwdevice_get_type_name(av->hw_device_type), codec->name);
-                } else {
-                    al_log_warn("ff_decoder", "Selected hardware decoder (%s) does not support %s.",
-                        av_hwdevice_get_type_name(av->hw_device_type), codec->name);
-                }
-                av->hw_device_type = AV_HWDEVICE_TYPE_NONE;
                 break;
             }
 
-            if (config->methods & AV_CODEC_HW_CONFIG_METHOD_HW_FRAMES_CTX) {
+            if (!hw_device_supported(av, config->device_type)) {
+                continue;
+            }
+
+            if (hw_device_priority(config->device_type) > hw_device_priority(av->hw_device_type)) {
+                av->hw_device_type = config->device_type;
                 av->hw_pix_fmt = config->pix_fmt;
-                av->use_frames_context = true;
+                av->use_frames_context = hw_config_needs_frames_ctx(config);
             }
+        }
 
-            if (config->methods & AV_CODEC_HW_CONFIG_METHOD_HW_DEVICE_CTX) {
-                if (config->device_type == av->hw_device_type) {
-                    av->hw_pix_fmt = config->pix_fmt;
-                    av->use_frames_context = false;
-                    break;
+        if (av->hw_device_type == AV_HWDEVICE_TYPE_NONE) {
+            const AVCodec *hw_codec;
+            al_array_foreach(av->supported_hw_codecs, i, hw_codec) {
+                if (hw_codec->id == codecpar->codec_id) {
+                    av->hw_device_type = hw_device_supported_by_name(av, hw_codec->wrapper_name);
+                    if (av->hw_device_type != AV_HWDEVICE_TYPE_NONE) {
+                        for (u32 j = 0; ; j++) {
+                            config = avcodec_get_hw_config(hw_codec, j);
+                            if (!config) {
+                                al_log_warn("ff_decoder", "Hardware codec has no hw_configs.");
+                                av->hw_device_type = AV_HWDEVICE_TYPE_NONE;
+                                break;
+                            }
+                            codec = hw_codec;
+                            av->hw_pix_fmt = config->pix_fmt;
+                            av->use_frames_context = hw_config_needs_frames_ctx(config);
+                            break;
+                        }
+                    }
+                    if (av->hw_device_type != AV_HWDEVICE_TYPE_NONE) {
+                        break;
+                    }
                 }
             }
         }
+
+        if (av->hw_device_type == AV_HWDEVICE_TYPE_NONE) {
+            al_log_warn("ff_decoder",
+                "Hardware accelerated video decoding of %s not supported.", codec->name);
+        }
     }
+#endif
 
     av->codec_context = avcodec_alloc_context3(codec);
     if (!av->codec_context) {
@@ -171,14 +229,23 @@ static bool ff_decoder_init(struct camu_decoder *dec, struct camu_renderer *rend
         goto err;
     }
 
+#ifdef CAMU_FF_DECODER_HWACCEL
+    if ((av->hw_device_type != AV_HWDEVICE_TYPE_NONE && !av->use_frames_context) &&
+        (init_hwdevice_context(av, av->codec_context) < 0)) {
+        av->hw_device_type = AV_HWDEVICE_TYPE_NONE;
+    }
+#endif
+
 #if !defined CAMU_SINK_NO_VIDEO
     if (codecpar->codec_type == AVMEDIA_TYPE_VIDEO && renderer && renderer->get_buffer2) {
         av->renderer = renderer;
         av->codec_context->opaque = av;
         av->codec_context->get_buffer2 = get_buffer2;
+#ifdef CAMU_FF_DECODER_HWACCEL
         if (av->hw_device_type != AV_HWDEVICE_TYPE_NONE) {
             av->codec_context->get_format = get_hw_format;
         }
+#endif
     }
 #else
     (void)renderer;
@@ -196,12 +263,6 @@ static bool ff_decoder_init(struct camu_decoder *dec, struct camu_renderer *rend
         al_log_debug("ff_decoder", "Using %i threads for decoder.", cpus);
     }
 
-    if ((av->hw_device_type != AV_HWDEVICE_TYPE_NONE) &&
-        (init_hwdevice_context(av, av->codec_context) < 0)) {
-        av->hw_pix_fmt = AV_PIX_FMT_NONE;
-        return false;
-    }
-
     if (avcodec_open2(av->codec_context, codec, NULL) < 0) {
         al_log_error("ff_decoder", "Failed to open codec (%s).", codec->name);
         goto err;
diff --git a/src/codec/ffmpeg/decoder.h b/src/codec/ffmpeg/decoder.h
index 827afd2..b1cbf19 100644
--- a/src/codec/ffmpeg/decoder.h
+++ b/src/codec/ffmpeg/decoder.h
@@ -14,9 +14,11 @@ struct camu_ff_decoder {
     struct camu_decoder dec;
     AVCodecContext *codec_context;
     struct camu_renderer *renderer;
+    array(const AVCodec *) supported_hw_codecs;
+    array(enum AVHWDeviceType) supported_hw_devices;
     AVBufferRef *hw_context;
-    enum AVPixelFormat hw_pix_fmt;
     enum AVHWDeviceType hw_device_type;
+    enum AVPixelFormat hw_pix_fmt;
     bool use_frames_context;
     void (*callback)(void *, struct camu_codec_frame *);
     void *userdata;
diff --git a/src/codec/meson.build b/src/codec/meson.build
index 2a72b55..2e89d95 100644
--- a/src/codec/meson.build
+++ b/src/codec/meson.build
@@ -39,6 +39,8 @@ if get_option('codecs').contains('ffmpeg')
       if is_windows
         # Extra static dependencies for Windows.
         ffmpeg_deps += [compiler.find_library('bcrypt')]
+      elif is_android
+        ffmpeg_deps += [compiler.find_library('mediandk')]
       endif
       soxr = compiler.find_library('soxr', required: false)
       if soxr.found()
diff --git a/src/render/renderer_libplacebo.c b/src/render/renderer_libplacebo.c
index a15adfd..e6cfa0c 100644
--- a/src/render/renderer_libplacebo.c
+++ b/src/render/renderer_libplacebo.c
@@ -294,7 +294,7 @@ static void renderer_lp_render(struct camu_renderer *renderer, struct camu_scree
 
     f64 tick = nn_get_tick();
     f64 frame_time = tick - lr->last_render_tick;
-    if (frame_time > 0.020) {
+    if (frame_time > 0.050) {
         al_log_info("render_libplacebo", "FRAME_TIME: %fs", frame_time);
     }
     lr->last_render_tick = tick;
diff --git a/subprojects/packagefiles/glfm/get_egl_internal.diff b/subprojects/packagefiles/glfm/get_egl_internal.diff
new file mode 100644
index 0000000..3c9098c
--- /dev/null
+++ b/subprojects/packagefiles/glfm/get_egl_internal.diff
@@ -0,0 +1,39 @@
+diff --git a/include/glfm.h b/include/glfm.h
+index ceace39..d18711c 100644
+--- a/include/glfm.h
++++ b/include/glfm.h
+@@ -892,6 +892,9 @@ ANativeActivity *glfmAndroidGetActivity(void) GLFM_DEPRECATED("Use glfmGetAndroi
+ /// ``GLFMAppFocusFunc``.
+ void *glfmGetAndroidActivity(const GLFMDisplay *display);
+ 
++void *glfmGetEglDisplay(const GLFMDisplay *display);
++void *glfmGetEglContext(const GLFMDisplay *display);
++
+ #endif // GLFM_EXPOSE_NATIVE_ANDROID
+ 
+ #ifdef __cplusplus
+diff --git a/src/glfm_android.c b/src/glfm_android.c
+index 67d1927..7942ddf 100644
+--- a/src/glfm_android.c
++++ b/src/glfm_android.c
+@@ -2800,4 +2800,20 @@ void *glfmGetAndroidActivity(const GLFMDisplay *display) {
+     return platformData->activity;
+ }
+ 
++void *glfmGetEglDisplay(const GLFMDisplay *display) {
++    if (!display || !display->platformData) {
++        return NULL;
++    }
++    GLFMPlatformData *platformData = display->platformData;
++    return platformData->eglDisplay;
++}
++
++void *glfmGetEglContext(const GLFMDisplay *display) {
++    if (!display || !display->platformData) {
++        return NULL;
++    }
++    GLFMPlatformData *platformData = display->platformData;
++    return platformData->eglContext;
++}
++
+ #endif // __ANDROID__