Server resource unload, many tweaks and fixes

 - Initial liana client preferences.
 - Hook up libplacebo dx11 backend.
 - Make usage of FFmpeg hardware decoding api make some sense.

Signed-off-by: Andrew Opalach <andrew@akon.city>

6 files changed, 156 insertions, 29 deletions

diff --git a/src/render/meson.build b/src/render/meson.build
index 75cde7c..f0fd2fd 100644
--- a/src/render/meson.build
+++ b/src/render/meson.build
@@ -16,13 +16,12 @@ elif get_option('renderer') == 'libplacebo'
     'glslang=disabled',
     'gl-proc-addr=disabled',
     'vk-proc-addr=disabled',
-    'd3d11=disabled',
     'lcms=disabled',
     'dovi=disabled',
     'libdovi=disabled'
   ]
 
-  if get_option('sink-use-vulkan')
+  if get_option('sink-use-vulkan') or get_option('sink-use-dx11')
     shaderc = dependency('shaderc', required: false, allow_fallback: false)
     if shaderc.found()
       render_deps += [shaderc]
@@ -48,9 +47,57 @@ elif get_option('renderer') == 'libplacebo'
         shaderc_proj.dependency('SPIRV-Tools-static')
       ]
     endif
-    libplacebo_opts += ['shaderc=enabled', 'vulkan=enabled', 'opengl=disabled']
+    #glslang = dependency('glslang', required: false, allow_fallback: false)
+    #if glslang.found()
+    #  render_deps += [glslang]
+    #else
+    #  glslang_opts = cmake.subproject_options()
+    #  reltype = is_minsize ? 'MinSizeRel' : 'Release'
+    #  glslang_opts.add_cmake_defines({ 'CMAKE_BUILD_TYPE': is_debug ? 'Debug' : reltype })
+    #  glslang_opts.add_cmake_defines({ 'ENABLE_OPT': '0' })
+    #  glslang_proj = cmake.subproject('glslang', options: glslang_opts)
+    #  message('CMake targets:\n - ' + '\n - '.join(glslang_proj.target_list()))
+    #  glslang = glslang_proj.dependency('glslang')
+    #  spirv = glslang_proj.dependency('SPIRV')
+    #  glslang_default_resource_limits = glslang_proj.dependency('glslang-default-resource-limits')
+    #  meson.override_dependency('glslang', glslang)
+    #  meson.override_dependency('SPIRV', spirv)
+    #  meson.override_dependency('glslang-default-resource-limits', glslang_default_resource_limits)
+    #  render_deps += [
+    #    glslang_proj.dependency('MachineIndependent'),
+    #    glslang_proj.dependency('OSDependent'),
+    #    glslang_proj.dependency('GenericCodeGen'),
+    #    glslang,
+    #    spirv,
+    #    glslang_default_resource_limits
+    #  ]
+    #endif
+  endif
+
+  if get_option('sink-use-vulkan')
+    libplacebo_opts += ['shaderc=enabled', 'vulkan=enabled', 'd3d11=disabled', 'opengl=disabled']
+  elif get_option('sink-use-dx11')
+    spirv_cross_opts = cmake.subproject_options()
+    reltype = is_minsize ? 'MinSizeRel' : 'Release'
+    spirv_cross_opts.add_cmake_defines({ 'CMAKE_BUILD_TYPE': is_debug ? 'Debug' : reltype })
+    spirv_cross_opts.add_cmake_defines({ 'SPIRV_CROSS_CLI': false })
+    spirv_cross_opts.add_cmake_defines({ 'SPIRV_CROSS_ENABLE_TESTS': false })
+    spirv_cross_opts.add_cmake_defines({ 'SPIRV_CROSS_ENABLE_MSL': false })
+    spirv_cross_opts.add_cmake_defines({ 'SPIRV_CROSS_ENABLE_CPP': false })
+    spirv_cross_opts.add_cmake_defines({ 'SPIRV_CROSS_ENABLE_REFLECT': false })
+    spirv_cross_opts.add_cmake_defines({ 'SPIRV_CROSS_ENABLE_UTIL': false })
+    spirv_cross_proj = cmake.subproject('SPIRV-Cross', options: spirv_cross_opts)
+    spirv_cross_c_shared = spirv_cross_proj.dependency('spirv-cross-c')
+    render_deps += [
+      spirv_cross_proj.dependency('spirv-cross-glsl'),
+      spirv_cross_proj.dependency('spirv-cross-hlsl'),
+      spirv_cross_proj.dependency('spirv-cross-core'),
+      spirv_cross_c_shared
+    ]
+    meson.override_dependency('spirv-cross-c-shared', spirv_cross_c_shared)
+    libplacebo_opts += ['shaderc=enabled', 'd3d11=enabled', 'vulkan=enabled', 'opengl=disabled']
   else
-    libplacebo_opts += ['shaderc=disabled', 'opengl=enabled', 'vulkan=disabled']
+    libplacebo_opts += ['shaderc=disabled', 'opengl=enabled', 'vulkan=disabled', 'd3d11=disabled']
   endif
 
   libplacebo = dependency('libplacebo', default_options: libplacebo_opts)
diff --git a/src/render/queue_libplacebo.c b/src/render/queue_libplacebo.c
index 165fc4b..b6a7477 100644
--- a/src/render/queue_libplacebo.c
+++ b/src/render/queue_libplacebo.c
@@ -10,8 +10,8 @@
 
 #include "queue_libplacebo.h"
 
-#ifdef CAMU_HAVE_FFMPEG
-static bool queue_lp_configure_subtitiles(struct camu_frame_queue *queue, u32 width, u32 height, AVCodecParameters *codecpar)
+static bool queue_lp_configure_subtitiles(struct camu_frame_queue *queue, u32 width, u32 height,
+    struct camu_codec_stream *stream)
 {
     struct camu_frame_queue_lp *lq = (struct camu_frame_queue_lp *)queue;
     if (!lq->ass) return false;
@@ -28,12 +28,12 @@ static bool queue_lp_configure_subtitiles(struct camu_frame_queue *queue, u32 wi
     ass_set_frame_size(lq->ass_renderer, width, height);
     ass_set_storage_size(lq->ass_renderer, width, height);
     ass_set_fonts(lq->ass_renderer, NULL, NULL, ASS_FONTPROVIDER_AUTODETECT, NULL, 0);
+    AVCodecParameters *codecpar = stream->av.stream->codecpar;
     ass_process_codec_private(lq->ass_track, (const char *)codecpar->extradata,
         codecpar->extradata_size);
     lq->have_subtitles = true;
     return true;
 }
-#endif
 
 static bool map_frame(pl_gpu gpu, pl_tex *tex, const struct pl_source_frame *src,
     struct pl_frame *out_frame)
@@ -206,15 +206,38 @@ static bool map_av_frame(pl_gpu gpu, pl_tex *tex, const struct pl_source_frame *
     struct camu_frame_queue_lp *lq = (struct camu_frame_queue_lp *)frame->opaque;
     AVStream *stream = lq->q.buf->stream->av.stream;
 
-    bool ok = pl_map_avframe_ex(gpu, out_frame, pl_avframe_params(
-        .frame = frame,
-        .tex = tex,
-        .map_dovi = false
-    ));
+    bool ok = false;
+
+    if (!lq->copy_frame_fallback) {
+        ok = pl_map_avframe_ex(gpu, out_frame, pl_avframe_params(
+            .frame = frame,
+            .tex = tex
+        ));
+    }
+
+    if (!ok) {
+        if (!lq->copy_frame_fallback) {
+            al_log_warn("frame_queue_libplacebo", "Falling back to software copy.");
+            lq->copy_frame_fallback = true;
+            lq->swframe = av_frame_alloc();
+        }
+        ok = av_hwframe_transfer_data(lq->swframe, frame, 0) == 0;
+        if (ok) {
+            av_frame_copy_props(lq->swframe, frame);
+        }
+    }
+
+    if (lq->copy_frame_fallback) {
+        ok = pl_map_avframe_ex(gpu, out_frame, pl_avframe_params(
+            .frame = lq->swframe,
+            .tex = tex
+        ));
+        av_frame_unref(lq->swframe);
+    }
 
     ((struct pl_source_frame *)src)->frame_data = NULL;
 
-    if (lq->have_subtitles) {
+    if (ok && lq->have_subtitles) {
         s64 now = av_rescale_q(frame->best_effort_timestamp, stream->time_base, (AVRational){ 1, 1000 });
         s32 change;
         ASS_Image *ass_frame = ass_render_frame(lq->ass_renderer, lq->ass_track, now, &change);
@@ -289,14 +312,15 @@ static void queue_lp_push_av_frame(struct camu_frame_queue *queue, AVFrame *fram
         .first_field = pl_field_from_avframe(frame)
     });
 }
+#endif
 
-static void queue_lp_push_subtitle(struct camu_frame_queue *queue, AVPacket *pkt)
+static void queue_lp_push_subtitle(struct camu_frame_queue *queue, struct camu_codec_packet *packet)
 {
     struct camu_frame_queue_lp *lq = (struct camu_frame_queue_lp *)queue;
     al_assert(lq->ass_track && lq->ass_renderer);
+    AVPacket *pkt = packet->av.pkt;
     ass_process_chunk(lq->ass_track, (const char *)pkt->data, pkt->size, pkt->pts, pkt->duration);
 }
-#endif
 
 static void queue_lp_flush(struct camu_frame_queue *queue)
 {
@@ -343,6 +367,9 @@ static void queue_lp_free(struct camu_frame_queue **queue)
         ass_free_track(lq->ass_track);
         ass_renderer_done(lq->ass_renderer);
     }
+    if (lq->copy_frame_fallback) {
+        av_frame_free(&lq->swframe);
+    }
     pl_queue_destroy(&lq->queue);
     al_free(lq);
     *queue = NULL;
@@ -353,9 +380,7 @@ struct camu_frame_queue *camu_frame_queue_lp_create(void)
     struct camu_frame_queue_lp *lq = al_alloc_object(struct camu_frame_queue_lp);
     al_array_init(lq->overlays);
     lq->have_subtitles = false;
-#ifdef CAMU_HAVE_FFMPEG
     lq->q.configure_subtitles = queue_lp_configure_subtitiles;
-#endif
     lq->q.push = queue_lp_push;
 #ifdef CAMU_HAVE_FFMPEG
     lq->q.push_av_frame = queue_lp_push_av_frame;
diff --git a/src/render/queue_libplacebo.h b/src/render/queue_libplacebo.h
index 4f5f644..f87a34d 100644
--- a/src/render/queue_libplacebo.h
+++ b/src/render/queue_libplacebo.h
@@ -18,6 +18,8 @@ struct camu_frame_queue_lp {
     struct camu_frame_queue q;
     pl_queue queue;
     struct pl_queue_params params;
+    bool copy_frame_fallback;
+    AVFrame *swframe;
     array(struct camu_overlay_lp *) overlays;
     ASS_Library *ass;
     ASS_Renderer *ass_renderer;
diff --git a/src/render/renderer.h b/src/render/renderer.h
index 101dda0..a4d3a53 100644
--- a/src/render/renderer.h
+++ b/src/render/renderer.h
@@ -13,8 +13,12 @@
 #endif
 #elif defined STELA_API_VULKAN
 #define CAMU_RENDERER_VULKAN
+#elif defined STELA_API_DX11
+#define CAMU_RENDERER_DX11
 #endif
 
+#include "../codec/codec.h"
+
 struct camu_frame_queue;
 struct camu_screen;
 struct camu_renderer {
@@ -24,6 +28,8 @@ struct camu_renderer {
         VKAPI_ATTR PFN_vkVoidFunction VKAPI_CALL (*get_vk_proc_address)(VkInstance, const char *),
         VkResult (*vk_create_surface)(void *, VkInstance, VkSurfaceKHR *),
         const char *const *(*vk_get_extensions)(u32 *),
+#elif defined CAMU_RENDERER_DX11
+        HWND win32_window,
 #elif defined CAMU_RENDERER_OPENGL
 #ifdef STELA_USE_EGL
         EGLDisplay display,
@@ -38,6 +44,7 @@ struct camu_renderer {
 #endif
         void *priv);
     struct camu_frame_queue *(*create_queue)(struct camu_renderer *);
+    void (*add_font)(struct camu_renderer *, struct camu_codec_stream *);
     u32 (*get_latency)(struct camu_renderer *);
     void (*resize)(struct camu_renderer *, u32 *, u32 *);
     void (*render)(struct camu_renderer *, struct camu_screen *, bool);
diff --git a/src/render/renderer_libplacebo.c b/src/render/renderer_libplacebo.c
index 2c632ee..0dcff45 100644
--- a/src/render/renderer_libplacebo.c
+++ b/src/render/renderer_libplacebo.c
@@ -50,6 +50,8 @@ static bool renderer_lp_create_renderer(struct camu_renderer *renderer, u32 *wid
     VKAPI_ATTR PFN_vkVoidFunction VKAPI_CALL (*get_vk_proc_address)(VkInstance, const char *),
     VkResult (*vk_create_surface)(void *, VkInstance, VkSurfaceKHR *),
     const char *const *(*vk_get_extensions)(u32 *),
+#elif defined CAMU_RENDERER_DX11
+    HWND win32_window,
 #elif defined CAMU_RENDERER_OPENGL
 #ifdef STELA_USE_EGL
     EGLDisplay display,
@@ -118,6 +120,28 @@ static bool renderer_lp_create_renderer(struct camu_renderer *renderer, u32 *wid
     }
 
     lr->gpu = lr->vk->gpu;
+
+    al_log_info("render_libplacebo", "Using Vulkan.");
+#elif defined CAMU_RENDERER_DX11
+    lr->d3d11 = pl_d3d11_create(lr->logger, pl_d3d11_params(
+        .debug = false
+    ));
+    if (!lr->d3d11) {
+        al_log_error("render_libplacebo", "Failed to create D3D11 device.");
+        goto err;
+    }
+
+    lr->swapchain = pl_d3d11_create_swapchain(lr->d3d11, pl_d3d11_swapchain_params(
+        .window = win32_window,
+    ));
+    if (!lr->swapchain) {
+        al_log_error("render_libplacebo", "Failed to create D3D11 swapchain.");
+        goto err;
+    }
+
+    lr->gpu = lr->d3d11->gpu;
+
+    al_log_info("render_libplacebo", "Using DirectX 11.");
 #elif defined CAMU_RENDERER_OPENGL
     (void)gl_load_loader;
     lr->gl = pl_opengl_create(lr->logger, pl_opengl_params(
@@ -139,7 +163,7 @@ static bool renderer_lp_create_renderer(struct camu_renderer *renderer, u32 *wid
     }
 
     lr->swapchain = pl_opengl_create_swapchain(lr->gl, pl_opengl_swapchain_params(
-        .max_swapchain_depth = 2,
+        //.max_swapchain_depth = 2,
         .swap_buffers = gl_swap_buffers,
         .priv = priv
     ));
@@ -149,6 +173,8 @@ static bool renderer_lp_create_renderer(struct camu_renderer *renderer, u32 *wid
     }
 
     lr->gpu = lr->gl->gpu;
+
+    al_log_info("render_libplacebo", "Using OpenGL.");
 #endif
 
     lr->renderer = pl_renderer_create(lr->logger, lr->gpu);
@@ -211,6 +237,15 @@ static struct camu_frame_queue *renderer_lp_create_queue(struct camu_renderer *r
     return queue;
 }
 
+static void renderer_lp_add_font(struct camu_renderer *renderer, struct camu_codec_stream *attachment)
+{
+    struct camu_renderer_lp *lr = (struct camu_renderer_lp *)renderer;
+    AVStream *stream = attachment->av.stream;
+    AVDictionaryEntry *entry = av_dict_get(stream->metadata, "filename", NULL, 0);
+    AVCodecParameters *codecpar = stream->codecpar;
+    ass_add_font(lr->ass, entry->value, (const char *)codecpar->extradata, codecpar->extradata_size);
+}
+
 static u32 renderer_lp_get_latency(struct camu_renderer *renderer)
 {
     struct camu_renderer_lp *lr = (struct camu_renderer_lp *)renderer;
@@ -229,7 +264,12 @@ static void renderer_lp_render(struct camu_renderer *renderer, struct camu_scree
 
     if (!lr->have_frame) {
         struct pl_swapchain_frame frame;
-        pl_swapchain_start_frame(lr->swapchain, &frame);
+        if (!pl_swapchain_start_frame(lr->swapchain, &frame)) {
+            if (force) {
+                pl_swapchain_swap_buffers(lr->swapchain);
+            }
+            return;
+        }
         pl_frame_from_swapchain(&lr->target, &frame);
         pl_frame_clear_rgba(lr->gpu, &lr->target, clear_color);
         lr->have_frame = true;
@@ -286,12 +326,13 @@ static void renderer_lp_render(struct camu_renderer *renderer, struct camu_scree
 
     if (!scr->videos.count && !force) {
         // Don't spin too hard on a potential error state.
-        //al_log_warn("render_libplacebo", "Spinning in renderer.");
-        nn_thread_sleep(NNWT_TS_FROM_USEC(128));
+        nn_thread_sleep(NNWT_TS_FROM_USEC(192));
         return;
     }
 
-    pl_swapchain_submit_frame(lr->swapchain);
+    if (!pl_swapchain_submit_frame(lr->swapchain)) {
+        al_log_error("render_libplacebo", "Failed to submit frame, likely fatal.");
+    }
     lr->have_frame = false;
 
     pl_swapchain_swap_buffers(lr->swapchain);
@@ -301,14 +342,14 @@ static void renderer_lp_render(struct camu_renderer *renderer, struct camu_scree
         pl_gpu_finish(lr->gpu);
     }
 
-    if (!force) {
-        f64 tick = nn_get_tick();
-        f64 frame_time = tick - lr->last_render_tick;
-        if (frame_time > (1.0 / 24.0)) {
-            al_log_info("render_libplacebo", "FRAME_TIME: %fs", frame_time);
-        }
-        lr->last_render_tick = tick;
+#if 0
+    f64 tick = nn_get_tick();
+    f64 frame_time = tick - lr->last_render_tick;
+    if (frame_time > (1.0 / 24.0)) { // 24fps is totally arbitrary.
+        al_log_info("render_libplacebo", "FRAME_TIME: %fs", frame_time);
     }
+    lr->last_render_tick = tick;
+#endif
 }
 
 void renderer_lp_free(struct camu_renderer **renderer)
@@ -331,6 +372,7 @@ struct camu_renderer *camu_renderer_lp_create(void)
     struct camu_renderer_lp *lr = al_alloc_object(struct camu_renderer_lp);
     lr->r.create_renderer = renderer_lp_create_renderer;
     lr->r.create_queue = renderer_lp_create_queue;
+    lr->r.add_font = renderer_lp_add_font;
     lr->r.get_latency = renderer_lp_get_latency;
     lr->r.resize = renderer_lp_resize;
     lr->r.render = renderer_lp_render;
diff --git a/src/render/renderer_libplacebo.h b/src/render/renderer_libplacebo.h
index 38407bb..8a284d5 100644
--- a/src/render/renderer_libplacebo.h
+++ b/src/render/renderer_libplacebo.h
@@ -10,6 +10,8 @@
 
 #if defined CAMU_RENDERER_VULKAN
 #include <libplacebo/vulkan.h>
+#elif defined CAMU_RENDERER_DX11
+#include <libplacebo/d3d11.h>
 #elif defined CAMU_RENDERER_OPENGL
 #include <libplacebo/opengl.h>
 #endif
@@ -20,6 +22,8 @@ struct camu_renderer_lp {
     VkSurfaceKHR surface;
     pl_vulkan vk;
     pl_vk_inst vk_inst;
+#elif defined CAMU_RENDERER_DX11
+    pl_d3d11 d3d11;
 #elif defined CAMU_RENDERER_OPENGL
     pl_opengl gl;
 #endif