path: root/src/render/renderer_libplacebo.c

#define AL_LOG_SECTION "renderer_libplacebo"
//#define AL_LOG_ENABLE_TRACE
#include <al/log.h>
#include <al/lib.h>
#include <nnwt/file.h>
#include <nnwt/time.h>

#include "../screen/screen.h"
#include "../util/color_palette.h"

#include "shaders/vr_video.h"

#include "renderer_libplacebo.h"
#include "queue_libplacebo.h"

#ifdef AL_DEBUG
// vkQueueSubmit2: VK_ERROR_VALIDATION_FAILED_EXT leads to an annoying deadlock.
#define RENDERER_DEBUG 0
#else
#define RENDERER_DEBUG 0
#endif

#define CRT_BACKGROUND
#define CRT_BACKGROUND_COLOR (1.f / 7.5f)

#ifdef CRT_BACKGROUND
static f32 clear_color[4] = { CRT_BACKGROUND_COLOR, CRT_BACKGROUND_COLOR, CRT_BACKGROUND_COLOR, 1.f };
#else
static f32 clear_color[4] = { 0.f, 0.f, 0.f, 1.f };
#endif

static struct pl_render_params default_params;

static inline void pl_swapchain_resize_compat(pl_swapchain swapchain, u32 *width, u32 *height)
{
    s32 req_width = (s32)*width;
    s32 req_height = (s32)*height;
    if (pl_swapchain_resize(swapchain, &req_width, &req_height)) {
        al_assert(req_width > 0 && req_height > 0);
        *width = (u32)req_width;
        *height = (u32)req_height;
    }
}

static void renderer_lp_resize(struct camu_renderer *renderer, u32 *width, u32 *height)
{
    struct camu_renderer_lp *lr = (struct camu_renderer_lp *)renderer;
    if (lr->swapchain) {
        if (lr->have_frame) {
            if (pl_swapchain_submit_frame(lr->swapchain)) {
                lr->have_frame = false;
            } else {
                log_error("Failed to submit frame on resize.");
            }
        }
        pl_swapchain_resize_compat(lr->swapchain, width, height);
    }
}

static void renderer_lp_set(struct camu_renderer *renderer, u8 option, u8 value)
{
    struct camu_renderer_lp *lr = (struct camu_renderer_lp *)renderer;
    switch (option) {
    case CAMU_RENDERER_FULLSCREEN:
#ifdef STELA_API_DX11
        IDXGISwapChain *d3d_swapchain = pl_d3d11_swapchain_unwrap(lr->swapchain);
        IDXGISwapChain_SetFullscreenState(d3d_swapchain, value, NULL);
        IDXGISwapChain_Release(d3d_swapchain);
#endif
        break;
    case CAMU_RENDERER_SCALING:
        switch (value) {
        case CAMU_SCALING_DEFAULT:
            lr->params.upscaler = default_params.upscaler;
            lr->params.downscaler = default_params.downscaler;
            lr->params.skip_anti_aliasing = default_params.skip_anti_aliasing;
            break;
        case CAMU_SCALING_NEAREST:
            lr->params.upscaler = &pl_filter_nearest;
            lr->params.downscaler = &pl_filter_nearest;
            lr->params.skip_anti_aliasing = true;
            break;
        }
        break;
    case CAMU_RENDERER_BACKGROUND:
        switch (value) {
        case CAMU_BACKGROUND_DEFAULT:
            clear_color[3] = 1.f;
            lr->params.background = PL_CLEAR_TILES;
            break;
        case CAMU_BACKGROUND_TRANSPARENT:
            clear_color[3] = 0.f;
            lr->params.background = PL_CLEAR_SKIP;
            break;
        }
        break;
    }
}

static void log_callback(void *userdata, enum pl_log_level level, const char *message)
{
    (void)userdata;
    if (level < PL_LOG_INFO) {
        // Console spam of unsupported format kills Windows performance, but the performance
        // will be shot anyway so allow it.
        log_info(message);
    } else {
        log_debug(message);
    }
}

#ifdef CAMU_HAVE_SUBTITLES
// level = 0(FATAL)-7(VERBOSE).
static void ass_log_callback(s32 level, const char *fmt, va_list args, void *userdata)
{
    (void)userdata;
    if (level <= 1) {
        al_logv(AL_LOG_ERROR, "ass", fmt, args);
    } else if (level <= 3) {
        al_logv(AL_LOG_WARN, "ass", fmt, args);
    } else if (level <= 5) {
#ifdef AL_DEBUG
        al_logv(AL_LOG_DEBUG, "ass", fmt, args);
#endif
    } else {
#ifdef AL_LOG_ENABLE_TRACE
        al_logv(AL_LOG_TRACE, "ass", fmt, args);
#endif
    }
}
#endif

static bool renderer_lp_create_renderer(struct camu_renderer *renderer, u32 *width, u32 *height,
#if defined STELA_API_VULKAN
    VKAPI_ATTR PFN_vkVoidFunction VKAPI_CALL (*get_vk_proc_address)(VkInstance, const char *),
    VkResult (*vk_create_surface)(void *, VkInstance, VkSurfaceKHR *),
    VkResult (*vk_destroy_surface)(VkInstance, VkSurfaceKHR),
    const char *const *(*vk_get_extensions)(u32 *),
#elif defined STELA_API_DX11
    HWND win32_window,
#elif defined STELA_API_OPENGL
#ifdef STELA_USE_EGL
    EGLDisplay display,
    EGLContext context,
#endif
    void (*get_gl_proc_address(char const *))(void),
    bool (*gl_load_loader)(void *),
    bool (*gl_make_current)(void *),
    void (*gl_release_current)(void *),
    void (*gl_swap_buffers)(void *),
#endif
    void *priv)
{
    struct camu_renderer_lp *lr = (struct camu_renderer_lp *)renderer;

#ifndef CRT_BACKGROUND
    clear_color[0] = ((global_color_palette.colors[0] >> 16) & 0xff) / 255.f;
    clear_color[1] = ((global_color_palette.colors[0] >> 8) & 0xff) / 255.f;
    clear_color[2] = ((global_color_palette.colors[0]) & 0xff) / 255.f;
#endif

    lr->logger = pl_log_create(PL_API_VER, pl_log_params(
        .log_cb = log_callback,
        .log_priv = NULL,
#ifdef AL_LOG_ENABLE_TRACE
        .log_level = PL_LOG_TRACE
#else
        .log_level = PL_LOG_INFO
#endif
    ));

#if defined STELA_API_VULKAN
    lr->vk_destroy_surface = vk_destroy_surface;

    u32 num;
    const char *const *extensions = vk_get_extensions(&num);
    lr->vk_inst = pl_vk_inst_create(lr->logger, pl_vk_inst_params(
        .debug = RENDERER_DEBUG,
        .get_proc_addr = get_vk_proc_address,
        .extensions = extensions,
        .num_extensions = num
    ));
    if (!lr->vk_inst) {
        log_error("Failed to create vulkan instance.");
        return false;
    }

    VkResult ret = vk_create_surface(priv, lr->vk_inst->instance, &lr->surface);
    if (ret != VK_SUCCESS) {
        log_error("Failed to create vulkan surface.");
        return false;
    }

    lr->vk = pl_vulkan_create(lr->logger, pl_vulkan_params(
        .allow_software = RENDERER_DEBUG,
        .instance = lr->vk_inst->instance,
        .get_proc_addr = lr->vk_inst->get_proc_addr,
        .surface = lr->surface
    ));
    if (!lr->vk) {
        log_error("Failed to create vulkan device.");
        return false;
    }

    lr->swapchain = pl_vulkan_create_swapchain(lr->vk, pl_vulkan_swapchain_params(
        .surface = lr->surface,
        .present_mode = VK_PRESENT_MODE_FIFO_KHR
        //.present_mode = VK_PRESENT_MODE_IMMEDIATE_KHR
    ));
    if (!lr->swapchain) {
        log_error("Failed to create vulkan swapchain.");
        return false;
    }

    lr->gpu = lr->vk->gpu;

    log_info("Using Vulkan.");
#elif defined STELA_API_DX11
    (void)priv;
    lr->d3d11 = pl_d3d11_create(lr->logger, pl_d3d11_params(
        .debug = RENDERER_DEBUG
    ));
    if (!lr->d3d11) {
        log_error("Failed to create D3D11 device.");
        return false;
    }

    // https://learn.microsoft.com/en-us/windows/win32/api/dxgi/ne-dxgi-dxgi_swap_chain_flag
    // https://learn.microsoft.com/en-us/windows/win32/api/dxgi/ne-dxgi-dxgi_swap_effect
    char *FLIP = getenv("SINK_DXGI_FLIP");
    if (!FLIP) FLIP = "0";
    lr->swapchain = pl_d3d11_create_swapchain(lr->d3d11, pl_d3d11_swapchain_params(
        .window = win32_window,
        .blit = al_strscmp(FLIP, "0") == 0,
        //.flags = DXGI_SWAP_CHAIN_FLAG_ALLOW_TEARING
    ));
    if (!lr->swapchain) {
        log_error("Failed to create D3D11 swapchain.");
        return false;
    }

    lr->gpu = lr->d3d11->gpu;

    log_info("Using DirectX 11.");
#elif defined STELA_API_OPENGL
    (void)gl_load_loader;
    lr->gl = pl_opengl_create(lr->logger, pl_opengl_params(
        .debug = RENDERER_DEBUG,
        .allow_software = RENDERER_DEBUG,
        .make_current = gl_make_current,
        .release_current = gl_release_current,
        .get_proc_addr = get_gl_proc_address,
        .get_proc_addr_ex = NULL,
#ifdef STELA_USE_EGL
        .egl_display = display,
        .egl_context = context,
#endif
        .priv = priv
    ));
    if (!lr->gl) {
        log_error("Failed to create GL device.");
        return false;
    }

    // max_swapchain_depth is returned as-is by pl_swapchain_latency().
    // Combined with eglSwapInterval(1), I don't think that's the value we're looking for.
    lr->swapchain = pl_opengl_create_swapchain(lr->gl, pl_opengl_swapchain_params(
        //.max_swapchain_depth = 1,
        .swap_buffers = gl_swap_buffers,
        .priv = priv
    ));
    if (!lr->swapchain) {
        log_error("Failed to create GL swapchain.");
        return false;
    }

    lr->gpu = lr->gl->gpu;

    log_info("Using OpenGL.");
#endif

    lr->renderer = pl_renderer_create(lr->logger, lr->gpu);

    pl_swapchain_resize_compat(lr->swapchain, width, height);
    lr->have_frame = false;

    //default_params = pl_render_high_quality_params;
    default_params = pl_render_default_params;
    //default_params = pl_render_fast_params;

    al_memset(&lr->params, 0, sizeof(struct pl_render_params));
    lr->params = default_params;
    lr->params.deband_params = NULL;
    //lr->params.frame_mixer = NULL;

    // Clear manually so we can draw multiple images per frame.
    lr->params.border = PL_CLEAR_SKIP;
    lr->params.background = PL_CLEAR_TILES;
    lr->params.blend_params = &pl_alpha_overlay;

    // Prioritize a more consistent image.
    lr->params.correct_subpixel_offsets = true;

    //lr->params.distort_params = pl_distort_params(.unscaled = false);
    lr->vr_emulation_hook[0] = pl_mpv_user_shader_parse(lr->gpu, vr_video_shader, sizeof(vr_video_shader) - 1);
    /*
    struct nn_file file;
    if (nn_file_open(&file, &al_str_c(""), 0)) {
        char *c_str;
        size_t length = nn_file_read_as_c_str(&file, &c_str);
        const struct pl_hook *hook = pl_mpv_user_shader_parse(lr->gpu, (const char *)c_str, length);
    }
    */

#ifdef CAMU_HAVE_FFMPEG
    lr->r.get_buffer2 = pl_get_buffer2;
    lr->r.opaque = &lr->gpu;
#endif

#ifdef CAMU_HAVE_SUBTITLES
    lr->ass = ass_library_init();
    if (!lr->ass) {
        log_error("Failed to initialize libass.");
        return false;
    }
    ass_set_message_cb(lr->ass, ass_log_callback, lr);
#endif

    lr->last_swap_tick = nn_get_tick();

    return true;
}

static struct camu_frame_queue *renderer_lp_create_queue(struct camu_renderer *renderer)
{
    struct camu_renderer_lp *lr = (struct camu_renderer_lp *)renderer;
    struct camu_frame_queue *queue = camu_frame_queue_lp_create();
    struct camu_frame_queue_lp *lq = (struct camu_frame_queue_lp *)queue;
    lq->gpu = lr->gpu;
    lq->queue = pl_queue_create(lr->gpu);
    al_memset(&lq->params, 0, sizeof(struct pl_queue_params));
    lq->params.interpolation_threshold = 0.01f;
    lq->params.radius = pl_frame_mix_radius(&lr->params);
    lq->params.timeout = 0;
#ifdef CAMU_HAVE_FFMPEG
    lq->copy_frame_fallback = false;
#endif
#ifdef CAMU_HAVE_SUBTITLES
    lq->subs.ass = lr->ass;
#endif
    return queue;
}

static void renderer_lp_add_font(struct camu_renderer *renderer, struct camu_codec_stream *attachment)
{
    struct camu_renderer_lp *lr = (struct camu_renderer_lp *)renderer;
    // @TODO: No FFmpeg + HAVE_SUBTITLES (no usecase yet).
#ifdef CAMU_HAVE_SUBTITLES
    AVStream *stream = attachment->av.stream;
    AVDictionaryEntry *entry = av_dict_get(stream->metadata, "filename", NULL, 0);
    AVCodecParameters *codecpar = stream->codecpar;
    ass_add_font(lr->ass, entry->value, (const char *)codecpar->extradata, codecpar->extradata_size);
#else
    (void)lr;
    (void)attachment;
#endif
}

static u32 renderer_lp_get_latency(struct camu_renderer *renderer)
{
    struct camu_renderer_lp *lr = (struct camu_renderer_lp *)renderer;
    return (u32)pl_swapchain_latency(lr->swapchain);
}

// This is a completely broken "hash" of the f64 pts value, but it works for now.
// https://www.virtualdub.org/blog2/entry_259.html
static inline intptr_t float_64_hash(f64 value)
{
    union { f64 f; u64 u; } fv = { .f = value };
    intptr_t hash = (intptr_t)fv.u;
#ifdef AL_WE_32BIT
    hash += (intptr_t)(fv.u >> 32);
#endif
    return hash;
}

static bool renderer_lp_render(struct camu_renderer *renderer, struct camu_screen *scr, bool force)
{
    struct camu_renderer_lp *lr = (struct camu_renderer_lp *)renderer;

#ifdef CAMU_SCREEN_THREADED
    if (atomic_load(bool)(&scr->queued, AL_ATOMIC_RELAXED)) {
        camu_screen_run_queue(scr);
    }
#endif

    if (!lr->have_frame) {
        struct pl_swapchain_frame frame;
        if (pl_swapchain_start_frame(lr->swapchain, &frame)) {
            pl_frame_from_swapchain(&lr->target, &frame);
            lr->have_frame = true;
#ifdef CRT_BACKGROUND
            lr->frame_cleared = false;
#else
            pl_frame_clear_rgba(lr->gpu, &lr->target, clear_color);
            lr->frame_cleared = true;
#endif
        } else {
            log_error("Failed to start frame, fatal.");
            return false;
        }
    }

    f64 mouse_x, mouse_y;
    bool vr_emulation = scr->vr_emulation && lr->vr_emulation_hook[0];
    if (vr_emulation) {
        mouse_x = (scr->last_pointer_x / scr->width) - (scr->vr_calibrate_x - 0.5);
        mouse_y = (scr->last_pointer_y / scr->height) - (scr->vr_calibrate_y - 0.5);
        if (scr->vr_emulation == CAMU_VR_360) {
            mouse_x = CLAMP(mouse_x * 2.0, -1.0, 3.0);
        } else if (scr->vr_left_eye) {
            mouse_x = CLAMP(mouse_x, -0.5, 1.5);
        } else {
            mouse_x = CLAMP(1.0 + mouse_x, -1.5, 2.5);
        }
        mouse_y = CLAMP(1.0 - mouse_y, -0.5, 1.5);
        lr->params.hooks = lr->vr_emulation_hook;
        lr->params.num_hooks = 1;
        lr->params.hooks[0]->parameters[0].data->f = (f32)mouse_x;
        lr->params.hooks[0]->parameters[1].data->f = (f32)mouse_y;
        lr->params.hooks[0]->parameters[3].data->f = (scr->vr_emulation == CAMU_VR_360) ? 2.0 : 1.0 - (f32)scr->vr_left_eye;
    }

    enum {
        RESULT_SUBMIT = 1,
        RESULT_SUBMIT_AND_WAIT = 1 << 1
    };

    u8 result = 0;
    struct pl_frame *target = &lr->target;
    struct pl_frame_mix mix;
    struct camu_screen_video *video;
    while (scr->videos.count > 0) {
        bool any_eof = false;
        al_array_foreach_ptr(scr->videos, i, video) {
            bool weighted = false;
            if (camu_video_buffer_read(video->buf, &mix, &weighted)) {
                // If mix.frames is NULL, read() returned QUEUE_MORE.
                if (mix.frames) {
                    // weighted is only set when read() returns QUEUE_OK.
                    result |= (1 | 1 << weighted);
                    // Terrible hack. Lets us distinguish static video buffers with the same dimensions.
                    // Tied to a libplacebo patch to consider info_priv in the hash.
                    intptr_t hash = (intptr_t)video->buf;
                    // Also add in seek_pts to account for frame queue resets (seek() while paused).
                    hash += float_64_hash(video->buf->seek_pts);
                    if (vr_emulation) {
                        hash += float_64_hash(mouse_x);
                        hash += float_64_hash(mouse_y);
                        hash += float_64_hash(video->view.fov);
                        lr->params.hooks[0]->parameters[2].data->f = (f32)video->view.fov;
                    }
#ifdef CAMU_HAVE_SUBTITLES
                    for (u32 j = 0; j < (u32)mix.num_frames; j++) {
                        struct pl_frame *mix_frame = (struct pl_frame *)mix.frames[j];
                        if (!mix_frame->overlays) continue;
                        struct camu_overlay *overlay = CAMU_OVERLAY_FROM_HEAD(mix_frame->overlays);
                        if (!overlay->num) continue;
                        if (scr->subtitles_enabled) {
                            mix_frame->num_overlays = overlay->num;
                            hash += overlay->num;
                        } else {
                            mix_frame->num_overlays = 0;
                        }
                    }
#endif
                    lr->params.info_priv = (void *)hash;
                    target->crop = mix.frames[0]->crop;
                    target->crop.x1 *= video->view.zoom / video->view.stretch;
                    target->crop.y1 *= video->view.zoom * video->view.stretch;
                    target->crop.x0 += video->view.x_offset;
                    target->crop.y0 += video->view.y_offset;
                    target->crop.x1 += video->view.x_offset;
                    target->crop.y1 += video->view.y_offset;
                    target->rotation = video->view.rotation;
                    /*
                    //lr->params.color_map_params = pl_color_map_params(.inverse_tone_mapping = true);
                    if () {
                        target->color = pl_color_space_srgb;
                        target->color.hdr.max_luma = PL_COLOR_SDR_WHITE - 50.f;
                        target->color.hdr.min_luma = (PL_COLOR_SDR_WHITE - 50.f) / PL_COLOR_SDR_CONTRAST;
                    }
                    */
#ifdef CRT_BACKGROUND
                    if (!lr->frame_cleared) {
                        clear_color[0] = 0.03f;
                        clear_color[1] = 0.03f;
                        clear_color[2] = 0.03f;
                        pl_frame_clear_rgba(lr->gpu, &lr->target, clear_color);
                        lr->frame_cleared = true;
                    }
#endif
                    pl_render_image_mix(lr->renderer, &mix, target, &lr->params);
                }
            } else {
                any_eof = true;
            }
        }
        if (any_eof) {
#ifdef CAMU_SCREEN_THREADED
            if (atomic_load(bool)(&scr->queued, AL_ATOMIC_RELAXED)) {
                camu_screen_run_queue(scr);
            }
#endif
        } else {
            break;
        }
    }

    lr->params.hooks = NULL;
    lr->params.num_hooks = 0;

    // Considerations about the result of the loop above.
    //  1. Any given call to video_buffer_read() may not produce a frame.
    //  2. If any buffer signals EOF, by the time run_queue() is called at the bottom of the loop,
    //     all previous buffers could have been queued for removal. Meaning scr->videos would be
    //     empty while there could still be data we want to display rendered to the frame.
    if (!(result & RESULT_SUBMIT) && !force) {
        nn_thread_sleep(NNWT_TS_FROM_USEC(256));
        return true;
    }

#ifdef CRT_BACKGROUND
    if (!lr->frame_cleared) {
        clear_color[0] = CRT_BACKGROUND_COLOR;
        clear_color[1] = CRT_BACKGROUND_COLOR;
        clear_color[2] = CRT_BACKGROUND_COLOR;
        pl_frame_clear_rgba(lr->gpu, &lr->target, clear_color);
        lr->frame_cleared = true;
    }
#endif

    if (pl_swapchain_submit_frame(lr->swapchain)) {
        lr->have_frame = false;
    } else {
        log_error("Failed to submit frame, possibly fatal.");
        return false;
    }

    pl_swapchain_swap_buffers(lr->swapchain);

    f64 tick = nn_get_tick();
    // This is mainly to log when render() is being called, not to be an accurate measurement.
    log_trace("swap_buffers(), (interval: %f).", tick - lr->last_swap_tick);
    lr->last_swap_tick = tick;

    if (result & RESULT_SUBMIT_AND_WAIT) {
        pl_gpu_finish(lr->gpu);
    }

    return true;
}

void renderer_lp_free(struct camu_renderer **renderer)
{
    struct camu_renderer_lp *lr = (struct camu_renderer_lp *)*renderer;
    if (lr->renderer) pl_renderer_destroy(&lr->renderer);
    if (lr->swapchain) pl_swapchain_destroy(&lr->swapchain);
#if defined STELA_API_VULKAN
    if (lr->vk) pl_vulkan_destroy(&lr->vk);
    if (lr->vk_inst) {
        lr->vk_destroy_surface(lr->vk_inst->instance, lr->surface);
        pl_vk_inst_destroy(&lr->vk_inst);
    }
#elif defined STELA_API_DX11
    if (lr->d3d11) pl_d3d11_destroy(&lr->d3d11);
#elif defined STELA_API_OPENGL
    if (lr->gl) pl_opengl_destroy(&lr->gl);
#endif
    if (lr->logger) pl_log_destroy(&lr->logger);
    if (lr->vr_emulation_hook[0]) pl_mpv_user_shader_destroy(&lr->vr_emulation_hook[0]);
#ifdef CAMU_HAVE_SUBTITLES
    if (lr->ass) ass_library_done(lr->ass);
#endif
    al_free(lr);
    *renderer = NULL;
}

struct camu_renderer *camu_renderer_libplacebo_create(void)
{
    struct camu_renderer_lp *lr = al_alloc_object(struct camu_renderer_lp);
    lr->r.create_renderer = renderer_lp_create_renderer;
    lr->r.create_queue = renderer_lp_create_queue;
    lr->r.add_font = renderer_lp_add_font;
    lr->r.get_latency = renderer_lp_get_latency;
    lr->r.resize = renderer_lp_resize;
    lr->r.set = renderer_lp_set;
    lr->r.render = renderer_lp_render;
    lr->r.free = renderer_lp_free;
    return (struct camu_renderer *)lr;
}