#define AL_LOG_SECTION "renderer_libplacebo" //#define AL_LOG_ENABLE_TRACE #include #include #include #include #include "../screen/screen.h" #include "../util/color_palette.h" #include "shaders/vr_video.h" #include "renderer_libplacebo.h" #include "queue_libplacebo.h" #ifdef AL_DEBUG // vkQueueSubmit2: VK_ERROR_VALIDATION_FAILED_EXT leads to an annoying deadlock. #define RENDERER_DEBUG 0 #else #define RENDERER_DEBUG 0 #endif static f32 clear_color[4] = { 0.f, 0.f, 0.f, 1.f }; static struct pl_render_params default_params; static inline void pl_swapchain_resize_compat(pl_swapchain swapchain, u32 *width, u32 *height) { s32 req_width = (s32)*width; s32 req_height = (s32)*height; if (pl_swapchain_resize(swapchain, &req_width, &req_height)) { al_assert(req_width > 0 && req_height > 0); *width = (u32)req_width; *height = (u32)req_height; } } static void renderer_lp_resize(struct camu_renderer *renderer, u32 *width, u32 *height) { struct camu_renderer_lp *lr = (struct camu_renderer_lp *)renderer; if (lr->swapchain) { if (lr->have_frame) { if (pl_swapchain_submit_frame(lr->swapchain)) { lr->have_frame = false; } else { log_error("Failed to submit frame on resize."); } } pl_swapchain_resize_compat(lr->swapchain, width, height); } } static void renderer_lp_set(struct camu_renderer *renderer, u8 option, u8 value) { struct camu_renderer_lp *lr = (struct camu_renderer_lp *)renderer; switch (option) { case CAMU_RENDERER_FULLSCREEN: #ifdef STELA_API_DX11 IDXGISwapChain *d3d_swapchain = pl_d3d11_swapchain_unwrap(lr->swapchain); IDXGISwapChain_SetFullscreenState(d3d_swapchain, value, NULL); IDXGISwapChain_Release(d3d_swapchain); #endif break; case CAMU_RENDERER_SCALING: switch (value) { case CAMU_SCALING_DEFAULT: lr->params.upscaler = default_params.upscaler; lr->params.downscaler = default_params.downscaler; lr->params.skip_anti_aliasing = default_params.skip_anti_aliasing; break; case CAMU_SCALING_NEAREST: lr->params.upscaler = &pl_filter_nearest; lr->params.downscaler = &pl_filter_nearest; lr->params.skip_anti_aliasing = true; break; } break; case CAMU_RENDERER_BACKGROUND: switch (value) { case CAMU_BACKGROUND_DEFAULT: clear_color[3] = 1.f; lr->params.background = PL_CLEAR_TILES; break; case CAMU_BACKGROUND_TRANSPARENT: clear_color[3] = 0.f; lr->params.background = PL_CLEAR_SKIP; break; } break; } } static void log_callback(void *userdata, enum pl_log_level level, const char *message) { (void)userdata; if (level < PL_LOG_INFO) { // Console spam of unsupported format kills Windows performance, but the performance // will be shot anyway so allow it. log_info(message); } else { log_debug(message); } } #ifdef CAMU_HAVE_SUBTITLES // level = 0(FATAL)-7(VERBOSE). static void ass_log_callback(s32 level, const char *fmt, va_list args, void *userdata) { (void)userdata; if (level <= 1) { al_logv(AL_LOG_ERROR, "ass", fmt, args); } else if (level <= 3) { al_logv(AL_LOG_WARN, "ass", fmt, args); } else if (level <= 5) { #ifdef AL_DEBUG al_logv(AL_LOG_DEBUG, "ass", fmt, args); #endif } else { #ifdef AL_LOG_ENABLE_TRACE al_logv(AL_LOG_TRACE, "ass", fmt, args); #endif } } #endif static bool renderer_lp_create_renderer(struct camu_renderer *renderer, u32 *width, u32 *height, #if defined STELA_API_VULKAN VKAPI_ATTR PFN_vkVoidFunction VKAPI_CALL (*get_vk_proc_address)(VkInstance, const char *), VkResult (*vk_create_surface)(void *, VkInstance, VkSurfaceKHR *), VkResult (*vk_destroy_surface)(VkInstance, VkSurfaceKHR), const char *const *(*vk_get_extensions)(u32 *), #elif defined STELA_API_DX11 HWND win32_window, #elif defined STELA_API_OPENGL #ifdef STELA_USE_EGL EGLDisplay display, EGLContext context, #endif void (*get_gl_proc_address(char const *))(void), bool (*gl_load_loader)(void *), bool (*gl_make_current)(void *), void (*gl_release_current)(void *), void (*gl_swap_buffers)(void *), #endif void *priv) { struct camu_renderer_lp *lr = (struct camu_renderer_lp *)renderer; clear_color[0] = ((global_color_palette.colors[0] >> 16) & 0xff) / 255.f; clear_color[1] = ((global_color_palette.colors[0] >> 8) & 0xff) / 255.f; clear_color[2] = ((global_color_palette.colors[0]) & 0xff) / 255.f; lr->logger = pl_log_create(PL_API_VER, pl_log_params( .log_cb = log_callback, .log_priv = NULL, #ifdef AL_LOG_ENABLE_TRACE .log_level = PL_LOG_TRACE #else .log_level = PL_LOG_INFO #endif )); #if defined STELA_API_VULKAN lr->vk_destroy_surface = vk_destroy_surface; u32 num; const char *const *extensions = vk_get_extensions(&num); lr->vk_inst = pl_vk_inst_create(lr->logger, pl_vk_inst_params( .debug = RENDERER_DEBUG, .get_proc_addr = get_vk_proc_address, .extensions = extensions, .num_extensions = num )); if (!lr->vk_inst) { log_error("Failed to create vulkan instance."); goto err; } VkResult ret = vk_create_surface(priv, lr->vk_inst->instance, &lr->surface); if (ret != VK_SUCCESS) { log_error("Failed to create vulkan surface."); goto err; } lr->vk = pl_vulkan_create(lr->logger, pl_vulkan_params( .allow_software = RENDERER_DEBUG, .instance = lr->vk_inst->instance, .get_proc_addr = lr->vk_inst->get_proc_addr, .surface = lr->surface )); if (!lr->vk) { log_error("Failed to create vulkan device."); goto err; } lr->swapchain = pl_vulkan_create_swapchain(lr->vk, pl_vulkan_swapchain_params( .surface = lr->surface, .present_mode = VK_PRESENT_MODE_FIFO_KHR //.present_mode = VK_PRESENT_MODE_IMMEDIATE_KHR )); if (!lr->swapchain) { log_error("Failed to create vulkan swapchain."); goto err; } lr->gpu = lr->vk->gpu; log_info("Using Vulkan."); #elif defined STELA_API_DX11 (void)priv; lr->d3d11 = pl_d3d11_create(lr->logger, pl_d3d11_params( .debug = RENDERER_DEBUG )); if (!lr->d3d11) { log_error("Failed to create D3D11 device."); goto err; } // https://learn.microsoft.com/en-us/windows/win32/api/dxgi/ne-dxgi-dxgi_swap_chain_flag lr->swapchain = pl_d3d11_create_swapchain(lr->d3d11, pl_d3d11_swapchain_params( .window = win32_window, //.blit = true, //.flags = DXGI_SWAP_CHAIN_FLAG_ALLOW_TEARING )); if (!lr->swapchain) { log_error("Failed to create D3D11 swapchain."); goto err; } lr->gpu = lr->d3d11->gpu; log_info("Using DirectX 11."); #elif defined STELA_API_OPENGL (void)gl_load_loader; lr->gl = pl_opengl_create(lr->logger, pl_opengl_params( .debug = RENDERER_DEBUG, .allow_software = RENDERER_DEBUG, .make_current = gl_make_current, .release_current = gl_release_current, .get_proc_addr = get_gl_proc_address, .get_proc_addr_ex = NULL, #ifdef STELA_USE_EGL .egl_display = display, .egl_context = context, #endif .priv = priv )); if (!lr->gl) { log_error("Failed to create GL device."); goto err; } // max_swapchain_depth is returned as-is by pl_swapchain_latency(). // Combined with eglSwapInterval(1), I don't think that's the value we're looking for. lr->swapchain = pl_opengl_create_swapchain(lr->gl, pl_opengl_swapchain_params( //.max_swapchain_depth = 1, .swap_buffers = gl_swap_buffers, .priv = priv )); if (!lr->swapchain) { log_error("Failed to create GL swapchain."); goto err; } lr->gpu = lr->gl->gpu; log_info("Using OpenGL."); #endif lr->renderer = pl_renderer_create(lr->logger, lr->gpu); pl_swapchain_resize_compat(lr->swapchain, width, height); lr->have_frame = false; //default_params = pl_render_high_quality_params; default_params = pl_render_default_params; //default_params = pl_render_fast_params; al_memset(&lr->params, 0, sizeof(struct pl_render_params)); lr->params = default_params; lr->params.deband_params = NULL; //lr->params.frame_mixer = NULL; // Clear manually so we can draw multiple images per frame. lr->params.border = PL_CLEAR_SKIP; lr->params.background = PL_CLEAR_TILES; lr->params.blend_params = &pl_alpha_overlay; // Prioritize a more consistent image. lr->params.correct_subpixel_offsets = true; //lr->params.distort_params = pl_distort_params(.unscaled = false); lr->vr_emulation_hook[0] = pl_mpv_user_shader_parse(lr->gpu, vr_video_shader, sizeof(vr_video_shader) - 1); /* struct nn_file file; if (nn_file_open(&file, &al_str_c(""), 0)) { char *c_str; size_t length = nn_file_read_as_c_str(&file, &c_str); const struct pl_hook *hook = pl_mpv_user_shader_parse(lr->gpu, (const char *)c_str, length); } */ #ifdef CAMU_HAVE_FFMPEG lr->r.get_buffer2 = pl_get_buffer2; lr->r.opaque = &lr->gpu; #endif #ifdef CAMU_HAVE_SUBTITLES lr->ass = ass_library_init(); ass_set_message_cb(lr->ass, ass_log_callback, lr); #endif lr->last_swap_tick = nn_get_tick(); return true; err: lr->r.free((struct camu_renderer **)&lr); return false; } static struct camu_frame_queue *renderer_lp_create_queue(struct camu_renderer *renderer) { struct camu_renderer_lp *lr = (struct camu_renderer_lp *)renderer; struct camu_frame_queue *queue = camu_frame_queue_lp_create(); struct camu_frame_queue_lp *lq = (struct camu_frame_queue_lp *)queue; lq->gpu = lr->gpu; lq->queue = pl_queue_create(lr->gpu); al_memset(&lq->params, 0, sizeof(struct pl_queue_params)); lq->params.interpolation_threshold = 0.01f; lq->params.radius = pl_frame_mix_radius(&lr->params); lq->params.timeout = 0; #ifdef CAMU_HAVE_FFMPEG lq->copy_frame_fallback = false; #endif #ifdef CAMU_HAVE_SUBTITLES lq->ass = lr->ass; #endif return queue; } static void renderer_lp_add_font(struct camu_renderer *renderer, struct camu_codec_stream *attachment) { struct camu_renderer_lp *lr = (struct camu_renderer_lp *)renderer; // @TODO: No FFmpeg + HAVE_SUBTITLES (no usecase yet). #ifdef CAMU_HAVE_SUBTITLES AVStream *stream = attachment->av.stream; AVDictionaryEntry *entry = av_dict_get(stream->metadata, "filename", NULL, 0); AVCodecParameters *codecpar = stream->codecpar; ass_add_font(lr->ass, entry->value, (const char *)codecpar->extradata, codecpar->extradata_size); #else (void)lr; (void)attachment; #endif } static u32 renderer_lp_get_latency(struct camu_renderer *renderer) { struct camu_renderer_lp *lr = (struct camu_renderer_lp *)renderer; return (u32)pl_swapchain_latency(lr->swapchain); } // This is a completely broken "hash" of the f64 pts value, but it works for now. // https://www.virtualdub.org/blog2/entry_259.html static inline intptr_t float_64_hash(f64 value) { union { f64 f; u64 u; } fv = { .f = value }; intptr_t hash = (intptr_t)fv.u; #ifdef AL_WE_32BIT hash += (intptr_t)(fv.u >> 32); #endif return hash; } static bool renderer_lp_render(struct camu_renderer *renderer, struct camu_screen *scr, bool force) { struct camu_renderer_lp *lr = (struct camu_renderer_lp *)renderer; #ifdef CAMU_SCREEN_THREADED if (atomic_load(bool)(&scr->queued, AL_ATOMIC_RELAXED)) { camu_screen_run_queue(scr); } #endif if (!lr->have_frame) { struct pl_swapchain_frame frame; if (pl_swapchain_start_frame(lr->swapchain, &frame)) { pl_frame_from_swapchain(&lr->target, &frame); pl_frame_clear_rgba(lr->gpu, &lr->target, clear_color); lr->have_frame = true; } else { log_error("Failed to start frame, fatal."); return false; } } f64 mouse_x, mouse_y; bool vr_emulation = scr->vr_emulation && lr->vr_emulation_hook[0]; if (vr_emulation) { mouse_x = (scr->last_pointer_x / scr->width) - (scr->vr_calibrate_x - 0.5); mouse_y = (scr->last_pointer_y / scr->height) - (scr->vr_calibrate_y - 0.5); if (scr->vr_emulation == CAMU_VR_360) { mouse_x = CLAMP(mouse_x * 2.0, -1.0, 3.0); } else if (scr->vr_left_eye) { mouse_x = CLAMP(mouse_x, -0.5, 1.5); } else { mouse_x = CLAMP(1.0 + mouse_x, -1.5, 2.5); } mouse_y = CLAMP(1.0 - mouse_y, -0.5, 1.5); lr->params.hooks = lr->vr_emulation_hook; lr->params.num_hooks = 1; lr->params.hooks[0]->parameters[0].data->f = (f32)mouse_x; lr->params.hooks[0]->parameters[1].data->f = (f32)mouse_y; lr->params.hooks[0]->parameters[3].data->f = (scr->vr_emulation == CAMU_VR_360) ? 2.0 : 1.0 - (f32)scr->vr_left_eye; } enum { RESULT_SUBMIT = 1, RESULT_SUBMIT_AND_WAIT = 1 << 1 }; u8 result = 0; struct pl_frame *target = &lr->target; struct pl_frame_mix mix; struct camu_screen_video *video; while (scr->videos.count > 0) { bool any_eof = false; al_array_foreach_ptr(scr->videos, i, video) { bool weighted = false; if (camu_video_buffer_read(video->buf, &mix, &weighted)) { // If mix.frames is NULL, read() returned QUEUE_MORE. if (mix.frames) { // weighted is only set when read() returns QUEUE_OK. result |= (1 | 1 << weighted); // Terrible hack. Lets us distinguish single frames with the same dimensions. // Tied to a libplacebo patch to consider info_priv in the hash. intptr_t hash = (intptr_t)video->buf; // Also add in seek_pts to account for frame queue resets (seek() while paused). hash += float_64_hash(video->buf->seek_pts); if (vr_emulation) { hash += float_64_hash(mouse_x); hash += float_64_hash(mouse_y); hash += float_64_hash(video->view.fov); lr->params.hooks[0]->parameters[2].data->f = (f32)video->view.fov; } #ifdef CAMU_HAVE_SUBTITLES for (u32 j = 0; j < (u32)mix.num_frames; j++) { struct pl_frame *mix_frame = (struct pl_frame *)mix.frames[j]; if (!mix_frame->overlays) continue; struct camu_overlay *overlay = CAMU_OVERLAY_FROM_HEAD(mix_frame->overlays); if (!overlay->num) continue; if (scr->subtitles_enabled) { mix_frame->num_overlays = overlay->num; hash += overlay->num; } else { mix_frame->num_overlays = 0; } } #endif lr->params.info_priv = (void *)hash; target->crop = mix.frames[0]->crop; target->crop.x1 *= video->view.zoom / video->view.stretch; target->crop.y1 *= video->view.zoom * video->view.stretch; target->crop.x0 += video->view.x_offset; target->crop.y0 += video->view.y_offset; target->crop.x1 += video->view.x_offset; target->crop.y1 += video->view.y_offset; target->rotation = video->view.rotation; /* //lr->params.color_map_params = pl_color_map_params(.inverse_tone_mapping = true); if () { target->color = pl_color_space_srgb; target->color.hdr.max_luma = PL_COLOR_SDR_WHITE - 50.f; target->color.hdr.min_luma = (PL_COLOR_SDR_WHITE - 50.f) / PL_COLOR_SDR_CONTRAST; } */ pl_render_image_mix(lr->renderer, &mix, target, &lr->params); } } else { any_eof = true; } } if (any_eof) { #ifdef CAMU_SCREEN_THREADED if (atomic_load(bool)(&scr->queued, AL_ATOMIC_RELAXED)) { camu_screen_run_queue(scr); } #endif } else { break; } } lr->params.hooks = NULL; lr->params.num_hooks = 0; // Considerations about the result of the loop above. // 1. Any given call to video_buffer_read() may not produce a frame. // 2. If any buffer signals EOF, by the time run_queue() is called at the bottom of the loop, // all previous buffers could have been queued for removal. Meaning scr->videos would be // empty while there could still be data we want to display rendered to the frame. if (!(result & RESULT_SUBMIT) && !force) { nn_thread_sleep(NNWT_TS_FROM_USEC(256)); return true; } if (pl_swapchain_submit_frame(lr->swapchain)) { lr->have_frame = false; } else { log_error("Failed to submit frame, possibly fatal."); return false; } pl_swapchain_swap_buffers(lr->swapchain); f64 tick = nn_get_tick(); // This is mainly to log when render() is being called, not to be an accurate measurement. log_trace("swap_buffers(), (interval: %f).", tick - lr->last_swap_tick); lr->last_swap_tick = tick; if (result & RESULT_SUBMIT_AND_WAIT) { pl_gpu_finish(lr->gpu); } return true; } void renderer_lp_free(struct camu_renderer **renderer) { struct camu_renderer_lp *lr = (struct camu_renderer_lp *)*renderer; if (lr->renderer) pl_renderer_destroy(&lr->renderer); if (lr->swapchain) pl_swapchain_destroy(&lr->swapchain); #if defined STELA_API_VULKAN if (lr->vk) pl_vulkan_destroy(&lr->vk); if (lr->vk_inst) { lr->vk_destroy_surface(lr->vk_inst->instance, lr->surface); pl_vk_inst_destroy(&lr->vk_inst); } #elif defined STELA_API_DX11 if (lr->d3d11) pl_d3d11_destroy(&lr->d3d11); #elif defined STELA_API_OPENGL if (lr->gl) pl_opengl_destroy(&lr->gl); #endif if (lr->logger) pl_log_destroy(&lr->logger); if (lr->vr_emulation_hook[0]) pl_mpv_user_shader_destroy(&lr->vr_emulation_hook[0]); #ifdef CAMU_HAVE_SUBTITLES if (lr->ass) ass_library_done(lr->ass); #endif al_free(lr); *renderer = NULL; } struct camu_renderer *camu_renderer_libplacebo_create(void) { struct camu_renderer_lp *lr = al_alloc_object(struct camu_renderer_lp); lr->r.create_renderer = renderer_lp_create_renderer; lr->r.create_queue = renderer_lp_create_queue; lr->r.add_font = renderer_lp_add_font; lr->r.get_latency = renderer_lp_get_latency; lr->r.resize = renderer_lp_resize; lr->r.set = renderer_lp_set; lr->r.render = renderer_lp_render; lr->r.free = renderer_lp_free; return (struct camu_renderer *)lr; }