path: root/src/render/queue_libplacebo.c

#define AL_LOG_SECTION "frame_queue_libplacebo"
#include <al/log.h>
#include <al/lib.h>

#ifdef CAMU_HAVE_FFMPEG
#include "../codec/ffmpeg/common.h"
#endif

#include "queue_libplacebo.h"

static bool queue_lp_configure_subtitles(struct camu_frame_queue *queue, u32 width, u32 height,
    struct camu_codec_stream *stream)
{
    struct camu_frame_queue_lp *lq = (struct camu_frame_queue_lp *)queue;
#ifdef CAMU_HAVE_SUBTITLES
    ASS_Library *ass = lq->subs.ass;
    if (!(lq->subs.renderer = ass_renderer_init(ass))) {
        log_error("Failed to initialize ass renderer.");
        return false;
    }
    if (!(lq->subs.track = ass_new_track(ass))) {
        log_error("Failed to create ass track.");
        return false;
    }
    ASS_Renderer *renderer = lq->subs.renderer;
    ASS_Track *track = lq->subs.track;
    ass_set_shaper(renderer, ASS_SHAPING_COMPLEX);
    ass_set_frame_size(renderer, width, height);
    ass_set_storage_size(renderer, width, height);
    ass_set_fonts(renderer, NULL, NULL, ASS_FONTPROVIDER_AUTODETECT, NULL, 0);
    ass_set_hinting(renderer, ASS_HINTING_NONE);
    AVCodecParameters *codecpar = stream->av.stream->codecpar;
    lq->subs.conversion_needed = codecpar->codec_id != AV_CODEC_ID_ASS;
    if (lq->subs.conversion_needed) {
        const AVCodecDescriptor *desc = avcodec_descriptor_get(codecpar->codec_id);
        if (!(desc->props & AV_CODEC_PROP_TEXT_SUB)) {
            log_error("Cannot convert non-text (bitmap) subtitles.");
            return false;
        }
        const AVCodec *codec = avcodec_find_decoder(codecpar->codec_id);
        if (!codec) {
            log_error("Failed to find subtitle decoder.");
            return false;
        }
        if (!(lq->subs.converter = camu_ff_alloc_codec_context(codec, codecpar))) {
            return false;
        }
        AVCodecContext *converter = lq->subs.converter;
        converter->sub_charenc_mode = FF_SUB_CHARENC_MODE_IGNORE;
        if (!camu_ff_open_avcodec(converter, codec, NULL)) {
            return false;
        }
        const AVCodecDescriptor *ass_desc = avcodec_descriptor_get(AV_CODEC_ID_ASS);
        log_info("Subtitle conversion: %s -> %s.", desc->long_name, ass_desc->long_name);
        // https://github.com/mpv-player/mpv/blob/266cb79f38fd1a5fd448b453dee5971795a145ca/sub/sd_ass.c#L634
#define MP_ASS_FONT_PLAYRESX 384.0
        track->PlayResX = track->PlayResY * (f64)width / MAX(height, (u32)1);
        f64 fix_margins = track->PlayResX / MP_ASS_FONT_PLAYRESX;
        f64 font_scale = 1.0;
        for (s32 n = 0; n < track->n_styles; n++) {
            track->styles[n].MarginL = track->styles[n].MarginL * fix_margins;
            track->styles[n].MarginR = track->styles[n].MarginR * fix_margins;
            track->styles[n].MarginV = track->styles[n].MarginV * font_scale;
        }
        ass_process_codec_private(track, (const char *)converter->subtitle_header,
            converter->subtitle_header_size);
    } else {
        ass_process_codec_private(track, (const char *)codecpar->extradata,
            codecpar->extradata_size);
    }
    nn_mutex_init(&lq->subs.lock);
    lq->subs.are_present = true;
    return true;
#else
    (void)lq;
    (void)width;
    (void)height;
    (void)stream;
    return false;
#endif
}

static bool map_frame(pl_gpu gpu, pl_tex *tex, const struct pl_source_frame *src,
    struct pl_frame *out_frame)
{
    struct camu_codec_frame *frame = (struct camu_codec_frame *)src->frame_data;

    u64 masks[4] = { 0 };
    s32 pixel_stride;
    switch (frame->format) {
    case CAMU_PIXEL_FORMAT_RGBA:
        masks[0] = 0x000000ff;
        masks[1] = 0x0000ff00;
        masks[2] = 0x00ff0000;
        masks[3] = 0xff000000;
        pixel_stride = 4;
        break;
    case CAMU_PIXEL_FORMAT_RGB24:
        masks[0] = 0x000000ff;
        masks[1] = 0x0000ff00;
        masks[2] = 0x00ff0000;
        pixel_stride = 3;
        break;
    case CAMU_PIXEL_FORMAT_GREY:
        masks[0] = 0x000000ff;
        pixel_stride = 1;
        break;
    default:
        log_error("Unsuppored pixel format.");
        al_assert_and_return(false);
    }

    if (frame->format == CAMU_PIXEL_FORMAT_GREY) {
        out_frame->repr.sys = PL_COLOR_SYSTEM_BT_601; // pl_system_from_av(AVCOL_SPC_BT470BG)
        out_frame->repr.levels = PL_COLOR_LEVELS_FULL; // pl_levels_from_av(AVCOL_RANGE_JPEG)
    } else {
        out_frame->repr = pl_color_repr_unknown;
    }
    out_frame->color = pl_color_space_unknown;
    out_frame->crop = (pl_rect2df){ 0.f, 0.f, frame->video.width, frame->video.height };

    struct pl_plane_data data[4] = { 0 };
    data[0].type = PL_FMT_UNORM;
    data[0].width = frame->video.width;
    data[0].height = frame->video.height;
    data[0].pixel_stride = pixel_stride;
    data[0].row_stride = frame->video.width * pixel_stride;
    data[0].pixels = frame->data;
    pl_plane_data_from_mask(&data[0], masks);

    struct pl_plane plane = { 0 };
    bool ok = pl_upload_plane(gpu, &plane, tex, &data[0]);

    camu_codec_frame_discard(frame);

    if (!ok) {
        log_error("Failed to upload texture.");
        return false;
    }

    out_frame->num_planes = 1;
    out_frame->planes[0] = plane;

    return true;
}

static void unmap_frame(pl_gpu gpu, struct pl_frame *frame, const struct pl_source_frame *src)
{
    (void)gpu;
    (void)frame;
    (void)src;
}

static void discard_frame(const struct pl_source_frame *src)
{
    struct camu_codec_frame *frame = (struct camu_codec_frame *)src->frame_data;
    camu_codec_frame_discard(frame);
}

static void queue_lp_push(struct camu_frame_queue *queue, struct camu_codec_frame *frame, f64 pts, u32 *count)
{
    struct camu_frame_queue_lp *lq = (struct camu_frame_queue_lp *)queue;
    pl_queue_push(lq->queue, &(struct pl_source_frame){
        .pts = pts,
        .duration = 0.0,
        .map = map_frame,
        .unmap = unmap_frame,
        .discard = discard_frame,
        .frame_data = frame
    });
    *count = (u32)pl_queue_num_frames(lq->queue);
}

#ifdef CAMU_HAVE_FFMPEG
#ifdef CAMU_HAVE_SUBTITLES
static void free_overlay(pl_gpu gpu, struct camu_overlay *overlay)
{
    al_assert(!overlay->ref);
    struct pl_overlay *current = CAMU_OVERLAY_OFFSET(overlay, 0);
    for (u32 i = 0; i < overlay->alloc; i++) {
        if (current->tex) {
            pl_tex_destroy(gpu, &current->tex);
        }
        current++;
    }
    al_free(overlay);
}

static struct camu_overlay *first_nonref_overlay(struct camu_frame_queue_lp *lq, u32 *index)
{
    struct camu_overlay *overlay;
    al_array_foreach(lq->subs.overlays, i, overlay) {
        if (!overlay->ref) {
            *index = i;
            return overlay;
        }
    }
    return NULL;
}

static struct camu_overlay *create_subtitle_overlay(struct camu_overlay *prev, pl_gpu gpu, ASS_Image *ass_frame)
{
    u16 frame_count = 1;
    ASS_Image *head_ass_frame = ass_frame;
    while (ass_frame->next) { al_assert(frame_count < UINT16_MAX); frame_count++; ass_frame = ass_frame->next; }
    ass_frame = head_ass_frame;

    u16 prev_alloc = prev ? prev->alloc : 0;
    struct camu_overlay *overlay = camu_overlay_alloc(prev, frame_count,
        sizeof(struct pl_overlay) + sizeof(struct pl_overlay_part));
    if (frame_count > prev_alloc) {
        // Zero any newly allocated pl_overlays.
        al_memset(CAMU_OVERLAY_OFFSET(overlay, prev_alloc * sizeof(struct pl_overlay)), 0,
            (frame_count - prev_alloc) * sizeof(struct pl_overlay));
    }

    // The list of pl_overlay's need to be contiguous in memory.
    struct pl_overlay *current = CAMU_OVERLAY_OFFSET(overlay, 0);
    struct pl_overlay_part *current_part = CAMU_OVERLAY_OFFSET(overlay,
        sizeof(struct pl_overlay) * overlay->alloc);
    overlay->num = 0;

    for (; ass_frame; ass_frame = ass_frame->next) {
        bool ok = pl_tex_recreate(gpu, &current->tex, pl_tex_params(
            .w = ass_frame->w,
            .h = ass_frame->h,
            .format = pl_find_named_fmt(gpu, "r8"),
            .sampleable = true,
            .host_writable = true
        ));
        if (ok) {
            ok = pl_tex_upload(gpu, pl_tex_transfer_params(
                .tex = current->tex,
                .row_pitch = ass_frame->stride,
                .ptr = ass_frame->bitmap
            ));
        }
        if (!ok) {
            frame_count--;
            log_error("Failed to upload subtitle bitmap.");
            continue;
        }

        // https://github.com/mpv-player/mpv/blob/70aaba71d6e3071a732069a1d222d1eb4293faf2/video/out/vo_gpu_next.c#L289
        // https://github.com/mpv-player/mpv/blob/70aaba71d6e3071a732069a1d222d1eb4293faf2/sub/ass_mp.c#L195
        current->color = *pl_color_space(
            .primaries = PL_COLOR_PRIM_BT_709,
            .transfer = PL_COLOR_TRC_SRGB
        );
        current->mode = PL_OVERLAY_MONOCHROME;
        current->repr = pl_color_repr_unknown;
        current->repr.alpha = PL_ALPHA_INDEPENDENT;

        current->coords = PL_OVERLAY_COORDS_SRC_FRAME;
        current_part->src = (pl_rect2df){ 0.f, 0.f, ass_frame->w, ass_frame->h };
        current_part->dst = (pl_rect2df){
            ass_frame->dst_x,
            ass_frame->dst_y,
            ass_frame->dst_x + ass_frame->w,
            ass_frame->dst_y + ass_frame->h
        };
        u32 c = ass_frame->color;
        current_part->color[0] = (c >> 24) / 255.f;
        current_part->color[1] = ((c >> 16) & 0xff) / 255.f;
        current_part->color[2] = ((c >> 8) & 0xff) / 255.f;
        current_part->color[3] = 1.f - (c & 0xff) / 255.f;

        current->parts = current_part;
        current->num_parts = 1;

        overlay->num++;

        current++;
        current_part++;
    }

    al_assert((u32)overlay->num == frame_count);

    return overlay;
}
#endif

static bool map_av_frame(pl_gpu gpu, pl_tex *tex, const struct pl_source_frame *src,
    struct pl_frame *out_frame)
{
    AVFrame *frame = (AVFrame *)src->frame_data;
    struct camu_frame_queue_lp *lq = (struct camu_frame_queue_lp *)frame->opaque;
    AVStream *stream = lq->q.buf->stream->av.stream;

    bool ok = false;

    if (!lq->copy_frame_fallback) {
        ok = pl_map_avframe_ex(gpu, out_frame, pl_avframe_params(
            .frame = frame,
            .tex = tex
        ));
    }

    // https://github.com/streetpea/chiaki-ng/blob/dd145ec2c802814e98fb833d3a9692169aa67315/gui/src/qmlmainwindow.cpp#L860
    // https://github.com/streetpea/chiaki-ng/blob/dd145ec2c802814e98fb833d3a9692169aa67315/gui/src/qmlbackend.cpp#L786
    // pl_map_avframe_derived() will fail under Xwayland because of a lack of EGL_EXT_image_dma_buf_import.
    if (!ok && frame->hw_frames_ctx) {
        if (!lq->copy_frame_fallback) {
            log_warn("Failed to map hwframe, falling back to software copy.");
            lq->copy_frame_fallback = true;
            lq->swframe = av_frame_alloc();
            enum AVPixelFormat *fmts;
            av_hwframe_transfer_get_formats(frame->hw_frames_ctx, AV_HWFRAME_TRANSFER_DIRECTION_FROM, &fmts, 0);
            lq->swframe->format = fmts[0];
            av_free(fmts);
        }
        s32 ret = av_hwframe_transfer_data(lq->swframe, frame, 0);
        ok = ret == 0;
        if (ok) {
            av_frame_copy_props(lq->swframe, frame);
            ok = pl_map_avframe_ex(gpu, out_frame, pl_avframe_params(
                .frame = lq->swframe,
                .tex = tex
            ));
            av_frame_unref(lq->swframe);
        } else {
            log_error("Failed to transfer hwframe from the device (%s).", av_err2str(ret));
        }
    }

    ((struct pl_source_frame *)src)->frame_data = NULL;

#ifdef CAMU_HAVE_SUBTITLES
    if (ok && lq->subs.are_present) {
        s64 now = av_rescale_q(frame->best_effort_timestamp, stream->time_base, (AVRational){ 1, 1000 });
        s32 change = -1; // 1 = different position, 2 = different content.
        nn_mutex_lock(&lq->subs.lock);
        ASS_Image *ass_frame = ass_render_frame(lq->subs.renderer, lq->subs.track, now, &change);
        al_assert(change >= 0);
        // ass_frame = NULL and lq->subs.track->n_events = 0 can also mean parsing completely failed.
        if (ass_frame) {
            struct camu_overlay *overlay = lq->subs.last;
            if (change != 0) {
                u32 reused = AL_ARRAY_NO_INDEX;
                overlay = create_subtitle_overlay(first_nonref_overlay(lq, &reused), gpu, ass_frame);
                // If all texture uploads failed, overlay->num will be 0.
                lq->subs.last = overlay;
                if (overlay) {
                    if (reused != AL_ARRAY_NO_INDEX) {
                        // Pointer may have changed due to reallocation.
                        al_array_at(lq->subs.overlays, reused) = overlay;
                    } else { // New overlay object.
                        overlay->lock = &lq->subs.lock;
                        al_array_push(lq->subs.overlays, overlay);
                    }
                }
            }
            if (overlay) {
                overlay->ref++;
                out_frame->overlays = CAMU_OVERLAY_OFFSET(overlay, 0);
                ((struct pl_source_frame *)src)->frame_data = overlay;
            } else {
               out_frame->overlays = NULL;
            }
        }
        nn_mutex_unlock(&lq->subs.lock);
    }
#endif

    av_frame_free(&frame);

    if (!ok) {
        log_error("Failed to map AVFrame.");
        return false;
    }

    pl_frame_copy_stream_props(out_frame, stream);

    return true;
}

static void unmap_av_frame(pl_gpu gpu, struct pl_frame *frame, const struct pl_source_frame *src)
{
    if (src->frame_data) {
        struct camu_overlay *overlay = (struct camu_overlay *)src->frame_data;
        nn_mutex_lock(overlay->lock);
        overlay->ref--;
        nn_mutex_unlock(overlay->lock);
    }
    pl_unmap_avframe(gpu, frame);
}

static void discard_av_frame(const struct pl_source_frame *src)
{
    AVFrame *frame = (AVFrame *)src->frame_data;
    av_frame_free(&frame);
    log_warn("Dropped frame with PTS %.3f.", src->pts);
}

static void queue_lp_push_av_frame(struct camu_frame_queue *queue, AVFrame *frame, f64 pts, u32 *count)
{
    struct camu_frame_queue_lp *lq = (struct camu_frame_queue_lp *)queue;
    AVStream *stream = lq->q.buf->stream->av.stream;
    frame->opaque = lq;
    pl_queue_push(lq->queue, &(struct pl_source_frame){
        .pts = pts,
        .duration = camu_ff_frame_duration(frame) * av_q2d(stream->time_base),
        .map = map_av_frame,
        .unmap = unmap_av_frame,
        .discard = discard_av_frame,
        .frame_data = frame,
        .first_field = pl_field_from_avframe(frame)
    });
    *count = (u32)pl_queue_num_frames(lq->queue);
}
#endif

#ifdef CAMU_HAVE_SUBTITLES
static bool convert_subtitle(struct camu_frame_queue_lp *lq, AVPacket *pkt)
{
    AVSubtitle sub = { 0 };
    s32 got_sub = 0;
    s32 ret = avcodec_decode_subtitle2(lq->subs.converter, &sub, &got_sub, pkt);
    if (ret < 0) {
        log_error("Failed to convert subtitle (%s).", av_err2str(ret));
    } else if (got_sub) {
        for (u32 i = 0; i < sub.num_rects; i++) {
            AVSubtitleRect *rect = sub.rects[i];
            size_t size = al_strlen(rect->ass);
            if (rect->ass[size - 1] == 0x02) size--; // Not sure what this is...
            ass_process_chunk(lq->subs.track, rect->ass, size, pkt->pts, pkt->duration);
        }
        avsubtitle_free(&sub);
        return true;
    }
    return false;
}
#endif

static void queue_lp_push_subtitle(struct camu_frame_queue *queue, struct camu_codec_packet *packet)
{
    struct camu_frame_queue_lp *lq = (struct camu_frame_queue_lp *)queue;
#ifdef CAMU_HAVE_SUBTITLES
    if (!lq->subs.are_present) return;
    AVPacket *pkt = packet->av.pkt;
    nn_mutex_lock(&lq->subs.lock);
    al_assert(lq->subs.track && lq->subs.renderer);
    if (lq->subs.conversion_needed) {
        convert_subtitle(lq, pkt);
    } else {
        ass_process_chunk(lq->subs.track, (const char *)pkt->data, pkt->size, pkt->pts, pkt->duration);
    }
    nn_mutex_unlock(&lq->subs.lock);
#else
    (void)lq;
    (void)packet;
#endif
}

static void queue_lp_flush(struct camu_frame_queue *queue, u32 *count)
{
    struct camu_frame_queue_lp *lq = (struct camu_frame_queue_lp *)queue;
    *count = (u32)pl_queue_num_frames(lq->queue);
    pl_queue_push(lq->queue, NULL);
#ifdef CAMU_HAVE_SUBTITLES
    if (lq->subs.are_present) {
        if (lq->subs.conversion_needed) {
            AVPacket *empty_pkt = av_packet_alloc();
            while (convert_subtitle(lq, empty_pkt)) { }
            av_packet_free(&empty_pkt);
        }
        ass_flush_events(lq->subs.track);
    }
#endif
}

static u8 queue_lp_read(struct camu_frame_queue *queue, f64 pts, void *out, u32 *count)
{
    struct camu_frame_queue_lp *lq = (struct camu_frame_queue_lp *)queue;
    struct pl_frame_mix *mix = (struct pl_frame_mix *)out;
    lq->params.pts = pts;
    enum pl_queue_status ret = pl_queue_update(lq->queue, mix, &lq->params);
    *count = (u32)pl_queue_num_frames(lq->queue);
    switch (ret) {
    case PL_QUEUE_OK:
        break;
    case PL_QUEUE_MORE:
        return CAMU_QUEUE_MORE;
    case PL_QUEUE_EOF:
        return CAMU_QUEUE_EOF;
    case PL_QUEUE_ERR:
    default:
        return CAMU_QUEUE_ERR;
    }
    return CAMU_QUEUE_OK;
}

static void queue_lp_reset(struct camu_frame_queue *queue)
{
    struct camu_frame_queue_lp *lq = (struct camu_frame_queue_lp *)queue;
    u32 count;
    queue_lp_flush(&lq->q, &count);
    pl_queue_reset(lq->queue);
}

static void queue_lp_free(struct camu_frame_queue **queue)
{
    struct camu_frame_queue_lp *lq = (struct camu_frame_queue_lp *)*queue;
    pl_queue_destroy(&lq->queue);
#ifdef CAMU_HAVE_FFMPEG
    if (lq->copy_frame_fallback) {
        av_frame_free(&lq->swframe);
    }
#endif
#ifdef CAMU_HAVE_SUBTITLES
    struct camu_overlay *overlay;
    al_array_foreach_rev(lq->subs.overlays, i, overlay) {
        free_overlay(lq->gpu, overlay);
    }
    al_array_free(lq->subs.overlays);
    if (lq->subs.are_present) {
        nn_mutex_destroy(&lq->subs.lock);
    }
    if (lq->subs.converter) {
        avcodec_free_context(&lq->subs.converter);
    }
    if (lq->subs.track) {
        ass_free_track(lq->subs.track);
    }
    if (lq->subs.renderer) {
        ass_renderer_done(lq->subs.renderer);
    }
#endif
    al_free(lq);
    *queue = NULL;
}

struct camu_frame_queue *camu_frame_queue_lp_create(void)
{
    struct camu_frame_queue_lp *lq = al_alloc_object(struct camu_frame_queue_lp);
#ifdef CAMU_HAVE_SUBTITLES
    lq->subs.are_present = false;
    lq->subs.last = NULL;
    al_array_init(lq->subs.overlays);
#endif
    lq->q.configure_subtitles = queue_lp_configure_subtitles;
    lq->q.push = queue_lp_push;
#ifdef CAMU_HAVE_FFMPEG
    lq->q.push_av_frame = queue_lp_push_av_frame;
#endif
    lq->q.push_subtitle = queue_lp_push_subtitle;
    lq->q.flush = queue_lp_flush;
    lq->q.read = queue_lp_read;
    lq->q.reset = queue_lp_reset;
    lq->q.free = queue_lp_free;
    return (struct camu_frame_queue *)lq;
}