From ae8822ec0327c4d5674f1dc79de926b230f27f1f Mon Sep 17 00:00:00 2001 From: Andrew Opalach Date: Wed, 16 Apr 2025 15:12:41 -0400 Subject: Highly wip waveform view Signed-off-by: Andrew Opalach --- src/buffer/audio.c | 1 - src/cache/backings/file.c | 6 +- src/cache/backings/file_mapped.c | 12 ++- src/cache/backings/memory.c | 4 +- src/codec/ffmpeg/decoder.c | 2 +- src/codec/ffmpeg/demuxer.c | 3 +- src/codec/ffmpeg/meson.build | 16 ++- src/codec/ffmpeg/packet_ext.c | 3 +- src/codec/meson.build | 6 +- src/codec/packet_ext.c | 15 +++ src/codec/packet_ext.h | 8 ++ src/fruits/cmc/cmc.c | 25 +++++ src/fruits/cmc/cmc.h | 8 ++ src/fruits/cmc/meson.build | 4 +- src/fruits/cmc/ui/pane_list.c | 82 +++++++++++++- src/fruits/cmc/ui/pane_log.c | 44 ++++++++ src/fruits/cmc/ui/ui.c | 38 ++++++- src/fruits/cmc/ui/ui.h | 15 +++ src/fruits/cmc/ui/waveform.c | 202 +++++++++++++++++++++++++++++++++++ src/fruits/cmc/ui/waveform.h | 13 ++- src/liana/handlers/codec_server.c | 4 +- src/liana/list.c | 8 +- src/liana/meson.build | 3 +- src/liana/process.c | 115 ++++++++++++++++++++ src/liana/process.h | 13 +++ src/liana/server.c | 13 ++- src/liana/server.h | 3 +- src/libclient/client.c | 25 ++++- src/libclient/client.h | 5 +- src/libclient/common.h | 3 +- src/portal/py/modules/pixiv_web.py | 15 +-- src/portal/py/tests/archive_query.py | 70 +----------- src/portal/py/tests/test.py | 19 ++-- src/screen/screen.c | 1 + src/server/common.h | 3 +- src/server/resource.h | 4 +- src/server/server.c | 53 ++++++++- 37 files changed, 736 insertions(+), 128 deletions(-) create mode 100644 src/codec/packet_ext.c create mode 100644 src/codec/packet_ext.h create mode 100644 src/fruits/cmc/ui/pane_log.c create mode 100644 src/fruits/cmc/ui/waveform.c create mode 100644 src/liana/process.c create mode 100644 src/liana/process.h (limited to 'src') diff --git a/src/buffer/audio.c b/src/buffer/audio.c index 8bfed20..d6041e4 100644 --- a/src/buffer/audio.c +++ b/src/buffer/audio.c @@ -184,7 +184,6 @@ static bool push_internal(struct camu_audio_buffer *buf, f64 pts, u8 **data, s32 return true; } - #ifdef CAMU_HAVE_FFMPEG static void push_av_frame_internal(struct camu_audio_buffer *buf, AVFrame *frame) { diff --git a/src/cache/backings/file.c b/src/cache/backings/file.c index 9b905ac..500e16f 100644 --- a/src/cache/backings/file.c +++ b/src/cache/backings/file.c @@ -84,10 +84,12 @@ struct cch_backing *cch_backing_file_create(str *path, size_t size) file->backing.free = file_backing_free; al_array_init(file->backing.available); if (!file_open_internal(file, path, size)) { - al_free(file); - return NULL; + goto err; } file->u.pointer = 0; nn_mutex_init(&file->mutex); return (struct cch_backing *)file; +err: + al_free(file); + return NULL; } diff --git a/src/cache/backings/file_mapped.c b/src/cache/backings/file_mapped.c index 17620d0..adb840c 100644 --- a/src/cache/backings/file_mapped.c +++ b/src/cache/backings/file_mapped.c @@ -76,10 +76,16 @@ struct cch_backing *cch_backing_file_create(str *path, size_t size) file->backing.resize = file_backing_resize; file->backing.free = file_backing_free; al_array_init(file->backing.available); - if (!file_open_internal(file, path, size) || !(file->u.map = nn_file_mmap(&file->file))) { - al_free(file); - return NULL; + if (!file_open_internal(file, path, size)) { + goto err; + } + if (!(file->u.map = nn_file_mmap(&file->file))) { + nn_file_close(&file->file); + goto err; } nn_mutex_init(&file->mutex); return (struct cch_backing *)file; +err: + al_free(file); + return NULL; } diff --git a/src/cache/backings/memory.c b/src/cache/backings/memory.c index aa5c1b4..8e0306a 100644 --- a/src/cache/backings/memory.c +++ b/src/cache/backings/memory.c @@ -21,7 +21,7 @@ static void memory_backing_write(struct cch_backing *backing, u8 *buf, off_t ind struct cch_backing_memory *mem = (struct cch_backing_memory *)backing; nn_mutex_lock(&mem->mutex); if (mem->size >= 0 && index + (off_t)*size >= mem->size) { - *size = MAX(mem->size - index, (off_t)0L); + *size = MAX(mem->size - index, (off_t)0); } ensure_alloced(mem, index + *size); al_memcpy(mem->data + index, buf, *size); @@ -34,7 +34,7 @@ static u8 *memory_backing_get_ptr(struct cch_backing *backing, off_t index, size struct cch_backing_memory *mem = (struct cch_backing_memory *)backing; nn_mutex_lock(&mem->mutex); if (mem->size >= 0 && index + (off_t)*size >= mem->size) { - *size = MAX(mem->size - index, (off_t)0L); + *size = MAX(mem->size - index, (off_t)0); } return (u8 *)(mem->data + index); } diff --git a/src/codec/ffmpeg/decoder.c b/src/codec/ffmpeg/decoder.c index a6431b1..17be6cc 100644 --- a/src/codec/ffmpeg/decoder.c +++ b/src/codec/ffmpeg/decoder.c @@ -3,7 +3,7 @@ #include #include -#ifndef CAMU_SINK_NO_VIDEO +#if !defined CAMU_SINK_NO_VIDEO && defined CAMU_FF_DECODER_HWACCEL #include "../../render/renderer.h" #endif diff --git a/src/codec/ffmpeg/demuxer.c b/src/codec/ffmpeg/demuxer.c index 058a423..b612b2d 100644 --- a/src/codec/ffmpeg/demuxer.c +++ b/src/codec/ffmpeg/demuxer.c @@ -14,7 +14,7 @@ static void close_internal(struct camu_ff_demuxer *av) if (av->format_context) { avformat_flush(av->format_context); avformat_close_input(&av->format_context); - avformat_free_context(av->format_context); + // avformat_close_input() should call avformat_free_context(). } } @@ -105,6 +105,7 @@ static bool ff_demuxer_init(struct camu_demuxer *demux, struct cch_handle *handl .mode = CAMU_FFMPEG_COMPAT, .type = type, .duration = duration, + .index = stream->index, .av.stream = stream })); } diff --git a/src/codec/ffmpeg/meson.build b/src/codec/ffmpeg/meson.build index dfbf624..eb68fad 100644 --- a/src/codec/ffmpeg/meson.build +++ b/src/codec/ffmpeg/meson.build @@ -12,9 +12,6 @@ ffmpeg_server_src = [ ] ffmpeg_server_deps = [] -# This is layed out based on how things work currently. -# Obviously the decoder, resampler, scaler, and encoder should -# be a part of a server-side transcoding function. ffmpeg_client_src = [ ffmpeg_src, 'decoder.c', @@ -24,6 +21,16 @@ ffmpeg_client_src = [ ] ffmpeg_client_deps = [] +server_transcode = not get_option('sink-only') +if server_transcode + ffmpeg_server_src += [ + 'decoder.c', + 'resampler.c', + 'scaler.c', + 'encoder.c' + ] +endif + libavutil = dependency('libavutil', required: false) libavformat = dependency('libavformat', required: false) libavcodec = dependency('libavcodec', required: false) @@ -60,6 +67,9 @@ if libavutil.found() and libavformat.found() and libavcodec.found() and libswres # This is probably meaningless compared to libav* versions, it's just something I wanted. ffmpeg_args += ['-DFFMPEG_VERSION="' + ffmpeg_version_string + '"'] ffmpeg_server_deps += [libavutil, libavformat, libavcodec] + if server_transcode + ffmpeg_server_deps += [libswresample, libswscale] + endif ffmpeg_client_deps += [libavutil, libavformat, libavcodec, libswresample, libswscale] soxr = compiler.find_library('soxr', required: false) if soxr.found() diff --git a/src/codec/ffmpeg/packet_ext.c b/src/codec/ffmpeg/packet_ext.c index e7b1da8..acf5bbd 100644 --- a/src/codec/ffmpeg/packet_ext.c +++ b/src/codec/ffmpeg/packet_ext.c @@ -165,8 +165,7 @@ void nn_packet_read_av_dictionary(struct nn_packet *packet, AVDictionary **dict) u32 len; NNWT_PACKET_READ_TYPE(packet, u32, len); NNWT_PACKET_READ_DATA(packet, len, key); - // Key and value MUST be allocated with av_malloc functions, - // just like all FFmpeg structures. + // Key and value MUST be allocated with av_malloc functions. key = av_strndup(key, len); NNWT_PACKET_READ_TYPE(packet, u32, len); NNWT_PACKET_READ_DATA(packet, len, value); diff --git a/src/codec/meson.build b/src/codec/meson.build index 4c9e16b..a739de0 100644 --- a/src/codec/meson.build +++ b/src/codec/meson.build @@ -1,3 +1,5 @@ +codec_src = ['packet_ext.c'] + codec_server_deps = [] codec_client_deps = [] @@ -21,5 +23,5 @@ if not no_video codec_client_deps += [stela] endif -codecs_server = declare_dependency(dependencies: codec_server_deps) -codecs_client = declare_dependency(dependencies: codec_client_deps) +codecs_server = declare_dependency(sources: codec_src, dependencies: codec_server_deps) +codecs_client = declare_dependency(sources: codec_src, dependencies: codec_client_deps) diff --git a/src/codec/packet_ext.c b/src/codec/packet_ext.c new file mode 100644 index 0000000..c41eadb --- /dev/null +++ b/src/codec/packet_ext.c @@ -0,0 +1,15 @@ +#include "packet_ext.h" + +void nn_packet_write_audio_format(struct nn_packet *packet, struct camu_audio_format *fmt) +{ + NNWT_PACKET_WRITE_TYPE(packet, s32, fmt->format); + NNWT_PACKET_WRITE_TYPE(packet, s32, fmt->sample_rate); + NNWT_PACKET_WRITE_TYPE(packet, s32, fmt->channel_count); +} + +void nn_packet_read_audio_format(struct nn_packet *packet, struct camu_audio_format *fmt) +{ + NNWT_PACKET_READ_TYPE(packet, s32, fmt->format); + NNWT_PACKET_READ_TYPE(packet, s32, fmt->sample_rate); + NNWT_PACKET_READ_TYPE(packet, s32, fmt->channel_count); +} diff --git a/src/codec/packet_ext.h b/src/codec/packet_ext.h new file mode 100644 index 0000000..6bf1b56 --- /dev/null +++ b/src/codec/packet_ext.h @@ -0,0 +1,8 @@ +#pragma once + +#include + +#include "codec.h" + +void nn_packet_write_audio_format(struct nn_packet *packet, struct camu_audio_format *fmt); +void nn_packet_read_audio_format(struct nn_packet *packet, struct camu_audio_format *fmt); diff --git a/src/fruits/cmc/cmc.c b/src/fruits/cmc/cmc.c index 6e62a4a..265550f 100644 --- a/src/fruits/cmc/cmc.c +++ b/src/fruits/cmc/cmc.c @@ -4,6 +4,7 @@ #include "../../server/common.h" #include "../../liana/list.h" +#include "../../codec/packet_ext.h" #include "../common.h" @@ -31,6 +32,7 @@ static struct cmc_search *get_search_by_id(struct cmc *c, s32 id) static void read_list_entry(struct nn_packet *packet, struct cmc_list_entry *entry, u32 id) { entry->id = id; + entry->node_id = nn_packet_read_u32(packet); entry->duration = nn_packet_read_u64(packet); entry->start = nn_packet_read_u64(packet); entry->paused_at = nn_packet_read_u64(packet); @@ -70,6 +72,9 @@ static void parse_initial_user_state(struct cmc *c, struct nn_packet *packet) u32 id = nn_packet_read_u32(packet); struct cmc_list_entry *entry = al_alloc_object(struct cmc_list_entry); read_list_entry(packet, entry, id); + if (list->current >= 0 && j == (u32)list->current) { + camu_client_request_visual_data(&c->client, entry->node_id); + } al_array_push(list->entries, entry); } al_array_push(c->lists, list); @@ -162,6 +167,7 @@ static void client_callback(void *userdata, u8 op, void *opaque) u32 id = nn_packet_read_u32(packet); struct cmc_list_entry *entry = al_array_at(list->entries, list->current); read_list_entry(packet, entry, id); + camu_client_request_visual_data(&c->client, entry->node_id); cmc_ui_queue_render(&c->ui); break; } @@ -193,6 +199,25 @@ static void client_callback(void *userdata, u8 op, void *opaque) } break; } + case CAMU_CLIENT_GOT_VISUAL_DATA: { + struct nn_packet *packet = (struct nn_packet *)opaque; + u32 node_id = nn_packet_read_u32(packet); + struct cmc_list *list = al_array_at(c->lists, 0); + struct cmc_list_entry *entry; + al_array_foreach(list->entries, i, entry) { + if (entry->node_id == node_id) { + if (entry->raw.data != NULL) break; + nn_packet_read_audio_format(packet, &entry->raw.fmt); + u32 size = nn_packet_read_u32(packet); + u8 *ptr; + NNWT_PACKET_READ_DATA(packet, size, ptr); + entry->raw.data = al_malloc(size); + al_memcpy(entry->raw.data, ptr, size); + entry->raw.sample_count = nn_packet_read_u32(packet); + } + } + break; + } } } diff --git a/src/fruits/cmc/cmc.h b/src/fruits/cmc/cmc.h index e2cdde3..ea714dc 100644 --- a/src/fruits/cmc/cmc.h +++ b/src/fruits/cmc/cmc.h @@ -6,6 +6,7 @@ #include "../../libclient/client.h" #include "../../portal/src/post_cache.h" +#include "../../codec/codec.h" #include "ui/ui.h" @@ -23,11 +24,18 @@ struct cmc_search { struct cmc_list_entry { u32 id; + u32 node_id; u64 duration; u64 start; u64 paused_at; u64 offset; wstr name; + struct { + struct camu_audio_format fmt; + u8 *data; + u32 sample_count; + char ***drawing; + } raw; }; struct cmc_list { diff --git a/src/fruits/cmc/meson.build b/src/fruits/cmc/meson.build index 6b7060d..a0fa097 100644 --- a/src/fruits/cmc/meson.build +++ b/src/fruits/cmc/meson.build @@ -12,8 +12,10 @@ if use_tui 'ui/ui.c', 'ui/pane_list.c', 'ui/pane_search.c', + 'ui/pane_log.c', + 'ui/waveform.c' ] - cmc_deps += [notcurses, notcurses_core] + cmc_deps += [notcurses, notcurses_core, codecs_client] endif if is_windows and meson.is_cross_build() diff --git a/src/fruits/cmc/ui/pane_list.c b/src/fruits/cmc/ui/pane_list.c index 695ff60..b62e0a6 100644 --- a/src/fruits/cmc/ui/pane_list.c +++ b/src/fruits/cmc/ui/pane_list.c @@ -7,6 +7,7 @@ #include "ui.h" #include "ext.h" +#include "waveform.h" void cmc_lp_init(struct cmc_ui *ui) { @@ -141,7 +142,7 @@ bool cmc_lp_handle_input(struct cmc_ui *ui, struct ncinput *input) CONSIDER_INPUT(); if (tab->move.active) { tab->move.active = false; - } else { + } else if (tab->selected != tab->list->current) { camu_client_skipto(&ui->c->client, &tab->list->name, tab->selected); } break; @@ -244,7 +245,7 @@ static void render_now_playing(struct cmc_ui *ui, struct cmc_list_tab *tab, u32 u64 pos = 0; u64 duration = 0; u64 remaining = 0; - u32 y_off = height - 6; + u32 y_off = height - 7; ncplane_set_styles(n, NCSTYLE_BOLD); if (entry) { @@ -288,10 +289,10 @@ static void render_now_playing(struct cmc_ui *ui, struct cmc_list_tab *tab, u32 ncplane_putstr_yx(n, y_off, 4 + offset, "โŽน"); if (entry) { - cmc_ui_putnwstr_yx(n, y_off, 6 + offset, MIN(entry->name.length, (u32)40), &entry->name); + cmc_ui_putnwstr_yx(n, y_off, 6 + offset, MIN(entry->name.length, (u32)60), &entry->name); } - bool show_remaining = true; + bool show_remaining = false; if (show_remaining) { offset = camu_print_time(timestr, sizeof(timestr), remaining, true, 4); } else { @@ -306,10 +307,81 @@ static void render_now_playing(struct cmc_ui *ui, struct cmc_list_tab *tab, u32 } ncplane_putstr_yx(n, y_off, width - 2 - offset, timestr); + /* ncplane_putchar_yx(n, height - 5, 2, 'A'); ncplane_putchar_yx(n, height - 4, 2, 'A'); ncplane_putchar_yx(n, height - 3, 2, 'A'); ncplane_putchar_yx(n, height - 2, 2, 'A'); + */ + + if (entry && entry->raw.data != NULL) { + u32 length = width - 2; + u32 top = height - 6; + u32 wave_height = 5; + if (entry->raw.drawing == NULL || entry != ui->lp.blitted) { + entry->raw.drawing = al_malloc(sizeof(char **) * wave_height); + for (u32 j = 0; j < wave_height; j++) { + entry->raw.drawing[j] = al_malloc(sizeof(char *) * length); + for (u32 i = 0; i < length; i++) { + entry->raw.drawing[j][i] = al_calloc(1, 6); + } + } + if (!ui->lp.w) { + struct ncplane_options opts = { 0 }; + opts.x = 1; + opts.y = top; + opts.cols = length; + opts.rows = wave_height; + ui->lp.w = ncplane_create(n, &opts); + } + /* + struct nccell ncl; + nccell_init(&ncl); + nccell_load(w, &ncl, " "); + nccell_set_bg_palindex(&ncl, 3); + ncplane_polyfill_yx(w, 0, 0, &ncl); + */ + u32 *rgba; + u32 pixel_width, pixel_height; + ncplane_pixel_geom(ui->lp.w, &pixel_height, &pixel_width, NULL, NULL, NULL, NULL); + //cmc_draw_waveform(entry, length, wave_height, false, entry->raw.drawing, &rgba); + cmc_draw_waveform(entry, pixel_width, pixel_height, false, entry->raw.drawing, &rgba); + if (ui->lp.vis) { + ncvisual_destroy(ui->lp.vis); + } + ui->lp.vis = ncvisual_from_rgba(rgba, pixel_height, pixel_width * 4, pixel_width); + } + + if (entry != ui->lp.blitted) { + ncplane_erase(ui->lp.w); + struct ncvisual_options vopts = { 0 }; + vopts.blitter = NCBLIT_PIXEL; + vopts.scaling = NCSCALE_STRETCH; + vopts.flags = NCVISUAL_OPTION_NOINTERPOLATE; + vopts.n = ui->lp.w; + vopts.x = 0; + vopts.y = 0; + ncvisual_blit(ui->nc, ui->lp.vis, &vopts); + ui->lp.blitted = entry; + } + + /* + ncplane_set_fg_palindex(n, 5); + for (u32 j = 0; j < wave_height; j++) { + for (u32 i = 0; i < length; i++) { + char *cell = entry->raw.drawing[j][i]; + if (cell[0] == 'R') { + ncplane_set_bg_palindex(n, 2); + } + ncplane_putstr_yx(n, top + j, 1 + i, cell + 1); + if (cell[0] == 'R') { + ncplane_set_bg_default(n); + } + } + } + ncplane_set_fg_default(n); + */ + } /* char buf[24]; @@ -330,7 +402,7 @@ void cmc_lp_render(struct cmc_ui *ui) if (ui->lp.tabs.count <= ui->lp.current) return; struct cmc_list_tab *tab = &al_array_at(ui->lp.tabs, ui->lp.current); - tab->page_length = height - 8; + tab->page_length = height - 9; render_list_entries(ui, tab, tab->page_length); render_now_playing(ui, tab, tab->page_length, height); } diff --git a/src/fruits/cmc/ui/pane_log.c b/src/fruits/cmc/ui/pane_log.c new file mode 100644 index 0000000..a35cf20 --- /dev/null +++ b/src/fruits/cmc/ui/pane_log.c @@ -0,0 +1,44 @@ +#include "ui.h" + +void cmc_logp_init(struct cmc_ui *ui) +{ + al_array_init(ui->logp.messages); +} + +void cmc_logp_layout(struct cmc_ui *ui, struct ncplane *parent) +{ + if (ui->logp.n) ncplane_destroy(ui->logp.n); + struct ncplane_options nopts = { 0 }; + nopts.rows = ncplane_dim_y(parent); + nopts.cols = ncplane_dim_x(parent); + ui->logp.n = ncplane_create(parent, &nopts); +} + +void cmc_logp_push_message(struct cmc_ui *ui, char *message) +{ + al_array_push(ui->logp.messages, message); +} + +void cmc_logp_erase(struct cmc_ui *ui) +{ + ncplane_erase(ui->logp.n); +} + +void cmc_logp_render(struct cmc_ui *ui) +{ + struct ncplane *n = ui->logp.n; + + u32 max_height = ncplane_dim_y(n); + + u32 size = ui->logp.messages.count; + u32 index = (size > max_height) ? size - max_height : 0; + for (u32 i = index; i < size; i++) { + // We can't use putnstr to control the width because these messages will have wide characters. + ncplane_putstr_yx(n, i - index, 0, al_array_at(ui->logp.messages, i)); + } + + for (u32 i = 0; i < index; i++) { + al_free(al_array_at(ui->logp.messages, i)); + } + al_array_remove_range(ui->logp.messages, 0, index); +} diff --git a/src/fruits/cmc/ui/ui.c b/src/fruits/cmc/ui/ui.c index 10357c0..b830042 100644 --- a/src/fruits/cmc/ui/ui.c +++ b/src/fruits/cmc/ui/ui.c @@ -1,4 +1,5 @@ #include +#include #include "ui.h" #include "ext.h" @@ -43,14 +44,23 @@ static void erase_previous_pane(struct cmc_ui *ui, u8 previous, u8 upcoming) case CMC_PANE_SEARCH: pn = ui->sp.n; break; + case CMC_PANE_LOG: + pn = ui->logp.n; + break; } ncplane_erase(pn); switch (upcoming) { case CMC_PANE_LIST: - ncplane_move_family_below(pn, ui->lp.n); + ncplane_move_top(ui->lp.n); + //ncplane_move_family_below(pn, ui->lp.n); break; case CMC_PANE_SEARCH: - ncplane_move_family_below(pn, ui->sp.n); + ncplane_move_top(ui->sp.n); + //ncplane_move_family_below(pn, ui->sp.n); + break; + case CMC_PANE_LOG: + ncplane_move_top(ui->logp.n); + //ncplane_move_family_below(pn, ui->logp.n); break; } } @@ -72,6 +82,9 @@ void cmc_ui_queue_render(struct cmc_ui *ui) case CMC_PANE_SEARCH: cmc_sp_erase(ui); break; + case CMC_PANE_LOG: + cmc_logp_erase(ui); + break; } if (ui->overlay_shown) { @@ -85,6 +98,9 @@ void cmc_ui_queue_render(struct cmc_ui *ui) case CMC_PANE_SEARCH: cmc_sp_render(ui); break; + case CMC_PANE_LOG: + cmc_logp_render(ui); + break; } if (ui->overlay_shown) { @@ -132,11 +148,12 @@ static void input_poll_callback(void *userdata, s32 revents) switch_to_pane(ui, CMC_PANE_LIST); break; case '2': - switch_to_pane(ui, CMC_PANE_SEARCH); break; case '3': + switch_to_pane(ui, CMC_PANE_SEARCH); break; case '4': + switch_to_pane(ui, CMC_PANE_LOG); break; case NCKEY_TAB: cmc_ui_toggle_overlay(ui); @@ -154,6 +171,9 @@ static void input_poll_callback(void *userdata, s32 revents) do_render |= true; } break; + case CMC_PANE_LOG: + do_render = true; + break; } if (do_render) { cmc_ui_queue_render(ui); @@ -187,6 +207,7 @@ static bool relayout(struct cmc_ui *ui) cmc_lp_layout(ui, ui->n); cmc_sp_layout(ui, ui->n); + cmc_logp_layout(ui, ui->n); if (ui->oln) ncplane_destroy(ui->oln); nopts.margin_r = MAX((u32)6, ncplane_dim_x(ui->n) / 2); @@ -206,6 +227,14 @@ static s32 resize_cb(struct ncplane *p) return 0; } +static s32 log_callback(void *userdata, u8 level, char *message) +{ + struct cmc_ui *ui = (struct cmc_ui *)userdata; + (void)level; + cmc_logp_push_message(ui, message); + return al_strnlen(message, AL_LOG_MESSAGE_SIZE); +} + bool cmc_ui_init(struct cmc_ui *ui, struct nn_event_loop *loop, struct cmc *c) { al_memset(ui, 0, sizeof(struct cmc_ui)); @@ -223,6 +252,9 @@ bool cmc_ui_init(struct cmc_ui *ui, struct nn_event_loop *loop, struct cmc *c) cmc_lp_init(ui); cmc_sp_init(ui); + cmc_logp_init(ui); + al_set_print(log_callback, ui); + ui->pending_layout = false; ui->pending_render = false; diff --git a/src/fruits/cmc/ui/ui.h b/src/fruits/cmc/ui/ui.h index b0ae730..302d735 100644 --- a/src/fruits/cmc/ui/ui.h +++ b/src/fruits/cmc/ui/ui.h @@ -10,6 +10,7 @@ enum { CMC_PANE_LIST = 0, CMC_PANE_SEARCH, + CMC_PANE_LOG }; struct cmc_list_tab { @@ -51,6 +52,9 @@ struct cmc_ui { u64 last_mouse_ts; struct { struct ncplane *n; + struct ncplane *w; + struct ncvisual *vis; + struct cmc_list_entry *blitted; array(struct cmc_list_tab) tabs; u32 current; } lp; // list pane. @@ -59,6 +63,10 @@ struct cmc_ui { array(struct cmc_search_tab) tabs; u32 current; } sp; // search pane. + struct { + struct ncplane *n; + array(char *) messages; + } logp; struct nn_poll input_poll; struct nn_timer render_timer; struct cmc *c; @@ -95,3 +103,10 @@ void cmc_sp_add_search(struct cmc_ui *ui, struct cmc_search *search); bool cmc_sp_handle_input(struct cmc_ui *ui, struct ncinput *input); void cmc_sp_erase(struct cmc_ui *ui); void cmc_sp_render(struct cmc_ui *ui); + +void cmc_logp_init(struct cmc_ui *ui); +void cmc_logp_layout(struct cmc_ui *ui, struct ncplane *parent); +void cmc_logp_push_message(struct cmc_ui *ui, char *message); +bool cmc_logp_handle_input(struct cmc_ui *ui, struct ncinput *input); +void cmc_logp_erase(struct cmc_ui *ui); +void cmc_logp_render(struct cmc_ui *ui); diff --git a/src/fruits/cmc/ui/waveform.c b/src/fruits/cmc/ui/waveform.c new file mode 100644 index 0000000..9bfc296 --- /dev/null +++ b/src/fruits/cmc/ui/waveform.c @@ -0,0 +1,202 @@ +#include +#include +#include "waveform.h" + +#define WITHIN_RMS(low, high, rms) \ + ((low >= 0 && low <= rms) || (high < 0 && high >= -rms) || (low < 0 && high > 0 && rms > 0)) + +#define OUTPUT_WAVEFORM_PNG +//#define WAVEFORM_REFERENCE_MODE +#ifdef OUTPUT_WAVEFORM_PNG +#include +#include +#define STB_IMAGE_WRITE_IMPLEMENTATION +#include + +// min/max: #BC615E, #F4EAE0 +// rms: #FF928F, #E4CAAF +// +// min/max: #D93A50, #b22f41 +// rms: #f44f7a, #c84064 +// extra: #f6ac7f, #F5F4FA, #5b524b +// left channel: n == 0, right channel: n == 1. +#define COLOR_FOR_PIXEL(n, rms) rms ? \ + n == 0 ? nn_htonl(0xE4CAAF99) : nn_htonl(0xFF928FFF) : \ + n == 0 ? nn_htonl(0xF4EAE099) : nn_htonl(0xBC615EFF) + +// ss: current sample, ns: next sample. +#define HIT_WAVE(low, high, ss, ns) \ + ((ss <= low && ns > low) || (ss > high && ns <= high) || (ss > low && ns <= high)) + +static void test_write_bitmap(struct cmc_list_entry *entry, u32 width, u32 height, u32 **rgba) +{ + f64 start = nn_get_tick(); + + *rgba = al_calloc(width * height, 4); + + u32 channels = entry->raw.fmt.channel_count; + u32 samples_per_column = entry->raw.sample_count / width; + + s32 wave_min = INT16_MIN; + s32 wave_max = INT16_MAX; + s32 step = (wave_max * 2 + (height - 1)) / height; + //s32 step = (wave_max * 2 + (height / 2)) / height; + + s16 *steps = al_malloc(sizeof(s16) * (height + 1)); + for (s32 i = 0; i <= (s32)height; i++) { + steps[i] = (s16)MAX(wave_max - step * i, wave_min); + } + //steps[height] = wave_min; + al_assert(steps[height] == wave_min); + + s64 rms = 0; + s16 min, max; + s16 low, high; + s16 *data = (s16 *)entry->raw.data; + const u32 samples_stride = samples_per_column * channels; + for (u32 i = 0; i < width; i++) { + u32 f = i * samples_stride; + for (u32 n = 0; n < channels; n++) { + rms = 0; + min = wave_max, max = wave_min; + for (u32 m = 0; m <= samples_stride; m += channels) { + s16 ss = data[f + m + n]; + rms += ss * ss; + min = MIN(min, ss); + max = MAX(max, ss); + } + rms /= samples_per_column; + rms = sqrt(rms); +#ifndef WAVEFORM_REFERENCE_MODE + for (u32 j = 0; j < height; j++) { + low = steps[j + 1], high = steps[j]; + if (low < max && high >= min) { + (*rgba)[j * width + i] = COLOR_FOR_PIXEL(n, WITHIN_RMS(low, high, rms)); + } + } +#else + for (u32 m = 0; m < samples_stride; m += channels) { + s16 ss = data[f + m + n]; + s16 ns = data[f + m + n + channels]; + for (u32 j = 0; j < height; j++) { + low = steps[j + 1], high = steps[j]; + if (HIT_WAVE(low, high, ss, ns)) { + (*rgba)[j * width + i] = COLOR_FOR_PIXEL(n, WITHIN_RMS(low, high, rms)); + } + } + } +#endif + } + } + + log_info("generating waveform took: %.2fs.", nn_get_tick() - start); + + //stbi_write_png("./waveform.png", width, height, 4, *rgba, width * 4); + + /* + struct nn_file file; + nn_file_open(&file, &al_str_c("./raw_samples.raw"), NNWT_FILE_CREATE); + size_t bytes = entry->raw.sample_count * entry->raw.fmt.channel_count * camu_audio_format_bytes_per_sample(&entry->raw.fmt); + nn_file_write(&file, entry->raw.data, bytes); + nn_file_close(&file); + */ +} +#endif + +static const char *octant_lut[] = { + " ", "๐œบจ", "๐œบซ", "๐Ÿฎ‚", "๐œด€", "โ–˜", "๐œด", "๐œด‚", "๐œดƒ", "๐œด„", "โ–", "๐œด…", "๐œด†", "๐œด‡", "๐œดˆ", "โ–€", + "๐œด‰", "๐œดŠ", "๐œด‹", "๐œดŒ", "๐Ÿฏฆ", "๐œด", "๐œดŽ", "๐œด", "๐œด", "๐œด‘", "๐œด’", "๐œด“", "๐œด”", "๐œด•", "๐œด–", "๐œด—", + "๐œด˜", "๐œด™", "๐œดš", "๐œด›", "๐œดœ", "๐œด", "๐œดž", "๐œดŸ", "๐Ÿฏง", "๐œด ", "๐œดก", "๐œดข", "๐œดฃ", "๐œดค", "๐œดฅ", "๐œดฆ", + "๐œดง", "๐œดจ", "๐œดฉ", "๐œดช", "๐œดซ", "๐œดฌ", "๐œดญ", "๐œดฎ", "๐œดฏ", "๐œดฐ", "๐œดฑ", "๐œดฒ", "๐œดณ", "๐œดด", "๐œดต", "๐Ÿฎ…", + "๐œบฃ", "๐œดถ", "๐œดท", "๐œดธ", "๐œดน", "๐œดบ", "๐œดป", "๐œดผ", "๐œดฝ", "๐œดพ", "๐œดฟ", "๐œต€", "๐œต", "๐œต‚", "๐œตƒ", "๐œต„", + "โ––", "๐œต…", "๐œต†", "๐œต‡", "๐œตˆ", "โ–Œ", "๐œต‰", "๐œตŠ", "๐œต‹", "๐œตŒ", "โ–ž", "๐œต", "๐œตŽ", "๐œต", "๐œต", "โ–›", + "๐œต‘", "๐œต’", "๐œต“", "๐œต”", "๐œต•", "๐œต–", "๐œต—", "๐œต˜", "๐œต™", "๐œตš", "๐œต›", "๐œตœ", "๐œต", "๐œตž", "๐œตŸ", "๐œต ", + "๐œตก", "๐œตข", "๐œตฃ", "๐œตค", "๐œตฅ", "๐œตฆ", "๐œตง", "๐œตจ", "๐œตฉ", "๐œตช", "๐œตซ", "๐œตฌ", "๐œตญ", "๐œตฎ", "๐œตฏ", "๐œตฐ", + "๐œบ ", "๐œตฑ", "๐œตฒ", "๐œตณ", "๐œตด", "๐œตต", "๐œตถ", "๐œตท", "๐œตธ", "๐œตน", "๐œตบ", "๐œตป", "๐œตผ", "๐œตฝ", "๐œตพ", "๐œตฟ", + "๐œถ€", "๐œถ", "๐œถ‚", "๐œถƒ", "๐œถ„", "๐œถ…", "๐œถ†", "๐œถ‡", "๐œถˆ", "๐œถ‰", "๐œถŠ", "๐œถ‹", "๐œถŒ", "๐œถ", "๐œถŽ", "๐œถ", + "โ–—", "๐œถ", "๐œถ‘", "๐œถ’", "๐œถ“", "โ–š", "๐œถ”", "๐œถ•", "๐œถ–", "๐œถ—", "โ–", "๐œถ˜", "๐œถ™", "๐œถš", "๐œถ›", "โ–œ", + "๐œถœ", "๐œถ", "๐œถž", "๐œถŸ", "๐œถ ", "๐œถก", "๐œถข", "๐œถฃ", "๐œถค", "๐œถฅ", "๐œถฆ", "๐œถง", "๐œถจ", "๐œถฉ", "๐œถช", "๐œถซ", + "โ–‚", "๐œถฌ", "๐œถญ", "๐œถฎ", "๐œถฏ", "๐œถฐ", "๐œถฑ", "๐œถฒ", "๐œถณ", "๐œถด", "๐œถต", "๐œถถ", "๐œถท", "๐œถธ", "๐œถน", "๐œถบ", + "๐œถป", "๐œถผ", "๐œถฝ", "๐œถพ", "๐œถฟ", "๐œท€", "๐œท", "๐œท‚", "๐œทƒ", "๐œท„", "๐œท…", "๐œท†", "๐œท‡", "๐œทˆ", "๐œท‰", "๐œทŠ", + "๐œท‹", "๐œทŒ", "๐œท", "๐œทŽ", "๐œท", "๐œท", "๐œท‘", "๐œท’", "๐œท“", "๐œท”", "๐œท•", "๐œท–", "๐œท—", "๐œท˜", "๐œท™", "๐œทš", + "โ–„", "๐œท›", "๐œทœ", "๐œท", "๐œทž", "โ–™", "๐œทŸ", "๐œท ", "๐œทก", "๐œทข", "โ–Ÿ", "๐œทฃ", "โ–†", "๐œทค", "๐œทฅ", "โ–ˆ" +}; + +#define RMS_BIT (1 << 8) + +void cmc_draw_waveform(struct cmc_list_entry *entry, u32 width, u32 height, bool include_rms, char ***buf, u32 **rgba) +{ +#ifdef OUTPUT_WAVEFORM_PNG + test_write_bitmap(entry, width, height, rgba); + return; +#endif + + u32 channels = entry->raw.fmt.channel_count; + u32 samples_per_column = entry->raw.sample_count / (width * 2); + + s32 pixel_height = height * 4; + s32 wave_min = INT16_MIN; + s32 wave_max = INT16_MAX; + s32 step = (wave_max * 2 + (pixel_height - 1)) / pixel_height; + + s16 *steps = al_malloc(sizeof(s16) * (pixel_height + 1)); + for (s32 i = 0; i <= (s32)pixel_height; i++) { + steps[i] = (s16)MAX(wave_max - step * i, wave_min); + } + al_assert(steps[pixel_height] == wave_min); + + s64 rms = 0; + s16 min, max; + s16 low, high; + s16 *data = (s16 *)entry->raw.data; + const u32 samples_stride = samples_per_column * channels; + u16 masks_for_chars[height]; + //u32 slen[height]; + //al_memset(slen, 0, sizeof(u32) * height); + for (u32 i = 0; i < width; i++) { + al_memset(masks_for_chars, 0, sizeof(u16) * height); + for (u32 k = 0; k < 2; k++) { + u32 f = (i * 2 + k) * samples_stride; + for (u32 n = 0; n < channels; n++) { + rms = 0; + min = wave_max, max = wave_min; + for (u32 m = 0; m <= samples_stride; m += channels) { + s16 ss = data[f + m + n]; + rms += ss * ss; + min = MIN(min, ss); + max = MAX(max, ss); + } + rms /= samples_per_column; + rms = sqrt(rms); + for (u32 j = 0; j < height; j++) { + for (u32 l = 0; l < 4; l++) { + u32 step = j * 4 + l; + low = steps[step + 1], high = steps[step]; + if (low < max && high >= min) { + if (include_rms && WITHIN_RMS(low, high, rms)) { + masks_for_chars[j] |= RMS_BIT; + } else { + masks_for_chars[j] |= 1 << (l * 2 + k); + } + } + } + } + } + } + for (u32 j = 0; j < height; j++) { + if (masks_for_chars[j] & RMS_BIT) { + buf[j][i][0] = 'R'; + masks_for_chars[j] &= ~RMS_BIT; + } + u32 len = masks_for_chars[j] == 0 ? 2 : 5; + al_memcpy(buf[j][i] + 1, octant_lut[masks_for_chars[j]], len); + //slen[j] += len; + } + } + + /* + for (u32 j = 0; j < height; j++) { + buf[j][slen[j]] = '\0'; + } + */ +} diff --git a/src/fruits/cmc/ui/waveform.h b/src/fruits/cmc/ui/waveform.h index f3803dd..464d589 100644 --- a/src/fruits/cmc/ui/waveform.h +++ b/src/fruits/cmc/ui/waveform.h @@ -1,5 +1,4 @@ /* - notcurses.h: Halves @@ -84,9 +83,15 @@ AAAAAAAAAAAAAAAA ๐œดข 0b00101011 +๐œบจ๐œบซ๐Ÿฎ‚๐œด€โ–˜๐œด๐œด‚๐œดƒ๐œด„โ–๐œด…๐œด†๐œด‡๐œดˆโ–€๐œด‰๐œดŠ๐œด‹๐œดŒ๐Ÿฏฆ๐œด๐œดŽ๐œด๐œด๐œด‘๐œด’๐œด“๐œด”๐œด•๐œด–๐œด—๐œด˜๐œด™๐œดš๐œด›๐œดœ๐œด๐œดž๐œดŸ๐Ÿฏง๐œด ๐œดก๐œดข๐œดฃ๐œดค๐œดฅ๐œดฆ๐œดง๐œดจ๐œดฉ๐œดช๐œดซ๐œดฌ๐œดญ๐œดฎ๐œดฏ๐œดฐ๐œดฑ๐œดฒ๐œดณ๐œดด๐œดต๐Ÿฎ…๐œบฃ๐œดถ๐œดท๐œดธ๐œดน๐œดบ๐œดป๐œดผ๐œดฝ๐œดพ๐œดฟ๐œต€๐œต๐œต‚๐œตƒ๐œต„โ––๐œต…๐œต†๐œต‡๐œตˆโ–Œ๐œต‰๐œตŠ๐œต‹๐œตŒโ–ž๐œต๐œตŽ๐œต๐œตโ–›๐œต‘๐œต’๐œต“๐œต”๐œต•๐œต–๐œต—๐œต˜๐œต™๐œตš๐œต›๐œตœ๐œต๐œตž๐œตŸ๐œต ๐œตก๐œตข๐œตฃ๐œตค๐œตฅ๐œตฆ๐œตง๐œตจ๐œตฉ๐œตช๐œตซ๐œตฌ๐œตญ๐œตฎ๐œตฏ๐œตฐ๐œบ ๐œตฑ๐œตฒ๐œตณ๐œตด๐œตต๐œตถ๐œตท๐œตธ๐œตน๐œตบ๐œตป๐œตผ๐œตฝ๐œตพ๐œตฟ๐œถ€๐œถ๐œถ‚๐œถƒ๐œถ„๐œถ…๐œถ†๐œถ‡๐œถˆ๐œถ‰๐œถŠ๐œถ‹๐œถŒ๐œถ๐œถŽ๐œถโ–—๐œถ๐œถ‘๐œถ’๐œถ“โ–š๐œถ”๐œถ•๐œถ–๐œถ—โ–๐œถ˜๐œถ™๐œถš๐œถ›โ–œ๐œถœ๐œถ๐œถž๐œถŸ๐œถ ๐œถก๐œถข๐œถฃ๐œถค๐œถฅ๐œถฆ๐œถง๐œถจ๐œถฉ๐œถช๐œถซโ–‚๐œถฌ๐œถญ๐œถฎ๐œถฏ๐œถฐ๐œถฑ๐œถฒ๐œถณ๐œถด๐œถต๐œถถ๐œถท๐œถธ๐œถน๐œถบ๐œถป๐œถผ๐œถฝ๐œถพ๐œถฟ๐œท€๐œท๐œท‚๐œทƒ๐œท„๐œท…๐œท†๐œท‡๐œทˆ๐œท‰๐œทŠ๐œท‹๐œทŒ๐œท๐œทŽ๐œท๐œท๐œท‘๐œท’๐œท“๐œท”๐œท•๐œท–๐œท—๐œท˜๐œท™๐œทšโ–„๐œท›๐œทœ๐œท๐œทžโ–™๐œทŸ๐œท ๐œทก๐œทขโ–Ÿ๐œทฃโ–†๐œทค๐œทฅโ–ˆ */ -static char *bar_lut[] = { +#include + +#include "../cmc.h" + +/* +static const char *bar_lut[] = { [0] = " ๐œบ โ–—๐œถ–โ–" , [1] = "๐œบฃโ–‚๐œท‹๐œท“๐œท•", [2] = "โ––๐œถปโ–„๐œทก๐œทฅ", @@ -94,11 +99,13 @@ static char *bar_lut[] = { [4] = "โ–Œ๐œท€โ–™๐œทคโ–ˆ" }; -static char *bar_bottom_lut[] = { +static const char *bar_bottom_lut[] = { [0] = " ๐œบซโ–๐œดกโ–" , [1] = "๐œบจ๐Ÿฎ‚๐œด…๐œดข๐œถ˜", [2] = "โ–˜๐œด‚โ–€๐œดฆโ–œ", [3] = "๐œด๐œด๐œด—๐Ÿฎ…๐œถซ", [4] = "โ–Œ๐œตŠโ–›๐œตฐโ–ˆ" }; +*/ +void cmc_draw_waveform(struct cmc_list_entry *entry, u32 width, u32 height, bool include_rms, char ***buf, u32 **rgba); diff --git a/src/liana/handlers/codec_server.c b/src/liana/handlers/codec_server.c index 7868a42..99db398 100644 --- a/src/liana/handlers/codec_server.c +++ b/src/liana/handlers/codec_server.c @@ -59,7 +59,9 @@ static void codec_server_write_info(struct lia_server_handler *handler, struct n nn_packet_write_u8(packet, stream->mode); nn_packet_write_u8(packet, stream->type); nn_packet_write_u64(packet, stream->duration); - nn_packet_write_s32(packet, i); + // stream->index = i is currently expected but not necessary. + al_assert(stream->index == (s32)i); + nn_packet_write_s32(packet, stream->index); switch (stream->mode) { case CAMU_NORMAL: { struct camu_video_format *fmt = &stream->video.fmt; diff --git a/src/liana/list.c b/src/liana/list.c index 1c7c6c9..0ceb591 100644 --- a/src/liana/list.c +++ b/src/liana/list.c @@ -148,7 +148,8 @@ static void pump_queue(struct lia_list *list); static bool handle_add_sink(struct lia_list *list, struct lia_list_sink *sink) { - if (list->current >= 0 && !list->idle) { + // The list being idle is not equivalent to current being unset. + if (list->current >= 0) { struct lia_list_entry *current = al_array_at(list->entries, list->current); u64 now = nn_get_timestamp(); u8 pause; @@ -243,9 +244,7 @@ static s32 get_sequence_from_entry_id(struct lia_list *list, u32 id) // Not returning the entry pointer here is a meaningful simplification. struct lia_list_entry *entry; al_array_foreach(list->entries, i, entry) { - if (entry->id == id) { - return (s32)i; - } + if (entry->id == id) return i; } return -1; } @@ -254,6 +253,7 @@ static bool handle_skipto(struct lia_list *list, s32 sequence, s32 index) { if (sequence == LIANA_SEQUENCE_ANY) sequence = list->current; if (sequence < 0) return true; // list->current = -1 + if (sequence == index) return true; if (sequence != list->current) { log_warn("Discarding out of date skip()."); return true; diff --git a/src/liana/meson.build b/src/liana/meson.build index 34abc66..12345c9 100644 --- a/src/liana/meson.build +++ b/src/liana/meson.build @@ -2,7 +2,8 @@ liana_server_src = [ 'list.c', 'server.c', 'handlers/codec_server.c', - 'handlers.c' + 'handlers.c', + 'process.c' ] liana_client_src = [ 'client.c', diff --git a/src/liana/process.c b/src/liana/process.c new file mode 100644 index 0000000..9a70210 --- /dev/null +++ b/src/liana/process.c @@ -0,0 +1,115 @@ +#ifdef CAMU_HAVE_FFMPEG +#include "../codec/ffmpeg/demuxer.h" +#include "../codec/ffmpeg/decoder.h" +#include "../codec/ffmpeg/resampler.h" +#endif + +#include "process.h" + +struct raw_samples_data { + struct camu_resampler *resampler; + u32 channels; + u32 bps; + struct nn_buffer buf; + u32 index; +}; + +static u32 append_samples(struct raw_samples_data *raw, u8 *data, u32 sample_count) +{ + u32 bytes = sample_count * raw->channels * raw->bps; + nn_buffer_write(&raw->buf, data, raw->index, bytes); + return bytes; +} + +static void data_callback(void *userdata, struct camu_codec_frame *frame) +{ + struct raw_samples_data *raw = (struct raw_samples_data *)userdata; + u8 **data = frame->av.frame->data; + s32 sample_count = frame->av.frame->nb_samples; + sample_count = raw->resampler->convert(raw->resampler, (const u8 **)data, sample_count); + data = raw->resampler->get_data(raw->resampler); + raw->index += append_samples(raw, data[0], (u32)sample_count); +} + +static void flush_resampler(struct camu_resampler *resampler, struct raw_samples_data *raw) +{ + s32 sample_count = resampler->flush(resampler); + u8 **data = resampler->get_data(resampler); + raw->index += append_samples(raw, data[0], (u32)sample_count); +} + +bool lia_prepare_visual_data(struct cch_handle *handle, struct lia_visual_data *data) +{ + struct camu_demuxer *demux = camu_ff_demuxer_create(); + struct camu_decoder *dec = camu_ff_decoder_create(); + struct camu_resampler *resampler = camu_ff_resampler_create(); + data->samples = NULL; + + if (!demux->init(demux, handle)) { + goto out; + } + + struct camu_codec_stream *stream; + struct camu_codec_stream *selected = NULL; + al_array_foreach_ptr(demux->streams, i, stream) { + if (stream->type == CAMU_STREAM_AUDIO) { + selected = stream; + break; + } + } + if (!selected) goto out; + demux->subscribed = 1 << selected->index; + + struct camu_resampler_format fmt; + AVCodecParameters *codecpar = selected->av.stream->codecpar; + fmt.in.format = codecpar->format; + fmt.req.format = CAMU_SAMPLE_FORMAT_S16; + fmt.in.sample_rate = codecpar->sample_rate; + fmt.req.sample_rate = codecpar->sample_rate; + av_channel_layout_copy(&fmt.in.channel_layout, &codecpar->ch_layout); + av_channel_layout_copy(&fmt.req.channel_layout, &codecpar->ch_layout); + fmt.in.channel_count = codecpar->ch_layout.nb_channels; + fmt.req.channel_count = codecpar->ch_layout.nb_channels; + + struct raw_samples_data raw; + raw.resampler = resampler; + raw.channels = fmt.req.channel_count; + raw.bps = camu_audio_format_bytes_per_sample(&fmt.req); + nn_buffer_init(&raw.buf); + raw.index = 0; + + if (!dec->init(dec, NULL, selected, data_callback, &raw)) { + goto out; + } + + fmt.resampler_needed = true; + if (!resampler->init(resampler, &fmt)) { + goto out; + } + + AVPacket *pkt = av_packet_alloc(); + struct camu_codec_packet packet = { .av.pkt = pkt }; + s32 status; + do { + status = demux->get_packet(demux, &packet); + if (status == CAMU_OK) { + dec->push_av_packet(dec, pkt); + dec->process(dec); + av_packet_unref(pkt); + } + } while (status == CAMU_OK); + dec->flush(dec); + flush_resampler(resampler, &raw); + + data->size = raw.index; + data->samples = raw.buf.data; + data->count = raw.index / raw.channels / raw.bps; + camu_audio_format_copy(&data->fmt, &fmt.req); + +out: + demux->free(&demux); + dec->free(&dec); + resampler->free(&resampler); + + return data->samples != NULL; +} diff --git a/src/liana/process.h b/src/liana/process.h new file mode 100644 index 0000000..a71728b --- /dev/null +++ b/src/liana/process.h @@ -0,0 +1,13 @@ +#pragma once + +#include "../codec/codec.h" +#include "../cache/handle.h" + +struct lia_visual_data { + struct camu_audio_format fmt; + u32 size; + u8 *samples; + u32 count; +}; + +bool lia_prepare_visual_data(struct cch_handle *handle, struct lia_visual_data *data); diff --git a/src/liana/server.c b/src/liana/server.c index abf4154..9d1f42a 100644 --- a/src/liana/server.c +++ b/src/liana/server.c @@ -1,6 +1,7 @@ #include "server.h" #include "handler.h" #include "handlers.h" +#include "process.h" #include "list.h" static inline u32 get_incremental_id(struct lia_server *server) @@ -445,16 +446,24 @@ static void duration_signal_callback(void *userdata) nn_signal_stop(&node->signal); nn_thread_join(&node->thread); node->handler->free(&node->handler); + bool got_visual_data = false; + struct lia_visual_data visual_data; + //cch_handle_seek(&node->handle, 0, SEEK_SET); + //got_visual_data = lia_prepare_visual_data(&node->handle, &visual_data); cch_entry_return_handle(node->entry, &node->handle); if (should_free_node(node)) { free_node(node); } else { if (node->errored) { - node->callback(node->userdata, LIANA_NODE_ERRORED, LIANA_TIMESTAMP_INVALID); + al_assert(node->duration == LIANA_TIMESTAMP_INVALID); + node->callback(node->userdata, LIANA_NODE_ERRORED, &node->duration); } else { - node->callback(node->userdata, LIANA_NODE_DURATION, node->duration); + node->callback(node->userdata, LIANA_NODE_DURATION, &node->duration); } } + if (got_visual_data) { + node->callback(node->userdata, LIANA_NODE_VISUAL_DATA, &visual_data); + } } void lia_node_get_duration(struct lia_node *node) diff --git a/src/liana/server.h b/src/liana/server.h index 679d607..13f4a6e 100644 --- a/src/liana/server.h +++ b/src/liana/server.h @@ -41,6 +41,7 @@ struct lia_node_connection { enum { LIANA_NODE_DURATION = 0, + LIANA_NODE_VISUAL_DATA, LIANA_NODE_ERRORED }; @@ -52,7 +53,7 @@ struct lia_node { bool closed; u64 duration; struct lia_server *server; - void (*callback)(void *, u8, u64); + void (*callback)(void *, u8, void *); void *userdata; // Temporary copy-paste from node_connection. bool errored; diff --git a/src/libclient/client.c b/src/libclient/client.c index 7350df2..ddcc1db 100644 --- a/src/libclient/client.c +++ b/src/libclient/client.c @@ -44,9 +44,24 @@ static bool meta_callback(void *userdata, struct nn_rpc_connection *conn, return false; } +static bool visual_data_callback(void *userdata, struct nn_rpc_connection *conn, + struct nn_packet *packet, struct nn_packet *rpacket) +{ + struct camu_client *client = (struct camu_client *)userdata; + (void)conn; + (void)rpacket; + + client->callback(client->userdata, CAMU_CLIENT_GOT_VISUAL_DATA, packet); + + nn_packet_stream_return_packet(conn->stream, packet); + + return false; +} + static struct nn_rpc_command commands[] = { { .op = CAMU_CLIENT_RESULTS, .callback = results_callback, .userdata = NULL }, - { .op = CAMU_CLIENT_META, .callback = meta_callback, .userdata = NULL } + { .op = CAMU_CLIENT_META, .callback = meta_callback, .userdata = NULL }, + { .op = CAMU_CLIENT_VISUAL_DATA, .callback = visual_data_callback, .userdata = NULL } }; static void idd_callback(void *userdata, struct nn_rpc_connection *conn, struct nn_packet *packet) @@ -135,6 +150,14 @@ void camu_client_get_page(struct camu_client *client, s32 id, u32 num) nn_rpc_connection_command(client->conn, packet, NULL, NULL); } +void camu_client_request_visual_data(struct camu_client *client, u32 node_id) +{ + struct nn_packet *packet = nn_rpc_get_packet(&client->client, CAMU_SERVER_CLIENT_COMMAND); + nn_packet_write_u8(packet, CAMU_CLIENT_REQUEST_VISUAL_DATA); + nn_packet_write_u32(packet, node_id); + nn_rpc_connection_command(client->conn, packet, NULL, NULL); +} + void camu_client_skipto(struct camu_client *client, str *list, s32 i) { struct nn_packet *packet = nn_rpc_get_packet(&client->client, CAMU_SERVER_LIST_ACTION); diff --git a/src/libclient/client.h b/src/libclient/client.h index 892bf0d..70b5b27 100644 --- a/src/libclient/client.h +++ b/src/libclient/client.h @@ -6,7 +6,8 @@ enum { CAMU_CLIENT_LOGGED_IN = 0, CAMU_CLIENT_SEARCH_CREATED, CAMU_CLIENT_GOT_RESULTS, - CAMU_CLIENT_GOT_META + CAMU_CLIENT_GOT_META, + CAMU_CLIENT_GOT_VISUAL_DATA }; struct camu_client { @@ -29,6 +30,8 @@ void camu_client_toggle_sink(struct camu_client *client, str *sink, str *list, b void camu_client_create_search(struct camu_client *client, str *module, str *query); void camu_client_get_page(struct camu_client *client, s32 id, u32 num); +void camu_client_request_visual_data(struct camu_client *client, u32 node_id); + void camu_client_skipto(struct camu_client *client, str *list, s32 i); void camu_client_seek(struct camu_client *client, str *list, u32 id, u64 pos); diff --git a/src/libclient/common.h b/src/libclient/common.h index b074277..677e6b2 100644 --- a/src/libclient/common.h +++ b/src/libclient/common.h @@ -2,5 +2,6 @@ enum { CAMU_CLIENT_RESULTS = 0, - CAMU_CLIENT_META + CAMU_CLIENT_META, + CAMU_CLIENT_VISUAL_DATA }; diff --git a/src/portal/py/modules/pixiv_web.py b/src/portal/py/modules/pixiv_web.py index c6a9c82..3c04202 100644 --- a/src/portal/py/modules/pixiv_web.py +++ b/src/portal/py/modules/pixiv_web.py @@ -97,16 +97,15 @@ class PixivWebBase(Search): if not ugoira: url = f'{BASE_URL}/illust/{illust_id}/ugoira_meta?lang={LANG}&version={VERSION}' ugoira = self.check_api_response(self.module.do_request(Method.GET, url)) - if not ugoira: - return None - kwargs['raw_responses']['ugoira'] = json.dumps(ugoira) - kwargs['media'] = { '0': self.parse_ugoira(ugoira, data['urls']['original']) } elif data['pageCount'] > 1: if not pages: url = f'{BASE_URL}/illust/{illust_id}/pages?lang={LANG}&version={VERSION}' pages = self.check_api_response(self.module.do_request(Method.GET, url)) - if not pages: - return None + # Retrieving ugoria or pages could return 404. + if ugoira: + kwargs['raw_responses']['ugoira'] = json.dumps(ugoira) + kwargs['media'] = { '0': self.parse_ugoira(ugoira, data['urls']['original']) } + elif pages: kwargs['raw_responses']['pages'] = json.dumps(pages) kwargs['media'] = self.parse_pages(pages) else: @@ -142,7 +141,7 @@ class PixivWebBase(Search): url = f'{BASE_URL}/illust/{illust_id}?lang={LANG}&version={VERSION}' obj = self.check_api_response(self.module.do_request(Method.GET, url)) if not obj: - return None + return self.create_tombstone({ 'id': illust_id }) return self.create_post(obj) def create_user(self, data: ParsedJson) -> User: @@ -249,6 +248,8 @@ class PixivWebBookmarks(PixivWebBase): self.pages[num] = [] for illust in obj['works']: if illust['userId'] == 0: + # Is it worth assuming this is a tombstone? Should run some tests + # to see if an illust object is ever returned anyway. post = self.create_tombstone(illust) else: post = self.request_illust(int(illust['id'])) diff --git a/src/portal/py/tests/archive_query.py b/src/portal/py/tests/archive_query.py index 9c4cf41..4cc1e1e 100644 --- a/src/portal/py/tests/archive_query.py +++ b/src/portal/py/tests/archive_query.py @@ -14,8 +14,9 @@ from tests.logger import Logger mode = 'pixiv_web' #mode = 'fanbox' #mode = 'patreon' -cmd = 'user' -#cmd = 'bookmarks' +#mode = 'instagram' +#cmd = 'user' +cmd = 'bookmarks' #cmd = 'search' module = ALL_MODULES[mode][0] if not module.init(): @@ -216,70 +217,9 @@ def download_args(): os.mkdir(f'{output_dir}/raw_responses') #args = module.search(f'following:{22781328}') - #args = [ - # '5838770', - # '44194940' - #] - # TODO: - #args = [ - # '68839843', - # '16105069', - # '67090304', - # '188984', - # '15041022', - # '3948', - # '2232374', - # '194668', - # '7835', - # '3067312', - # '59375709', - # '9077832', - # '199750', - # '5806160', - # '33726', - # '15261563', - # '7246', - # '8146', - # '40259810', - # '28865', - # '18362', - # '2750098', - # '2157729', - # '65523978', - # '154858', - # '37625681', - # '34642665', - # '24023', - # '3564336', - # '2658856', - # '990017', - # '55392960', - # '8967182', - # '8965979', - # '14305327', - # '469378', - # '8885157', - # '1107939', - # '18159122', - # '57304923', - # '6534119', - # '91521', - # '1074517', - # '14394', - # '46563991', - # '33694842', - # '84663', - # '4028970', - # '8876470', - # '1829995', - # '14113571', - # '56349311', - # '143555', - # '36388895', - # '14188490' - #] + # @TODO: + args = [] # twitter test: butcha_u, sep__rina, nagayori000, sattinittas, - threads = [] start = True diff --git a/src/portal/py/tests/test.py b/src/portal/py/tests/test.py index 2fe2eda..ef166d9 100644 --- a/src/portal/py/tests/test.py +++ b/src/portal/py/tests/test.py @@ -57,22 +57,21 @@ from twitter.scraper import Scraper # print('Skipping file') -module = ALL_MODULES['pixiv_web'][0] -module.init() -search = module.search('illust:91352621') -page = search.get_page(0) -for unique_id in page: - post = module.get_item(unique_id) - print(json.dumps(post, indent=4, cls=PostEncoder)) - -#module = ALL_MODULES['instagram'][0] +#module = ALL_MODULES['pixiv_web'][0] #module.init() -#search = module.search('plottttwistttttt') +#search = module.search('illust:91352621') #page = search.get_page(0) #for unique_id in page: # post = module.get_item(unique_id) # print(json.dumps(post, indent=4, cls=PostEncoder)) +module = ALL_MODULES['instagram'][0] +module.init() +search = module.search('plottttwistttttt') +page = search.get_page(0) +for unique_id in page: + post = module.get_item(unique_id) + print(json.dumps(post, indent=4, cls=PostEncoder)) #d = Date(DateType.EDITED, (0.0, 0.0), DateMeta.EDITED_AT_UNKNOWN_TIME) #d = Date(DateType.EDITED, (0.0, 0.0), DateMeta.NONE) diff --git a/src/screen/screen.c b/src/screen/screen.c index 1633a49..4d5aee5 100644 --- a/src/screen/screen.c +++ b/src/screen/screen.c @@ -194,6 +194,7 @@ static bool key_callback(void *userdata, u8 state, u16 button) break; } // fallthrough + case STELA_KEY_TAB: case STELA_KEY_D: { s32 n = 1; scr->callback(scr->userdata, CAMU_SCREEN_SKIP, &n); diff --git a/src/server/common.h b/src/server/common.h index 4232de8..9cb9501 100644 --- a/src/server/common.h +++ b/src/server/common.h @@ -24,7 +24,8 @@ enum { CAMU_CLIENT_CREATE_LIST = 0, CAMU_CLIENT_TOGGLE_SINK, CAMU_CLIENT_CREATE_SEARCH, - CAMU_CLIENT_GET_PAGE + CAMU_CLIENT_GET_PAGE, + CAMU_CLIENT_REQUEST_VISUAL_DATA }; enum { diff --git a/src/server/resource.h b/src/server/resource.h index 2999523..3158479 100644 --- a/src/server/resource.h +++ b/src/server/resource.h @@ -1,7 +1,7 @@ #pragma once -#include "../cache/entry.h" #include "../liana/server.h" +#include "../liana/process.h" struct camu_resource { u8 type; @@ -11,6 +11,8 @@ struct camu_resource { u64 duration; u32 ref; array(struct lia_list_entry *) pending; + struct lia_visual_data visual_data; + struct camu_server *server; }; struct camu_resource_file { diff --git a/src/server/server.c b/src/server/server.c index 47f9995..dfeee78 100644 --- a/src/server/server.c +++ b/src/server/server.c @@ -12,6 +12,7 @@ #ifdef CAMU_HAVE_PORTAL #include "../portal/src/packet_ext.h" #endif +#include "../codec/packet_ext.h" #include "server.h" #include "common.h" @@ -55,9 +56,25 @@ static struct camu_server_client *get_client_by_connection(struct camu_server *s return NULL; } +static struct camu_resource *get_resource_by_node_id(struct camu_server *server, u32 node_id) +{ + struct camu_resource *resource; + al_array_foreach(server->data.resources, i, resource) { + if (resource->node && resource->node->id == node_id) return resource; + } + return NULL; +} + static void write_list_entry(struct lia_list_entry *entry, struct nn_packet *packet) { + struct camu_resource *resource = (struct camu_resource *)entry->opaque; nn_packet_write_u32(packet, entry->id); + // For now this will get implicitly synced, probably on CURRENT_CHANGED. + if (resource->node) { + nn_packet_write_u32(packet, resource->node->id); + } else { + nn_packet_write_u32(packet, 0); + } nn_packet_write_u64(packet, entry->duration); nn_packet_write_u64(packet, entry->start); nn_packet_write_u64(packet, entry->paused_at); @@ -219,14 +236,34 @@ static void process_pending(struct camu_resource *resource) al_array_free(pending); } -static void node_callback(void *userdata, u8 op, u64 duration) +static void send_clients_visual_data(struct camu_server *server, u32 node_id, struct lia_visual_data *visual_data) +{ + struct camu_server_client *client; + al_array_foreach(server->clients, i, client) { + struct nn_packet *packet = nn_rpc_get_packet(client->conn->rpc, CAMU_CLIENT_VISUAL_DATA); + nn_packet_write_u32(packet, node_id); + nn_packet_write_audio_format(packet, &visual_data->fmt); + nn_packet_write_u32(packet, visual_data->size); + NNWT_PACKET_WRITE_DATA(packet, visual_data->samples, visual_data->size); + nn_packet_write_u32(packet, visual_data->count); + nn_rpc_connection_command(client->conn, packet, NULL, NULL); + } +} + +static void node_callback(void *userdata, u8 op, void *opaque) { struct camu_resource *resource = (struct camu_resource *)userdata; switch (op) { - case LIANA_NODE_DURATION: + case LIANA_NODE_DURATION: { + u64 duration = *(u64 *)opaque; resource->load = LIANA_ENTRY_LOADED; resource->duration = duration; break; + } + case LIANA_NODE_VISUAL_DATA: { + resource->visual_data = *(struct lia_visual_data *)opaque; + break; + } case LIANA_NODE_ERRORED: resource->load = LIANA_ENTRY_ERRORED; break; @@ -554,6 +591,14 @@ static bool client_command_callback(void *userdata, struct nn_rpc_connection *co break; } #endif + case CAMU_CLIENT_REQUEST_VISUAL_DATA: { + u32 node_id = nn_packet_read_u32(packet); + struct camu_resource *resource = get_resource_by_node_id(server, node_id); + if (resource) { + send_clients_visual_data(resource->server, resource->node->id, &resource->visual_data); + } + break; + } } out: @@ -667,7 +712,9 @@ static void handle_add_command(struct camu_server *server, struct lia_list *list #endif } al_assert(resource); + resource->server = server; al_array_push(server->data.resources, resource); + resource->entry = NULL; resource->node = NULL; resource->duration = LIANA_TIMESTAMP_INVALID; al_array_init(resource->pending); @@ -847,8 +894,8 @@ void camu_server_init(struct camu_server *server, struct nn_event_loop *loop) nn_rpc_add_command(&server->server, &commands[i]); } - al_array_init(server->data.resources); lia_server_init(&server->data.server, server->loop); + al_array_init(server->data.resources); #ifdef CAMU_HAVE_PORTAL camu_post_cache_init(&server->cache); -- cgit v1.2.3-101-g0448