diff options
172 files changed, 10070 insertions, 2962 deletions
@@ -6,8 +6,11 @@ compile_commands.json data/ doc/**.html env/bin/ -old/ -old_old/ +old*/ +src/portal/vendor/* +src/portal/cpy/portal.c +src/portal/cpy/portal.h +src/portal/py/config.py subprojects/BLAKE3/ subprojects/c89atomic/ subprojects/c89atomic.wrap @@ -37,7 +40,6 @@ subprojects/miniaudio/ subprojects/packagecache/ subprojects/sekihi subprojects/shaderc/ -subprojects/shoki subprojects/stb/ subprojects/vulkan/ subprojects/vulkan.wrap @@ -5,11 +5,11 @@ "systems": "systems" }, "locked": { - "lastModified": 1705309234, - "narHash": "sha256-uNRRNRKmJyCRC/8y1RqBkqWBLM034y4qN7EprSdmgyA=", + "lastModified": 1710146030, + "narHash": "sha256-SZ5L6eA7HJ/nmkzGG7/ISclqe6oZdOZTNoesiInkXPQ=", "owner": "numtide", "repo": "flake-utils", - "rev": "1ef2e671c3b0c19053962c07dbda38332dcebf26", + "rev": "b1d9ab70662946ef0850d488da1c9019f3a9752a", "type": "github" }, "original": { @@ -20,11 +20,11 @@ }, "nixpkgs": { "locked": { - "lastModified": 1707546158, - "narHash": "sha256-nYYJTpzfPMDxI8mzhQsYjIUX+grorqjKEU9Np6Xwy/0=", + "lastModified": 1712439257, + "narHash": "sha256-aSpiNepFOMk9932HOax0XwNxbA38GOUVOiXfUVPOrck=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "d934204a0f8d9198e1e4515dd6fec76a139c87f0", + "rev": "ff0dbd94265ac470dda06a657d5fe49de93b4599", "type": "github" }, "original": { @@ -7,7 +7,15 @@ flake-utils.lib.eachDefaultSystem (system: let - overlays = []; + overlays = [ + (self: super: { + libplacebo = super.libplacebo.overrideAttrs (old: { + patches = [ + ./subprojects/packagefiles/libplacebo/stretch.diff + ]; + }); + }) + ]; pkgs = import nixpkgs { inherit overlays; inherit system; }; in with pkgs; @@ -50,6 +58,7 @@ lcms shaderc libGL + vulkan-headers libxkbcommon glib dbus @@ -77,11 +86,11 @@ clang-tools_17 perf-tools valgrind - python3 - python3.pkgs.requests - python3.pkgs.pip - python3.pkgs.cython_3 - python3.pkgs.setuptools + ((pkgs.python312.override { enableOptimizations = true; reproducibleBuild = false; }).withPackages (python-pkgs: [ + python-pkgs.setuptools + python-pkgs.pip + python-pkgs.cython_3 + ])) nodePackages.pyright perl asciidoctor @@ -124,7 +133,8 @@ meson ninja cmake - python3 + wineWowPackages.staging + glslang ]; buildInputs = with pkgsCross.ucrt64; [ (zlib.override { shared = false; static = true; }) @@ -136,6 +146,25 @@ exec ${pkgs.zsh}/bin/zsh ''; }; + + devShells.mingw64-pthreads = let + gcc = pkgsCross.mingwW64.buildPackages.wrapCC (pkgsCross.mingwW64.buildPackages.gcc-unwrapped.override ({ + threadsCross = { + model = "posix"; + package = null; + }; + })); + gccWin32Stdenv = overrideCC pkgsCross.mingwW64.stdenv gcc; + in mkShell.override { stdenv = gccWin32Stdenv; } { + NIX_HARDENING_ENABLE = ""; + nativeBuildInputs = devShells.mingw64.nativeBuildInputs ++ []; + buildInputs = []; + shellHook = '' + export NIX_SHELL="mingw64-pthreads" + unset SOURCE_DATE_EPOCH + exec ${pkgs.zsh}/bin/zsh + ''; + }; } ); } diff --git a/meson.build b/meson.build index 47ffce7..81fc271 100644 --- a/meson.build +++ b/meson.build @@ -24,7 +24,8 @@ common_deps += [util] subdir('src/cache') subdir('src/codec') -subdir('src/bimu') +subdir('src/list') +subdir('src/shrub') sink_use_vulkan = false if get_option('sink').enabled() @@ -37,14 +38,14 @@ if get_option('sink').enabled() endif if get_option('server').enabled() or get_option('client').enabled() - shoki = subproject('shoki').get_variable('shoki') + subdir('src/portal') endif if get_option('server').enabled() - subdir('src/tree') + subdir('src/server') endif if get_option('client').enabled() - subdir('src/libclient') - subdir('src/fruits/cmc') + #subdir('src/libclient') + #subdir('src/fruits/cmc') endif diff --git a/pyrightconfig.json b/pyrightconfig.json new file mode 100644 index 0000000..5d93693 --- /dev/null +++ b/pyrightconfig.json @@ -0,0 +1,10 @@ +{ + "reportUnusedVariable": "none", + "extraPaths": [ + "./src/portal/py", + "./src/portal/vendor/vendor" + ], + "pythonVersion": "3.12", + "pythonPlatform": "Linux", + "executionEnvironments": [ { "root": "./src/portal/py" } ] +} diff --git a/scripts/line_count.sh b/scripts/line_count.sh index 2ecb7f5..19844b9 100755 --- a/scripts/line_count.sh +++ b/scripts/line_count.sh @@ -1,2 +1,8 @@ #! /usr/bin/env sh -find ./src -type f | xargs wc -l +find ./src -type f \ + -not -path "./src/portal/vendor/*" \ + -not -path "./src/portal/patches/*" \ + -not -path "./src/portal/cpy/portal.c" \ + -not -path "./src/portal/cpy/portal.h" \ + -not -path "*__pycache__*" \ + | xargs wc -l diff --git a/scripts/run_server.sh b/scripts/run_server.sh new file mode 100755 index 0000000..1c2cbbf --- /dev/null +++ b/scripts/run_server.sh @@ -0,0 +1,4 @@ +#! /usr/bin/env sh +export PYTHONDONTWRITEBYTECODE=1 +export PYTHONPATH=$HOME/c/camu/src/portal/vendor/vendor +rm -f /tmp/camu_sock && gdb -ex run --args ./src/server/server $@ diff --git a/scripts/run_tree.sh b/scripts/run_tree.sh deleted file mode 100755 index 274ec5c..0000000 --- a/scripts/run_tree.sh +++ /dev/null @@ -1,4 +0,0 @@ -#! /usr/bin/env sh -export PYTHONDONTWRITEBYTECODE=1 -export PYTHONPATH=$HOME/c/shoki/vendor/vendor -rm -f /tmp/tree_sock && gdb -ex run --args ./src/tree/tree $@ diff --git a/src/bimu/client.h b/src/bimu/client.h deleted file mode 100644 index db1f9e5..0000000 --- a/src/bimu/client.h +++ /dev/null @@ -1,31 +0,0 @@ -#pragma once - -#include <al/types.h> -#include <aki/packet_stream.h> - -#include "vcr.h" - -struct bmu_client { - struct aki_event_loop *loop; - u16 node_id; - u16 connection_id; - str addr; - s32 port; - struct aki_packet_stream control; - struct aki_packet_stream data; - bool closed; - u32 level; - bool corked; - u64 duration; - u64 seek_pos; - array(s32) subscribed; - struct bmu_vcr vcr; - void (*callback)(void *, u8, struct bmu_vcr_stream *stream, void *); - void *userdata; -}; - -void bmu_client_connect(struct bmu_client *client, struct aki_event_loop *loop, str *addr, s32 port, u16 node_id); -void bmu_client_reseek(struct bmu_client *client); -void bmu_client_seek(struct bmu_client *client, f64 percent); -void bmu_client_close(struct bmu_client *client); -void bmu_client_free(struct bmu_client *client); diff --git a/src/bimu/common.h b/src/bimu/common.h deleted file mode 100644 index 485d454..0000000 --- a/src/bimu/common.h +++ /dev/null @@ -1,23 +0,0 @@ -#pragma once - -enum { - BIMU_STREAM_AUDIO = 0, - BIMU_STREAM_VIDEO, - BIMU_STREAM_UNKNOWN, -}; - -enum { - BIMU_CONNECTION_CONTROL = 0, - BIMU_CONNECTION_DATA -}; - -enum { - BIMU_CONTROL_INFO = 0, - BIMU_CONTROL_SUBSCRIBE, - BIMU_CONTROL_RECONNECT, - BIMU_CONTROL_START, - BIMU_CONTROL_TOGGLE_PAUSE, - BIMU_CONTROL_PAUSE, - BIMU_CONTROL_RESUME, - BIMU_CONTROL_SEEK -}; diff --git a/src/bimu/handler.h b/src/bimu/handler.h deleted file mode 100644 index c6b482d..0000000 --- a/src/bimu/handler.h +++ /dev/null @@ -1,52 +0,0 @@ -#pragma once - -#include <aki/packet_pool.h> - -#include "../codec/codec.h" -#include "../cache/handle.h" - -#include "vcr.h" - -struct bmu_server_handler { - bool (*init)(struct bmu_server_handler *, struct cch_handle *, struct aki_packet_pool *); - void (*write_info)(struct bmu_server_handler *, struct aki_packet *); - void (*subscribe)(struct bmu_server_handler *, s32, bool); - u64 (*get_duration)(struct bmu_server_handler *); - bool (*seek)(struct bmu_server_handler *, u64); - bool (*step)(struct bmu_server_handler *); - void (*free)(struct bmu_server_handler **); - void *userdata; -}; - -enum { - BIMU_PACKET_DATA = 0, - BIMU_PACKET_EOF, - BIMU_PACKET_ERROR -}; - -enum { - BIMU_CLIENT_CONFIGURE = 0, - BIMU_CLIENT_SET, - BIMU_CLIENT_DATA, - BIMU_CLIENT_REMOVE_BUFFERS, - BIMU_CLIENT_EOF, - BIMU_CLIENT_CLOSED -}; - -struct bmu_seek_req { - u64 base; - u64 ts; - u64 delay; -}; - -struct bmu_client_stream; -struct bmu_client_handler { - bool (*init)(struct bmu_client_handler *, struct camu_renderer *, struct bmu_vcr_stream *); - void (*handle_data_packet)(struct bmu_client_handler *, struct aki_packet *); - void (*handle_eof)(struct bmu_client_handler *); - void (*flush)(struct bmu_client_handler *); - void (*free)(struct bmu_client_handler **); - struct bmu_vcr_stream *stream; - void (*callback)(void *, u8, struct bmu_vcr_stream *stream, void *); - void *userdata; -}; diff --git a/src/bimu/handlers/codec.h b/src/bimu/handlers/codec.h deleted file mode 100644 index a8031b1..0000000 --- a/src/bimu/handlers/codec.h +++ /dev/null @@ -1,20 +0,0 @@ -#pragma once - -#include "../handler.h" - -#include "../../codec/codec.h" - -struct bmu_codec_server { - struct bmu_server_handler handler; - struct camu_demuxer *demux; - struct camu_packet packet; - struct aki_packet_pool *pool; -}; - -struct bmu_codec_client { - struct bmu_client_handler handler; - struct camu_decoder *dec; -}; - -struct bmu_server_handler *bmu_codec_server_create(void); -struct bmu_client_handler *bmu_codec_client_create(void); diff --git a/src/bimu/handlers/codec_client.c b/src/bimu/handlers/codec_client.c deleted file mode 100644 index 79ef27d..0000000 --- a/src/bimu/handlers/codec_client.c +++ /dev/null @@ -1,104 +0,0 @@ -#include "../../codec/codec.h" -#include "../../codec/libav/decoder.h" -#include "../../codec/libav/packet_ext.h" - -#include "../../bimu/vcr.h" - -#include "codec.h" - -static void data_callback(void *userdata, struct camu_frame *frame) -{ - struct bmu_codec_client *codec = (struct bmu_codec_client *)userdata; - codec->handler.callback(codec->handler.userdata, BIMU_CLIENT_DATA, codec->handler.stream, frame); -} - -static bool codec_client_init(struct bmu_client_handler *handler, struct camu_renderer *renderer, - struct bmu_vcr_stream *stream) -{ - struct bmu_codec_client *codec = (struct bmu_codec_client *)handler; - codec->dec = camu_lav_decoder_create(); - codec->handler.stream = stream; - if (!codec->dec->init(codec->dec, renderer, &stream->stream)) { - return false; - } - codec->dec->stream = &stream->stream; - codec->handler.callback(codec->handler.userdata, BIMU_CLIENT_CONFIGURE, codec->handler.stream, NULL); - codec->dec->set_callback(codec->dec, data_callback, codec); - return true; -} - -#ifdef HAVE_FFMPEG -static void push_av_packet(struct bmu_codec_client *codec, AVPacket *pkt) -{ - struct camu_packet packet; - packet.av.pkt = pkt; - s32 ret = codec->dec->push(codec->dec, &packet); - av_packet_unref(pkt); - av_packet_free(&pkt); - al_assert(ret == CAMU_OK); -} -#endif - -static void push_packet(struct bmu_codec_client *codec, struct aki_buffer *buffer) -{ - struct camu_packet packet; - packet.buffer = buffer; - s32 ret = codec->dec->push(codec->dec, &packet); - al_assert(ret == CAMU_OK); -} - -static void codec_client_handle_data_packet(struct bmu_client_handler *handler, struct aki_packet *packet) -{ - struct bmu_codec_client *codec = (struct bmu_codec_client *)handler; - switch (aki_packet_read_u8(packet)) { - case CAMU_NORMAL: { - struct aki_buffer buffer; - aki_packet_read_buffer(packet, &buffer); - push_packet(codec, &buffer); - break; - } -#ifdef HAVE_FFMPEG - case CAMU_FFMPEG_COMPAT: { - push_av_packet(codec, aki_packet_read_av_packet(packet)); - break; - } -#endif - } - s32 ret = codec->dec->process(codec->dec); - al_assert(ret == CAMU_ERR_AGAIN || ret == CAMU_ERR_EOF); -} - -static void codec_client_handle_eof(struct bmu_client_handler *handler) -{ - struct bmu_codec_client *codec = (struct bmu_codec_client *)handler; - s32 ret = codec->dec->push(codec->dec, NULL); - // Flush returns success. - ret = codec->dec->process(codec->dec); - al_assert(ret == CAMU_ERR_EOF); - codec->handler.callback(codec->handler.userdata, BIMU_CLIENT_EOF, codec->handler.stream, NULL); -} - -static void codec_client_flush(struct bmu_client_handler *handler) -{ - struct bmu_codec_client *codec = (struct bmu_codec_client *)handler; - codec->dec->flush(codec->dec); -} - -static void codec_client_free(struct bmu_client_handler **handler) -{ - struct bmu_codec_client *codec = (struct bmu_codec_client *)*handler; - codec->dec->free(&codec->dec); - al_free(codec); - *handler = NULL; -} - -struct bmu_client_handler *bmu_codec_client_create(void) -{ - struct bmu_codec_client *codec = al_alloc_object(struct bmu_codec_client); - codec->handler.init = codec_client_init; - codec->handler.handle_data_packet = codec_client_handle_data_packet; - codec->handler.handle_eof = codec_client_handle_eof; - codec->handler.flush = codec_client_flush; - codec->handler.free = codec_client_free; - return (struct bmu_client_handler *)codec; -} diff --git a/src/bimu/local.c b/src/bimu/local.c deleted file mode 100644 index 76824ed..0000000 --- a/src/bimu/local.c +++ /dev/null @@ -1,281 +0,0 @@ -#include "../codec/libav/packet_ext.h" - -#include "local.h" - -#define PACKETS 1400 - -static struct bmu_local_stream *stream_at_index(struct bmu_local *runner, s32 index) -{ - struct bmu_local_stream *stream = NULL; - al_array_foreach(runner->streams, i, stream) { - if (stream->s.index == index) { - return stream; - } - } - return NULL; -} - -static aki_thread_result AKI_THREADCALL stream_thread(void *userdata) -{ - struct bmu_local_stream *stream = (struct bmu_local_stream *)userdata; - do { - if (!aki_packet_cache_wait(&stream->cache)) break; - struct aki_packet *packet = aki_packet_cache_pop(&stream->cache); - if (!packet) break; - aki_mutex_lock(&stream->mutex); - if (al_atomic_s32_load(&stream->state, AL_ATOMIC_RELAXED) == BIMU_LOCAL_STOPPED) { - aki_cond_wait(&stream->cond, &stream->mutex); - } - s32 state = al_atomic_s32_load(&stream->state, AL_ATOMIC_RELAXED); - aki_mutex_unlock(&stream->mutex); - if (state == BIMU_LOCAL_CLOSED) { - aki_packet_pool_return(&stream->runner->pool, packet); - break; - } - switch (aki_packet_read_u8(packet)) { - case 0: - stream->s.client->handle_data_packet(stream->s.client, packet); - break; - case 1: - stream->s.client->handle_eof(stream->s.client); - break; - } - aki_packet_pool_return(&stream->runner->pool, packet); - } while (1); - return 0; -} - -static u8 packet_pool_callback(void *userdata, struct aki_packet *packet) -{ - struct bmu_local *runner = (struct bmu_local *)userdata; - if (!packet) { - aki_event_loop_break(&runner->loop); - return AKI_PACKET_POOL_CLOSED; - } - aki_packet_write_size(packet); - u8 close = aki_packet_read_u8(packet); - if (close) return AKI_PACKET_POOL_DISABLE; - struct bmu_local_stream *stream = NULL; - s32 index = aki_packet_read_s32(packet); - if (index >= 0) { - stream = stream_at_index(runner, index); - if (!stream) return AKI_PACKET_POOL_RETURN; - if (al_atomic_s32_load(&stream->state, AL_ATOMIC_RELAXED) != BIMU_LOCAL_CLOSED) { - if (aki_packet_cache_send_packet(&stream->cache, packet)) { - return AKI_PACKET_POOL_KEEP; - } - } - } - return AKI_PACKET_POOL_RETURN; -} - -bool bmu_local_init(struct bmu_local *runner, struct cch_entry *entry) -{ - runner->entry = entry; - runner->server = bmu_codec_server_create(); - cch_entry_get_handle(runner->entry, &runner->handle); - aki_event_loop_init(&runner->loop); - aki_packet_pool_init(&runner->pool, PACKETS, &runner->loop, packet_pool_callback, runner); - if (!runner->server->init(runner->server, &runner->handle, &runner->pool, NULL)) { - return false; - } - al_array_init(runner->streams); - al_atomic_bool_store(&runner->closed, false, AL_ATOMIC_RELAXED); - return true; -} - -bool bmu_local_prepare_clients(struct bmu_local *runner, struct camu_renderer *renderer) -{ - struct aki_packet *packet = aki_packet_create(); - runner->server->write_info(runner->server, packet); - aki_packet_write_size(packet); - u32 count = aki_packet_read_u32(packet); - for (u32 i = 0; i < count; i++) { - struct bmu_local_stream *local = al_alloc_object(struct bmu_local_stream); - struct bmu_client_stream *stream = &local->s; - local->runner = runner; - local->s.client = bmu_codec_client_create(); - local->s.client->callback = runner->callback; - local->s.client->userdata = runner->userdata; - aki_cond_init(&local->cond); - aki_mutex_init(&local->mutex); - aki_packet_cache_init(&local->cache, PACKETS); - al_atomic_s32_store(&local->state, BIMU_LOCAL_RUNNING, AL_ATOMIC_RELAXED); - stream->stream.type = aki_packet_read_u8(packet); - switch (stream->stream.type) { - case CAMU_NORMAL: - stream->index = 0; - stream->type = BIMU_STREAM_VIDEO; - stream->stream.video.width = aki_packet_read_s32(packet); - stream->stream.video.height = aki_packet_read_s32(packet); - break; -#ifdef HAVE_FFMPEG - case CAMU_FFMPEG_COMPAT: { - stream->stream.type = CAMU_FFMPEG_COMPAT; - stream->stream.av.format_context = avformat_alloc_context(); - const AVCodec *codec = avcodec_find_decoder(aki_packet_read_av_codec_id(packet)); - stream->stream.av.stream = aki_packet_read_av_stream(stream->stream.av.format_context, codec, packet); - stream->index = stream->stream.av.stream->index; - switch (stream->stream.av.stream->codecpar->codec_type) { - case AVMEDIA_TYPE_AUDIO: - stream->type = BIMU_STREAM_AUDIO; - break; - case AVMEDIA_TYPE_VIDEO: - stream->type = BIMU_STREAM_VIDEO; - break; - case AVMEDIA_TYPE_SUBTITLE: - default: - stream->type = BIMU_STREAM_UNKNOWN; - break; - } - } -#endif - } - if (!local->s.client->init(local->s.client, renderer, stream)) { - aki_packet_free(packet); - return false; - } - aki_thread_create(&local->thread, stream_thread, stream); - al_array_push(runner->streams, local); - } - aki_packet_free(packet); - return true; -} - -void bmu_local_stream_stop(struct bmu_local_stream *stream) -{ - aki_mutex_lock(&stream->mutex); - if (al_atomic_s32_load(&stream->state, AL_ATOMIC_RELAXED) == BIMU_LOCAL_RUNNING) { - al_atomic_s32_store(&stream->state, BIMU_LOCAL_STOPPED, AL_ATOMIC_RELAXED); - } - aki_mutex_unlock(&stream->mutex); -} - -void bmu_local_stream_continue(struct bmu_local_stream *stream) -{ - aki_mutex_lock(&stream->mutex); - al_atomic_s32_store(&stream->state, BIMU_LOCAL_RUNNING, AL_ATOMIC_RELAXED); - if (aki_cond_is_waiting(&stream->cond)) { - aki_cond_signal(&stream->cond); - } - aki_mutex_unlock(&stream->mutex); -} - -void bmu_local_stream_close(struct bmu_local_stream *stream) -{ - aki_mutex_lock(&stream->mutex); - u8 state = al_atomic_s32_load(&stream->state, AL_ATOMIC_RELAXED); - if (state == BIMU_LOCAL_CLOSED) { - aki_mutex_unlock(&stream->mutex); - return; - } - al_atomic_s32_store(&stream->state, BIMU_LOCAL_CLOSED, AL_ATOMIC_RELAXED); - aki_packet_cache_disable(&stream->cache); - if (aki_cond_is_waiting(&stream->cond)) { - aki_cond_signal(&stream->cond); - } - aki_mutex_unlock(&stream->mutex); - aki_thread_join(&stream->thread); -} - -static aki_thread_result AKI_THREADCALL decode_thread(void *userdata) -{ - struct bmu_local *runner = (struct bmu_local *)userdata; - while (!al_atomic_bool_load(&runner->closed, AL_ATOMIC_RELAXED) - && runner->server->step(runner->server)) {} - return 0; -} - -static aki_thread_result AKI_THREADCALL event_loop_thread(void *userdata) -{ - struct bmu_local *runner = (struct bmu_local *)userdata; - aki_event_loop_run(&runner->loop); - return 0; -} - -void bmu_local_run(struct bmu_local *runner) -{ - aki_thread_create(&runner->decode_thread, decode_thread, runner); - aki_thread_create(&runner->event_loop_thread, event_loop_thread, runner); -} - -f64 bmu_local_get_duration(struct bmu_local *runner) -{ - return runner->server->get_duration(runner->server) / 1000000.0; -} - -void bmu_local_seek(struct bmu_local *runner, f64 percent) -{ - s64 pos = runner->server->get_duration(runner->server) * percent; - f64 fpos = pos / 1000000.0; - al_atomic_bool_store(&runner->closed, true, AL_ATOMIC_RELAXED); - aki_packet_pool_disable(&runner->pool); - struct bmu_local_stream *stream = NULL; - al_array_foreach(runner->streams, i, stream) { - bmu_local_stream_close(stream); - struct aki_packet *packet; - while ((packet = aki_packet_cache_pop(&stream->cache))) { - aki_packet_pool_return(&stream->runner->pool, packet); - } - } - aki_thread_join(&runner->decode_thread); - runner->callback(runner->userdata, BIMU_CLIENT_SEEK, NULL, &fpos); - runner->server->seek(runner->server, pos); - al_atomic_bool_store(&runner->closed, false, AL_ATOMIC_RELAXED); - aki_packet_pool_enable(&runner->pool); - al_array_foreach(runner->streams, i, stream) { - stream->s.client->flush(stream->s.client); - al_atomic_s32_store(&stream->state, BIMU_LOCAL_RUNNING, AL_ATOMIC_RELAXED); - aki_packet_cache_enable(&stream->cache); - aki_thread_create(&stream->thread, stream_thread, stream); - } - aki_thread_create(&runner->decode_thread, decode_thread, runner); -} - -void bmu_local_stop(struct bmu_local *runner) -{ - al_atomic_bool_store(&runner->closed, true, AL_ATOMIC_RELAXED); - cch_handle_disable(&runner->handle); - struct bmu_local_stream *stream = NULL; - al_array_foreach(runner->streams, i, stream) { - bmu_local_stream_close(stream); - } - aki_thread_join(&runner->decode_thread); - cch_entry_return_handle(runner->entry, &runner->handle); - struct aki_packet *packet = aki_packet_pool_get(&runner->pool); - if (!packet) al_assert(false); - aki_packet_write_u8(packet, 1); - aki_packet_pool_submit(&runner->pool, packet); - aki_thread_join(&runner->event_loop_thread); - runner->callback(runner->userdata, BIMU_CLIENT_CLOSED, NULL, NULL); -} - -void bmu_local_close(struct bmu_local *runner) -{ - struct bmu_local_stream *stream = NULL; - al_array_foreach(runner->streams, i, stream) { - stream->s.client->free(&stream->s.client); - switch (stream->s.stream.type) { - case CAMU_NORMAL: - break; -#ifdef HAVE_FFMPEG - case CAMU_FFMPEG_COMPAT: - avformat_free_context(stream->s.stream.av.format_context); - break; -#endif - } - struct aki_packet *packet; - while ((packet = aki_packet_cache_pop(&stream->cache))) { - aki_packet_pool_return(&stream->runner->pool, packet); - } - aki_mutex_destroy(&stream->mutex); - aki_cond_destroy(&stream->cond); - aki_packet_cache_free(&stream->cache); - al_free(stream); - al_array_remove_at_iter(runner->streams, i); - } - al_array_free(runner->streams); - runner->server->free(&runner->server); - aki_packet_pool_free(&runner->pool); - aki_event_loop_destroy(&runner->loop); -} diff --git a/src/bimu/local.h b/src/bimu/local.h deleted file mode 100644 index 10c0cc8..0000000 --- a/src/bimu/local.h +++ /dev/null @@ -1,52 +0,0 @@ -#pragma once - -#include <al/atomic.h> -#include <aki/packet_pool.h> -#include <aki/packet_cache.h> -#include <aki/signal.h> - -#include "../cache/entry.h" - -#include "client.h" -#include "common.h" - -enum { - BIMU_LOCAL_RUNNING = 0, - BIMU_LOCAL_STOPPED, - BIMU_LOCAL_CLOSED -}; - -struct bmu_local_stream { - struct bmu_client_stream s; - atomic_s32 state; - struct aki_cond cond; - struct aki_mutex mutex; - struct aki_packet_cache cache; - struct aki_thread thread; - struct bmu_local *runner; -}; - -struct bmu_local { - struct cch_entry *entry; - struct cch_handle handle; - struct bmu_server_handler *server; - struct aki_thread decode_thread; - struct aki_event_loop loop; - struct aki_packet_pool pool; - struct aki_thread event_loop_thread; - atomic_bool closed; - array(struct bmu_local_stream *) streams; - void (*callback)(void *, u8, struct bmu_client_stream *stream, void *); - void *userdata; -}; - -bool bmu_local_init(struct bmu_local *runner, struct cch_entry *entry); -bool bmu_local_prepare_clients(struct bmu_local *runner, struct camu_renderer *renderer); -void bmu_local_stream_stop(struct bmu_local_stream *stream); -void bmu_local_stream_continue(struct bmu_local_stream *stream); -void bmu_local_stream_close(struct bmu_local_stream *stream); -void bmu_local_run(struct bmu_local *runner); -f64 bmu_local_get_duration(struct bmu_local *runner); -void bmu_local_seek(struct bmu_local *runner, f64 percent); -void bmu_local_stop(struct bmu_local *runner); -void bmu_local_close(struct bmu_local *runner); diff --git a/src/bimu/server.h b/src/bimu/server.h deleted file mode 100644 index 20f73c6..0000000 --- a/src/bimu/server.h +++ /dev/null @@ -1,59 +0,0 @@ -#pragma once - -#include <al/array.h> -#include <aki/packet_stream.h> -#include <aki/packet_pool.h> - -#include "../cache/entry.h" -#include "../cache/handle.h" - -#include "handler.h" - -enum { - BIMU_PLAYING = 0, - BIMU_NOT_PAUSED, - BIMU_PAUSED, - BIMU_PAUSED_AND_RUNNING -}; - -struct bmu_node_connection { - u16 id; - struct bmu_node *node; - struct aki_packet_stream *control; - struct aki_packet_stream *data; - struct aki_packet_pool pool; - s32 running; - u8 paused; - u64 seek_pos; - struct cch_handle handle; - struct bmu_server_handler *handler; - struct aki_thread thread; -}; - -enum { - BIMU_NODE_STARTED = 1, - BIMU_NODE_SEEKED = 1 << 1, - BIMU_NODE_PAUSED = 1 << 2 -}; - -struct bmu_node { - u16 id; - struct cch_entry *entry; - u8 flags; - u64 start; - u64 seek_pos; - u16 connection_id; - array(struct bmu_node_connection *) connections; - struct bmu_server *server; -}; - -struct bmu_server { - struct aki_event_loop *loop; - struct aki_packet_stream server; - array(struct bmu_node *) nodes; -}; - -bool bmu_server_init(struct bmu_server *server); -void bmu_server_listen(struct bmu_server *server, struct aki_event_loop *loop, str *addr, s32 port); -u16 bmu_server_create_node(struct bmu_server *server, struct cch_entry *entry); -void bmu_server_close(struct bmu_server *server); diff --git a/src/bimu/vcr.h b/src/bimu/vcr.h deleted file mode 100644 index 0f81c12..0000000 --- a/src/bimu/vcr.h +++ /dev/null @@ -1,45 +0,0 @@ -#pragma once - -#include <aki/packet_cache.h> -#include <al/atomic.h> -#include <aki/packet_stream.h> -#include <aki/signal.h> - -#include "../codec/codec.h" - -enum { - BIMU_STREAM_RUNNING = 0, - BIMU_STREAM_STOPPED, - BIMU_STREAM_CLOSED -}; - -struct bmu_vcr_stream { - u8 type; - s32 index; - struct camu_stream stream; - struct bmu_client_handler *client; - atomic_s32 state; - struct aki_packet_cache cache; - s32 buffered; - struct aki_cond cond; - struct aki_mutex mutex; - bool running; - struct aki_thread thread; - struct bmu_vcr *vcr; -}; - -struct bmu_vcr { - array(struct bmu_vcr_stream *) streams; - atomic_u64 count; - struct aki_packet_stream *data; - struct aki_signal signal; -}; - -void bmu_vcr_init(struct bmu_vcr *vcr, struct aki_event_loop *loop, struct aki_packet_stream *data); -void bmu_vcr_add_stream(struct bmu_vcr *vcr, struct bmu_vcr_stream *stream); -bool bmu_vcr_push_packet(struct bmu_vcr *vcr, struct aki_packet *packet); -void bmu_vcr_stream_close(struct bmu_vcr_stream *stream); -void bmu_vcr_flush(struct bmu_vcr *vcr); -void bmu_vcr_stream_cork(struct bmu_vcr_stream *stream); -void bmu_vcr_stream_uncork(struct bmu_vcr_stream *stream); -void bmu_vcr_free(struct bmu_vcr *vcr); diff --git a/src/buffer/audio.c b/src/buffer/audio.c index e32ef1f..bfefed7 100644 --- a/src/buffer/audio.c +++ b/src/buffer/audio.c @@ -2,6 +2,7 @@ #include "audio.h" #include "common.h" +#include "common_internal.h" #define BUFFER_USEC (12 * 1000000L) #define BUFFER_WATERMARK_LOW (4 * 1000000L) // Must be a most half of the buffer size. @@ -11,18 +12,11 @@ #ifdef CAMU_AUDIO_BUFFER_FADE #define FADE 1.0 -#define FADE_LENGTH 18000 -#define FADE_STEP(rate) (FADE / FADE_LENGTH) -//#define FADE_STEP(rate) ((FADE / FADE_LENGTH) / (FADE_LENGTH - 1.0)) +#define FADE_LENGTH(rate) (rate * 0.4) +#define FADE_STEP(rate) (FADE / FADE_LENGTH(rate)) #endif enum { - FLOWING = 0, - FLUSHED, - SIGNALED -}; - -enum { PAUSE_PRE = 0, PAUSE_PAUSED, #ifdef CAMU_AUDIO_BUFFER_FADE @@ -31,8 +25,8 @@ enum { PAUSE_PLAYING }; -static bool setup_optimal_resampler(struct camu_audio_buffer *buf, struct camu_lav_resampler *resamp, - struct camu_lav_resample_fmt *fmt) +static bool setup_optimal_resampler(struct camu_audio_buffer *buf, struct camu_ff_resampler *resamp, + struct camu_ff_resample_fmt *fmt) { /* switch ((enum AVSampleFormat)fmt->in_format) { @@ -63,9 +57,9 @@ static bool setup_optimal_resampler(struct camu_audio_buffer *buf, struct camu_l */ buf->fade_rate = buf->fmt.req_sample_rate / 10; - buf->bytes_per_sample = (s32)camu_lav_resample_fmt_bytes_per_sample(fmt); + buf->bytes_per_sample = (s32)camu_ff_resample_fmt_bytes_per_sample(fmt); - return camu_lav_resampler_init(resamp, fmt); + return camu_ff_resampler_init(resamp, fmt); } bool camu_audio_buffer_init(struct camu_audio_buffer *buf, struct camu_clock *clock, @@ -73,15 +67,16 @@ bool camu_audio_buffer_init(struct camu_audio_buffer *buf, struct camu_clock *cl { buf->mixer = mixer; buf->clock = clock; - buf->pause = PAUSE_PRE; buf->ignore_desync = false; + al_atomic_size_t_store(&buf->continue_mark, 0, AL_ATOMIC_RELAXED); buf->buffered = false; + al_atomic_u8_store(&buf->flow, FLOWING, AL_ATOMIC_RELAXED); + buf->pts = -1.0; + buf->pause = PAUSE_PRE; #ifdef CAMU_AUDIO_BUFFER_FADE buf->fade_period = 0; buf->fade_offset = 0; #endif - al_atomic_size_t_store(&buf->continue_mark, 0, AL_ATOMIC_RELAXED); - al_atomic_u8_store(&buf->flow, FLOWING, AL_ATOMIC_RELAXED); #ifdef CAMU_MIXER_THREADED al_atomic_bool_store(&buf->ref, false, AL_ATOMIC_RELAXED); #endif @@ -106,12 +101,12 @@ bool camu_audio_buffer_configure(struct camu_audio_buffer *buf, struct camu_stre in_format_name, buf->fmt.in_channel_layout.nb_channels, buf->fmt.in_sample_rate, req_format_name, buf->fmt.req_channel_layout.nb_channels, buf->fmt.req_sample_rate); - buf->size = camu_lav_resample_fmt_usec_to_bytes(&buf->fmt, BUFFER_USEC); + buf->size = camu_ff_resample_fmt_usec_to_bytes(&buf->fmt, BUFFER_USEC); buf->data = (u8 *)al_malloc(buf->size); al_ring_buffer_init(&buf->rb, buf->data, buf->size); - buf->watermark.low = camu_lav_resample_fmt_usec_to_bytes(&buf->fmt, BUFFER_WATERMARK_LOW); - buf->watermark.high = camu_lav_resample_fmt_usec_to_bytes(&buf->fmt, BUFFER_WATERMARK_HIGH); + buf->watermark.low = camu_ff_resample_fmt_usec_to_bytes(&buf->fmt, BUFFER_WATERMARK_LOW); + buf->watermark.high = camu_ff_resample_fmt_usec_to_bytes(&buf->fmt, BUFFER_WATERMARK_HIGH); camu_peak_buffer_init(&buf->peak); buf->stream = stream; @@ -119,35 +114,43 @@ bool camu_audio_buffer_configure(struct camu_audio_buffer *buf, struct camu_stre return true; } +#ifdef _DEBUG_ +#define BUFFER_OCCUPIED_DEBUG(buf) \ + camu_ff_resample_fmt_bytes_to_sec(&buf->fmt, buf->size - al_ring_buffer_space(&buf->rb)) +#else +#define BUFFER_OCCUPIED_DEBUG(buf) 0 +#endif + static bool push_internal(struct camu_audio_buffer *buf, u8 **data, s32 sample_count, f64 pts) { if (buf->fmt.resampler_needed) { - sample_count = camu_lav_resampler_convert(&buf->resamp, (const u8 **)data, sample_count); - data = camu_lav_resampler_get_data(&buf->resamp); + sample_count = camu_ff_resampler_convert(&buf->resamp, (const u8 **)data, sample_count); + data = camu_ff_resampler_get_data(&buf->resamp); } + if (sample_count <= 0) return false; - f64 duration = camu_lav_resample_fmt_samples_to_sec(&buf->fmt, sample_count); + f64 duration = camu_ff_resample_fmt_samples_to_sec(&buf->fmt, sample_count); if (pts + duration < camu_clock_get_base_pts(buf->clock)) { return false; } + if (buf->pts == -1.0) buf->pts = pts; - if (sample_count <= 0) return false; - - size_t size, have = camu_lav_resample_fmt_samples_to_bytes(&buf->fmt, (size_t)sample_count); + size_t size, have = camu_ff_resample_fmt_samples_to_bytes(&buf->fmt, (size_t)sample_count); size_t space = al_ring_buffer_space(&buf->rb); if (!buf->buffered && buf->size - space > buf->watermark.high) { buf->callback(buf->userdata, CAMU_BUFFER_BUFFERED); buf->buffered = true; + al_log_debug("audio_buffer", "Buffered (watermark: %.2fs).", BUFFER_OCCUPIED_DEBUG(buf)); } size_t peak = buf->peak.buf.size; if (space < (have + peak)) { camu_peak_buffer_push(&buf->peak, data[0], have); if (peak >= buf->watermark.low) { - // Discard the peak buffer. If this happens the user of this buffer is - // taking _way_ too long to stop. + // Discard the peak buffer. If this happens the writer of this buffer + // is taking _way_ too long to stop. camu_peak_buffer_flush(&buf->peak, &size); } - // Continuing based on an outdated continue_mark value is safe as long as the + // Continuing based on an outdated continue mark value is safe as long as the // peak buffer is smaller than the low watermark and the low watermark is // less than or equal to half the buffer size. al_atomic_size_t_store(&buf->continue_mark, buf->watermark.low + peak, AL_ATOMIC_RELAXED); @@ -165,7 +168,7 @@ static bool push_internal(struct camu_audio_buffer *buf, u8 **data, s32 sample_c return true; } -#ifdef HAVE_FFMPEG +#ifdef CAMU_HAVE_FFMPEG static void push_av_frame_internal(struct camu_audio_buffer *buf, AVFrame *frame) { // It doesn't matter if there's a race here with flush(), as long @@ -185,7 +188,7 @@ void camu_audio_buffer_push(struct camu_audio_buffer *buf, struct camu_frame *fr switch (frame->type) { case CAMU_NORMAL: break; -#ifdef HAVE_FFMPEG +#ifdef CAMU_HAVE_FFMPEG case CAMU_FFMPEG_COMPAT: { push_av_frame_internal(buf, frame->av.frame); break; @@ -195,23 +198,32 @@ void camu_audio_buffer_push(struct camu_audio_buffer *buf, struct camu_frame *fr al_free(frame); } +void camu_audio_buffer_unpause(struct camu_audio_buffer *buf) +{ + buf->pause = PAUSE_PRE; +} + +// Not thread-safe, must be called while the buffer is not be read or written to. void camu_audio_buffer_reset(struct camu_audio_buffer *buf) { al_atomic_size_t_store(&buf->continue_mark, 0, AL_ATOMIC_RELAXED); + buf->buffered = false; al_atomic_u8_store(&buf->flow, FLOWING, AL_ATOMIC_RELAXED); - buf->pts = camu_clock_get_base_pts(buf->clock) + camu_mixer_get_latency(buf->mixer); + buf->pts = -1.0; buf->pause = PAUSE_PRE; - buf->buffered = false; al_ring_buffer_reset(&buf->rb); } +// flush() always comes from the same thread as push(). void camu_audio_buffer_flush(struct camu_audio_buffer *buf) { - al_atomic_u8_store(&buf->flow, FLUSHED, AL_ATOMIC_RELAXED); if (!buf->buffered) { buf->callback(buf->userdata, CAMU_BUFFER_BUFFERED); buf->buffered = true; + al_log_debug("audio_buffer", "Buffered (watermark: %.2fs).", BUFFER_OCCUPIED_DEBUG(buf)); } + al_atomic_u8_store(&buf->flow, FLUSHED, AL_ATOMIC_RELAXED); + al_log_debug("audio_buffer", "Flush requested."); } #ifdef CAMU_AUDIO_BUFFER_FADE @@ -263,13 +275,12 @@ static void handle_fade(struct camu_audio_buffer *buf, u8 *data, size_t size, bo // The sole purpose of PAUSE_PRE is to avoid a fade in at the start of the stream. size_t camu_audio_buffer_read(struct camu_audio_buffer *buf, u8 *data, size_t req) { - f64 pts = camu_clock_get_pts(buf->clock, camu_mixer_get_latency(buf->mixer)); - if (pts < 0.0) { + if (camu_clock_is_paused(buf->clock)) { #ifdef CAMU_AUDIO_BUFFER_FADE if (buf->pause == PAUSE_PLAYING) { // Fade out. Signified by >0 fade_period and pause = PAUSE_FADING. buf->volume = FADE; - buf->fade_period = FADE_LENGTH; + buf->fade_period = FADE_LENGTH(buf->fade_rate); buf->fade_offset = 0; buf->pause = PAUSE_FADING; } else if (buf->fade_period == 0) { // Fade out done. @@ -283,7 +294,7 @@ size_t camu_audio_buffer_read(struct camu_audio_buffer *buf, u8 *data, size_t re } else if (buf->pause == PAUSE_FADING || buf->pause == PAUSE_PAUSED) { // Fade in. Signified by >0 fade_period and pause = PAUSE_PLAYING. buf->volume = 1.0 - FADE; - buf->fade_period = FADE_LENGTH; + buf->fade_period = FADE_LENGTH(buf->fade_rate); buf->fade_offset = 0; // We don't want to consider sync if reversing an active // fade but we do if pause = PAUSE_PAUSED. @@ -297,6 +308,7 @@ size_t camu_audio_buffer_read(struct camu_audio_buffer *buf, u8 *data, size_t re return req; #endif } + f64 pts = camu_clock_get_pts(buf->clock, camu_mixer_get_latency(buf->mixer)); size_t ret, signal = req; size_t size = al_ring_buffer_occupied(&buf->rb); // Cut off fade if it's reaching too far. @@ -309,20 +321,20 @@ size_t camu_audio_buffer_read(struct camu_audio_buffer *buf, u8 *data, size_t re // Possibly attempt syncing to the clock. For this to work the mixer must // report a reasonably accurate value for latency. if (UNLIKELY(!buf->ignore_desync && (buf->pause == PAUSE_PRE || buf->pause == PAUSE_PAUSED))) { - if (fabs(pts) >= 0.322) { + if (UNLIKELY(fabs(pts) >= 0.322 && buf->pause != PAUSE_PRE)) { al_log_warn("audio_buffer", "Abnormally large audio desync of %.5fs", pts); } if (pts > 0.0) { - ret = camu_lav_resample_fmt_sec_to_bytes(&buf->fmt, pts); + ret = camu_ff_resample_fmt_sec_to_bytes(&buf->fmt, pts); ret = AL_MIN(ret, size); al_log_debug("audio_buffer", "Skipping %.5fs of audio (%zu bytes).", pts, ret); ret = al_ring_buffer_discard(&buf->rb, ret); size -= ret; - buf->pts += camu_lav_resample_fmt_bytes_to_sec(&buf->fmt, ret); + buf->pts += camu_ff_resample_fmt_bytes_to_sec(&buf->fmt, ret); // Could go on to underrun. } else if (pts < 0.0) { pts = -pts; - ret = camu_lav_resample_fmt_sec_to_bytes(&buf->fmt, pts); + ret = camu_ff_resample_fmt_sec_to_bytes(&buf->fmt, pts); ret = AL_MIN(ret, req); al_log_debug("audio_buffer", "Delaying audio by %.5fs (%zu bytes).", pts, ret); al_memset(data, 0, ret); @@ -341,9 +353,9 @@ size_t camu_audio_buffer_read(struct camu_audio_buffer *buf, u8 *data, size_t re // If flow = FLUSHED, we know we've recieved all the data // this stream wants to send. Signal that we are done. signal = size; + buf->callback(buf->userdata, CAMU_BUFFER_EOF); al_atomic_u8_store(&buf->flow, SIGNALED, AL_ATOMIC_RELAXED); al_log_debug("audio_buffer", "Flushed (signal: %zu).", signal); - buf->callback(buf->userdata, CAMU_BUFFER_EOF); } else { // Put silence into the remainder of the request buffer. al_memset(data + size, 0, req - size); @@ -368,7 +380,7 @@ size_t camu_audio_buffer_read(struct camu_audio_buffer *buf, u8 *data, size_t re #endif // Read as much as we can from the ring buffer. ret = al_ring_buffer_read(&buf->rb, data, req); - buf->pts += camu_lav_resample_fmt_bytes_to_sec(&buf->fmt, ret); + buf->pts += camu_ff_resample_fmt_bytes_to_sec(&buf->fmt, ret); if (flow == FLOWING) { ret = al_atomic_size_t_load(&buf->continue_mark, AL_ATOMIC_RELAXED); // Request to uncork if we are below the mark. @@ -378,9 +390,7 @@ size_t camu_audio_buffer_read(struct camu_audio_buffer *buf, u8 *data, size_t re } #ifdef CAMU_AUDIO_BUFFER_FADE } - if (buf->fade_period > 0) { - handle_fade(buf, data, req, buf->pause == PAUSE_FADING); - } + if (buf->fade_period > 0) handle_fade(buf, data, req, buf->pause == PAUSE_FADING); #endif } // Never return less than req, unless we hit the end of the stream. @@ -390,7 +400,7 @@ size_t camu_audio_buffer_read(struct camu_audio_buffer *buf, u8 *data, size_t re void camu_audio_buffer_free(struct camu_audio_buffer *buf) { if (buf->fmt.resampler_needed) { - camu_lav_resampler_close(&buf->resamp); + camu_ff_resampler_close(&buf->resamp); } if (buf->data) al_free(buf->data); camu_peak_buffer_free(&buf->peak); diff --git a/src/buffer/audio.h b/src/buffer/audio.h index 1a8baf0..c4ba142 100644 --- a/src/buffer/audio.h +++ b/src/buffer/audio.h @@ -4,7 +4,7 @@ #include <al/ring_buffer.h> #include "../codec/codec.h" -#include "../codec/libav/resampler.h" +#include "../codec/ffmpeg/resampler.h" #include "../mixer/mixer.h" #include "clock.h" @@ -27,10 +27,8 @@ struct camu_audio_buffer { s32 fade_rate; s32 bytes_per_sample; - struct camu_lav_resampler resamp; - struct camu_lav_resample_fmt fmt; - - atomic_u8 flow; + struct camu_ff_resampler resamp; + struct camu_ff_resample_fmt fmt; u8 *data; size_t size; @@ -41,6 +39,8 @@ struct camu_audio_buffer { struct camu_peak_buffer peak; atomic_size_t continue_mark; + atomic_u8 flow; + #ifdef CAMU_MIXER_THREADED atomic_bool ref; #endif @@ -52,6 +52,7 @@ struct camu_audio_buffer { bool camu_audio_buffer_init(struct camu_audio_buffer *buf, struct camu_clock *clock, struct camu_mixer *mixer); bool camu_audio_buffer_configure(struct camu_audio_buffer *buf, struct camu_stream *stream); void camu_audio_buffer_push(struct camu_audio_buffer *buf, struct camu_frame *frame); +void camu_audio_buffer_unpause(struct camu_audio_buffer *buf); void camu_audio_buffer_reset(struct camu_audio_buffer *buf); void camu_audio_buffer_flush(struct camu_audio_buffer *buf); size_t camu_audio_buffer_read(struct camu_audio_buffer *buf, u8 *data, size_t req); diff --git a/src/buffer/clock.c b/src/buffer/clock.c index f8284ec..032e44a 100644 --- a/src/buffer/clock.c +++ b/src/buffer/clock.c @@ -4,57 +4,72 @@ void camu_clock_set(struct camu_clock *clock, u64 base, u64 ts, u64 delay) { - al_atomic_f64_store(&clock->base, base / 1000000.f, AL_ATOMIC_RELAXED); + al_atomic_f64_store(&clock->base, base / 1000000.0, AL_ATOMIC_RELAXED); al_atomic_f64_store(&clock->tick, 0.0, AL_ATOMIC_RELAXED); - al_atomic_u64_store(&clock->start, ts, AL_ATOMIC_RELAXED); + al_atomic_u64_store(&clock->start, ts + delay, AL_ATOMIC_RELAXED); clock->delay = delay; + clock->pause_at = -1.0; } -bool camu_clock_calc_tick(struct camu_clock *clock) +f64 calc_tick_internal(struct camu_clock *clock) { - u64 start = al_atomic_u64_load(&clock->start, AL_ATOMIC_RELAXED); - if (start == 0L) return false; - f64 tick = al_atomic_f64_load(&clock->tick, AL_ATOMIC_RELAXED); - u64 ts = (tick == 0.0 && clock->delay == 0) ? start : aki_get_timestamp(); - al_assert(start <= ts); - u64 diff = ts - start; - if (tick == 0.0) { - if (!(diff >= clock->delay && clock->delay >= (diff - clock->delay))) { - // We are past the requested start time. - return false; - } - diff = clock->delay - (diff - clock->delay); - tick = aki_get_tick() - al_atomic_f64_load(&clock->base, AL_ATOMIC_RELAXED); - } - al_atomic_f64_store(&clock->tick, tick + (diff / 1000000.f), AL_ATOMIC_RELAXED); - return true; + s64 diff = clock->delay == 0 ? 0 : ((al_atomic_u64_load(&clock->start, AL_ATOMIC_ACQUIRE)) - aki_get_timestamp()); + f64 tick = aki_get_tick() - al_atomic_f64_load(&clock->base, AL_ATOMIC_RELAXED); + tick += diff / 1000000.0; + al_atomic_f64_store(&clock->tick, tick, AL_ATOMIC_RELAXED); + al_atomic_u64_store(&clock->start, 0, AL_ATOMIC_RELEASE); + return tick; +} + +bool camu_clock_is_late(struct camu_clock *clock) +{ + s64 diff = (s64)(al_atomic_u64_load(&clock->start, AL_ATOMIC_RELAXED)) - aki_get_timestamp(); + return clock->delay > 0 && diff < 0; } void camu_clock_resume(struct camu_clock *clock) { - u64 start = al_atomic_u64_load(&clock->start, AL_ATOMIC_ACQUIRE); - if (start == 0L) return; - al_atomic_u64_store(&clock->start, 0L, AL_ATOMIC_RELEASE); + f64 tick = al_atomic_f64_load(&clock->tick, AL_ATOMIC_ACQUIRE); + if (tick != 0.0) return; + u64 start = al_atomic_u64_load(&clock->start, AL_ATOMIC_RELAXED); + if (start != 0L) { + // No longer report paused, calculate tick on first call to get_pts(). + al_atomic_f64_store(&clock->tick, -1.0, AL_ATOMIC_RELEASE); + } else { + tick = al_atomic_f64_load(&clock->base, AL_ATOMIC_RELAXED); + al_atomic_f64_store(&clock->tick, aki_get_tick() - tick, AL_ATOMIC_RELEASE); + } } -static f64 get_pts_internal(struct camu_clock *clock) +void camu_clock_arm_resume(struct camu_clock *clock, u64 ts) { - f64 tick = al_atomic_f64_load(&clock->tick, AL_ATOMIC_RELAXED); - return aki_get_tick() - tick; + al_atomic_u64_store(&clock->start, ts, AL_ATOMIC_RELAXED); + al_atomic_f64_store(&clock->tick, -1.0, AL_ATOMIC_RELAXED); } void camu_clock_pause(struct camu_clock *clock) { - u64 start = al_atomic_u64_load(&clock->start, AL_ATOMIC_ACQUIRE); - if (start != 0L) return; - al_atomic_f64_store(&clock->base, get_pts_internal(clock), AL_ATOMIC_RELAXED); - al_atomic_u64_store(&clock->start, aki_get_timestamp(), AL_ATOMIC_RELEASE); + f64 tick = al_atomic_f64_load(&clock->tick, AL_ATOMIC_ACQUIRE); + if (tick == 0.0) return; + al_atomic_f64_store(&clock->base, aki_get_tick() - tick, AL_ATOMIC_RELAXED); + al_atomic_f64_store(&clock->tick, 0.0, AL_ATOMIC_RELEASE); +} + +void camu_clock_arm_pause(struct camu_clock *clock, f64 pts) +{ + clock->pause_at = pts; } bool camu_clock_is_paused(struct camu_clock *clock) { - u64 start = al_atomic_u64_load(&clock->start, AL_ATOMIC_RELAXED); - return start != 0L; + f64 tick = al_atomic_f64_load(&clock->tick, AL_ATOMIC_RELAXED); + if (clock->pause_at >= 0.0 && (aki_get_tick() - tick) >= clock->pause_at) { + al_atomic_f64_store(&clock->base, clock->pause_at, AL_ATOMIC_RELAXED); + al_atomic_f64_store(&clock->tick, 0.0, AL_ATOMIC_RELAXED); + clock->pause_at = -1.0; + return true; + } + return tick == 0.0; } f64 camu_clock_get_base_pts(struct camu_clock *clock) @@ -64,10 +79,8 @@ f64 camu_clock_get_base_pts(struct camu_clock *clock) f64 camu_clock_get_pts(struct camu_clock *clock, f64 offset) { - u64 start = al_atomic_u64_load(&clock->start, AL_ATOMIC_RELAXED); - if (start != 0L) return -1.0; - f64 base = al_atomic_f64_load(&clock->base, AL_ATOMIC_RELAXED); - f64 pts = get_pts_internal(clock) + offset; - if (pts < base) return -1.0; + f64 tick = al_atomic_f64_load(&clock->tick, AL_ATOMIC_RELAXED); + if (tick == -1.0) tick = calc_tick_internal(clock); + f64 pts = (aki_get_tick() - tick) + offset; return pts; } diff --git a/src/buffer/clock.h b/src/buffer/clock.h index a7399b5..bdd50d8 100644 --- a/src/buffer/clock.h +++ b/src/buffer/clock.h @@ -7,12 +7,15 @@ struct camu_clock { atomic_f64 base, tick; atomic_u64 start; u64 delay; + f64 pause_at; }; void camu_clock_set(struct camu_clock *clock, u64 base, u64 ts, u64 delay); -bool camu_clock_calc_tick(struct camu_clock *clock); +bool camu_clock_is_late(struct camu_clock *clock); void camu_clock_resume(struct camu_clock *clock); +void camu_clock_arm_resume(struct camu_clock *clock, u64 ts); void camu_clock_pause(struct camu_clock *clock); +void camu_clock_arm_pause(struct camu_clock *clock, f64 pts); bool camu_clock_is_paused(struct camu_clock *clock); f64 camu_clock_get_base_pts(struct camu_clock *clock); f64 camu_clock_get_pts(struct camu_clock *clock, f64 offset); diff --git a/src/buffer/common_internal.h b/src/buffer/common_internal.h new file mode 100644 index 0000000..0dc1e5f --- /dev/null +++ b/src/buffer/common_internal.h @@ -0,0 +1,10 @@ +#pragma once + +enum { + // Flowing. + FLOWING, + // Got request to flush. + FLUSHED, + // Realized flush. + SIGNALED +}; diff --git a/src/buffer/frame_queue.h b/src/buffer/frame_queue.h index df58497..512b7f2 100644 --- a/src/buffer/frame_queue.h +++ b/src/buffer/frame_queue.h @@ -1,6 +1,6 @@ #pragma once -#ifdef HAVE_FFMPEG +#ifdef CAMU_HAVE_FFMPEG #include <libavutil/frame.h> #endif @@ -15,7 +15,7 @@ enum { struct camu_frame_queue { void (*push)(struct camu_frame_queue *, struct camu_frame *, f64); -#ifdef HAVE_FFMPEG +#ifdef CAMU_HAVE_FFMPEG void (*push_av_frame)(struct camu_frame_queue *, AVFrame *, f64); #endif void (*flush)(struct camu_frame_queue *); diff --git a/src/buffer/video.c b/src/buffer/video.c index 15af78e..ca8baab 100644 --- a/src/buffer/video.c +++ b/src/buffer/video.c @@ -2,6 +2,7 @@ #include "video.h" #include "common.h" +#include "common_internal.h" //#define CAMU_VIDEO_BUFFER_FORCE_SCALER @@ -10,12 +11,14 @@ #define BUFFER_WATERMARK_HIGH 14 #define BUFFER_WATERMARK_RESET BUFFER_WATERMARK_HIGH + 10. -bool camu_video_buffer_init(struct camu_video_buffer *buf, struct camu_clock *clock, +bool camu_video_buffer_init(struct camu_video_buffer *buf, struct camu_clock *clock, f64 offset, struct camu_renderer *renderer) { buf->clock = clock; + buf->offset = offset; buf->queue = renderer->create_queue(renderer); buf->buffered = false; + al_atomic_u8_store(&buf->flow, FLOWING, AL_ATOMIC_RELAXED); #ifdef CAMU_SCREEN_THREADED al_atomic_bool_store(&buf->ref, false, AL_ATOMIC_RELAXED); #endif @@ -31,7 +34,7 @@ bool camu_video_buffer_configure(struct camu_video_buffer *buf, struct camu_stre buf->avg_frame_duration = 0.0; break; } -#ifdef HAVE_FFMPEG +#ifdef CAMU_HAVE_FFMPEG case CAMU_FFMPEG_COMPAT: { s32 width = stream->av.stream->codecpar->width; s32 height = stream->av.stream->codecpar->height; @@ -50,7 +53,7 @@ bool camu_video_buffer_configure(struct camu_video_buffer *buf, struct camu_stre buf->fmt.req_width = width; buf->fmt.req_height = height; buf->fmt.req_format = AV_PIX_FMT_RGBA; - if (camu_lav_scaler_init(&buf->scale, &buf->fmt) && buf->fmt.scaler_needed) { + if (camu_ff_scaler_init(&buf->scale, &buf->fmt) && buf->fmt.scaler_needed) { format_name = av_get_pix_fmt_name(buf->fmt.req_format); al_log_info("video_buffer", "Scaling stream to: %s (%dx%d)", format_name, buf->fmt.req_width, buf->fmt.req_height); @@ -65,19 +68,27 @@ bool camu_video_buffer_configure(struct camu_video_buffer *buf, struct camu_stre return true; } +bool camu_video_buffer_is_single_frame(struct camu_video_buffer *buf) +{ + return buf->single_frame; +} + static void after_push_internal(struct camu_video_buffer *buf) { s32 count = buf->queue->count(buf->queue); if (!buf->buffered && (buf->single_frame || count >= BUFFER_WATERMARK_BUFFERED)) { + // Preserve order of, flush -> callback -> set flow, for single frames. + if (buf->single_frame) buf->queue->flush(buf->queue); buf->callback(buf->userdata, CAMU_BUFFER_BUFFERED); buf->buffered = true; + al_log_debug("video_buffer", "Buffered (watermark: %d frames).", count); + if (buf->single_frame) al_atomic_u8_store(&buf->flow, FLUSHED, AL_ATOMIC_RELAXED); } if (count >= BUFFER_WATERMARK_RESET) buf->queue->reset(buf->queue); else if (count >= BUFFER_WATERMARK_HIGH) buf->callback(buf->userdata, CAMU_BUFFER_CORK); - if (buf->single_frame) camu_video_buffer_flush(buf); } -#ifdef HAVE_FFMPEG +#ifdef CAMU_HAVE_FFMPEG static void push_av_frame_internal(struct camu_video_buffer *buf, AVFrame *frame) { f64 pts = 0.0; @@ -91,7 +102,7 @@ static void push_av_frame_internal(struct camu_video_buffer *buf, AVFrame *frame } #ifdef CAMU_VIDEO_BUFFER_FORCE_SCALER if (buf->fmt.scaler_needed) { - if (!camu_lav_scaler_scale(&buf->scale, (const u8 **)frame->data, frame->linesize)) { + if (!camu_ff_scaler_scale(&buf->scale, (const u8 **)frame->data, frame->linesize)) { return; } av_frame_free(&frame); @@ -105,11 +116,16 @@ static void push_av_frame_internal(struct camu_video_buffer *buf, AVFrame *frame void camu_video_buffer_push(struct camu_video_buffer *buf, struct camu_frame *frame) { + // A single frame will be sent again after a seek, discard it here (for now). + if (buf->single_frame && buf->buffered) { + camu_frame_discard(frame); + return; + } switch (frame->type) { case CAMU_NORMAL: buf->queue->push(buf->queue, frame, 0.0); break; -#ifdef HAVE_FFMPEG +#ifdef CAMU_HAVE_FFMPEG case CAMU_FFMPEG_COMPAT: push_av_frame_internal(buf, frame->av.frame); al_free(frame); @@ -119,34 +135,41 @@ void camu_video_buffer_push(struct camu_video_buffer *buf, struct camu_frame *fr after_push_internal(buf); } +// Not thread-safe, must be called while the buffer is not be read or written to. void camu_video_buffer_reset(struct camu_video_buffer *buf) { buf->queue->reset(buf->queue); buf->buffered = false; + al_atomic_u8_store(&buf->flow, FLOWING, AL_ATOMIC_RELAXED); } -bool camu_video_buffer_is_single_frame(struct camu_video_buffer *buf) -{ - return buf->single_frame; -} - +// flush() always comes from the same thread as push(). void camu_video_buffer_flush(struct camu_video_buffer *buf) { buf->queue->flush(buf->queue); if (!buf->buffered) { buf->callback(buf->userdata, CAMU_BUFFER_BUFFERED); buf->buffered = true; + al_log_debug("video_buffer", "Buffered (watermark: %d frames).", buf->queue->count(buf->queue)); } + al_atomic_u8_store(&buf->flow, FLUSHED, AL_ATOMIC_RELAXED); } bool camu_video_buffer_read(struct camu_video_buffer *buf, void *out) { - f64 pts = buf->single_frame ? 0.0 : camu_clock_get_pts(buf->clock, 0); - if (pts < 0.0) pts = camu_clock_get_base_pts(buf->clock); - u8 ret = buf->queue->read(buf->queue, pts, out); - if (ret == CAMU_QUEUE_EOF || (ret == CAMU_QUEUE_OK && buf->single_frame)) { + bool paused = buf->single_frame || camu_clock_is_paused(buf->clock); + f64 base = camu_clock_get_base_pts(buf->clock); + if (!paused) { + f64 pts = camu_clock_get_pts(buf->clock, buf->offset); + if (pts > base) base = pts; + } + u8 ret = buf->queue->read(buf->queue, base, out); + u8 flow = al_atomic_u8_load(&buf->flow, AL_ATOMIC_RELAXED); + if (flow == FLUSHED && (ret == CAMU_QUEUE_EOF || (buf->single_frame && ret == CAMU_QUEUE_OK))) { buf->callback(buf->userdata, CAMU_BUFFER_EOF); - } else if (buf->queue->count(buf->queue) <= BUFFER_WATERMARK_LOW) { + al_atomic_u8_store(&buf->flow, SIGNALED, AL_ATOMIC_RELAXED); + al_log_debug("video_buffer", "Flushed."); + } else if (flow == FLOWING && buf->queue->count(buf->queue) <= BUFFER_WATERMARK_LOW) { buf->callback(buf->userdata, CAMU_BUFFER_UNCORK); } return ret == CAMU_QUEUE_OK || ret == CAMU_QUEUE_MORE; @@ -155,7 +178,7 @@ bool camu_video_buffer_read(struct camu_video_buffer *buf, void *out) void camu_video_buffer_free(struct camu_video_buffer *buf) { #ifdef CAMU_VIDEO_BUFFER_FORCE_SCALER - if (buf->fmt.scaler_needed) camu_lav_scaler_close(&buf->scale); + if (buf->fmt.scaler_needed) camu_ff_scaler_close(&buf->scale); #endif buf->queue->free(&buf->queue); } diff --git a/src/buffer/video.h b/src/buffer/video.h index a6349f8..5b92dc6 100644 --- a/src/buffer/video.h +++ b/src/buffer/video.h @@ -3,7 +3,7 @@ #include <al/types.h> #include "../codec/codec.h" -#include "../codec/libav/scaler.h" +#include "../codec/ffmpeg/scaler.h" #include "../render/renderer.h" #include "../screen/screen.h" @@ -16,15 +16,18 @@ struct camu_video_buffer { struct camu_clock *clock; f32 start_time; f32 avg_frame_duration; + f64 offset; bool single_frame; - struct camu_lav_scaler scale; - struct camu_lav_scale_fmt fmt; + struct camu_ff_scaler scale; + struct camu_ff_scale_fmt fmt; struct camu_frame_queue *queue; bool buffered; + atomic_u8 flow; + #ifdef CAMU_SCREEN_THREADED atomic_bool ref; #endif @@ -33,11 +36,12 @@ struct camu_video_buffer { void *userdata; }; -bool camu_video_buffer_init(struct camu_video_buffer *buf, struct camu_clock *clock, struct camu_renderer *renderer); +bool camu_video_buffer_init(struct camu_video_buffer *buf, struct camu_clock *clock, f64 offset, + struct camu_renderer *renderer); bool camu_video_buffer_configure(struct camu_video_buffer *buf, struct camu_stream *stream); +bool camu_video_buffer_is_single_frame(struct camu_video_buffer *buf); void camu_video_buffer_push(struct camu_video_buffer *buf, struct camu_frame *frame); void camu_video_buffer_reset(struct camu_video_buffer *buf); -bool camu_video_buffer_is_single_frame(struct camu_video_buffer *buf); void camu_video_buffer_flush(struct camu_video_buffer *buf); bool camu_video_buffer_read(struct camu_video_buffer *buf, void *out); void camu_video_buffer_free(struct camu_video_buffer *buf); diff --git a/src/codec/codec.h b/src/codec/codec.h index f9ba660..fd847b8 100644 --- a/src/codec/codec.h +++ b/src/codec/codec.h @@ -6,12 +6,12 @@ #include "../cache/handle.h" -#ifdef HAVE_FFMPEG +#ifdef CAMU_HAVE_FFMPEG #include <libavutil/pixdesc.h> #include <libavformat/avformat.h> #endif -#ifdef HAVE_FFMPEG +#ifdef CAMU_HAVE_FFMPEG enum { CAMU_OK = 0, CAMU_ERR_EOF = AVERROR_EOF, @@ -42,7 +42,7 @@ enum { enum { CAMU_NORMAL = 0, -#ifdef HAVE_FFMPEG +#ifdef CAMU_HAVE_FFMPEG CAMU_FFMPEG_COMPAT #endif }; @@ -62,7 +62,7 @@ struct camu_stream { struct camu_icc_profile icc; } video; //struct { } audio; -#ifdef HAVE_FFMPEG +#ifdef CAMU_HAVE_FFMPEG struct { AVStream *stream; AVFormatContext *format_context; @@ -73,7 +73,7 @@ struct camu_stream { struct camu_packet { u8 type; struct aki_buffer *buffer; -#ifdef HAVE_FFMPEG +#ifdef CAMU_HAVE_FFMPEG struct { AVPacket *pkt; } av; #endif }; @@ -86,7 +86,7 @@ struct camu_frame { s32 width; s32 height; f64 pts; -#ifdef HAVE_FFMPEG +#ifdef CAMU_HAVE_FFMPEG struct { AVFrame *frame; } av; #endif void *opaque; @@ -113,3 +113,13 @@ struct camu_decoder { void (*free)(struct camu_decoder **); struct camu_stream *stream; }; + +static inline void camu_frame_discard(struct camu_frame *frame) +{ +#ifdef CAMU_HAVE_FFMPEG + if (frame->type == CAMU_FFMPEG_COMPAT) { + av_frame_free(&frame->av.frame); + } +#endif + al_free(frame); +} diff --git a/src/codec/common.h b/src/codec/common.h index f3db182..adf503d 100644 --- a/src/codec/common.h +++ b/src/codec/common.h @@ -4,7 +4,7 @@ static inline void *camu_page_alloc(size_t size) { -#ifdef HAVE_POSIX_MEMALIGN +#ifdef AL_HAVE_POSIX_MEMALIGN void *ptr = NULL; al_posix_memalign(&ptr, al_page_size, size); return ptr; @@ -15,7 +15,7 @@ static inline void *camu_page_alloc(size_t size) static inline void *camu_page_realloc(void *ptr, size_t old_size, size_t size) { -#ifdef HAVE_POSIX_MEMALIGN +#ifdef AL_HAVE_POSIX_MEMALIGN void *old_ptr = ptr; al_posix_memalign(&ptr, al_page_size, size); if (old_ptr) { diff --git a/src/codec/libav/avio.c b/src/codec/ffmpeg/avio.c index 97f66da..12dd468 100644 --- a/src/codec/libav/avio.c +++ b/src/codec/ffmpeg/avio.c @@ -1,5 +1,7 @@ #include "avio.h" +#include "../cache/handle.h" + s32 camu_avio_read(void *data, u8 *buf, s32 buf_size) { struct cch_handle *handle = (struct cch_handle *)data; diff --git a/src/codec/libav/avio.h b/src/codec/ffmpeg/avio.h index 09df4ec..b57cbcf 100644 --- a/src/codec/libav/avio.h +++ b/src/codec/ffmpeg/avio.h @@ -1,10 +1,8 @@ #pragma once +#include <al/types.h> #include <libavutil/error.h> #include <libavformat/avio.h> -#include "../../codec/codec.h" -#include "../../cache/handle.h" - s32 camu_avio_read(void *data, u8 *buf, s32 buf_size); s64 camu_avio_seek(void *data, s64 offset, s32 whence); diff --git a/src/codec/ffmpeg/common.c b/src/codec/ffmpeg/common.c new file mode 100644 index 0000000..57caff7 --- /dev/null +++ b/src/codec/ffmpeg/common.c @@ -0,0 +1,6 @@ +#include "common.h" + +void camu_ff_set_log_callback(void (*callback)(void *, int, const char *, va_list)) +{ + av_log_set_callback(callback); +} diff --git a/src/codec/ffmpeg/common.h b/src/codec/ffmpeg/common.h new file mode 100644 index 0000000..ded7b35 --- /dev/null +++ b/src/codec/ffmpeg/common.h @@ -0,0 +1,6 @@ +#pragma once + +#include <al/types.h> +#include <libavutil/log.h> + +void camu_ff_set_log_callback(void (*callback)(void *, int, const char *, va_list)); diff --git a/src/codec/libav/decoder.c b/src/codec/ffmpeg/decoder.c index e732be9..e5236b5 100644 --- a/src/codec/libav/decoder.c +++ b/src/codec/ffmpeg/decoder.c @@ -6,35 +6,32 @@ #include "decoder.h" -static void close_internal(struct camu_lav_decoder *av) +static void close_internal(struct camu_ff_decoder *av) { - if (av->codec_context) { - avcodec_close(av->codec_context); - avcodec_free_context(&av->codec_context); - } + if (av->codec_context) avcodec_free_context(&av->codec_context); } -static bool lav_decoder_init(struct camu_decoder *dec, struct camu_renderer *renderer, struct camu_stream *stream) +static bool ff_decoder_init(struct camu_decoder *dec, struct camu_renderer *renderer, struct camu_stream *stream) { - struct camu_lav_decoder *av = (struct camu_lav_decoder *)dec; + struct camu_ff_decoder *av = (struct camu_ff_decoder *)dec; AVCodecParameters *codecpar = stream->av.stream->codecpar; const AVCodec *codec = avcodec_find_decoder(codecpar->codec_id); if (!codec) { - al_log_error("lav_decoder", "Failed to find decoder."); + al_log_error("ff_decoder", "Failed to find decoder."); goto err; } av->codec_context = avcodec_alloc_context3(codec); if (!av->codec_context) { - al_log_error("lav_decoder", "Failed to alloc codec context."); + al_log_error("ff_decoder", "Failed to alloc codec context."); goto err; } if (avcodec_parameters_to_context(av->codec_context, codecpar) < 0) { - al_log_error("lav_decoder", "Failed to copy codec parameters."); + al_log_error("ff_decoder", "Failed to copy codec parameters."); goto err; } @@ -43,7 +40,7 @@ static bool lav_decoder_init(struct camu_decoder *dec, struct camu_renderer *ren cpus = (cpus > 16) ? 4 : cpus / 4; if (cpus == 0) cpus = 1; av->codec_context->thread_count = cpus; - al_log_debug("lav_decoder", "Using %i threads for decoder.", cpus); + al_log_debug("ff_decoder", "Using %i threads for decoder.", cpus); */ (void)renderer; @@ -55,11 +52,11 @@ static bool lav_decoder_init(struct camu_decoder *dec, struct camu_renderer *ren */ if (avcodec_open2(av->codec_context, codec, NULL) < 0) { - al_log_error("lav_decoder", "Failed to open codec (%s).", codec->name); + al_log_error("ff_decoder", "Failed to open codec (%s).", codec->name); goto err; } - al_log_info("lav_decoder", "Codec: %s (%s) %ldkbps.", codec->name, codec->long_name, codecpar->bit_rate / 1000); + al_log_info("ff_decoder", "Codec: %s (%s) %ldkbps.", codec->name, codec->long_name, codecpar->bit_rate / 1000); av->time_base = stream->av.stream->time_base; av->last_pts = 0; @@ -72,14 +69,14 @@ err: return false; } -static void lav_decoder_set_callback(struct camu_decoder *dec, void (*callback)(void *, struct camu_frame *), void *userdata) +static void ff_decoder_set_callback(struct camu_decoder *dec, void (*callback)(void *, struct camu_frame *), void *userdata) { - struct camu_lav_decoder *av = (struct camu_lav_decoder *)dec; + struct camu_ff_decoder *av = (struct camu_ff_decoder *)dec; av->callback = callback; av->userdata = userdata; } -static s32 send_packet(struct camu_lav_decoder *av, AVPacket *pkt) +static s32 send_packet(struct camu_ff_decoder *av, AVPacket *pkt) { /* if (pkt && av->seek_pos >= 0 && pkt->pts + pkt->duration < av->seek_pos) { @@ -91,25 +88,25 @@ static s32 send_packet(struct camu_lav_decoder *av, AVPacket *pkt) s32 ret = avcodec_send_packet(av->codec_context, pkt); if (ret < 0 && ret != AVERROR(EAGAIN) && ret != AVERROR_EOF) { - al_log_error("lav_decoder", "Error sending packet to decoder: (%s).", av_err2str(ret)); + al_log_error("ff_decoder", "Error sending packet to decoder: (%s).", av_err2str(ret)); } return ret; } -static s32 lav_decoder_push(struct camu_decoder *dec, struct camu_packet *packet) +static s32 ff_decoder_push(struct camu_decoder *dec, struct camu_packet *packet) { - struct camu_lav_decoder *av = (struct camu_lav_decoder *)dec; + struct camu_ff_decoder *av = (struct camu_ff_decoder *)dec; return send_packet(av, (packet) ? packet->av.pkt : NULL); } -static void lav_decoder_flush(struct camu_decoder *dec) +static void ff_decoder_flush(struct camu_decoder *dec) { - struct camu_lav_decoder *av = (struct camu_lav_decoder *)dec; + struct camu_ff_decoder *av = (struct camu_ff_decoder *)dec; avcodec_flush_buffers(av->codec_context); } -static s32 receive_frames(struct camu_lav_decoder *av) +static s32 receive_frames(struct camu_ff_decoder *av) { s32 ret; do { @@ -121,7 +118,7 @@ static s32 receive_frames(struct camu_lav_decoder *av) av_frame_free(&frame->av.frame); al_free(frame); if (ret == AVERROR(EAGAIN) || ret == AVERROR_EOF) break; - al_log_error("lav_decoder", "Error receiving packet from decoder: (%s).", av_err2str(ret)); + al_log_error("ff_decoder", "Error receiving packet from decoder: (%s).", av_err2str(ret)); continue; } // Track pts and duration of the previous frame so we can handle multiple frames @@ -139,28 +136,28 @@ static s32 receive_frames(struct camu_lav_decoder *av) return ret; } -static s32 lav_decoder_process(struct camu_decoder *dec) +static s32 ff_decoder_process(struct camu_decoder *dec) { - struct camu_lav_decoder *av = (struct camu_lav_decoder *)dec; + struct camu_ff_decoder *av = (struct camu_ff_decoder *)dec; return receive_frames(av); } -static void lav_decoder_free(struct camu_decoder **dec) +static void ff_decoder_free(struct camu_decoder **dec) { - struct camu_lav_decoder *av = (struct camu_lav_decoder *)*dec; + struct camu_ff_decoder *av = (struct camu_ff_decoder *)*dec; close_internal(av); al_free(av); *dec = NULL; } -struct camu_decoder *camu_lav_decoder_create(void) +struct camu_decoder *camu_ff_decoder_create(void) { - struct camu_lav_decoder *av = al_alloc_object(struct camu_lav_decoder); - av->dec.init = lav_decoder_init; - av->dec.set_callback = lav_decoder_set_callback; - av->dec.push = lav_decoder_push; - av->dec.process = lav_decoder_process; - av->dec.flush = lav_decoder_flush; - av->dec.free = lav_decoder_free; + struct camu_ff_decoder *av = al_alloc_object(struct camu_ff_decoder); + av->dec.init = ff_decoder_init; + av->dec.set_callback = ff_decoder_set_callback; + av->dec.push = ff_decoder_push; + av->dec.process = ff_decoder_process; + av->dec.flush = ff_decoder_flush; + av->dec.free = ff_decoder_free; return (struct camu_decoder *)av; } diff --git a/src/codec/libav/decoder.h b/src/codec/ffmpeg/decoder.h index 44105ce..45cca00 100644 --- a/src/codec/libav/decoder.h +++ b/src/codec/ffmpeg/decoder.h @@ -6,7 +6,7 @@ #include "../codec.h" -struct camu_lav_decoder { +struct camu_ff_decoder { struct camu_decoder dec; AVCodecContext *codec_context; AVRational time_base; @@ -17,4 +17,4 @@ struct camu_lav_decoder { void *userdata; }; -struct camu_decoder *camu_lav_decoder_create(void); +struct camu_decoder *camu_ff_decoder_create(void); diff --git a/src/codec/libav/demuxer.c b/src/codec/ffmpeg/demuxer.c index 2169816..9b26596 100644 --- a/src/codec/libav/demuxer.c +++ b/src/codec/ffmpeg/demuxer.c @@ -5,7 +5,7 @@ #define BUF_SIZE 16384 -static void close_internal(struct camu_lav_demuxer *av) +static void close_internal(struct camu_ff_demuxer *av) { if (av->io_context) { av_free(av->io_context->buffer); @@ -18,20 +18,20 @@ static void close_internal(struct camu_lav_demuxer *av) } } -static bool lav_demuxer_init(struct camu_demuxer *demux, struct cch_handle *handle) +static bool ff_demuxer_init(struct camu_demuxer *demux, struct cch_handle *handle) { - struct camu_lav_demuxer *av = (struct camu_lav_demuxer *)demux; + struct camu_ff_demuxer *av = (struct camu_ff_demuxer *)demux; av->format_context = avformat_alloc_context(); if (!av->format_context) { - al_log_error("lav_demuxer", "Failed to create format context."); + al_log_error("ff_demuxer", "Failed to create format context."); goto err; } u8 *buf = (u8 *)av_malloc(BUF_SIZE); av->io_context = avio_alloc_context(buf, BUF_SIZE, 0, handle, camu_avio_read, NULL, camu_avio_seek); if (!av->io_context) { - al_log_error("lav_demuxer", "Failed to create custom io context."); + al_log_error("ff_demuxer", "Failed to create custom io context."); goto err; } @@ -48,17 +48,17 @@ static bool lav_demuxer_init(struct camu_demuxer *demux, struct cch_handle *hand av->format_context->flags |= AVFMT_FLAG_CUSTOM_IO; if (avformat_open_input(&av->format_context, "", 0, 0) < 0) { - al_log_error("lav_demuxer", "Failed to open input."); + al_log_error("ff_demuxer", "Failed to open input."); goto err; } if (avformat_find_stream_info(av->format_context, 0) < 0) { - al_log_error("lav_demux", "Failed to find stream info."); + al_log_error("ff_demuxer", "Failed to find stream info."); goto err; } if (!av->format_context->nb_streams) { - al_log_error("lav_demux", "No streams found."); + al_log_error("ff_demuxer", "No streams found."); goto err; } @@ -72,16 +72,16 @@ static bool lav_demuxer_init(struct camu_demuxer *demux, struct cch_handle *hand enum AVMediaType type = stream->codecpar->codec_type; // Try to detect attached images. if (type == AVMEDIA_TYPE_VIDEO && GUESS_STREAM_IS_IMAGE(stream)) { - al_log_info("lav_demux", "Guessing that stream #%u is an image.", i); + al_log_info("ff_demuxer", "Guessing that stream #%u is an image.", i); stream->duration = 0; } else if (stream->duration < 0) { - al_log_warn("lav_demux", "Stream #%u has an invalid duration (%ld).", i, stream->duration); + al_log_warn("ff_demuxer", "Stream #%u has an invalid duration (%ld).", i, stream->duration); if (av->format_context->duration < 0) { stream->duration = 0; } else { stream->duration = av_rescale_q(av->format_context->duration, AV_TIME_BASE_Q, stream->time_base); } - al_log_info("lav_demux", "Setting stream #%u to a duration of %.3fs.", i, stream->duration * av_q2d(stream->time_base)); + al_log_info("ff_demuxer", "Setting stream #%u to a duration of %.3fs.", i, stream->duration * av_q2d(stream->time_base)); } if (stream->start_time < 0) stream->start_time = 0; s64 duration = av_rescale_q(stream->duration, stream->time_base, AV_TIME_BASE_Q); @@ -97,6 +97,7 @@ static bool lav_demuxer_init(struct camu_demuxer *demux, struct cch_handle *hand //av_dump_format(av->format_context, 0, "", 0); + av->seeked = false; av->eof = false; return true; @@ -105,6 +106,17 @@ err: return false; } +/* +static AVStream *av_stream_at_index(struct camu_ff_demuxer *av, s32 index) +{ + struct camu_stream *stream; + al_array_foreach_ptr(av->demux.streams, i, stream) { + if (stream->av.stream->index == index) return stream->av.stream; + } + return NULL; +} +*/ + static bool subscribed_to_index(struct camu_demuxer *demux, s32 index) { s32 subscribed; @@ -114,9 +126,9 @@ static bool subscribed_to_index(struct camu_demuxer *demux, s32 index) return false; } -static s32 lav_demuxer_get_packet(struct camu_demuxer *demux, struct camu_packet *packet) +static s32 ff_demuxer_get_packet(struct camu_demuxer *demux, struct camu_packet *packet) { - struct camu_lav_demuxer *av = (struct camu_lav_demuxer *)demux; + struct camu_ff_demuxer *av = (struct camu_ff_demuxer *)demux; packet->type = CAMU_FFMPEG_COMPAT; s32 ret = 0; do { @@ -126,7 +138,7 @@ static s32 lav_demuxer_get_packet(struct camu_demuxer *demux, struct camu_packet break; } if (ret < 0) { - al_log_error("lav_demuxer", "Failed to read frame (%s).", av_err2str(ret)); + al_log_error("ff_demuxer", "Failed to read frame (%s).", av_err2str(ret)); continue; } if (!subscribed_to_index(demux, packet->av.pkt->stream_index)) { @@ -138,46 +150,47 @@ static s32 lav_demuxer_get_packet(struct camu_demuxer *demux, struct camu_packet return ret; } -static u64 lav_demuxer_get_duration(struct camu_demuxer *demux) +static u64 ff_demuxer_get_duration(struct camu_demuxer *demux) { - struct camu_lav_demuxer *av = (struct camu_lav_demuxer *)demux; + struct camu_ff_demuxer *av = (struct camu_ff_demuxer *)demux; return av->duration; } -static bool lav_demuxer_seek(struct camu_demuxer *demux, u64 pos) +static bool ff_demuxer_seek(struct camu_demuxer *demux, u64 pos) { - struct camu_lav_demuxer *av = (struct camu_lav_demuxer *)demux; + struct camu_ff_demuxer *av = (struct camu_ff_demuxer *)demux; if (pos > av->duration) pos = av->duration; s64 ts = av_rescale_q((s64)pos, (AVRational){ 1, 1000000 }, AV_TIME_BASE_Q); if (av->eof) { avformat_flush(av->format_context); av->eof = false; } - if (avformat_seek_file(av->format_context, -1, INT64_MIN, ts, ts, AVSEEK_FLAG_BACKWARD) >= 0) { - return true; + if (avformat_seek_file(av->format_context, -1, INT64_MIN, ts, ts, AVSEEK_FLAG_BACKWARD) < 0) { + return false; } - return false; + av->seeked = true; + return true; } -static void lav_demuxer_free(struct camu_demuxer **demux) +static void ff_demuxer_free(struct camu_demuxer **demux) { - struct camu_lav_demuxer *av = (struct camu_lav_demuxer *)*demux; + struct camu_ff_demuxer *av = (struct camu_ff_demuxer *)*demux; close_internal(av); al_array_free(av->demux.streams); al_free(av); *demux = NULL; } -struct camu_demuxer *camu_lav_demuxer_create(void) +struct camu_demuxer *camu_ff_demuxer_create(void) { - struct camu_lav_demuxer *av = al_alloc_object(struct camu_lav_demuxer); + struct camu_ff_demuxer *av = al_alloc_object(struct camu_ff_demuxer); av->demux.type = CAMU_FFMPEG_COMPAT; al_array_init(av->demux.streams); al_array_init(av->demux.subscribed); - av->demux.init = lav_demuxer_init; - av->demux.get_packet = lav_demuxer_get_packet; - av->demux.get_duration = lav_demuxer_get_duration; - av->demux.seek = lav_demuxer_seek; - av->demux.free = lav_demuxer_free; + av->demux.init = ff_demuxer_init; + av->demux.get_packet = ff_demuxer_get_packet; + av->demux.get_duration = ff_demuxer_get_duration; + av->demux.seek = ff_demuxer_seek; + av->demux.free = ff_demuxer_free; return (struct camu_demuxer *)av; } diff --git a/src/codec/libav/demuxer.h b/src/codec/ffmpeg/demuxer.h index 9b2d093..8327534 100644 --- a/src/codec/libav/demuxer.h +++ b/src/codec/ffmpeg/demuxer.h @@ -8,12 +8,13 @@ #include "../codec.h" -struct camu_lav_demuxer { +struct camu_ff_demuxer { struct camu_demuxer demux; AVIOContext *io_context; AVFormatContext *format_context; u64 duration; + bool seeked; bool eof; }; -struct camu_demuxer *camu_lav_demuxer_create(void); +struct camu_demuxer *camu_ff_demuxer_create(void); diff --git a/src/codec/libav/encoder.c b/src/codec/ffmpeg/encoder.c index 4f3fa80..9ceaa99 100644 --- a/src/codec/libav/encoder.c +++ b/src/codec/ffmpeg/encoder.c @@ -15,21 +15,21 @@ #define FRAME_SIZE 20 #define CHANNELS 2 -bool camu_lav_encoder_init(struct camu_lav_encoder *enc, char *encoder_name, char *ext) +bool camu_ff_encoder_init(struct camu_ff_encoder *enc, char *encoder_name, char *ext) { - al_memset(enc, 0, sizeof(struct camu_lav_encoder)); + al_memset(enc, 0, sizeof(struct camu_ff_encoder)); const AVCodec *codec = avcodec_find_encoder_by_name(encoder_name); if (!codec) { - al_log_error("lav_encoder", "Failed to find codec with name: %s.", encoder_name); + al_log_error("ff_encoder", "Failed to find codec with name: %s.", encoder_name); goto err; } enc->codec_context = avcodec_alloc_context3(codec); if (!enc->codec_context) { - al_log_error("lav_encoder", "Failed to alloc codec context."); + al_log_error("ff_encoder", "Failed to alloc codec context."); goto err; } @@ -54,7 +54,7 @@ bool camu_lav_encoder_init(struct camu_lav_encoder *enc, char *encoder_name, cha av_dict_set(&opts, "application", "audio", 0); if (avcodec_open2(enc->codec_context, codec, &opts) < 0) { - al_log_error("lav_encoder", "Failed to open codec."); + al_log_error("ff_encoder", "Failed to open codec."); av_dict_free(&opts); goto err; } @@ -71,7 +71,7 @@ bool camu_lav_encoder_init(struct camu_lav_encoder *enc, char *encoder_name, cha enc->fmt.req_channel_layout.nb_channels, 1); if (!enc->audio_fifo) { - al_log_error("lav_encoder", "Failed to alloc fifo."); + al_log_error("ff_encoder", "Failed to alloc fifo."); goto err; } @@ -84,11 +84,11 @@ bool camu_lav_encoder_init(struct camu_lav_encoder *enc, char *encoder_name, cha return true; err: - camu_lav_encoder_close(enc); + camu_ff_encoder_close(enc); return false; } -void camu_lav_encoder_push(struct camu_lav_encoder *enc, u8 **data, s32 sample_count) +void camu_ff_encoder_push(struct camu_ff_encoder *enc, u8 **data, s32 sample_count) { s32 alloc_size = av_audio_fifo_size(enc->audio_fifo) + sample_count; if (av_audio_fifo_space(enc->audio_fifo) < alloc_size) { @@ -99,12 +99,12 @@ void camu_lav_encoder_push(struct camu_lav_encoder *enc, u8 **data, s32 sample_c av_audio_fifo_write(enc->audio_fifo, (void **)data, sample_count); } -static void free_frame_data(struct camu_lav_encoder *enc) +static void free_frame_data(struct camu_ff_encoder *enc) { if (enc->frame && enc->frame->nb_samples > 0) av_freep(&enc->frame->data[0]); } -static void alloc_frame_data(struct camu_lav_encoder *enc, s32 frame_size) +static void alloc_frame_data(struct camu_ff_encoder *enc, s32 frame_size) { if (enc->frame->nb_samples >= frame_size) return; free_frame_data(enc); @@ -113,7 +113,7 @@ static void alloc_frame_data(struct camu_lav_encoder *enc, s32 frame_size) enc->frame->nb_samples = frame_size; } -static bool receive_packets(struct camu_lav_encoder *enc) +static bool receive_packets(struct camu_ff_encoder *enc) { s32 ret; AVPacket pkt = { 0 }; @@ -125,23 +125,23 @@ static bool receive_packets(struct camu_lav_encoder *enc) } while (1); if (ret != AVERROR(EAGAIN) && ret != AVERROR_EOF) { - al_log_error("lav_encoder", "Error receiving packet from encoder: (%s).", av_err2str(ret)); + al_log_error("ff_encoder", "Error receiving packet from encoder: (%s).", av_err2str(ret)); return false; } return true; } -static s32 send_frame(struct camu_lav_encoder *enc, AVFrame *in_frame) +static s32 send_frame(struct camu_ff_encoder *enc, AVFrame *in_frame) { s32 ret = avcodec_send_frame(enc->codec_context, in_frame); if (ret < 0 && ret != AVERROR_EOF) { - al_log_error("lav_encoder", "Error sending frame to encoder: (%s).", av_err2str(ret)); + al_log_error("ff_encoder", "Error sending frame to encoder: (%s).", av_err2str(ret)); } return ret; } -bool camu_lav_encoder_process(struct camu_lav_encoder *enc, bool flush) +bool camu_ff_encoder_process(struct camu_ff_encoder *enc, bool flush) { s32 samples_available = av_audio_fifo_size(enc->audio_fifo); @@ -177,7 +177,7 @@ bool camu_lav_encoder_process(struct camu_lav_encoder *enc, bool flush) return ret == 0; } -void camu_lav_encoder_close(struct camu_lav_encoder *enc) +void camu_ff_encoder_close(struct camu_ff_encoder *enc) { if (enc->codec_context) avcodec_free_context(&enc->codec_context); if (enc->audio_fifo) av_audio_fifo_free(enc->audio_fifo); diff --git a/src/codec/ffmpeg/encoder.h b/src/codec/ffmpeg/encoder.h new file mode 100644 index 0000000..6cba9ac --- /dev/null +++ b/src/codec/ffmpeg/encoder.h @@ -0,0 +1,26 @@ +#pragma once + +#include <libavcodec/avcodec.h> +#include <libavformat/avformat.h> +#include <libavutil/audio_fifo.h> +#include <libavutil/rational.h> + +#include "resampler.h" + +struct camu_ff_encoder { + AVCodecContext *codec_context; + const AVOutputFormat *output_format; + + struct camu_ff_resample_fmt fmt; + + AVFrame *frame; + AVAudioFifo *audio_fifo; + + void (*callback)(void *, AVPacket *); + void *userdata; +}; + +bool camu_ff_encoder_init(struct camu_ff_encoder *enc, char *encoder_name, char *ext); +void camu_ff_encoder_push(struct camu_ff_encoder *enc, u8 **data, s32 sample_count); +bool camu_ff_encoder_process(struct camu_ff_encoder *enc, bool flush); +void camu_ff_encoder_close(struct camu_ff_encoder *enc); diff --git a/src/codec/libav/packet_ext.c b/src/codec/ffmpeg/packet_ext.c index 8fe4c39..8fe4c39 100644 --- a/src/codec/libav/packet_ext.c +++ b/src/codec/ffmpeg/packet_ext.c diff --git a/src/codec/libav/packet_ext.h b/src/codec/ffmpeg/packet_ext.h index ff6e337..ff6e337 100644 --- a/src/codec/libav/packet_ext.h +++ b/src/codec/ffmpeg/packet_ext.h diff --git a/src/codec/libav/resampler.c b/src/codec/ffmpeg/resampler.c index 229c68c..d4b281a 100644 --- a/src/codec/libav/resampler.c +++ b/src/codec/ffmpeg/resampler.c @@ -8,21 +8,21 @@ #include "resampler.h" -static bool in_req_matches(struct camu_lav_resample_fmt *fmt) +static bool in_req_matches(struct camu_ff_resample_fmt *fmt) { return (fmt->in_format == fmt->req_format) && (av_channel_layout_compare(&fmt->in_channel_layout, &fmt->req_channel_layout) == 0) && (fmt->in_sample_rate == fmt->req_sample_rate); } -bool camu_lav_resampler_init(struct camu_lav_resampler *resamp, struct camu_lav_resample_fmt *fmt) +bool camu_ff_resampler_init(struct camu_ff_resampler *resamp, struct camu_ff_resample_fmt *fmt) { - al_memset(resamp, 0, sizeof(struct camu_lav_resampler)); + al_memset(resamp, 0, sizeof(struct camu_ff_resampler)); fmt->resampler_needed = !in_req_matches(fmt); if (!fmt->resampler_needed) return true; - camu_lav_resample_fmt_copy(&resamp->fmt, fmt); + camu_ff_resample_fmt_copy(&resamp->fmt, fmt); SwrContext *ctx = NULL; s32 ret = swr_alloc_set_opts2(&ctx, @@ -54,19 +54,19 @@ bool camu_lav_resampler_init(struct camu_lav_resampler *resamp, struct camu_lav_ return true; } -s32 camu_lav_resampler_sample_count(struct camu_lav_resampler *resamp, s32 in_samples) +s32 camu_ff_resampler_sample_count(struct camu_ff_resampler *resamp, s32 in_samples) { return swr_get_out_samples(resamp->resample_context, in_samples); //return av_rescale_rnd(swr_get_delay(resamp->resample_context, resamp->fmt.req_sample_rate) + // in_samples, resamp->fmt.in_sample_rate, resamp->fmt.req_sample_rate, AV_ROUND_UP); } -static inline void free_sample_data(struct camu_lav_resampler *resamp) +static inline void free_sample_data(struct camu_ff_resampler *resamp) { if (resamp->data[0]) av_freep(&resamp->data[0]); } -static inline void alloc_sample_data(struct camu_lav_resampler *resamp, s32 sample_count) +static inline void alloc_sample_data(struct camu_ff_resampler *resamp, s32 sample_count) { if (resamp->sample_count >= sample_count) return; free_sample_data(resamp); @@ -75,10 +75,10 @@ static inline void alloc_sample_data(struct camu_lav_resampler *resamp, s32 samp resamp->sample_count = sample_count; } -s32 camu_lav_resampler_convert(struct camu_lav_resampler *resamp, const u8 **in_data, s32 in_samples) +s32 camu_ff_resampler_convert(struct camu_ff_resampler *resamp, const u8 **in_data, s32 in_samples) { al_assert(in_samples > 0); - s32 max_resample_count = camu_lav_resampler_sample_count(resamp, in_samples); + s32 max_resample_count = camu_ff_resampler_sample_count(resamp, in_samples); alloc_sample_data(resamp, max_resample_count); s32 resample_count = swr_convert(resamp->resample_context, resamp->data, max_resample_count, in_data, in_samples); @@ -88,12 +88,12 @@ s32 camu_lav_resampler_convert(struct camu_lav_resampler *resamp, const u8 **in_ return resample_count; } -u8 **camu_lav_resampler_get_data(struct camu_lav_resampler *resamp) +u8 **camu_ff_resampler_get_data(struct camu_ff_resampler *resamp) { return (u8 **)resamp->data; } -void camu_lav_resampler_close(struct camu_lav_resampler *resamp) +void camu_ff_resampler_close(struct camu_ff_resampler *resamp) { if (resamp->resample_context) { swr_close(resamp->resample_context); @@ -102,7 +102,7 @@ void camu_lav_resampler_close(struct camu_lav_resampler *resamp) free_sample_data(resamp); } -void camu_lav_resample_fmt_copy(struct camu_lav_resample_fmt *dest, struct camu_lav_resample_fmt *src) +void camu_ff_resample_fmt_copy(struct camu_ff_resample_fmt *dest, struct camu_ff_resample_fmt *src) { dest->resampler_needed = src->resampler_needed; dest->in_format = src->in_format; @@ -113,50 +113,49 @@ void camu_lav_resample_fmt_copy(struct camu_lav_resample_fmt *dest, struct camu_ dest->req_sample_rate = src->req_sample_rate; } -size_t camu_lav_resample_fmt_bytes_per_sample(struct camu_lav_resample_fmt *fmt) +size_t camu_ff_resample_fmt_bytes_per_sample(struct camu_ff_resample_fmt *fmt) { return (size_t)av_get_bytes_per_sample(fmt->req_format); } -size_t camu_lav_resample_fmt_samples_to_bytes(struct camu_lav_resample_fmt *fmt, size_t samples) +size_t camu_ff_resample_fmt_samples_to_bytes(struct camu_ff_resample_fmt *fmt, size_t samples) { - s32 size, nb_channels = fmt->req_channel_layout.nb_channels; - if (av_samples_get_buffer_size(&size, nb_channels, (s32)samples, fmt->req_format, 1) < 0) { - return 0; - } + s32 size; + s32 ret = av_samples_get_buffer_size(&size, fmt->req_channel_layout.nb_channels, (s32)samples, fmt->req_format, 1); + al_assert(ret >= 0); return (size_t)size; } -size_t camu_lav_resample_fmt_samples_to_usec(struct camu_lav_resample_fmt *fmt, size_t samples) +size_t camu_ff_resample_fmt_samples_to_usec(struct camu_ff_resample_fmt *fmt, size_t samples) { return samples / (fmt->req_sample_rate / 1000000.0); } -f64 camu_lav_resample_fmt_samples_to_sec(struct camu_lav_resample_fmt *fmt, size_t samples) +f64 camu_ff_resample_fmt_samples_to_sec(struct camu_ff_resample_fmt *fmt, size_t samples) { return samples / (f64)fmt->req_sample_rate; } -size_t camu_lav_resample_fmt_usec_to_bytes(struct camu_lav_resample_fmt *fmt, size_t usec) +size_t camu_ff_resample_fmt_usec_to_bytes(struct camu_ff_resample_fmt *fmt, size_t usec) { size_t samples = usec * (fmt->req_sample_rate / 1000000.0); - return samples * (camu_lav_resample_fmt_bytes_per_sample(fmt) * fmt->req_channel_layout.nb_channels); + return samples * (camu_ff_resample_fmt_bytes_per_sample(fmt) * fmt->req_channel_layout.nb_channels); } -size_t camu_lav_resample_fmt_sec_to_bytes(struct camu_lav_resample_fmt *fmt, f64 sec) +size_t camu_ff_resample_fmt_sec_to_bytes(struct camu_ff_resample_fmt *fmt, f64 sec) { size_t samples = sec * fmt->req_sample_rate; - return samples * (camu_lav_resample_fmt_bytes_per_sample(fmt) * fmt->req_channel_layout.nb_channels); + return samples * (camu_ff_resample_fmt_bytes_per_sample(fmt) * fmt->req_channel_layout.nb_channels); } -size_t camu_lav_resample_fmt_bytes_to_usec(struct camu_lav_resample_fmt *fmt, size_t bytes) +size_t camu_ff_resample_fmt_bytes_to_usec(struct camu_ff_resample_fmt *fmt, size_t bytes) { - size_t samples = bytes / (camu_lav_resample_fmt_bytes_per_sample(fmt) * fmt->req_channel_layout.nb_channels); + size_t samples = bytes / (camu_ff_resample_fmt_bytes_per_sample(fmt) * fmt->req_channel_layout.nb_channels); return samples / (fmt->req_sample_rate / 1000000.0); } -f64 camu_lav_resample_fmt_bytes_to_sec(struct camu_lav_resample_fmt *fmt, size_t bytes) +f64 camu_ff_resample_fmt_bytes_to_sec(struct camu_ff_resample_fmt *fmt, size_t bytes) { - size_t samples = bytes / (camu_lav_resample_fmt_bytes_per_sample(fmt) * fmt->req_channel_layout.nb_channels); + size_t samples = bytes / (camu_ff_resample_fmt_bytes_per_sample(fmt) * fmt->req_channel_layout.nb_channels); return samples / ((f64)fmt->req_sample_rate); } diff --git a/src/codec/ffmpeg/resampler.h b/src/codec/ffmpeg/resampler.h new file mode 100644 index 0000000..4f7b07d --- /dev/null +++ b/src/codec/ffmpeg/resampler.h @@ -0,0 +1,40 @@ +#pragma once + +#include <libswresample/swresample.h> +#include <libavutil/opt.h> +#include <al/types.h> + +struct camu_ff_resample_fmt { + bool resampler_needed; + enum AVSampleFormat in_format; + enum AVSampleFormat req_format; + s32 in_channel_count; + s32 req_channel_count; + AVChannelLayout in_channel_layout; + AVChannelLayout req_channel_layout; + s32 in_sample_rate; + s32 req_sample_rate; +}; + +struct camu_ff_resampler { + SwrContext *resample_context; + struct camu_ff_resample_fmt fmt; + u8 *data[AV_NUM_DATA_POINTERS]; + s32 sample_count; +}; + +bool camu_ff_resampler_init(struct camu_ff_resampler *resamp, struct camu_ff_resample_fmt *fmt); +s32 camu_ff_resampler_sample_count(struct camu_ff_resampler *resamp, s32 in_samples); +s32 camu_ff_resampler_convert(struct camu_ff_resampler *resamp, const u8 **in_data, s32 in_samples); +u8 **camu_ff_resampler_get_data(struct camu_ff_resampler *resamp); +void camu_ff_resampler_close(struct camu_ff_resampler *resamp); + +void camu_ff_resample_fmt_copy(struct camu_ff_resample_fmt *dest, struct camu_ff_resample_fmt *src); +size_t camu_ff_resample_fmt_bytes_per_sample(struct camu_ff_resample_fmt *fmt); +size_t camu_ff_resample_fmt_samples_to_bytes(struct camu_ff_resample_fmt *fmt, size_t samples); +size_t camu_ff_resample_fmt_samples_to_usec(struct camu_ff_resample_fmt *fmt, size_t samples); +f64 camu_ff_resample_fmt_samples_to_sec(struct camu_ff_resample_fmt *fmt, size_t samples); +size_t camu_ff_resample_fmt_usec_to_bytes(struct camu_ff_resample_fmt *fmt, size_t usec); +size_t camu_ff_resample_fmt_sec_to_bytes(struct camu_ff_resample_fmt *fmt, f64 sec); +size_t camu_ff_resample_fmt_bytes_to_usec(struct camu_ff_resample_fmt *fmt, size_t bytes); +f64 camu_ff_resample_fmt_bytes_to_sec(struct camu_ff_resample_fmt *fmt, size_t bytes); diff --git a/src/codec/libav/scaler.c b/src/codec/ffmpeg/scaler.c index 882116f..9b5c5e0 100644 --- a/src/codec/libav/scaler.c +++ b/src/codec/ffmpeg/scaler.c @@ -11,7 +11,7 @@ static AVFrame *alloc_picture(enum AVPixelFormat pix_fmt, s32 width, s32 height) if (!frame) return NULL; s32 ret = av_image_alloc(frame->data, frame->linesize, width, height, pix_fmt, 32); if (ret < 0) { - al_log_error("lav_scaler", "Failed to allocate picture buffer (%s).", av_err2str(ret)); + al_log_error("ff_scaler", "Failed to allocate picture buffer (%s).", av_err2str(ret)); return NULL; } /* @@ -23,7 +23,7 @@ static AVFrame *alloc_picture(enum AVPixelFormat pix_fmt, s32 width, s32 height) } s32 ret = av_image_fill_arrays(frame->data, frame->linesize, buffer, pix_fmt, width, height, 1); if (ret < 0) { - al_log_error("lav_scaler", "Failed to allocate picture buffer (%s).", av_err2str(ret)); + al_log_error("ff_scaler", "Failed to allocate picture buffer (%s).", av_err2str(ret)); av_free(frame); return NULL; } @@ -34,9 +34,9 @@ static AVFrame *alloc_picture(enum AVPixelFormat pix_fmt, s32 width, s32 height) return frame; } -bool camu_lav_scaler_init(struct camu_lav_scaler *scale, struct camu_lav_scale_fmt *fmt) +bool camu_ff_scaler_init(struct camu_ff_scaler *scale, struct camu_ff_scale_fmt *fmt) { - al_memset(scale, 0, sizeof(struct camu_lav_scaler)); + al_memset(scale, 0, sizeof(struct camu_ff_scaler)); if (fmt->in_width == fmt->req_width && fmt->in_height == fmt->req_height && fmt->in_format == fmt->req_format) { @@ -48,13 +48,13 @@ bool camu_lav_scaler_init(struct camu_lav_scaler *scale, struct camu_lav_scale_f scale->scaler_context = sws_getContext(fmt->in_width, fmt->in_height, fmt->in_format, fmt->req_width, fmt->req_height, fmt->req_format, flags, NULL, NULL, NULL); if (!scale->scaler_context) { - al_log_error("lav_scaler", "Failed to create scaler context."); + al_log_error("ff_scaler", "Failed to create scaler context."); return false; } scale->frame = alloc_picture(fmt->req_format, fmt->req_width, fmt->req_height); if (!scale->frame) { - al_log_error("lav_scaler", "Failed to allocate frame."); + al_log_error("ff_scaler", "Failed to allocate frame."); return false; } @@ -65,18 +65,18 @@ bool camu_lav_scaler_init(struct camu_lav_scaler *scale, struct camu_lav_scale_f return true; } -bool camu_lav_scaler_scale(struct camu_lav_scaler *scale, const u8 **in_slice, s32 *in_strides) +bool camu_ff_scaler_scale(struct camu_ff_scaler *scale, const u8 **in_slice, s32 *in_strides) { s32 ret = sws_scale(scale->scaler_context, in_slice, in_strides, 0, scale->fmt.in_height, scale->frame->data, scale->frame->linesize); if (ret != scale->fmt.req_height) { - al_log_error("lav_scaler", "Failed to scale frame (%s).", av_err2str(ret)); + al_log_error("ff_scaler", "Failed to scale frame (%s).", av_err2str(ret)); return false; } return true; } -void camu_lav_scaler_close(struct camu_lav_scaler *scale) +void camu_ff_scaler_close(struct camu_ff_scaler *scale) { av_freep(&scale->frame->data[0]); av_frame_free(&scale->frame); diff --git a/src/codec/ffmpeg/scaler.h b/src/codec/ffmpeg/scaler.h new file mode 100644 index 0000000..3e53fbb --- /dev/null +++ b/src/codec/ffmpeg/scaler.h @@ -0,0 +1,24 @@ +#pragma once + +#include <libswscale/swscale.h> +#include <al/types.h> + +struct camu_ff_scale_fmt { + s32 in_width; + s32 in_height; + enum AVPixelFormat in_format; + s32 req_width; + s32 req_height; + enum AVPixelFormat req_format; + bool scaler_needed; +}; + +struct camu_ff_scaler { + struct SwsContext *scaler_context; + struct camu_ff_scale_fmt fmt; + AVFrame *frame; +}; + +bool camu_ff_scaler_init(struct camu_ff_scaler *scale, struct camu_ff_scale_fmt *fmt); +bool camu_ff_scaler_scale(struct camu_ff_scaler *scale, const u8 **in_slice, s32 *in_strides); +void camu_ff_scaler_close(struct camu_ff_scaler *scale); diff --git a/src/codec/libav/common.c b/src/codec/libav/common.c deleted file mode 100644 index 8ce6667..0000000 --- a/src/codec/libav/common.c +++ /dev/null @@ -1,6 +0,0 @@ -#include "common.h" - -void camu_lav_set_log_callback(void (*callback)(void *, int, const char *, va_list)) -{ - av_log_set_callback(callback); -} diff --git a/src/codec/libav/common.h b/src/codec/libav/common.h deleted file mode 100644 index 854e4c7..0000000 --- a/src/codec/libav/common.h +++ /dev/null @@ -1,6 +0,0 @@ -#pragma once - -#include <al/types.h> -#include <libavutil/log.h> - -void camu_lav_set_log_callback(void (*callback)(void *, int, const char *, va_list)); diff --git a/src/codec/libav/encoder.h b/src/codec/libav/encoder.h deleted file mode 100644 index b5a707a..0000000 --- a/src/codec/libav/encoder.h +++ /dev/null @@ -1,26 +0,0 @@ -#pragma once - -#include <libavcodec/avcodec.h> -#include <libavformat/avformat.h> -#include <libavutil/audio_fifo.h> -#include <libavutil/rational.h> - -#include "resampler.h" - -struct camu_lav_encoder { - AVCodecContext *codec_context; - const AVOutputFormat *output_format; - - struct camu_lav_resample_fmt fmt; - - AVFrame *frame; - AVAudioFifo *audio_fifo; - - void (*callback)(void *, AVPacket *); - void *userdata; -}; - -bool camu_lav_encoder_init(struct camu_lav_encoder *enc, char *encoder_name, char *ext); -void camu_lav_encoder_push(struct camu_lav_encoder *enc, u8 **data, s32 sample_count); -bool camu_lav_encoder_process(struct camu_lav_encoder *enc, bool flush); -void camu_lav_encoder_close(struct camu_lav_encoder *enc); diff --git a/src/codec/libav/resampler.h b/src/codec/libav/resampler.h deleted file mode 100644 index 92525b2..0000000 --- a/src/codec/libav/resampler.h +++ /dev/null @@ -1,40 +0,0 @@ -#pragma once - -#include <libswresample/swresample.h> -#include <libavutil/opt.h> -#include <al/types.h> - -struct camu_lav_resample_fmt { - bool resampler_needed; - enum AVSampleFormat in_format; - enum AVSampleFormat req_format; - s32 in_channel_count; - s32 req_channel_count; - AVChannelLayout in_channel_layout; - AVChannelLayout req_channel_layout; - s32 in_sample_rate; - s32 req_sample_rate; -}; - -struct camu_lav_resampler { - SwrContext *resample_context; - struct camu_lav_resample_fmt fmt; - u8 *data[AV_NUM_DATA_POINTERS]; - s32 sample_count; -}; - -bool camu_lav_resampler_init(struct camu_lav_resampler *resamp, struct camu_lav_resample_fmt *fmt); -s32 camu_lav_resampler_sample_count(struct camu_lav_resampler *resamp, s32 in_samples); -s32 camu_lav_resampler_convert(struct camu_lav_resampler *resamp, const u8 **in_data, s32 in_samples); -u8 **camu_lav_resampler_get_data(struct camu_lav_resampler *resamp); -void camu_lav_resampler_close(struct camu_lav_resampler *resamp); - -void camu_lav_resample_fmt_copy(struct camu_lav_resample_fmt *dest, struct camu_lav_resample_fmt *src); -size_t camu_lav_resample_fmt_bytes_per_sample(struct camu_lav_resample_fmt *fmt); -size_t camu_lav_resample_fmt_samples_to_bytes(struct camu_lav_resample_fmt *fmt, size_t samples); -size_t camu_lav_resample_fmt_samples_to_usec(struct camu_lav_resample_fmt *fmt, size_t samples); -f64 camu_lav_resample_fmt_samples_to_sec(struct camu_lav_resample_fmt *fmt, size_t samples); -size_t camu_lav_resample_fmt_usec_to_bytes(struct camu_lav_resample_fmt *fmt, size_t usec); -size_t camu_lav_resample_fmt_sec_to_bytes(struct camu_lav_resample_fmt *fmt, f64 sec); -size_t camu_lav_resample_fmt_bytes_to_usec(struct camu_lav_resample_fmt *fmt, size_t bytes); -f64 camu_lav_resample_fmt_bytes_to_sec(struct camu_lav_resample_fmt *fmt, size_t bytes); diff --git a/src/codec/libav/scaler.h b/src/codec/libav/scaler.h deleted file mode 100644 index 4d54085..0000000 --- a/src/codec/libav/scaler.h +++ /dev/null @@ -1,24 +0,0 @@ -#pragma once - -#include <libswscale/swscale.h> -#include <al/types.h> - -struct camu_lav_scale_fmt { - s32 in_width; - s32 in_height; - enum AVPixelFormat in_format; - s32 req_width; - s32 req_height; - enum AVPixelFormat req_format; - bool scaler_needed; -}; - -struct camu_lav_scaler { - struct SwsContext *scaler_context; - struct camu_lav_scale_fmt fmt; - AVFrame *frame; -}; - -bool camu_lav_scaler_init(struct camu_lav_scaler *scale, struct camu_lav_scale_fmt *fmt); -bool camu_lav_scaler_scale(struct camu_lav_scaler *scale, const u8 **in_slice, s32 *in_strides); -void camu_lav_scaler_close(struct camu_lav_scaler *scale); diff --git a/src/codec/meson.build b/src/codec/meson.build index 4ca9c53..f3b604f 100644 --- a/src/codec/meson.build +++ b/src/codec/meson.build @@ -1,26 +1,26 @@ -av_src = [ - 'libav/common.c', - 'libav/packet_ext.c' +ffmpeg_src = [ + 'ffmpeg/common.c', + 'ffmpeg/packet_ext.c' ] -av_server_src = [ - av_src, - 'libav/demuxer.c', - 'libav/encoder.c', - 'libav/avio.c' +ffmpeg_server_src = [ + ffmpeg_src, + 'ffmpeg/demuxer.c', + 'ffmpeg/encoder.c', + 'ffmpeg/avio.c' ] -av_client_src = [ - av_src, - 'libav/decoder.c', - 'libav/resampler.c' +ffmpeg_client_src = [ + ffmpeg_src, + 'ffmpeg/decoder.c', + 'ffmpeg/resampler.c' ] -av_inc = [] -av_args = [] -av_server_deps = [] -av_client_deps = [] -av_client_use_scaler = false +ffmpeg_inc = [] +ffmpeg_args = [] +ffmpeg_server_deps = [] +ffmpeg_client_deps = [] +ffmpeg_client_use_scaler = false -if av_client_use_scaler - av_client_src += ['libav/scaler.c'] +if ffmpeg_client_use_scaler + ffmpeg_client_src += ['ffmpeg/scaler.c'] endif libavutil = dependency('libavutil', required: false) @@ -29,40 +29,40 @@ libavcodec = dependency('libavcodec', required: false) libswresample = dependency('libswresample', required: false) libswscale = dependency('libswscale', required: false) if not (libavutil.found() and libavformat.found() and libavcodec.found() and libswresample.found() and libswscale.found()) - av_proj = subproject('ffmpeg', required: false) - if av_proj.found() - libavutil = av_proj.get_variable('avutil') - libavformat = av_proj.get_variable('avformat') - libavcodec = av_proj.get_variable('avcodec') - libswresample = av_proj.get_variable('swresample') - libswscale = av_proj.get_variable('swscale') - # Only used to set include dirs for libav. - av_inc_dep = av_proj.get_variable('av_inc_dep') - av_server_deps += [av_inc_dep] - av_client_deps += [av_inc_dep, dependency('zlib')] + ffmpeg_proj = subproject('ffmpeg', required: false) + if ffmpeg_proj.found() + libavutil = ffmpeg_proj.get_variable('avutil') + libavformat = ffmpeg_proj.get_variable('avformat') + libavcodec = ffmpeg_proj.get_variable('avcodec') + libswresample = ffmpeg_proj.get_variable('swresample') + libswscale = ffmpeg_proj.get_variable('swscale') + # Only used to set include dirs for ffmpeg. + ffmpeg_inc_dep = ffmpeg_proj.get_variable('ffmpeg_inc_dep') + ffmpeg_server_deps += [ffmpeg_inc_dep] + ffmpeg_client_deps += [ffmpeg_inc_dep, dependency('zlib')] soxr = compiler.find_library('soxr', required: false) if soxr.found() - av_client_deps += [soxr] + ffmpeg_client_deps += [soxr] endif - av_client_deps += [compiler.find_library('bcrypt')] + ffmpeg_client_deps += [compiler.find_library('bcrypt')] endif endif if libavutil.found() and libavformat.found() and libavcodec.found() and libswresample.found() and libswscale.found() - av_args += ['-DHAVE_FFMPEG'] - av_server_deps += [libavutil, libavformat, libavcodec] - av_client_deps += [libavutil, libavformat, libavcodec, libswresample] - if av_client_use_scaler - av_client_deps += [libswscale] + ffmpeg_args += ['-DCAMU_HAVE_FFMPEG'] + ffmpeg_server_deps += [libavutil, libavformat, libavcodec] + ffmpeg_client_deps += [libavutil, libavformat, libavcodec, libswresample] + if ffmpeg_client_use_scaler + ffmpeg_client_deps += [libswscale] endif - av_server = declare_dependency(sources: av_server_src, dependencies: av_server_deps, - include_directories: av_inc, compile_args: av_args) - av_client = declare_dependency(sources: av_client_src, dependencies: av_client_deps, - include_directories: av_inc, compile_args: av_args) + ffmpeg_server = declare_dependency(sources: ffmpeg_server_src, + dependencies: ffmpeg_server_deps, include_directories: ffmpeg_inc, compile_args: ffmpeg_args) + ffmpeg_client = declare_dependency(sources: ffmpeg_client_src, + dependencies: ffmpeg_client_deps, include_directories: ffmpeg_inc, compile_args: ffmpeg_args) endif -stb_image = declare_dependency(sources: ['stb_image/impl.c'], - dependencies: subproject('stb').get_variable('stb')) +#stb_image = declare_dependency(sources: ['stb_image/impl.c'], +# dependencies: subproject('stb').get_variable('stb')) #wuffs = declare_dependency(sources: ['wuffs/impl.c'], # dependencies: subproject('wuffs').get_variable('wuffs')) diff --git a/src/fruits/cap/cap.c b/src/fruits/cap/cap.c deleted file mode 100644 index 3f5f5e8..0000000 --- a/src/fruits/cap/cap.c +++ /dev/null @@ -1,267 +0,0 @@ -#include <al/random.h> - -#include "../../cache/handlers/file.h" -#ifdef AKIYO_HAS_CURL -#include "../../cache/handlers/http.h" -#endif - -#include "cap.h" - -void cap_init(struct cap_runner *cap, bool (*callback)(void *, u8, str *, str *, void *), void *userdata) -{ - cap->callback = callback; - cap->userdata = userdata; - al_array_init(cap->lists); -#if CAP_USE_PYTHON - sho_client_init(&cap->search, NULL); -#endif -} - -static struct cap_list *list_from_name(struct cap_runner *cap, str *name) -{ - struct cap_list *list; - al_array_foreach_ptr(cap->lists, i, list) { - if (al_str_eq(&list->name, name)) { - return list; - } - } - return NULL; -} - -static struct cch_entry *entry_for_unique_id(struct cap_runner *cap, str *unique_id) -{ - struct cch_entry *entry = NULL; -#ifdef AKIYO_HAS_CURL - bool is_search = al_str_at(unique_id, 0) == ';'; - if (is_search || al_str_cmp(unique_id, al_str_c("https://"), 0, 8) == 0 || - al_str_cmp(unique_id, al_str_c("http://"), 0, 7) == 0) { -#if CAP_USE_PYTHON - str provider; - str query; - al_str_from(&provider, ""); - al_str_from(&query, ""); - if (al_str_cmp(unique_id, al_str_c("https://twitter.com"), 0, 19) == 0 || - al_str_cmp(unique_id, al_str_c("https://x.com"), 0, 13) == 0) { - al_str_cat(&query, al_str_c("tweet:")); - al_str_cat(&query, unique_id); - al_str_cat(&provider, al_str_c("twitter")); - } else if (al_str_cmp(unique_id, al_str_c("https://instagram.com"), 0, 21) == 0) { - s32 slash = al_str_rfind(unique_id, '/'); - if (slash >= 0) { - al_str_cat(&query, al_str_substr(unique_id, slash + 1, unique_id->len)); - } - al_str_cat(&provider, al_str_c("instagram")); - } else { - if (is_search) { - al_str_cat(&query, al_str_substr(unique_id, 1, unique_id->len)); - } else { - al_str_cat(&query, al_str_c("link:")); - al_str_cat(&query, unique_id); - } - al_str_cat(&provider, al_str_c("youtube")); - } - s32 id = sho_client_create_search(&cap->search, &provider, &query); - al_str_free(&provider); - al_str_free(&query); - if (id < 0) return NULL; - struct sho_search *search = sho_client_get_search(&cap->search, id); - if (!search || !sho_search_from_page(search, 0)) return NULL; - struct sho_result_page *page = &al_array_at(search->pages, 0); - struct sho_post *post; - al_array_foreach_ptr(page->posts, i, post) { - struct sho_post_media *media; - al_array_foreach_ptr(post->media, j, media) { - if (media->url.len > 0) { - entry = cch_handler_http_create(&media->url); - break; - } - } - if (entry) break; - } - sho_client_discard_search(&cap->search, id); -#else - if (!is_search) entry = cch_handler_http_create(unique_id); -#endif - if (entry) entry->handler->maybe_spawn_worker(entry->handler, 0); - } else // { -#endif - entry = cch_handler_file_create(unique_id); - // } - return entry; -} - -static bool buffer_entry(struct cap_runner *cap, struct cap_list *list, struct cap_list_entry *entry) -{ - if (entry->errored) return false; - if (entry->buffer_requested) return true; - entry->entry = entry_for_unique_id(cap, &entry->unique_id); - if (!entry->entry || !cap->callback(cap->userdata, CAP_BUFFER, &list->name, &entry->unique_id, entry->entry)) { - entry->errored = true; - return false; - } - entry->buffer_requested = true; - return true; -} - -static void maybe_cleanup_entries(struct cap_runner *cap, struct cap_list *list) -{ - struct cap_list_entry *entry; - al_array_foreach_ptr(list->entries, i, entry) { - if (entry->buffer_requested && abs(list->current - (s32)i) >= 3) { - cap->callback(cap->userdata, CAP_UNLOAD, &list->name, &entry->unique_id, NULL); - entry->buffer_requested = false; - } - } -} - -static bool list_skip_internal(struct cap_runner *cap, struct cap_list *list, s32 n); - -static void list_pump_internal(struct cap_runner *cap, struct cap_list *list, bool buffer) -{ - s32 size = (s32)list->entries.size; - if (size <= list->current) return; - if (buffer) list->buffer = true; - if (list->set != list->current) { - struct cap_list_entry *entry = &al_array_at(list->entries, list->current); - if (list->swap && !entry->buffer_requested) { - list->swap = false; - return; - } - if (!buffer_entry(cap, list, entry)) { - list_skip_internal(cap, list, list->forward ? 1 : -1); - return; - } - cap->callback(cap->userdata, list->swap ? CAP_SWAP : CAP_SET, &list->name, &entry->unique_id, NULL); - list->set = list->current; - } - if (list->buffer) { - for (s32 i = 1; i <= 2; i++) { - if (list->current + i < size) { - buffer_entry(cap, list, &al_array_at(list->entries, list->current + i)); - list->buffer = false; - } - if (list->current - i >= 0) { - buffer_entry(cap, list, &al_array_at(list->entries, list->current - i)); - } - } - } - maybe_cleanup_entries(cap, list); -} - -bool list_skip_internal(struct cap_runner *cap, struct cap_list *list, s32 n) -{ - s32 size = (s32)list->entries.size; - if (list->current + n < 0 || list->current + n >= size) { - return false; - } - list->current += n; - list->forward = n >= 0; - list_pump_internal(cap, list, false); - return true; -} - -void cap_make_list(struct cap_runner *cap, str *name) -{ - struct cap_list list; - al_str_clone(&list.name, name); - list.current = 0; - list.forward = true; - list.set = -1; - list.idle = false; - list.buffer = false; - list.swap = false; - aki_mutex_init(&list.mutex); - al_array_init(list.entries); - al_array_push(cap->lists, list); -} - -void cap_list_add(struct cap_runner *cap, str *name, str *unique_id) -{ - struct cap_list *list = list_from_name(cap, name); - if (!list) return; - aki_mutex_lock(&list->mutex); - struct cap_list_entry entry; - al_str_clone(&entry.unique_id, unique_id); - entry.entry = NULL; - entry.errored = false; - entry.buffer_requested = false; - al_array_push(list->entries, entry); - if (list->idle && list_skip_internal(cap, list, 1)) { - list->idle = false; - } else { - list_pump_internal(cap, list, false); - } - aki_mutex_unlock(&list->mutex); -} - -bool cap_list_skip(struct cap_runner *cap, str *name, s32 n) -{ - struct cap_list *list = list_from_name(cap, name); - if (!list) return false; - aki_mutex_lock(&list->mutex); - bool ret = list_skip_internal(cap, list, n); - aki_mutex_unlock(&list->mutex); - return ret; -} - -void cap_list_pump(struct cap_runner *cap, str *name, bool buffer) -{ - struct cap_list *list = list_from_name(cap, name); - if (!list) return; - aki_mutex_lock(&list->mutex); - list_pump_internal(cap, list, buffer); - aki_mutex_unlock(&list->mutex); -} - -bool cap_list_set_completed(struct cap_runner *cap, str *name) -{ - struct cap_list *list = list_from_name(cap, name); - if (!list) return false; - aki_mutex_lock(&list->mutex); - list->swap = true; - list->idle = !list_skip_internal(cap, list, 1); - bool ret = !list->idle && list->swap; - list->swap = false; - aki_mutex_unlock(&list->mutex); - return ret; -} - -// https://benpfaff.org/writings/clc/shuffle.html -void cap_list_shuffle(struct cap_runner *cap, str *name) -{ - struct cap_list *list = list_from_name(cap, name); - if (!list) return; - aki_mutex_lock(&list->mutex); - u32 size = list->entries.size; - if (size > 0) { - for (u32 i = 0; i < size - 1; i++) { - u32 j = i + al_rand() / (AL_RAND_MAX / (size - i) + 1); - struct cap_list_entry tmp = al_array_at(list->entries, j); - al_array_at(list->entries, j) = al_array_at(list->entries, i); - al_array_at(list->entries, i) = tmp; - } - list->set = -1; - list_pump_internal(cap, list, false); - } - aki_mutex_unlock(&list->mutex); -} - -static void list_close_internal(struct cap_list *list) -{ - struct cap_list_entry *entry; - al_array_foreach_ptr(list->entries, i, entry) { - al_str_free(&entry->unique_id); - } - al_array_free(list->entries); - aki_mutex_destroy(&list->mutex); - al_str_free(&list->name); -} - -void cap_close(struct cap_runner *cap) -{ - struct cap_list *list; - al_array_foreach_ptr(cap->lists, i, list) { - list_close_internal(list); - } - al_array_free(cap->lists); -} diff --git a/src/fruits/cap/cap.h b/src/fruits/cap/cap.h deleted file mode 100644 index 69d4d6d..0000000 --- a/src/fruits/cap/cap.h +++ /dev/null @@ -1,55 +0,0 @@ -#pragma once - -#define CAP_USE_PYTHON 0 - -#include <al/array.h> -#include <al/str.h> -#include <aki/thread.h> - -#if CAP_USE_PYTHON -#include "../../shoki/src/search.h" -#endif - -struct cap_list_entry { - str unique_id; - struct cch_entry *entry; - bool buffer_requested; - bool errored; -}; - -struct cap_list { - str name; - s32 current; - bool forward; - s32 set; - bool idle; - bool buffer; - bool swap; - struct aki_mutex mutex; - array(struct cap_list_entry) entries; -}; - -enum { - CAP_BUFFER = 0, - CAP_SET, - CAP_SWAP, - CAP_UNLOAD -}; - -struct cap_runner { - array(struct cap_list) lists; -#if CAP_USE_PYTHON - struct sho_client search; -#endif - bool (*callback)(void *, u8, str *, str *, void *); - void *userdata; -}; - -void cap_init(struct cap_runner *cap, bool (*callback)(void *, u8, str *, str *, void *), void *userdata); -void cap_make_list(struct cap_runner *cap, str *name); -void cap_list_add(struct cap_runner *cap, str *name, str *unique_id); -bool cap_list_skip(struct cap_runner *cap, str *name, s32 n); -void cap_list_pump(struct cap_runner *cap, str *name, bool buffer); -bool cap_list_set_completed(struct cap_runner *cap, str *name); -void cap_list_shuffle(struct cap_runner *cap, str *name); -void cap_close(struct cap_runner *cap); diff --git a/src/fruits/cap/meson.build b/src/fruits/cap/meson.build deleted file mode 100644 index f85ab00..0000000 --- a/src/fruits/cap/meson.build +++ /dev/null @@ -1,8 +0,0 @@ -cap_src = ['cap.c'] -cap_deps = [common_deps] -use_python = true -if use_python - cap_deps += [shoki] -endif -cap = declare_dependency(sources: cap_src, - dependencies: cap_deps) diff --git a/src/fruits/cmc/cmc.c b/src/fruits/cmc/cmc.c index 97ae15d..5bbdb0b 100644 --- a/src/fruits/cmc/cmc.c +++ b/src/fruits/cmc/cmc.c @@ -5,7 +5,7 @@ #include <aki/common.h> #include <aki/event_loop.h> -#include "../../tree/common.h" +#include "../../server/common.h" #include "cmc.h" @@ -78,7 +78,7 @@ s32 main(s32 argc, char *argv[]) str module; str query; - str *cmd = al_str_c(argv[1]); + str *cmd = al_str_cr(argv[1]); if (al_str_eq(cmd, al_str_c("search"))) { c.type = CMC_SEARCH; if (argc == 2) { @@ -104,7 +104,7 @@ s32 main(s32 argc, char *argv[]) aki_event_loop_init(&c.loop); camu_client_init(&c.client, &c.loop, client_callback, &c); - if (!camu_client_login(&c.client, al_str_c("andrew"), TREE_SERVER_IP, TREE_PORT)) { + if (!camu_client_login(&c.client, al_str_c("andrew"), CAMU_SERVER_IP, CAMU_PORT)) { goto err; } diff --git a/src/fruits/cmc/meson.build b/src/fruits/cmc/meson.build index 6dd3960..57ded9c 100644 --- a/src/fruits/cmc/meson.build +++ b/src/fruits/cmc/meson.build @@ -4,6 +4,7 @@ cmc_src = [ 'tui/resources.c', 'tui/pager.c', 'tui/view_twitter.c', + 'tui/pane_lists.c', 'tui/pane_search.c' ] cmc_deps = [common_deps, libclient, subproject('stb').get_variable('stb')] diff --git a/src/fruits/cmc/tui/pager.c b/src/fruits/cmc/tui/pager.c index 00c5665..a672315 100644 --- a/src/fruits/cmc/tui/pager.c +++ b/src/fruits/cmc/tui/pager.c @@ -70,7 +70,7 @@ bool cmc_pager_iterate_to_height(struct cmc_pager *pager, u32 height, u32 anchor { u32 y = 0, offset; struct cmc_view_page *page = cmc_pager_page_containing_index(pager, anchor, &offset); - al_assert(page); + if (!page) return false; struct cmc_view *view; do { // If the page is not processed, return out of space as to not display a discontinuity. diff --git a/src/fruits/cmc/tui/pane_lists.c b/src/fruits/cmc/tui/pane_lists.c new file mode 100644 index 0000000..6367e57 --- /dev/null +++ b/src/fruits/cmc/tui/pane_lists.c @@ -0,0 +1,93 @@ +#include "../cmc.h" + +#include "pane_lists.h" +#include "ui.h" + +void cmc_pane_lists_init(struct cmc_pane_lists *pane, struct cmc_ui *ui) +{ + pane->offset = 0; + pane->selected = 0; + pane->list = al_array_at(ui->c->client.state.lists, 0); + pane->ui = ui; +} + +bool cmc_pane_lists_handle_input(struct cmc_pane_lists *pane, struct ncinput *input) +{ + if (!(input->evtype == NCTYPE_PRESS || input->evtype == NCTYPE_UNKNOWN)) return false; + s32 rows = pane->ui->rows - 1; + s32 size = (s32)pane->list->entries.size; + if (input->id == 'j') { + if (pane->selected + 1 < size) { + pane->selected++; + if (pane->selected - pane->offset >= rows) { + pane->offset += rows; + } + return true; + } + } else if (input->id == 'k') { + if (pane->selected - 1 >= 0) { + pane->selected--; + if (pane->selected - pane->offset < 0) { + pane->offset -= rows; + } + return true; + } + } else if (input->id == 'n') { + if (pane->offset + rows < size) { + pane->offset += rows; + pane->selected += rows; + if (pane->selected >= size) { + pane->selected = size - 1; + } + return true; + } + } else if (input->id == 'p') { + if (pane->offset - rows >= 0) { + pane->offset -= rows; + pane->selected -= rows; + return true; + } + } else if (input->id == NCKEY_RETURN) { + camu_client_skipto(&pane->ui->c->client, pane->selected); + } + return false; +} + +bool cmc_pane_lists_handle_input_overlay(struct cmc_pane_lists *pane, struct ncinput *input) +{ + (void)pane; + (void)input; + return false; +} + +void cmc_pane_lists_draw_overlay(struct cmc_pane_lists *pane) +{ + (void)pane; +} + +void cmc_pane_lists_refresh(struct cmc_pane_lists *pane) +{ + struct cmc_ui *ui = pane->ui; + + ncplane_cursor_move_yx(ui->p, 0, 0); + ncplane_erase(ui->p); + notcurses_refresh(ui->nc, NULL, NULL); + notcurses_render(ui->nc); + + str *unique_id; + u32 rows = AL_MIN(pane->list->entries.size - pane->offset, ui->rows - 1); + for (u32 i = pane->offset; i < pane->offset + rows; i++) { + unique_id = &al_array_at(pane->list->entries, i); + if ((s32)i == pane->selected) { + ncplane_set_fg_palindex(ui->p, 4); + } else { + ncplane_set_fg_default(ui->p); + } + u32 cols = AL_MIN(unique_id->len, ui->cols); + for (u32 j = 0; j < cols; j++) { + ncplane_putchar_yx(ui->p, i - pane->offset, j + 1, al_str_at(unique_id, j)); + } + } + + notcurses_render(ui->nc); +} diff --git a/src/fruits/cmc/tui/pane_lists.h b/src/fruits/cmc/tui/pane_lists.h new file mode 100644 index 0000000..323eb52 --- /dev/null +++ b/src/fruits/cmc/tui/pane_lists.h @@ -0,0 +1,20 @@ +#pragma once + +#include <al/types.h> +#include <notcurses/notcurses.h> + +#include "../../libclient/client.h" + +struct cmc_ui; +struct cmc_pane_lists { + s32 offset; + s32 selected; + struct camu_list *list; + struct cmc_ui *ui; +}; + +void cmc_pane_lists_init(struct cmc_pane_lists *pane, struct cmc_ui *ui); +bool cmc_pane_lists_handle_input(struct cmc_pane_lists *pane, struct ncinput *input); +bool cmc_pane_lists_handle_input_overlay(struct cmc_pane_lists *pane, struct ncinput *input); +void cmc_pane_lists_draw_overlay(struct cmc_pane_lists *pane); +void cmc_pane_lists_refresh(struct cmc_pane_lists *pane); diff --git a/src/fruits/cmc/tui/pane_search.c b/src/fruits/cmc/tui/pane_search.c index 4c14939..002a4d1 100644 --- a/src/fruits/cmc/tui/pane_search.c +++ b/src/fruits/cmc/tui/pane_search.c @@ -27,7 +27,6 @@ void cmc_pane_search_init(struct cmc_pane_search *pane, struct cmc_ui *ui) } if (pane->searches.size > 0) { pane->search = al_array_at(pane->searches, 0); - request_page_internal(pane->ui->c, pane->search, pane->search->search->page); } } @@ -55,8 +54,7 @@ static void add_selected_from_index(struct cmc *c, struct cmc_pager *pager, u32 { struct cmc_view *view = get_view_by_visual_index(pager, pager->selected); if (view && view->post->media.size > index) { - camu_client_add(&c->client, &view->post->unique_id, index); - camu_client_skip(&c->client, 1); + camu_client_add(&c->client, &view->post->unique_id, index, true); } } @@ -71,6 +69,7 @@ bool cmc_pane_search_handle_input(struct cmc_pane_search *pane, struct ncinput * if (pager->selected + 1 >= (s32)pager->count) { if (pager->pending_requests > 0 || pager->find_anchor != PAGE_NONE) return false; if (ON_LAST_PAGE(pager)) return false; + if (pager->count == 0) pager->count = 1; u32 index = pager->anchor + pager->count; pager->find_anchor = PAGE_DOWN; struct cmc_view_page *page = cmc_pager_page_containing_index(pager, index, NULL); @@ -116,7 +115,9 @@ bool cmc_pane_search_handle_input_overlay(struct cmc_pane_search *pane, struct n if (!(input->evtype == NCTYPE_PRESS || input->evtype == NCTYPE_UNKNOWN)) return false; struct cmc_ui *ui = pane->ui; struct cmc_pager *pager = &pane->search->pager; - if (input->id == 'j') { + if (input->id == NCKEY_TAB) { + pager->redraw = true; + } else if (input->id == 'j') { if (ui->overlay.selected + 1 < (s32)pane->searches.size) { ui->overlay.selected++; return true; @@ -137,12 +138,7 @@ bool cmc_pane_search_handle_input_overlay(struct cmc_pane_search *pane, struct n pane->search = al_array_at(pane->searches, ui->overlay.selected); pager = &pane->search->pager; pager->redraw = true; - u32 page = pane->search->search->page; - if (!cmc_get_page_from_num(pager, page)) { - request_page_internal(ui->c, pane->search, page); - } else { - return true; - } + return true; } return false; } @@ -197,6 +193,14 @@ static void draw_selection_line(struct cmc_view *view, u32 y, u32 rows) void cmc_pane_search_refresh(struct cmc_pane_search *pane) { struct cmc_pager *pager = &pane->search->pager; + if (pager->pending_requests > 0) return; + + u32 page = pane->search->search->page; + if (!cmc_get_page_from_num(pager, page)) { + request_page_internal(pane->ui->c, pane->search, page); + return; + } + if (pager->find_anchor == PAGE_UP) { u32 count; // If find_anchor = PAGE_UP, search->anchor is index of the requested view. @@ -204,6 +208,7 @@ void cmc_pane_search_refresh(struct cmc_pane_search *pane) while (anchor >= 0) { count = 0; cmc_pager_iterate_to_height(pager, pane->ui->rows, anchor, &count, NULL, NULL); + // TODO: Handle count == 0. // Is the requested view still visible? if ((s32)(anchor + count) - 1 < pager->anchor) break; anchor--; @@ -249,7 +254,7 @@ void cmc_pane_search_refresh(struct cmc_pane_search *pane) notcurses_refresh(pane->ui->nc, NULL, NULL); } notcurses_render(pane->ui->nc); - if (pager->pending_requests == 0 && pager->count > 0 && !out_of_space && pager->end_of_page == -1) { + if (pager->count > 0 && !out_of_space && pager->end_of_page == -1) { request_page_at_page_offset(pane, pager, (pager->anchor + pager->count) - 1, 1); } } diff --git a/src/fruits/cmc/tui/ui.c b/src/fruits/cmc/tui/ui.c index dac84a7..2c406bd 100644 --- a/src/fruits/cmc/tui/ui.c +++ b/src/fruits/cmc/tui/ui.c @@ -1,4 +1,4 @@ -#include "../../tree/common.h" +#include "../../server/common.h" #include "../cmc.h" @@ -31,6 +31,22 @@ void cmc_ui_toggle_overlay(struct cmc_ui *ui) ui->overlay.active = !ui->overlay.active; } +static void refresh_internal(struct cmc_ui *ui) +{ + switch (ui->pane) { + case CMC_PANE_LIBRARY: + break; + case CMC_PANE_LISTS: + cmc_pane_lists_refresh(&ui->lists); + break; + case CMC_PANE_BROWSE: + break; + case CMC_PANE_SEARCH: + cmc_pane_search_refresh(&ui->search); + break; + } +} + static void input_poll_callback(void *userdata, s32 revents) { struct cmc_ui *ui = (struct cmc_ui *)userdata; @@ -48,13 +64,16 @@ static void input_poll_callback(void *userdata, s32 revents) } else if (input.id == NCKEY_TAB) { cmc_ui_toggle_overlay(ui); do_refresh = true; - ui->search.search->pager.redraw = true; - continue; } switch (ui->pane) { case CMC_PANE_LIBRARY: break; case CMC_PANE_LISTS: + if (ui->overlay.active) { + do_refresh |= cmc_pane_lists_handle_input_overlay(&ui->lists, &input); + } else { + do_refresh |= cmc_pane_lists_handle_input(&ui->lists, &input); + } break; case CMC_PANE_BROWSE: break; @@ -68,18 +87,7 @@ static void input_poll_callback(void *userdata, s32 revents) } } } while (1); - if (!do_refresh) return; - switch (ui->pane) { - case CMC_PANE_LIBRARY: - break; - case CMC_PANE_LISTS: - break; - case CMC_PANE_BROWSE: - break; - case CMC_PANE_SEARCH: - cmc_pane_search_refresh(&ui->search); - break; - } + if (do_refresh) refresh_internal(ui); } static s32 resize_cb(struct ncplane *p) @@ -110,11 +118,12 @@ bool cmc_ui_open(struct cmc_ui *ui, struct cmc *c) ui->c = c; + cmc_pane_lists_init(&ui->lists, ui); cmc_pane_search_init(&ui->search, ui); - if (!cmc_resources_connect(&ui->resources, ui->nc, &ui->c->loop, TREE_SERVER_IP, TREE_RESOURCE_PORT)) { - return false; - } + //if (!cmc_resources_connect(&ui->resources, ui->nc, &ui->c->loop, CAMU_SERVER_IP, CAMU_RESOURCE_PORT)) { + // return false; + //} cmc_resources_set_callback(&ui->resources, cmc_ui_resources_callback, &ui->search); aki_poll_init(&ui->input, input_poll_callback, ui); @@ -129,5 +138,7 @@ bool cmc_ui_open(struct cmc_ui *ui, struct cmc *c) ncplane_set_userptr(stdplane, ui); resize_cb(stdplane); + refresh_internal(ui); + return true; } diff --git a/src/fruits/cmc/tui/ui.h b/src/fruits/cmc/tui/ui.h index 2f81392..76ea353 100644 --- a/src/fruits/cmc/tui/ui.h +++ b/src/fruits/cmc/tui/ui.h @@ -4,6 +4,7 @@ #include <notcurses/notcurses.h> #include "resources.h" +#include "pane_lists.h" #include "pane_search.h" enum { @@ -29,6 +30,7 @@ struct cmc_ui { } overlay; struct aki_poll input; struct cmc_resources resources; + struct cmc_pane_lists lists; struct cmc_pane_search search; struct cmc *c; }; diff --git a/src/fruits/sink/sink.c b/src/fruits/sink/sink.c index 08d2886..cbd82bd 100644 --- a/src/fruits/sink/sink.c +++ b/src/fruits/sink/sink.c @@ -8,7 +8,7 @@ //#include "../../render/renderer_tiger.h" #include "../../libsink/sink.h" -#include "../../tree/common.h" +#include "../../server/common.h" struct cmv { s32 quit; @@ -117,6 +117,9 @@ static void screen_callback(void *userdata, u8 op, f64 float0) case CAMU_SCREEN_SEEK: camu_sink_seek(&c->sink, float0); break; + case CAMU_SCREEN_RESEEK: + camu_sink_reseek(&c->sink); + break; case CAMU_SCREEN_CLOSE: c->quit = 1; break; @@ -145,6 +148,8 @@ s32 main(s32 argc, char *argv[]) s32 wmain(s32 argc, wchar_t **argv) #endif { + (void)argc; + (void)argv; if (!aki_common_init()) return EXIT_FAILURE; signal(SIGINT, sigint_handler); @@ -170,7 +175,7 @@ s32 wmain(s32 argc, wchar_t **argv) camu_sink_init(&c.sink, &c.loop, &c.mixer, c.renderer); c.sink.callback = sink_callback; c.sink.userdata = &c; - if (!camu_sink_connect(&c.sink, TREE_SERVER_IP, TREE_PORT)) { + if (!camu_sink_connect(&c.sink, CAMU_SERVER_IP, CAMU_PORT)) { goto err; } diff --git a/src/libclient/client.c b/src/libclient/client.c index bf22fee..cd0bdf5 100644 --- a/src/libclient/client.c +++ b/src/libclient/client.c @@ -1,6 +1,6 @@ #include <al/log.h> -#include "../tree/common.h" +#include "../server/common.h" #include "client.h" @@ -15,8 +15,8 @@ static void connection_callback(void *userdata, struct aki_rpc_connection *conn) { struct camu_client *client = (struct camu_client *)userdata; client->conn = conn; - struct aki_packet *packet = aki_rpc_get_packet(&client->client, TREE_CMD_IDENTIFY); - aki_packet_write_u8(packet, TREE_CLIENT); + struct aki_packet *packet = aki_rpc_get_packet(&client->client, CAMU_SRV_IDENTIFY); + aki_packet_write_u8(packet, CAMU_CLIENT); aki_packet_write_str(packet, &client->username); aki_rpc_connection_command(client->conn, packet, identifed_callback, client); } @@ -78,7 +78,7 @@ static bool update_state_command_callback(void *userdata, struct aki_rpc_connect } static struct aki_rpc_command commands[] = { - { .op = TREE_CMD_UPDATE_STATE, .callback = update_state_command_callback, .userdata = NULL } + { .op = CAMU_CONN_STATE, .callback = update_state_command_callback, .userdata = NULL } }; bool camu_client_init(struct camu_client *client, struct aki_event_loop *loop, @@ -125,11 +125,13 @@ static void create_search_callback(void *userdata, struct aki_packet *packet) void camu_client_create_search(struct camu_client *client, str *module, str *query) { + /* struct aki_packet *packet = aki_rpc_get_packet(&client->client, TREE_CMD_CREATE_SEARCH); aki_packet_write_str(packet, module); aki_packet_write_str(packet, query); packet->userdata = create_search_callback; aki_packet_pool_submit(&client->pool, packet); + */ } static struct camu_search *get_search_from_id(struct camu_client *client, s32 id) @@ -145,6 +147,7 @@ static void results_callback(void *userdata, struct aki_packet *packet) { struct camu_client *client = (struct camu_client *)userdata; + // We still get the requested id even if there was an error. s32 id = aki_packet_read_s32(packet); s32 ok = aki_packet_read_s32(packet); @@ -192,28 +195,48 @@ static void results_callback(void *userdata, struct aki_packet *packet) void camu_client_get_page(struct camu_client *client, struct camu_search *search, u32 num) { + /* struct aki_packet *packet = aki_rpc_get_packet(&client->client, TREE_CMD_GET_PAGE); aki_packet_write_s32(packet, search->id); aki_packet_write_u32(packet, num); packet->userdata = results_callback; aki_packet_pool_submit(&client->pool, packet); + */ } -void camu_client_add(struct camu_client *client, str *unique_id, u32 index) +void camu_client_add(struct camu_client *client, str *unique_id, u32 index, bool set) { - struct aki_packet *packet = aki_rpc_get_packet(&client->client, TREE_CMD_ADD); + /* + struct aki_packet *packet = aki_rpc_get_packet(&client->client, TREE_CMD_LIST_ACTION); + aki_packet_write_u8(packet, TREE_LIST_ADD); aki_packet_write_str(packet, unique_id); aki_packet_write_u32(packet, index); + aki_packet_write_u8(packet, set); packet->userdata = NULL; aki_packet_pool_submit(&client->pool, packet); + */ } void camu_client_skip(struct camu_client *client, s32 n) { - struct aki_packet *packet = aki_rpc_get_packet(&client->client, TREE_CMD_SKIP); + /* + struct aki_packet *packet = aki_rpc_get_packet(&client->client, TREE_CMD_LIST_ACTION); + aki_packet_write_u8(packet, TREE_LIST_SKIP); aki_packet_write_s32(packet, n); packet->userdata = NULL; aki_packet_pool_submit(&client->pool, packet); + */ +} + +void camu_client_skipto(struct camu_client *client, s32 i) +{ + /* + struct aki_packet *packet = aki_rpc_get_packet(&client->client, TREE_CMD_LIST_ACTION); + aki_packet_write_u8(packet, TREE_LIST_SKIPTO); + aki_packet_write_s32(packet, i); + packet->userdata = NULL; + aki_packet_pool_submit(&client->pool, packet); + */ } void camu_client_close(struct camu_client *client) diff --git a/src/libclient/client.h b/src/libclient/client.h index 68b0826..98276f2 100644 --- a/src/libclient/client.h +++ b/src/libclient/client.h @@ -54,7 +54,9 @@ bool camu_client_init(struct camu_client *client, struct aki_event_loop *loop, void (*callback)(void *, u8, void *), void *userdata); bool camu_client_login(struct camu_client *client, str *username, str *addr, s32 port); void camu_client_create_search(struct camu_client *client, str *module, str *query); +void camu_client_create_browse(struct camu_client *client); void camu_client_get_page(struct camu_client *client, struct camu_search *search, u32 num); -void camu_client_add(struct camu_client *client, str *unique_id, u32 index); +void camu_client_add(struct camu_client *client, str *unique_id, u32 index, bool set); void camu_client_skip(struct camu_client *client, s32 n); +void camu_client_skipto(struct camu_client *client, s32 i); void camu_client_close(struct camu_client *client); diff --git a/src/libclient/meson.build b/src/libclient/meson.build index a2f23b0..dff4014 100644 --- a/src/libclient/meson.build +++ b/src/libclient/meson.build @@ -1,3 +1,3 @@ libclient_src = ['client.c', 'resource_client.c'] -libclient_deps = [shoki] +libclient_deps = [portal] libclient = declare_dependency(sources: libclient_src, dependencies: libclient_deps) diff --git a/src/libsink/common.h b/src/libsink/common.h index 48e43e8..a5ad25c 100644 --- a/src/libsink/common.h +++ b/src/libsink/common.h @@ -1,7 +1,6 @@ #pragma once enum { - CAMU_SINK_CMD_BUFFER = 0, - CAMU_SINK_CMD_SET, - CAMU_SINK_CMD_QUEUE, + CAMU_SINK_SET = 0, + CAMU_SINK_QUEUE }; diff --git a/src/libsink/meson.build b/src/libsink/meson.build index 0e6dfb3..83cfdd7 100644 --- a/src/libsink/meson.build +++ b/src/libsink/meson.build @@ -1,3 +1,3 @@ libsink_src = ['sink.c'] -libsink_deps = [bimu_client] +libsink_deps = [shrub_client] libsink = declare_dependency(sources: libsink_src, dependencies: libsink_deps) diff --git a/src/libsink/sink.c b/src/libsink/sink.c index 3468a48..65a082e 100644 --- a/src/libsink/sink.c +++ b/src/libsink/sink.c @@ -1,12 +1,14 @@ #include <al/log.h> -#include "../tree/common.h" +#include "../server/common.h" -#include "../bimu/common.h" -#include "../bimu/handler.h" +#include "../shrub/common.h" +#include "../shrub/handler.h" #include "../buffer/common.h" +#include "../list/list.h" + #include "sink.h" #include "common.h" @@ -17,35 +19,25 @@ enum { }; enum { - ENTRY_LOADED = 0, - ENTRY_BUFFERED, - ENTRY_DISREGUARDED -}; - -enum { BUFFER_INIT = 0, BUFFER_QUEUED, BUFFER_CONFIGURED, BUFFER_SET_OR_BUFFERED, - BUFFER_ADDED, + BUFFER_ADDED }; enum { START, STOP, TOGGLE_PAUSE, + SEEK, SKIP, + FINISHED, RESEEK, - SEEK, SET_BUFFERED, // Currently set entry is buffered. - CLOSE, - // Internal. - CORK, - UNCORK, + CLOSE }; -#define BIMU_DELAY_IGNORE 0 - #define ENTRY_AUDIO_BUFFER_HELD(entry) \ (al_atomic_bool_load(&(entry)->audio.buf.ref, AL_ATOMIC_RELAXED)) #ifdef CAMU_SINK_NO_VIDEO @@ -56,10 +48,14 @@ enum { #define ENTRY_BUFFERS_HELD(entry) (ENTRY_AUDIO_BUFFER_HELD(entry) || ENTRY_VIDEO_BUFFER_HELD(entry)) #endif -#define ENTRY_VIDEO_EMPTY(entry) \ - (entry->video.state == BUFFER_INIT || entry->video.state == BUFFER_QUEUED) -#define ENTRY_AUDIO_EMPTY(entry) \ - (entry->audio.state == BUFFER_INIT || entry->audio.state == BUFFER_QUEUED) +#ifdef CAMU_SINK_NO_VIDEO +#define ENTRY_VIDEO_READY_OR_EMPTY(entry) true +#else +#define ENTRY_VIDEO_READY_OR_EMPTY(entry) \ + (entry->video.state == BUFFER_INIT || entry->video.state == BUFFER_QUEUED || entry->video.state == BUFFER_ADDED) +#endif +#define ENTRY_AUDIO_READY_OR_EMPTY(entry) \ + (entry->audio.state == BUFFER_INIT || entry->audio.state == BUFFER_QUEUED || entry->audio.state == BUFFER_ADDED) static void remove_entry_buffers(struct camu_sink *sink, struct camu_sink_entry *entry) { @@ -132,10 +128,19 @@ static void handle_sink_cmd(struct camu_sink *sink, struct camu_sink_cmd *cmd) } break; } + case SKIP: { + struct aki_packet *packet = aki_rpc_get_packet(&sink->client, CAMU_SRV_LIST_ACTION); + aki_packet_write_u8(packet, CAMU_LIST_SKIP); + s32 sequence = sink->current ? sink->current->sequence : CAMU_SEQUENCE_INVALID; + aki_packet_write_s32(packet, sequence); + aki_packet_write_s32(packet, cmd->value.i); + aki_rpc_connection_command(sink->conn, packet, NULL, NULL); + break; + } case TOGGLE_PAUSE: { struct camu_sink_entry *entry = (struct camu_sink_entry *)cmd->opaque; +#ifdef CAMU_SINK_LOCAL_PAUSE if (camu_clock_is_paused(&entry->clock)) { - camu_clock_calc_tick(&entry->clock); camu_clock_resume(&entry->clock); if (sink->audio.state == SINK_PAUSED) { sink->callback(sink->userdata, CAMU_SINK_START, CAMU_SINK_AUDIO, NULL); @@ -156,22 +161,40 @@ static void handle_sink_cmd(struct camu_sink *sink, struct camu_sink_cmd *cmd) } #endif } +#else + f64 base = camu_clock_get_base_pts(&entry->clock); + if (!camu_clock_is_paused(&entry->clock)) { + f64 pts = camu_clock_get_pts(&entry->clock, 0.0); + if (pts > base) base = pts; + } + struct aki_packet *packet = aki_rpc_get_packet(&sink->client, CAMU_SRV_LIST_ACTION); + aki_packet_write_u8(packet, CAMU_LIST_TOGGLE_PAUSE); + s32 sequence = sink->current ? sink->current->sequence : CAMU_SEQUENCE_INVALID; + aki_packet_write_s32(packet, sequence); + aki_packet_write_u64(packet, base * 1000000.0); + aki_rpc_connection_command(sink->conn, packet, NULL, NULL); +#endif break; } - case SKIP: { - struct aki_packet *packet = aki_rpc_get_packet(&sink->client, TREE_CMD_SKIP); - aki_packet_write_s32(packet, cmd->value.i); + case SEEK: { + struct aki_packet *packet = aki_rpc_get_packet(&sink->client, CAMU_SRV_LIST_ACTION); + aki_packet_write_u8(packet, CAMU_LIST_SEEK); + s32 sequence = sink->current ? sink->current->sequence : CAMU_SEQUENCE_INVALID; + aki_packet_write_s32(packet, sequence); + aki_packet_write_f64(packet, cmd->value.f); aki_rpc_connection_command(sink->conn, packet, NULL, NULL); break; } - case RESEEK: { - struct camu_sink_entry *entry = (struct camu_sink_entry *)cmd->opaque; - bmu_client_reseek(&entry->client); + case FINISHED: { + struct aki_packet *packet = aki_rpc_get_packet(&sink->client, CAMU_SRV_LIST_ACTION); + aki_packet_write_u8(packet, CAMU_LIST_FINISHED); + aki_packet_write_s32(packet, cmd->value.i); + aki_rpc_connection_command(sink->conn, packet, NULL, NULL); break; } - case SEEK: { + case RESEEK: { struct camu_sink_entry *entry = (struct camu_sink_entry *)cmd->opaque; - bmu_client_seek(&entry->client, cmd->value.f); + shrb_client_reseek(&entry->client); break; } // case SET_BUFFERED: { @@ -192,34 +215,6 @@ static void handle_sink_cmd(struct camu_sink *sink, struct camu_sink_cmd *cmd) sink->callback(sink->userdata, CAMU_SINK_EXIT, 0, NULL); return; } - case CORK: { - struct camu_sink_entry *entry = (struct camu_sink_entry *)cmd->opaque; - switch (cmd->value.i) { - case CAMU_SINK_AUDIO: - bmu_vcr_stream_cork(entry->audio.stream); - break; -#ifndef CAMU_SINK_NO_VIDEO - case CAMU_SINK_VIDEO: - bmu_vcr_stream_cork(entry->video.stream); - break; -#endif - } - break; - } - case UNCORK: { - struct camu_sink_entry *entry = (struct camu_sink_entry *)cmd->opaque; - switch (cmd->value.i) { - case CAMU_SINK_AUDIO: - bmu_vcr_stream_uncork(entry->audio.stream); - break; -#ifndef CAMU_SINK_NO_VIDEO - case CAMU_SINK_VIDEO: - bmu_vcr_stream_uncork(entry->video.stream); - break; -#endif - } - break; - } } } @@ -241,31 +236,30 @@ static void queue_cmd(struct camu_sink *sink, struct camu_sink_cmd cmd) aki_signal_send(&sink->signal); } +static void maybe_remove_previous(struct camu_sink *sink) +{ + struct camu_sink_entry *previous; + al_array_foreach(sink->previous, i, previous) { + remove_entry_buffers(sink, previous); + } + sink->previous.size = 0; +} + void add_audio_if_set_and_buffered(struct camu_sink_entry *entry) { u8 state = entry->audio.state; if (state == BUFFER_SET_OR_BUFFERED) { - // TODO: should_resume is not robust. CAMU_SINK_NO_VIDEO doesn't work but that's - // the least of our problems. -#ifndef CAMU_SINK_NO_VIDEO - bool should_resume = ENTRY_VIDEO_EMPTY(entry) || entry->video.state == BUFFER_ADDED; -#endif - if (should_resume && !camu_clock_calc_tick(&entry->clock)) { - queue_cmd(entry->sink, (struct camu_sink_cmd){ - .op = RESEEK, - .opaque = entry - }); - return; - } + bool can_resume = ENTRY_VIDEO_READY_OR_EMPTY(entry); queue_cmd(entry->sink, (struct camu_sink_cmd){ .op = START, .value.i = CAMU_SINK_AUDIO }); - if (should_resume) { + camu_audio_buffer_unpause(&entry->audio.buf); + entry->sink->callback(entry->sink->userdata, CAMU_SINK_ADD_BUFFER, CAMU_SINK_AUDIO, &entry->audio.buf); + if (can_resume) { camu_clock_resume(&entry->clock); - entry->state = ENTRY_BUFFERED; + maybe_remove_previous(entry->sink); } - entry->sink->callback(entry->sink->userdata, CAMU_SINK_ADD_BUFFER, CAMU_SINK_AUDIO, &entry->audio.buf); //queue_cmd(entry->sink, (struct camu_sink_cmd){ // .op = SET_BUFFERED, // .value.i = CAMU_SINK_AUDIO @@ -282,23 +276,16 @@ void add_video_if_set_and_buffered(struct camu_sink_entry *entry) { u8 state = entry->video.state; if (state == BUFFER_SET_OR_BUFFERED) { - bool should_resume = ENTRY_AUDIO_EMPTY(entry) || entry->audio.state == BUFFER_ADDED; - if (should_resume && !camu_clock_calc_tick(&entry->clock)) { - queue_cmd(entry->sink, (struct camu_sink_cmd){ - .op = RESEEK, - .opaque = entry - }); - return; - } + bool can_resume = ENTRY_AUDIO_READY_OR_EMPTY(entry); queue_cmd(entry->sink, (struct camu_sink_cmd){ .op = START, .value.i = CAMU_SINK_VIDEO }); - if (should_resume) { + entry->sink->callback(entry->sink->userdata, CAMU_SINK_ADD_BUFFER, CAMU_SINK_VIDEO, &entry->video.buf); + if (can_resume) { camu_clock_resume(&entry->clock); - entry->state = ENTRY_BUFFERED; + maybe_remove_previous(entry->sink); } - entry->sink->callback(entry->sink->userdata, CAMU_SINK_ADD_BUFFER, CAMU_SINK_VIDEO, &entry->video.buf); //queue_cmd(entry->sink, (struct camu_sink_cmd){ // .op = SET_BUFFERED, // .value.i = CAMU_SINK_VIDEO @@ -321,35 +308,27 @@ static void audio_buffer_callback(void *userdata, u8 op) aki_mutex_unlock(&entry->sink->mutex); break; case CAMU_BUFFER_CORK: - bmu_vcr_stream_cork(entry->audio.stream); - /* - queue_cmd(entry->sink, (struct camu_sink_cmd){ - .op = CORK, - .value.i = CAMU_SINK_AUDIO, - .opaque = entry - }); - */ + shrb_vcr_stream_cork(entry->audio.stream); break; case CAMU_BUFFER_UNCORK: - bmu_vcr_stream_uncork(entry->audio.stream); - /* - queue_cmd(entry->sink, (struct camu_sink_cmd){ - .op = UNCORK, - .value.i = CAMU_SINK_AUDIO, - .opaque = entry - }); - */ + shrb_vcr_stream_uncork(entry->audio.stream); break; case CAMU_BUFFER_PAUSED: - queue_cmd(entry->sink, (struct camu_sink_cmd){ - .op = STOP, - .value.i = CAMU_SINK_AUDIO - }); + aki_mutex_lock(&entry->sink->mutex); + if (!entry->audio.armed) { + queue_cmd(entry->sink, (struct camu_sink_cmd){ + .op = STOP, + .value.i = CAMU_SINK_AUDIO + }); + } + aki_mutex_unlock(&entry->sink->mutex); break; case CAMU_BUFFER_EOF: { + // TODO: Video only case not handled (video not image). struct camu_sink *sink = entry->sink; bool swapped = false; aki_mutex_lock(&sink->mutex); + s32 sequence = sink->current->sequence; if (sink->queued) { swap_buffers_internal(sink); swapped = true; @@ -368,6 +347,10 @@ static void audio_buffer_callback(void *userdata, u8 op) //}); #endif } + queue_cmd(entry->sink, (struct camu_sink_cmd){ + .op = FINISHED, + .value.i = sequence + }); break; } } @@ -384,24 +367,10 @@ static void video_buffer_callback(void *userdata, u8 op) aki_mutex_unlock(&entry->sink->mutex); break; case CAMU_BUFFER_CORK: - bmu_vcr_stream_cork(entry->video.stream); - /* - queue_cmd(entry->sink, (struct camu_sink_cmd){ - .op = CORK, - .value.i = CAMU_SINK_VIDEO, - .opaque = entry - }); - */ + shrb_vcr_stream_cork(entry->video.stream); break; case CAMU_BUFFER_UNCORK: - bmu_vcr_stream_uncork(entry->video.stream); - /* - queue_cmd(entry->sink, (struct camu_sink_cmd){ - .op = UNCORK, - .value.i = CAMU_SINK_VIDEO, - .opaque = entry - }); - */ + shrb_vcr_stream_uncork(entry->video.stream); break; case CAMU_BUFFER_EOF: queue_cmd(entry->sink, (struct camu_sink_cmd){ @@ -413,14 +382,15 @@ static void video_buffer_callback(void *userdata, u8 op) } #endif -static void client_callback(void *userdata, u8 op, struct bmu_vcr_stream *stream, void *opaque) +static void client_callback(void *userdata, u8 op, struct shrb_vcr_stream *stream, void *opaque) { struct camu_sink_entry *entry = (struct camu_sink_entry *)userdata; + struct camu_sink *sink = entry->sink; switch (op) { - case BIMU_CLIENT_CONFIGURE: { - // No data can be sent until all active streams are configured. + case SHRUB_CLIENT_CONFIGURE: { + // No data can be sent until all selected streams are configured. switch (stream->type) { - case BIMU_STREAM_AUDIO: + case SHRUB_STREAM_AUDIO: entry->audio.stream = stream; camu_audio_buffer_configure(&entry->audio.buf, &stream->stream); aki_mutex_lock(&entry->sink->mutex); @@ -432,7 +402,7 @@ static void client_callback(void *userdata, u8 op, struct bmu_vcr_stream *stream aki_mutex_unlock(&entry->sink->mutex); break; #ifndef CAMU_SINK_NO_VIDEO - case BIMU_STREAM_VIDEO: + case SHRUB_STREAM_VIDEO: entry->video.stream = stream; camu_video_buffer_configure(&entry->video.buf, &stream->stream); aki_mutex_lock(&entry->sink->mutex); @@ -447,46 +417,51 @@ static void client_callback(void *userdata, u8 op, struct bmu_vcr_stream *stream } break; } - case BIMU_CLIENT_SET: { - u64 ts = aki_get_timestamp(); - struct bmu_seek_req *req = (struct bmu_seek_req *)opaque; - req->delay = BIMU_DELAY_IGNORE; - req->ts -= req->delay; - bool late = ts <= req->ts; - if (late) {} + case SHRUB_CLIENT_SET: { + struct shrb_seek_req *req = (struct shrb_seek_req *)opaque; + //req->delay = SHRUB_DELAY_IGNORE; camu_clock_set(&entry->clock, req->base, req->ts, req->delay); - camu_audio_buffer_reset(&entry->audio.buf); - if (!late) { - // This is a fail-safe for the case where receiving data is extremely slow. - // It should not be depended on for checking if an entry buffers in time. - aki_timer_set_repeat(&entry->timer, AKI_TS_FROM_USEC(ts - req->ts)); - aki_timer_again(&entry->timer); + break; + } + case SHRUB_CLIENT_PAUSE: { + f64 pts = *(f64 *)opaque; + aki_mutex_lock(&entry->sink->mutex); + camu_clock_arm_pause(&entry->clock, pts); + entry->audio.armed = false; + aki_mutex_unlock(&entry->sink->mutex); + break; + } + case SHRUB_CLIENT_RESUME: { + u64 ts = *(u64 *)opaque; + aki_mutex_lock(&entry->sink->mutex); + camu_clock_arm_resume(&entry->clock, ts); + if (sink->audio.state == SINK_PAUSED) { + sink->callback(sink->userdata, CAMU_SINK_START, CAMU_SINK_AUDIO, NULL); + sink->audio.state = SINK_PLAYING; + } else { + entry->audio.armed = true; } + aki_mutex_unlock(&entry->sink->mutex); break; } - case BIMU_CLIENT_DATA: { + case SHRUB_CLIENT_DATA: { struct camu_frame *frame = (struct camu_frame *)opaque; switch (stream->type) { - case BIMU_STREAM_AUDIO: + case SHRUB_STREAM_AUDIO: camu_audio_buffer_push(&entry->audio.buf, frame); break; #ifndef CAMU_SINK_NO_VIDEO - case BIMU_STREAM_VIDEO: + case SHRUB_STREAM_VIDEO: camu_video_buffer_push(&entry->video.buf, frame); break; #endif default: -#ifdef HAVE_FFMPEG - if (frame->type == CAMU_FFMPEG_COMPAT) { - av_frame_free(&frame->av.frame); - } -#endif - al_free(frame); + camu_frame_discard(frame); break; } break; } - case BIMU_CLIENT_REMOVE_BUFFERS: { + case SHRUB_CLIENT_REMOVE_BUFFERS: { aki_mutex_lock(&entry->sink->mutex); if (entry->audio.state == BUFFER_ADDED) { entry->sink->callback(entry->sink->userdata, CAMU_SINK_REMOVE_BUFFER, CAMU_SINK_AUDIO, &entry->audio.buf); @@ -502,32 +477,25 @@ static void client_callback(void *userdata, u8 op, struct bmu_vcr_stream *stream aki_mutex_unlock(&entry->sink->mutex); #ifndef CAMU_SINK_NO_VIDEO if (single_frame) { - while (ENTRY_AUDIO_BUFFER_HELD(entry)) { - aki_thread_sleep(AKI_TS_FROM_USEC(200)); - } + while (ENTRY_AUDIO_BUFFER_HELD(entry)) { aki_thread_sleep(AKI_TS_FROM_USEC(200)); } } else { #endif - while (ENTRY_BUFFERS_HELD(entry)) { - aki_thread_sleep(AKI_TS_FROM_USEC(200)); - } + while (ENTRY_BUFFERS_HELD(entry)) { aki_thread_sleep(AKI_TS_FROM_USEC(200)); } #ifndef CAMU_SINK_NO_VIDEO - } -#endif -#ifndef CAMU_SINK_NO_VIDEO - if (!single_frame) { camu_video_buffer_reset(&entry->video.buf); } #endif + camu_audio_buffer_reset(&entry->audio.buf); break; } - case BIMU_CLIENT_EOF: { + case SHRUB_CLIENT_EOF: { switch (stream->type) { - case BIMU_STREAM_AUDIO: { + case SHRUB_STREAM_AUDIO: { camu_audio_buffer_flush(&entry->audio.buf); break; } #ifndef CAMU_SINK_NO_VIDEO - case BIMU_STREAM_VIDEO: { + case SHRUB_STREAM_VIDEO: { bool single_frame = camu_video_buffer_is_single_frame(&entry->video.buf); if (!single_frame) { camu_video_buffer_flush(&entry->video.buf); @@ -538,9 +506,8 @@ static void client_callback(void *userdata, u8 op, struct bmu_vcr_stream *stream } break; } - case BIMU_CLIENT_CLOSED: { - bmu_client_free(&entry->client); - aki_timer_stop(&entry->timer); + case SHRUB_CLIENT_CLOSED: { + shrb_client_free(&entry->client); camu_audio_buffer_free(&entry->audio.buf); #ifndef CAMU_SINK_NO_VIDEO camu_video_buffer_free(&entry->video.buf); @@ -566,6 +533,7 @@ bool camu_sink_init(struct camu_sink *sink, struct aki_event_loop *loop, aki_mutex_init(&sink->mutex); sink->queued = NULL; sink->current = NULL; + al_array_init(sink->previous); al_array_init(sink->entries); sink->audio.mixer = mixer; sink->audio.state = SINK_PAUSED; @@ -585,16 +553,8 @@ static struct camu_sink_entry *entry_from_node_id(struct camu_sink *sink, u16 no return NULL; } -static void buffer_timer_callback(void *userdata, struct aki_timer *timer) -{ - struct camu_sink_entry *entry = (struct camu_sink_entry *)userdata; - (void)timer; - //aki_mutex_lock(&entry->sink->mutex); - //aki_mutex_unlock(&entry->sink->mutex); - aki_timer_stop(&entry->timer); -} - -static struct camu_sink_entry *ensure_entry_buffered_internal(struct camu_sink *sink, str *addr, s32 port, u32 node_id) +static struct camu_sink_entry *ensure_entry_buffered_internal(struct camu_sink *sink, + str *addr, s32 port, u32 node_id) { struct camu_sink_entry *entry = entry_from_node_id(sink, node_id); if (entry) return entry; @@ -602,9 +562,9 @@ static struct camu_sink_entry *ensure_entry_buffered_internal(struct camu_sink * entry = al_alloc_object(struct camu_sink_entry); al_array_push(sink->entries, entry); entry->sink = sink; - entry->state = ENTRY_LOADED; entry->audio.state = BUFFER_INIT; + entry->audio.armed = false; camu_audio_buffer_init(&entry->audio.buf, &entry->clock, sink->audio.mixer); entry->audio.buf.callback = audio_buffer_callback; entry->audio.buf.userdata = entry; @@ -613,41 +573,18 @@ static struct camu_sink_entry *ensure_entry_buffered_internal(struct camu_sink * #ifndef CAMU_SINK_NO_VIDEO renderer = sink->video.renderer; entry->video.state = BUFFER_INIT; - camu_video_buffer_init(&entry->video.buf, &entry->clock, renderer); + camu_video_buffer_init(&entry->video.buf, &entry->clock, 0.0, renderer); entry->video.buf.callback = video_buffer_callback; entry->video.buf.userdata = entry; #endif - aki_timer_init(&entry->timer, sink->loop, buffer_timer_callback, entry); - entry->client.callback = client_callback; entry->client.userdata = entry; - bmu_client_connect(&entry->client, sink->loop, addr, port, node_id); + shrb_client_connect(&entry->client, sink->loop, addr, port, node_id); return entry; } -static bool buffer_command_callback(void *userdata, struct aki_rpc_connection *conn, - struct aki_packet *packet, struct aki_packet *rpacket) -{ - struct camu_sink *sink = (struct camu_sink *)userdata; - (void)conn; - (void)rpacket; - - str addr; - aki_packet_read_str(packet, &addr); - s32 port = aki_packet_read_s32(packet); - u16 node_id = aki_packet_read_u16(packet); - - aki_mutex_lock(&sink->mutex); - ensure_entry_buffered_internal(sink, &addr, port, node_id); - aki_mutex_unlock(&sink->mutex); - - aki_packet_free(packet); - - return false; -} - static bool set_command_callback(void *userdata, struct aki_rpc_connection *conn, struct aki_packet *packet, struct aki_packet *rpacket) { @@ -659,11 +596,20 @@ static bool set_command_callback(void *userdata, struct aki_rpc_connection *conn aki_packet_read_str(packet, &addr); s32 port = aki_packet_read_s32(packet); u16 node_id = aki_packet_read_u16(packet); + s32 sequence = aki_packet_read_s32(packet); + u64 start = aki_packet_read_u64(packet); aki_mutex_lock(&sink->mutex); struct camu_sink_entry *entry = ensure_entry_buffered_internal(sink, &addr, port, node_id); + entry->sequence = sequence; if (entry == sink->current) goto out; - struct camu_sink_entry *previous = sink->current; + // This will only have an effect if the clock is already set. + camu_clock_arm_resume(&entry->clock, start + SHRUB_BASE_DELAY); + if (sink->current) { + // TODOODO, pause_at sent from list calc difference in clock and set. + camu_clock_pause(&sink->current->clock); + al_array_push(sink->previous, sink->current); + } sink->current = entry; if (sink->queued) sink->queued = NULL; if (entry->audio.state == BUFFER_INIT) { @@ -678,11 +624,6 @@ static bool set_command_callback(void *userdata, struct aki_rpc_connection *conn add_video_if_set_and_buffered(entry); } #endif - if (previous) { - camu_clock_pause(&previous->clock); - // TODO: sink->previous that gets swapped in add_x_if_set_and_buffered? - remove_entry_buffers(sink, previous); - } out: aki_mutex_unlock(&sink->mutex); aki_packet_free(packet); @@ -700,9 +641,11 @@ static bool queue_command_callback(void *userdata, struct aki_rpc_connection *co aki_packet_read_str(packet, &addr); s32 port = aki_packet_read_s32(packet); u16 node_id = aki_packet_read_u16(packet); + s32 sequence = aki_packet_read_s32(packet); aki_mutex_lock(&sink->mutex); sink->queued = ensure_entry_buffered_internal(sink, &addr, port, node_id); + sink->queued->sequence = sequence; aki_mutex_unlock(&sink->mutex); aki_packet_free(packet); @@ -711,17 +654,16 @@ static bool queue_command_callback(void *userdata, struct aki_rpc_connection *co } static struct aki_rpc_command commands[] = { - { .op = CAMU_SINK_CMD_BUFFER, .callback = buffer_command_callback, .userdata = NULL }, - { .op = CAMU_SINK_CMD_SET, .callback = set_command_callback, .userdata = NULL }, - { .op = CAMU_SINK_CMD_QUEUE, .callback = queue_command_callback, .userdata = NULL } + { .op = CAMU_SINK_SET, .callback = set_command_callback, .userdata = NULL }, + { .op = CAMU_SINK_QUEUE, .callback = queue_command_callback, .userdata = NULL } }; static void connection_callback(void *userdata, struct aki_rpc_connection *conn) { struct camu_sink *sink = (struct camu_sink *)userdata; sink->conn = conn; - struct aki_packet *packet = aki_rpc_get_packet(&sink->client, TREE_CMD_IDENTIFY); - aki_packet_write_u8(packet, TREE_SINK); + struct aki_packet *packet = aki_rpc_get_packet(&sink->client, CAMU_SRV_IDENTIFY); + aki_packet_write_u8(packet, CAMU_SINK); aki_rpc_connection_command(sink->conn, packet, NULL, NULL); } @@ -744,6 +686,25 @@ bool camu_sink_connect(struct camu_sink *sink, str *addr, s32 port) return aki_rpc_connect(&sink->client, sink->loop, addr, port); } +struct camu_sink_entry *camu_sink_get_current(struct camu_sink *sink) +{ + aki_mutex_lock(&sink->mutex); + return sink->current; +} + +void camu_sink_return_current(struct camu_sink *sink) +{ + aki_mutex_unlock(&sink->mutex); +} + +void camu_sink_skip(struct camu_sink *sink, s32 n) +{ + queue_cmd(sink, (struct camu_sink_cmd){ + .op = SKIP, + .value.i = n + }); +} + void camu_sink_toggle_pause(struct camu_sink *sink) { aki_mutex_lock(&sink->mutex); @@ -771,23 +732,15 @@ void camu_sink_seek(struct camu_sink *sink, f64 precent) } } -void camu_sink_skip(struct camu_sink *sink, s32 n) -{ - queue_cmd(sink, (struct camu_sink_cmd){ - .op = SKIP, - .value.i = n - }); -} - -struct camu_sink_entry *camu_sink_get_current(struct camu_sink *sink) +void camu_sink_reseek(struct camu_sink *sink) { aki_mutex_lock(&sink->mutex); - return sink->current; -} - -void camu_sink_return_current(struct camu_sink *sink) -{ + struct camu_sink_entry *current = sink->current; aki_mutex_unlock(&sink->mutex); + queue_cmd(sink, (struct camu_sink_cmd){ + .op = RESEEK, + .opaque = current + }); } void camu_sink_stop(struct camu_sink *sink) @@ -814,7 +767,7 @@ void camu_sink_close(struct camu_sink *sink) aki_mutex_lock(&sink->mutex); struct camu_sink_entry *entry; al_array_foreach(sink->entries, i, entry) { - bmu_client_close(&entry->client); + shrb_client_close(&entry->client); } sink->entries.size = 0; aki_mutex_unlock(&sink->mutex); diff --git a/src/libsink/sink.h b/src/libsink/sink.h index 40d20d4..4d7dc06 100644 --- a/src/libsink/sink.h +++ b/src/libsink/sink.h @@ -1,12 +1,12 @@ #pragma once //#define CAMU_SINK_NO_VIDEO +//#define CAMU_SINK_LOCAL_PAUSE #include <al/str.h> #include <al/array.h> #include <aki/rpc2.h> #include <aki/signal.h> -#include <aki/timer.h> #include "../util/queue.h" @@ -16,7 +16,7 @@ #include "../buffer/video.h" #endif -#include "../bimu/client.h" +#include "../shrub/client.h" enum { CAMU_SINK_AUDIO = 0, @@ -36,20 +36,20 @@ enum { }; struct camu_sink_entry { - u8 state; - struct bmu_client client; + s32 sequence; struct camu_clock clock; - struct aki_timer timer; + struct shrb_client client; struct { u8 state; + bool armed; struct camu_audio_buffer buf; - struct bmu_vcr_stream *stream; + struct shrb_vcr_stream *stream; } audio; #ifndef CAMU_SINK_NO_VIDEO struct { u8 state; struct camu_video_buffer buf; - struct bmu_vcr_stream *stream; + struct shrb_vcr_stream *stream; } video; #endif struct camu_sink *sink; @@ -75,6 +75,7 @@ struct camu_sink { struct aki_mutex mutex; struct camu_sink_entry *queued; struct camu_sink_entry *current; + array(struct camu_sink_entry *) previous; array(struct camu_sink_entry *) entries; struct { u8 state; @@ -97,11 +98,12 @@ bool camu_sink_init(struct camu_sink *sink, struct aki_event_loop *loop, #endif ); bool camu_sink_connect(struct camu_sink *sink, str *addr, s32 port); -void camu_sink_toggle_pause(struct camu_sink *sink); -void camu_sink_seek(struct camu_sink *sink, f64 pos); -void camu_sink_skip(struct camu_sink *sink, s32 n); struct camu_sink_entry *camu_sink_get_current(struct camu_sink *sink); void camu_sink_return_current(struct camu_sink *sink); +void camu_sink_skip(struct camu_sink *sink, s32 n); +void camu_sink_toggle_pause(struct camu_sink *sink); +void camu_sink_seek(struct camu_sink *sink, f64 pos); +void camu_sink_reseek(struct camu_sink *sink); void camu_sink_stop(struct camu_sink *sink); void camu_sink_close(struct camu_sink *sink); void camu_sink_free(struct camu_sink *sink); diff --git a/src/list/list.c b/src/list/list.c new file mode 100644 index 0000000..3af75e4 --- /dev/null +++ b/src/list/list.c @@ -0,0 +1,172 @@ +#include <al/random.h> +#include <aki/thread.h> + +#include "../libsink/common.h" + +#include "list.h" + +void camu_list_init(struct camu_list *list, str *name) +{ + al_str_clone(&list->name, name); + list->current = -1; + list->idle = true; + list->sequence = 0; + al_array_init(list->entries); + al_array_init(list->sinks); +} + +static void camu_list_pump(struct camu_list *list) +{ + s32 size = (s32)list->entries.size; + if (list->current == -1 || size <= list->current) return; + struct camu_list_entry *entry = al_array_at(list->entries, list->current); + struct camu_list_entry *upcoming = NULL; + if (list->current + 1 < size) { + upcoming = al_array_at(list->entries, list->current + 1); + } + struct camu_list_sink *sink; + al_array_foreach(list->sinks, i, sink) { + if (sink->set != list->current) { + sink->callback(sink->userdata, CAMU_SINK_SET, entry->opaque, entry->sequence); + sink->set = list->current; + } + if (upcoming) { + sink->callback(sink->userdata, CAMU_SINK_QUEUE, upcoming->opaque, upcoming->sequence); + } + } +} + +void camu_list_add_sink(struct camu_list *list, void (*callback)(void *, u8, void *, s32), + void *userdata) +{ + struct camu_list_sink *sink = al_alloc_object(struct camu_list_sink); + sink->set = -1; + sink->callback = callback; + sink->userdata = userdata; + al_array_push(list->sinks, sink); + camu_list_pump(list); +} + +void camu_list_remove_sink(struct camu_list *list, void *userdata) +{ + struct camu_list_sink *sink; + al_array_foreach(list->sinks, i, sink) { + if (sink->userdata == userdata) { + al_array_remove_at_iter(list->sinks, i); + break; + } + } +} + +void camu_list_add(struct camu_list *list, void *opaque, void (*callback)(void *, u8, void *), + u64 duration, bool set) +{ + s32 size = (s32)list->entries.size; + struct camu_list_entry *entry = al_alloc_object(struct camu_list_entry); + entry->sequence = list->sequence++; + entry->opaque = opaque; + entry->callback = callback; + entry->duration = duration; + entry->start = aki_get_timestamp(); + if (!list->idle) { + struct camu_list_entry *prev = al_array_at(list->entries, size - 1); + u64 end = prev->start + prev->duration; + if (end > entry->start) { + u64 diff = end - entry->start; + entry->start += diff; + } + } + al_array_push(list->entries, entry); + if (set) list->current = size; + else if (list->idle) { + list->current++; + list->idle = false; + } + camu_list_pump(list); + struct camu_list_timing timing = { + .start = entry->start + }; + entry->callback(entry->opaque, CAMU_LIST_ENTRY_IMPULSE, &timing); +} + +static s32 index_of_entry_from_sequence(struct camu_list *list, s32 sequence) +{ + if (sequence != CAMU_SEQUENCE_INVALID) { + struct camu_list_entry *entry; + al_array_foreach(list->entries, i, entry) { + if (entry->sequence == sequence) return i; + } + } + return CAMU_SEQUENCE_INVALID; +} + +void camu_list_skip(struct camu_list *list, s32 sequence, s32 n) +{ + s32 size = (s32)list->entries.size; + s32 index = index_of_entry_from_sequence(list, sequence); + if (index == CAMU_SEQUENCE_INVALID) index = list->current; + if (index + n < 0 || index + n >= size || index + n == list->current) return; + list->current = index + n; + struct camu_list_entry *entry = al_array_at(list->entries, list->current); + struct camu_list_timing timing = { + .start = aki_get_timestamp() + }; + entry->callback(entry->opaque, CAMU_LIST_ENTRY_IMPULSE, &timing); + camu_list_pump(list); +} + +void camu_list_skipto(struct camu_list *list, s32 i) +{ + s32 size = (s32)list->entries.size; + if (i < 0 || i >= size) return; + list->current = i; + camu_list_pump(list); +} + +void camu_list_toggle_pause(struct camu_list *list, s32 sequence, u64 pos) +{ + s32 index = index_of_entry_from_sequence(list, sequence); + if (index == CAMU_SEQUENCE_INVALID) return; + struct camu_list_entry *entry = al_array_at(list->entries, index); + entry->callback(entry->opaque, CAMU_LIST_ENTRY_TOGGLE_PAUSE, &pos); +} + +void camu_list_seek(struct camu_list *list, s32 sequence, f64 percent) +{ + s32 index = index_of_entry_from_sequence(list, sequence); + if (index == CAMU_SEQUENCE_INVALID) return; + list->idle = false; + struct camu_list_entry *entry = al_array_at(list->entries, index); + u64 pos = percent * entry->duration; + entry->callback(entry->opaque, CAMU_LIST_ENTRY_SEEK, &pos); +} + +void camu_list_finished(struct camu_list *list, s32 sequence) +{ + s32 size = (s32)list->entries.size; + s32 index = index_of_entry_from_sequence(list, sequence); + if (index == CAMU_SEQUENCE_INVALID) return; + if (index + 1 >= size) { + list->idle = true; + return; + } + list->current = index + 1; + camu_list_pump(list); +} + +void camu_list_shuffle(struct camu_list *list) +{ + u32 size = list->entries.size; + if (size == 0) return; + for (u32 i = 0; i < size - 1; i++) { + u32 j = i + al_rand() / (AL_RAND_MAX / (size - i) + 1); + struct camu_list_entry *tmp = al_array_at(list->entries, j); + al_array_at(list->entries, j) = al_array_at(list->entries, i); + al_array_at(list->entries, i) = tmp; + } + struct camu_list_sink *sink; + al_array_foreach(list->sinks, i, sink) { + sink->set = -1; + } + camu_list_pump(list); +} diff --git a/src/list/list.h b/src/list/list.h new file mode 100644 index 0000000..edd0218 --- /dev/null +++ b/src/list/list.h @@ -0,0 +1,54 @@ +#pragma once + +#include <al/str.h> +#include <al/array.h> + +#define CAMU_SEQUENCE_INVALID INT32_MAX + +enum { + CAMU_LIST_ENTRY_IMPULSE = 0, + CAMU_LIST_ENTRY_TOGGLE_PAUSE, + CAMU_LIST_ENTRY_SEEK +}; + +struct camu_list_timing { + u64 start; +}; + +struct camu_list_entry { + s32 sequence; + void *opaque; + void (*callback)(void *, u8, void *); + u64 duration; + u64 start; + u64 paused_at; +}; + +struct camu_list_sink { + s32 set; + bool (*action)(void *, u8, void *); + void (*callback)(void *, u8, void *, s32); + void *userdata; +}; + +struct camu_list { + str name; + s32 current; + bool idle; + s32 sequence; // Next entry sequence. + array(struct camu_list_entry *) entries; + array(struct camu_list_sink *) sinks; +}; + +void camu_list_init(struct camu_list *list, str *name); +void camu_list_add_sink(struct camu_list *list, void (*callback)(void *, u8, void *, s32), + void *userdata); +void camu_list_remove_sink(struct camu_list *list, void *userdata); +void camu_list_add(struct camu_list *list, void *opaque, void (*callback)(void *, u8, void *), + u64 duration, bool set); +void camu_list_skip(struct camu_list *list, s32 sequence, s32 n); +void camu_list_skipto(struct camu_list *list, s32 i); +void camu_list_toggle_pause(struct camu_list *list, s32 sequence, u64 pos); +void camu_list_seek(struct camu_list *list, s32 sequence, f64 percent); +void camu_list_finished(struct camu_list *list, s32 sequence); +void camu_list_shuffle(struct camu_list *list); diff --git a/src/list/meson.build b/src/list/meson.build new file mode 100644 index 0000000..c61ac39 --- /dev/null +++ b/src/list/meson.build @@ -0,0 +1,5 @@ +list_src = [ + 'list.c' +] +list_deps = [common_deps] +list = declare_dependency(sources: list_src, dependencies: list_deps) diff --git a/src/mixer/audio.h b/src/mixer/audio.h index e460536..0e8e105 100644 --- a/src/mixer/audio.h +++ b/src/mixer/audio.h @@ -2,30 +2,12 @@ #include <al/str.h> -#include "../codec/libav/resampler.h" - -enum { - // Not initialized. - CAMU_AUDIO_STATE_NULL = 0, - // Context created, waiting for stream. - CAMU_AUDIO_STATE_INITIALIZED, - // Stream created, and running. - CAMU_AUDIO_STATE_STARTED, - // Error occured, no data will be requested. - CAMU_AUDIO_STATE_ERROR, - // Not built. - CAMU_AUDIO_STATE_UNSUPPORTED -}; +#include "../codec/ffmpeg/resampler.h" struct camu_audio { - u8 state; - - s32 (*data_callback)(void *, u8 *, s32); - void *userdata; - bool (*init)(struct camu_audio *, str *); - void (*pick_format)(struct camu_audio *, struct camu_lav_resample_fmt *); + void (*pick_format)(struct camu_audio *, struct camu_ff_resample_fmt *); bool (*configure_stream)(struct camu_audio *, void *); u64 (*get_latency)(struct camu_audio *); @@ -34,4 +16,7 @@ struct camu_audio { void (*stop)(struct camu_audio *); void (*free)(struct camu_audio **); + + s32 (*data_callback)(void *, u8 *, s32); + void *userdata; }; diff --git a/src/mixer/audio_miniaudio.c b/src/mixer/audio_miniaudio.c index dde742a..a7a1889 100644 --- a/src/mixer/audio_miniaudio.c +++ b/src/mixer/audio_miniaudio.c @@ -7,7 +7,6 @@ static ma_backend backends[] = { ma_backend_pulseaudio, #else ma_backend_wasapi -// ma_backend_dsound #endif }; @@ -42,7 +41,7 @@ static void miniaudio_log_callback(void *userdata, u32 level, const char *messag if (level < MA_LOG_LEVEL_WARNING) { al_log_debug("audio_miniaudio", message); } else { - al_log_warn("audio_miniaudio", message); + al_log_debug("audio_miniaudio", message); } } @@ -103,7 +102,7 @@ static enum AVSampleFormat av_sample_fmt_from_miniaudio(s32 fmt) } } -static void audio_miniaudio_pick_format(struct camu_audio *audio, struct camu_lav_resample_fmt *fmt) +static void audio_miniaudio_pick_format(struct camu_audio *audio, struct camu_ff_resample_fmt *fmt) { struct camu_audio_miniaudio *ma = (struct camu_audio_miniaudio *)audio; fmt->req_format = av_sample_fmt_from_miniaudio(ma->device.playback.internalFormat); @@ -119,12 +118,12 @@ static void data_callback(ma_device *device, void *output, const void *input, u3 } #define DEFAULT_PERIODS 3 -#define DEFAULT_PERIOD_SIZE_IN_MILLISECONDS 100 +#define DEFAULT_PERIOD_SIZE_IN_MILLISECONDS 75 static bool audio_miniaudio_configure_stream(struct camu_audio *audio, void *opaque) { struct camu_audio_miniaudio *ma = (struct camu_audio_miniaudio *)audio; - struct camu_lav_resample_fmt *fmt = (struct camu_lav_resample_fmt *)opaque; + struct camu_ff_resample_fmt *fmt = (struct camu_ff_resample_fmt *)opaque; (void)fmt; ma->config = ma_device_config_init(ma_device_type_playback); //ma->config.playback.pDeviceID = &ma->selected_device->id; @@ -190,16 +189,15 @@ static void audio_miniaudio_free(struct camu_audio **audio) struct camu_audio_miniaudio audio_plugin_miniaudio = { .a = { - .state = CAMU_AUDIO_STATE_NULL, - .data_callback = NULL, - .userdata = NULL, .init = audio_miniaudio_init, .pick_format = audio_miniaudio_pick_format, .configure_stream = audio_miniaudio_configure_stream, .get_latency = audio_miniaudio_get_latency, .start = audio_miniaudio_start, .stop = audio_miniaudio_stop, - .free = audio_miniaudio_free + .free = audio_miniaudio_free, + .data_callback = NULL, + .userdata = NULL }, .log = { }, .context_config = { 0 }, diff --git a/src/mixer/meson.build b/src/mixer/meson.build index ab14b6f..71c003b 100644 --- a/src/mixer/meson.build +++ b/src/mixer/meson.build @@ -21,7 +21,6 @@ if not is_windows miniaudio_args += ['-DMA_ENABLE_PULSEAUDIO', '-DMA_NO_RUNTIME_LINKING'] else miniaudio_args += ['-DMA_ENABLE_WASAPI'] - #miniaudio_args += ['-DMA_ENABLE_DSOUND'] endif mixer = declare_dependency(sources: mixer_src, dependencies: mixer_deps, diff --git a/src/mixer/mixer.c b/src/mixer/mixer.c index a4c0ef5..31a67b0 100644 --- a/src/mixer/mixer.c +++ b/src/mixer/mixer.c @@ -6,31 +6,35 @@ static s32 data_callback(void *userdata, u8 *data, s32 frame_count) { struct camu_mixer *mixer = (struct camu_mixer *)userdata; + size_t req = camu_ff_resample_fmt_samples_to_bytes(&mixer->fmt, (size_t)frame_count); + if (UNLIKELY(mixer->paused)) { + al_memset(data, 0, req); + } else { #ifdef CAMU_MIXER_THREADED - if (al_atomic_bool_load(&mixer->queued, AL_ATOMIC_RELAXED)) { - camu_mixer_run_queue(mixer); - } + if (al_atomic_bool_load(&mixer->queued, AL_ATOMIC_RELAXED)) { + camu_mixer_run_queue(mixer); + } #endif - size_t req = camu_lav_resample_fmt_samples_to_bytes(&mixer->fmt, (size_t)frame_count); - do { - if (mixer->buffers.size == 0) { - al_memset(data, 0, req); - } else { - struct camu_audio_buffer *buf = al_array_last(mixer->buffers); - size_t signal; - if ((signal = camu_audio_buffer_read(buf, data, req)) < req) { + do { + if (mixer->buffers.size == 0) { + al_memset(data, 0, req); + } else { + struct camu_audio_buffer *buf = al_array_last(mixer->buffers); + size_t signal; + if ((signal = camu_audio_buffer_read(buf, data, req)) < req) { #ifdef CAMU_MIXER_THREADED - if (al_atomic_bool_load(&mixer->queued, AL_ATOMIC_RELAXED)) { - camu_mixer_run_queue(mixer); - } + if (al_atomic_bool_load(&mixer->queued, AL_ATOMIC_RELAXED)) { + camu_mixer_run_queue(mixer); + } #endif - data += signal; - req = req - signal; - continue; + data += signal; + req = req - signal; + continue; + } } - } - break; - } while (1); + break; + } while (1); + } return frame_count; } @@ -55,7 +59,7 @@ bool camu_mixer_init(struct camu_mixer *mixer, struct camu_audio *audio) return true; } -void camu_mixer_pick_format(struct camu_mixer *mixer, struct camu_lav_resample_fmt *fmt) +void camu_mixer_pick_format(struct camu_mixer *mixer, struct camu_ff_resample_fmt *fmt) { mixer->audio->pick_format(mixer->audio, fmt); av_channel_layout_default(&fmt->req_channel_layout, fmt->req_channel_count); @@ -63,7 +67,6 @@ void camu_mixer_pick_format(struct camu_mixer *mixer, struct camu_lav_resample_f f64 camu_mixer_get_latency(struct camu_mixer *mixer) { - if (!mixer->audio) return 0; return mixer->audio->get_latency(mixer->audio) / 1000000.0; } @@ -144,18 +147,40 @@ void camu_mixer_run_queue(struct camu_mixer *mixer) void camu_mixer_pause(struct camu_mixer *mixer) { +#ifdef CAMU_MIXER_THREADED + aki_mutex_lock(&mixer->mutex); +#endif if (!mixer->paused) { mixer->audio->stop(mixer->audio); mixer->paused = true; } +#ifdef CAMU_MIXER_THREADED + aki_mutex_unlock(&mixer->mutex); +#endif } void camu_mixer_resume(struct camu_mixer *mixer) { +#ifdef CAMU_MIXER_THREADED + aki_mutex_lock(&mixer->mutex); +#endif if (mixer->paused) { +//#ifdef CAMU_MIXER_THREADED +// bool queued = al_atomic_bool_load(&mixer->queued, AL_ATOMIC_RELAXED); +// // Disable queue when starting audio in case start() directly calls data_callback(). +// // Locking here is necessary to 100% know the state of the audio backend at +// // any time. This allows us to act on the value of paused, like in remove_buffer(). +// if (queued) al_atomic_bool_store(&mixer->queued, false, AL_ATOMIC_RELAXED); +//#endif mixer->audio->start(mixer->audio); +//#ifdef CAMU_MIXER_THREADED +// if (queued) al_atomic_bool_store(&mixer->queued, true, AL_ATOMIC_RELAXED); +//#endif mixer->paused = false; } +#ifdef CAMU_MIXER_THREADED + aki_mutex_unlock(&mixer->mutex); +#endif } void camu_mixer_close(struct camu_mixer *mixer) diff --git a/src/mixer/mixer.h b/src/mixer/mixer.h index 464431e..113db62 100644 --- a/src/mixer/mixer.h +++ b/src/mixer/mixer.h @@ -9,12 +9,12 @@ #include <aki/thread.h> #endif -#include "../codec/libav/resampler.h" +#include "../codec/ffmpeg/resampler.h" struct camu_mixer { struct camu_audio *audio; bool paused; - struct camu_lav_resample_fmt fmt; + struct camu_ff_resample_fmt fmt; array(struct camu_audio_buffer *) buffers; #ifdef CAMU_MIXER_THREADED array(struct camu_audio_buffer *) add_queue; @@ -25,7 +25,7 @@ struct camu_mixer { }; bool camu_mixer_init(struct camu_mixer *mixer, struct camu_audio *audio); -void camu_mixer_pick_format(struct camu_mixer *mixer, struct camu_lav_resample_fmt *fmt); +void camu_mixer_pick_format(struct camu_mixer *mixer, struct camu_ff_resample_fmt *fmt); f64 camu_mixer_get_latency(struct camu_mixer *mixer); void camu_mixer_add_buffer(struct camu_mixer *mixer, struct camu_audio_buffer *buf); void camu_mixer_remove_buffer(struct camu_mixer *mixer, struct camu_audio_buffer *buf); diff --git a/src/portal/cpy/bridge.pxd b/src/portal/cpy/bridge.pxd new file mode 100644 index 0000000..ac14811 --- /dev/null +++ b/src/portal/cpy/bridge.pxd @@ -0,0 +1,8 @@ +include "post.pxd" + +cdef extern from "../src/search.h": + cdef struct camu_search: + pass + + void camu_search_add_post(camu_search *search, u32 page, camu_post *post) + void camu_search_add_to_list(camu_search *search, u32 page, str *unique_id) diff --git a/src/portal/cpy/build.sh b/src/portal/cpy/build.sh new file mode 100755 index 0000000..45a47f5 --- /dev/null +++ b/src/portal/cpy/build.sh @@ -0,0 +1,3 @@ +#! /usr/bin/env sh +export PYTHONDONTWRITEBYTECODE=1 +python3.12 setup.py build_ext -i diff --git a/src/portal/cpy/lib.pxd b/src/portal/cpy/lib.pxd new file mode 100644 index 0000000..5749141 --- /dev/null +++ b/src/portal/cpy/lib.pxd @@ -0,0 +1,13 @@ +include "types.pxd" + +cdef extern from "<al/lib.h>": + void al_free(void *ptr) + void al_malloc(size_t size) + void al_strlen(char *s) + void al_bzero(void *s, size_t n) + +cdef extern from "<al/log.h>": + s32 al_log_info(const char *ns, const char *, ...) + s32 al_log_warn(const char *ns, const char *, ...) + s32 al_log_error(const char *ns, const char *, ...) + s32 al_log_debug(const char *ns, const char *, ...) diff --git a/src/portal/cpy/portal.pyx b/src/portal/cpy/portal.pyx new file mode 100644 index 0000000..3ddde80 --- /dev/null +++ b/src/portal/cpy/portal.pyx @@ -0,0 +1,203 @@ +cimport lib +cimport str +cimport post +cimport bridge + +import sys +sys.path.append('./py') + +cdef public int log_info(char *msg) except -1: + return lib.al_log_info("portal", "%s", msg) + +cdef public int log_warn(char *msg) except -1: + return lib.al_log_warn("portal", "%s", msg) + +cdef public int log_error(char *msg) except -1: + return lib.al_log_error("portal", "%s", msg) + +cdef public int log_debug(char *msg) except -1: + return lib.al_log_debug("portal", "%s", msg) + +def encode_str(str): + return str.encode('UTF-8') + +def decode_str(str): + return str.decode('UTF-8') + +class LogOutputHook(object): + def __init__(self): + self.fd = sys.__stdout__ + + def __getattr__(self, attr): + return getattr(self.fd, attr) + + def write(self, msg): + if len(msg) > 0 and msg != '\n': + log_info(encode_str(msg)) + +hook = LogOutputHook() +sys.stdout = hook +sys.stderr = hook + +import log + +class PortalLogger(): + def debug(self, msg): + log_debug(encode_str(msg)) + + def info(self, msg): + log_info(encode_str(msg)) + + def warn(self, msg): + log_warn(encode_str(msg)) + + def error(self, msg): + log_error(encode_str(msg)) + +log.set_logger(PortalLogger()) + +def exception_handler(exception_type, exception, traceback): + log.error(repr(exception)) +sys.excepthook = exception_handler + +from random import randrange + +class Portal(): + def __init__(self, modules): + self.modules = modules + self.searches = {} + + def print_modules(self): + for name in ALL_MODULES.keys(): + log.info('Registered module \'{}\'.'.format(name)) + + def new_id(self): + while True: + id = randrange(16 ** 5) + if id not in self.searches: + return id + + def search(self, module, query): + if module not in self.modules: + return -1 + mod = self.modules[module] + search = mod[0].search(query, *mod[1]) + if not search: + return -1 + id = self.new_id() + self.searches[id] = search + return id + + def get_page(self, id, num): + if id not in self.searches: + return None + try: + return self.searches[id].get_page(num) + except Exception as e: + log.error(repr(e)) + return None + + def get_item(self, id, unique_id): + if id not in self.searches: + return None + try: + return self.searches[id].module.get_item(unique_id) + except Exception as e: + log.error(repr(e)) + return None + +from modules import ALL_MODULES + +portal = Portal(ALL_MODULES) +portal.print_modules() + +from post import PostType, DateType + +cdef public int portal_bridge_search(str.str *module, str.str *query) except -1: + cdef char *c_str0 = str.al_str_to_c_str(module) + cdef char *c_str1 = str.al_str_to_c_str(query) + ret = portal.search(decode_str(c_str0), decode_str(c_str1)) + lib.al_free(c_str0) + lib.al_free(c_str1) + return ret + +cdef public int py_str_to_str(str.str *s, char *py_c_str) except -1: + str.al_str_from(s, py_c_str) + return 0 + +cdef public int py_str_to_wstr(str.wstr *w, char *py_c_str) except -1: + str.al_wstr_from_cstr(w, py_c_str) + return 0 + +cdef public int add_post_internal(bridge.camu_search *search, int id, unsigned int num, char *py_unique_id) except -1: + cdef post.camu_post cpost + cdef str.str key + cdef str.str url + cdef str.str ext + cdef str.str thumbnail_url + cdef str.str thumbnail_ext + py_post = portal.get_item(id, decode_str(py_unique_id)) + if not py_post: + return 0 + post.camu_post_reset(&cpost) + cpost.version = py_post.version + cpost.type = py_post.type.value + py_str_to_str(&cpost.unique_id, encode_str(py_post.unique_id)) + if cpost.type == PostType.TOMBSTONE.value: + bridge.camu_search_add_post(search, num, &cpost) + return 0 + py_str_to_str(&cpost.url, encode_str(py_post.url)) + for type, date in py_post.dates.items(): + post.camu_post_add_date(&cpost, type.value, date) + py_str_to_str(&cpost.author.unique_id, encode_str(py_post.author.unique_id)) + py_str_to_wstr(&cpost.author.username, encode_str(py_post.author.username)) + py_str_to_wstr(&cpost.author.display_name, encode_str(py_post.author.display_name)) + py_str_to_str(&cpost.author.profile_picture_url, encode_str(py_post.author.profile_picture_url.url)) + if cpost.type != PostType.REPOST.value: + py_str_to_wstr(&cpost.title, encode_str(py_post.title)) + py_str_to_wstr(&cpost.text, encode_str(py_post.text)) + cpost.likes.set = py_post.likes != None + if cpost.likes.set: + cpost.likes.i = py_post.likes + cpost.bookmarks.set = py_post.bookmarks != None + if cpost.bookmarks.set: + cpost.bookmarks.i = py_post.bookmarks + cpost.reposts.set = py_post.reposts != None + if cpost.reposts.set: + cpost.reposts.i = py_post.reposts + cpost.quotes.set = py_post.quotes != None + if cpost.quotes.set: + cpost.quotes.i = py_post.quotes + cpost.comments.set = py_post.comments != None + if cpost.comments.set: + cpost.comments.i = py_post.comments + cpost.views.set = py_post.views != None + if cpost.views.set: + cpost.views.i = py_post.views + for k, media in py_post.media.items(): + py_str_to_str(&key, encode_str(k)) + py_str_to_str(&url, encode_str(media.url.url)) + py_str_to_str(&ext, encode_str(media.url.ext)) + py_str_to_str(&thumbnail_url, encode_str(media.thumbnail_url.url)) + py_str_to_str(&thumbnail_ext, encode_str(media.thumbnail_url.ext)) + post.camu_post_add_media(&cpost, media.type.value, &key, &url, &ext, &thumbnail_url, &thumbnail_ext) + py_str_to_str(&cpost.post.unique_id, encode_str(py_post.post.unique_id)) + py_str_to_str(&cpost.quoted.unique_id, encode_str(py_post.quoted.unique_id)) + py_str_to_str(&cpost.in_reply_to.unique_id, encode_str(py_post.in_reply_to.unique_id)) + if py_post.post.unique_id: + add_post_internal(search, id, num, encode_str(py_post.post.unique_id)) + if py_post.quoted.unique_id: + add_post_internal(search, id, num, encode_str(py_post.quoted.unique_id)) + bridge.camu_search_add_post(search, num, &cpost) + return 0 + +cdef public int portal_bridge_get_page(bridge.camu_search *search, int id, unsigned int num) except -1: + page = portal.get_page(id, num) + if not page: + return -1 + cdef str.str unique_id + for py_unique_id in page: + py_str_to_str(&unique_id, encode_str(py_unique_id)) + bridge.camu_search_add_to_list(search, num, &unique_id) + add_post_internal(search, id, num, encode_str(py_unique_id)) + return 0 diff --git a/src/portal/cpy/post.pxd b/src/portal/cpy/post.pxd new file mode 100644 index 0000000..3f66508 --- /dev/null +++ b/src/portal/cpy/post.pxd @@ -0,0 +1,61 @@ +include "str.pxd" + +cdef extern from "../src/post.h": + ctypedef struct optional_int: + s64 i + bool set + + cdef struct camu_post_date: + u8 type + f32 timestamp + + ctypedef struct camu_dates_array: + u32 size + u32 alloc + camu_post_date *data + + cdef struct camu_post_media: + u8 type + str key + str url + str ext + str thumbnail_url + str thumbnail_ext + + ctypedef struct camu_media_array: + u32 size + u32 alloc + camu_post_media *data + + cdef struct camu_post_user: + str unique_id + wstr username + wstr display_name + str profile_picture_url + + cdef struct camu_post_ref: + str unique_id + + cdef struct camu_post: + u16 version + u8 type + str unique_id + str url + camu_dates_array dates + camu_post_user author + wstr title + wstr text + optional_int likes + optional_int bookmarks + optional_int reposts + optional_int quotes + optional_int comments + optional_int views + camu_media_array media + camu_post_ref post + camu_post_ref quoted + camu_post_ref in_reply_to + + void camu_post_reset(camu_post *post) + void camu_post_add_date(camu_post *post, u8 type, f32 timestamp) + void camu_post_add_media(camu_post *post, u8 type, str *key, str *url, str *ext, str *thumbnail_url, str *thumbnail_ext) diff --git a/src/portal/cpy/setup.py b/src/portal/cpy/setup.py new file mode 100644 index 0000000..1a3e39c --- /dev/null +++ b/src/portal/cpy/setup.py @@ -0,0 +1,6 @@ +from setuptools import Extension +from Cython.Build import cythonize +from Cython.Compiler import Options +Options.embed = True +alabaster_inc = "../../../subprojects/libalabaster/include" +cythonize([Extension("portal", ["portal.pyx"], include_dirs=[alabaster_inc], extra_link_args=[])], language_level=3) diff --git a/src/portal/cpy/str.pxd b/src/portal/cpy/str.pxd new file mode 100644 index 0000000..3ef1e90 --- /dev/null +++ b/src/portal/cpy/str.pxd @@ -0,0 +1,18 @@ +include "types.pxd" + +cdef extern from "<al/str.h>": + ctypedef struct str: + u32 len + u32 alloc + char *data + + void al_str_from(str *s, const char *str) + char *al_str_to_c_str(str *s) + +cdef extern from "<al/wstr.h>": + ctypedef struct wstr: + u32 len + u32 alloc + wchar_t *data + + void al_wstr_from_cstr(wstr *s, const char *str) diff --git a/src/portal/cpy/types.pxd b/src/portal/cpy/types.pxd new file mode 100644 index 0000000..09cb224 --- /dev/null +++ b/src/portal/cpy/types.pxd @@ -0,0 +1,23 @@ +from libc.stdint cimport uint8_t +from libc.stdint cimport int8_t +from libc.stdint cimport uint16_t +from libc.stdint cimport int16_t +from libc.stdint cimport uint32_t +from libc.stdint cimport int32_t +from libc.stdint cimport uint64_t +from libc.stdint cimport int64_t +from libc.stddef cimport wchar_t + +ctypedef bint bool + +ctypedef uint8_t u8 +ctypedef int8_t s8 +ctypedef uint16_t u16 +ctypedef int16_t s16 +ctypedef uint32_t u32 +ctypedef int32_t s32 +ctypedef uint64_t u64 +ctypedef int64_t s64 + +ctypedef float f32 +ctypedef double f64 diff --git a/src/portal/meson.build b/src/portal/meson.build new file mode 100644 index 0000000..0df4366 --- /dev/null +++ b/src/portal/meson.build @@ -0,0 +1,12 @@ +portal_src = [ + 'src/post.c', + 'src/search.c', + 'src/post_cache.c', + 'src/packet_ext.c' +] +portal_deps = [] + +python3_embed = import('python').find_installation('python3.12').dependency(embed: true) +portal_deps += [python3_embed] + +portal = declare_dependency(sources: portal_src, dependencies: portal_deps) diff --git a/src/portal/patches/instagrapi_no_android.diff b/src/portal/patches/instagrapi_no_android.diff new file mode 100644 index 0000000..6f4fd34 --- /dev/null +++ b/src/portal/patches/instagrapi_no_android.diff @@ -0,0 +1,17 @@ +diff --git a/instagrapi/mixins/private.py b/instagrapi/mixins/private.py +index 9db04b0..2b17d75 100644 +--- a/instagrapi/mixins/private.py ++++ b/instagrapi/mixins/private.py +@@ -129,9 +129,9 @@ class PrivateRequestMixin: + # X-IG-WWW-Claim: hmac.AR3zruvyGTlwHvVd2ACpGCWLluOppXX4NAVDV-iYslo9CaDd + "X-Bloks-Is-Layout-RTL": "false", + "X-Bloks-Is-Panorama-Enabled": "true", +- "X-IG-Device-ID": self.uuid, +- "X-IG-Family-Device-ID": self.phone_id, +- "X-IG-Android-ID": self.android_device_id, ++ #"X-IG-Device-ID": self.uuid, ++ #"X-IG-Family-Device-ID": self.phone_id, ++ #"X-IG-Android-ID": self.android_device_id, + "X-IG-Timezone-Offset": str(self.timezone_offset), + "X-IG-Connection-Type": "WIFI", + "X-IG-Capabilities": "3brTvx0=", # "3brTvwE=" in instabot diff --git a/src/portal/patches/pixivpy_reauth.diff b/src/portal/patches/pixivpy_reauth.diff new file mode 100644 index 0000000..d9ce0d6 --- /dev/null +++ b/src/portal/patches/pixivpy_reauth.diff @@ -0,0 +1,96 @@ +diff --git a/pixivpy3/aapi.py b/pixivpy3/aapi.py +index 1492385..ffc88c0 100644 +--- a/pixivpy3/aapi.py ++++ b/pixivpy3/aapi.py +@@ -76,12 +76,11 @@ class AppPixivAPI(BasePixivAPI): + ) -> Response: + headers_ = CaseInsensitiveDict(headers or {}) + if self.hosts != "https://app-api.pixiv.net": +- headers_["host"] = "app-api.pixiv.net" +- if "user-agent" not in headers_: +- # Set User-Agent if not provided +- headers_["app-os"] = "ios" +- headers_["app-os-version"] = "14.6" +- headers_["user-agent"] = "PixivIOSApp/7.13.3 (iOS 14.6; iPhone13,2)" ++ headers_["Host"] = "app-api.pixiv.net" ++ headers_["App-OS"] = "ios" ++ headers_["App-OS-Version"] = "16.7.2" ++ headers_["App-Version"] = "7.19.6" ++ headers_["User-Agent"] = "PixivIOSApp/7.19.6 (iOS 16.7.2; iPhone13,2)" + + if not req_auth: + return self.requests_call(method, url, headers_, params, data) +diff --git a/pixivpy3/api.py b/pixivpy3/api.py +index c26dda2..d0e1c0f 100644 +--- a/pixivpy3/api.py ++++ b/pixivpy3/api.py +@@ -26,6 +26,7 @@ class BasePixivAPI: + self.user_id: int | str = 0 + self.access_token: str | None = None + self.refresh_token: str | None = None ++ self.token_expiration: int | None = None + self.hosts = "https://app-api.pixiv.net" + + # self.requests = requests.Session() +@@ -62,6 +63,11 @@ class BasePixivAPI: + stream: bool = False, + ) -> Response: + """requests http/https call for Pixiv API""" ++ if self.token_expiration is not None and datetime.utcnow().timestamp() >= self.token_expiration: ++ # Token expired refresh auth ++ print('Token expired, refreshing auth') ++ self.token_expiration = None ++ self.auth() + merged_headers = self.additional_headers.copy() + if headers: + # Use the headers in the parameter to override the +@@ -118,23 +124,22 @@ class BasePixivAPI: + headers: ParamDict = None, + ) -> ParsedJson: + """Login with password, or use the refresh_token to acquire a new bearer token""" +- local_time = datetime.utcnow().strftime("%Y-%m-%dT%H:%M:%S+00:00") ++ local_time = datetime.utcnow() ++ local_time_str = local_time.strftime("%Y-%m-%dT%H:%M:%S+00:00") + headers_ = CaseInsensitiveDict(headers or {}) +- headers_["x-client-time"] = local_time +- headers_["x-client-hash"] = hashlib.md5((local_time + self.hash_secret).encode("utf-8")).hexdigest() +- # Allow mock UA due to #171: https://github.com/upbit/pixivpy/issues/171 +- if "user-agent" not in headers_: +- headers_["app-os"] = "ios" +- headers_["app-os-version"] = "14.6" +- headers_["user-agent"] = "PixivIOSApp/7.13.3 (iOS 14.6; iPhone13,2)" +- ++ headers_["X-Client-Time"] = local_time_str ++ headers_["X-Client-Hash"] = hashlib.md5((local_time_str + self.hash_secret).encode("utf-8")).hexdigest() ++ headers_["App-OS"] = "ios" ++ headers_["App-OS-Version"] = "16.7.2" ++ headers_["App-Version"] = "7.19.6" ++ headers_["User-Agent"] = "PixivIOSApp/7.19.6 (iOS 16.7.2; iPhone13,2)" + # noinspection PyUnresolvedReferences + if not hasattr(self, "hosts") or self.hosts == "https://app-api.pixiv.net": + auth_hosts = "https://oauth.secure.pixiv.net" + else: + # noinspection PyUnresolvedReferences + auth_hosts = self.hosts # BAPI解析成IP的场景 +- headers_["host"] = "oauth.secure.pixiv.net" ++ headers_["Host"] = "oauth.secure.pixiv.net" + url = "%s/auth/token" % auth_hosts + data = { + "get_secure_url": 1, +@@ -149,6 +154,8 @@ class BasePixivAPI: + elif refresh_token or self.refresh_token: + data["grant_type"] = "refresh_token" + data["refresh_token"] = refresh_token or self.refresh_token ++ if self.access_token: ++ data["access_token"] = self.access_token + else: + raise PixivError("[ERROR] auth() but no password or refresh_token is set.") + +@@ -174,6 +181,7 @@ class BasePixivAPI: + self.user_id = token.response.user.id + self.access_token = token.response.access_token + self.refresh_token = token.response.refresh_token ++ self.token_expiration = local_time.timestamp() + token.response.expires_in + except json.JSONDecodeError: + raise PixivError( + "Get access_token error! Response: %s" % token, diff --git a/src/portal/patches/searxng_disable_engines_default.diff b/src/portal/patches/searxng_disable_engines_default.diff new file mode 100644 index 0000000..923519c --- /dev/null +++ b/src/portal/patches/searxng_disable_engines_default.diff @@ -0,0 +1,3675 @@ +diff --git a/searx/settings.yml b/searx/settings.yml +index 6946b89d..92378207 100644 +--- a/searx/settings.yml ++++ b/searx/settings.yml +@@ -294,468 +294,6 @@ categories_as_tabs: + social media: + + engines: +- - name: 9gag +- engine: 9gag +- shortcut: 9g +- disabled: true +- +- - name: annas archive +- engine: annas_archive +- disabled: true +- shortcut: aa +- +- # - name: annas articles +- # engine: annas_archive +- # shortcut: aaa +- # # https://docs.searxng.org/dev/engines/online/annas_archive.html +- # aa_content: 'journal_article' # book_any .. magazine, standards_document +- # aa_ext: 'pdf' # pdf, epub, .. +- # aa_sort: 'newest' # newest, oldest, largest, smallest +- +- - name: apk mirror +- engine: apkmirror +- timeout: 4.0 +- shortcut: apkm +- disabled: true +- +- - name: apple app store +- engine: apple_app_store +- shortcut: aps +- disabled: true +- +- # Requires Tor +- - name: ahmia +- engine: ahmia +- categories: onions +- enable_http: true +- shortcut: ah +- +- - name: anaconda +- engine: xpath +- paging: true +- first_page_num: 0 +- search_url: https://anaconda.org/search?q={query}&page={pageno} +- results_xpath: //tbody/tr +- url_xpath: ./td/h5/a[last()]/@href +- title_xpath: ./td/h5 +- content_xpath: ./td[h5]/text() +- categories: it +- timeout: 6.0 +- shortcut: conda +- disabled: true +- +- - name: arch linux wiki +- engine: archlinux +- shortcut: al +- +- - name: artic +- engine: artic +- shortcut: arc +- timeout: 4.0 +- +- - name: arxiv +- engine: arxiv +- shortcut: arx +- timeout: 4.0 +- +- # tmp suspended: dh key too small +- # - name: base +- # engine: base +- # shortcut: bs +- +- - name: bandcamp +- engine: bandcamp +- shortcut: bc +- categories: music +- +- - name: wikipedia +- engine: wikipedia +- shortcut: wp +- # add "list" to the array to get results in the results list +- display_type: ["infobox"] +- base_url: 'https://{language}.wikipedia.org/' +- categories: [general] +- +- - name: bilibili +- engine: bilibili +- shortcut: bil +- disabled: true +- +- - name: bing +- engine: bing +- shortcut: bi +- disabled: true +- +- - name: bing images +- engine: bing_images +- shortcut: bii +- +- - name: bing news +- engine: bing_news +- shortcut: bin +- +- - name: bing videos +- engine: bing_videos +- shortcut: biv +- +- - name: bitbucket +- engine: xpath +- paging: true +- search_url: https://bitbucket.org/repo/all/{pageno}?name={query} +- url_xpath: //article[@class="repo-summary"]//a[@class="repo-link"]/@href +- title_xpath: //article[@class="repo-summary"]//a[@class="repo-link"] +- content_xpath: //article[@class="repo-summary"]/p +- categories: [it, repos] +- timeout: 4.0 +- disabled: true +- shortcut: bb +- about: +- website: https://bitbucket.org/ +- wikidata_id: Q2493781 +- official_api_documentation: https://developer.atlassian.com/bitbucket +- use_official_api: false +- require_api_key: false +- results: HTML +- +- - name: btdigg +- engine: btdigg +- shortcut: bt +- disabled: true +- +- - name: ccc-tv +- engine: xpath +- paging: false +- search_url: https://media.ccc.de/search/?q={query} +- url_xpath: //div[@class="caption"]/h3/a/@href +- title_xpath: //div[@class="caption"]/h3/a/text() +- content_xpath: //div[@class="caption"]/h4/@title +- categories: videos +- disabled: true +- shortcut: c3tv +- about: +- website: https://media.ccc.de/ +- wikidata_id: Q80729951 +- official_api_documentation: https://github.com/voc/voctoweb +- use_official_api: false +- require_api_key: false +- results: HTML +- # We don't set language: de here because media.ccc.de is not just +- # for a German audience. It contains many English videos and many +- # German videos have English subtitles. +- +- - name: openverse +- engine: openverse +- categories: images +- shortcut: opv +- +- - name: chefkoch +- engine: chefkoch +- shortcut: chef +- # to show premium or plus results too: +- # skip_premium: false +- +- # - name: core.ac.uk +- # engine: core +- # categories: science +- # shortcut: cor +- # # get your API key from: https://core.ac.uk/api-keys/register/ +- # api_key: 'unset' +- +- - name: crossref +- engine: crossref +- shortcut: cr +- timeout: 30 +- disabled: true +- +- - name: crowdview +- engine: json_engine +- shortcut: cv +- categories: general +- paging: false +- search_url: https://crowdview-next-js.onrender.com/api/search-v3?query={query} +- results_query: results +- url_query: link +- title_query: title +- content_query: snippet +- disabled: true +- about: +- website: https://crowdview.ai/ +- +- - name: yep +- engine: json_engine +- shortcut: yep +- categories: general +- disabled: true +- paging: false +- content_html_to_text: true +- title_html_to_text: true +- search_url: https://api.yep.com/fs/1/?type=web&q={query}&no_correct=false&limit=100 +- results_query: 1/results +- title_query: title +- url_query: url +- content_query: snippet +- about: +- website: https://yep.com +- use_official_api: false +- require_api_key: false +- results: JSON +- +- - name: curlie +- engine: xpath +- shortcut: cl +- categories: general +- disabled: true +- paging: true +- lang_all: '' +- search_url: https://curlie.org/search?q={query}&lang={lang}&start={pageno}&stime=92452189 +- page_size: 20 +- results_xpath: //div[@id="site-list-content"]/div[@class="site-item"] +- url_xpath: ./div[@class="title-and-desc"]/a/@href +- title_xpath: ./div[@class="title-and-desc"]/a/div +- content_xpath: ./div[@class="title-and-desc"]/div[@class="site-descr"] +- about: +- website: https://curlie.org/ +- wikidata_id: Q60715723 +- use_official_api: false +- require_api_key: false +- results: HTML +- +- - name: currency +- engine: currency_convert +- categories: general +- shortcut: cc +- +- - name: deezer +- engine: deezer +- shortcut: dz +- disabled: true +- +- - name: deviantart +- engine: deviantart +- shortcut: da +- timeout: 3.0 +- +- - name: ddg definitions +- engine: duckduckgo_definitions +- shortcut: ddd +- weight: 2 +- disabled: true +- tests: *tests_infobox +- +- # cloudflare protected +- # - name: digbt +- # engine: digbt +- # shortcut: dbt +- # timeout: 6.0 +- # disabled: true +- +- - name: docker hub +- engine: docker_hub +- shortcut: dh +- categories: [it, packages] +- +- - name: erowid +- engine: xpath +- paging: true +- first_page_num: 0 +- page_size: 30 +- search_url: https://www.erowid.org/search.php?q={query}&s={pageno} +- url_xpath: //dl[@class="results-list"]/dt[@class="result-title"]/a/@href +- title_xpath: //dl[@class="results-list"]/dt[@class="result-title"]/a/text() +- content_xpath: //dl[@class="results-list"]/dd[@class="result-details"] +- categories: [] +- shortcut: ew +- disabled: true +- about: +- website: https://www.erowid.org/ +- wikidata_id: Q1430691 +- official_api_documentation: +- use_official_api: false +- require_api_key: false +- results: HTML +- +- # - name: elasticsearch +- # shortcut: es +- # engine: elasticsearch +- # base_url: http://localhost:9200 +- # username: elastic +- # password: changeme +- # index: my-index +- # # available options: match, simple_query_string, term, terms, custom +- # query_type: match +- # # if query_type is set to custom, provide your query here +- # #custom_query_json: {"query":{"match_all": {}}} +- # #show_metadata: false +- # disabled: true +- +- - name: wikidata +- engine: wikidata +- shortcut: wd +- timeout: 3.0 +- weight: 2 +- # add "list" to the array to get results in the results list +- display_type: ["infobox"] +- tests: *tests_infobox +- categories: [general] +- +- - name: duckduckgo +- engine: duckduckgo +- shortcut: ddg +- +- - name: duckduckgo images +- engine: duckduckgo_images +- shortcut: ddi +- timeout: 3.0 +- disabled: true +- +- - name: duckduckgo weather +- engine: duckduckgo_weather +- shortcut: ddw +- disabled: true +- +- - name: apple maps +- engine: apple_maps +- shortcut: apm +- disabled: true +- timeout: 5.0 +- +- - name: emojipedia +- engine: emojipedia +- timeout: 4.0 +- shortcut: em +- disabled: true +- +- - name: tineye +- engine: tineye +- shortcut: tin +- timeout: 9.0 +- disabled: true +- +- - name: etymonline +- engine: xpath +- paging: true +- search_url: https://etymonline.com/search?page={pageno}&q={query} +- url_xpath: //a[contains(@class, "word__name--")]/@href +- title_xpath: //a[contains(@class, "word__name--")] +- content_xpath: //section[contains(@class, "word__defination")] +- first_page_num: 1 +- shortcut: et +- categories: [dictionaries] +- about: +- website: https://www.etymonline.com/ +- wikidata_id: Q1188617 +- official_api_documentation: +- use_official_api: false +- require_api_key: false +- results: HTML +- +- # - name: ebay +- # engine: ebay +- # shortcut: eb +- # base_url: 'https://www.ebay.com' +- # disabled: true +- # timeout: 5 +- +- - name: 1x +- engine: www1x +- shortcut: 1x +- timeout: 3.0 +- disabled: true +- +- - name: fdroid +- engine: fdroid +- shortcut: fd +- disabled: true +- +- - name: flickr +- categories: images +- shortcut: fl +- # You can use the engine using the official stable API, but you need an API +- # key, see: https://www.flickr.com/services/apps/create/ +- # engine: flickr +- # api_key: 'apikey' # required! +- # Or you can use the html non-stable engine, activated by default +- engine: flickr_noapi +- +- - name: free software directory +- engine: mediawiki +- shortcut: fsd +- categories: [it, software wikis] +- base_url: https://directory.fsf.org/ +- search_type: title +- timeout: 5.0 +- disabled: true +- about: +- website: https://directory.fsf.org/ +- wikidata_id: Q2470288 +- +- # - name: freesound +- # engine: freesound +- # shortcut: fnd +- # disabled: true +- # timeout: 15.0 +- # API key required, see: https://freesound.org/docs/api/overview.html +- # api_key: MyAPIkey +- +- - name: frinkiac +- engine: frinkiac +- shortcut: frk +- disabled: true +- +- - name: genius +- engine: genius +- shortcut: gen +- +- - name: gentoo +- engine: gentoo +- shortcut: ge +- timeout: 10.0 +- +- - name: gitlab +- engine: json_engine +- paging: true +- search_url: https://gitlab.com/api/v4/projects?search={query}&page={pageno} +- url_query: web_url +- title_query: name_with_namespace +- content_query: description +- page_size: 20 +- categories: [it, repos] +- shortcut: gl +- timeout: 10.0 +- disabled: true +- about: +- website: https://about.gitlab.com/ +- wikidata_id: Q16639197 +- official_api_documentation: https://docs.gitlab.com/ee/api/ +- use_official_api: false +- require_api_key: false +- results: JSON +- +- - name: github +- engine: github +- shortcut: gh +- +- # This a Gitea service. If you would like to use a different instance, +- # change codeberg.org to URL of the desired Gitea host. Or you can create a +- # new engine by copying this and changing the name, shortcut and search_url. +- +- - name: codeberg +- engine: json_engine +- search_url: https://codeberg.org/api/v1/repos/search?q={query}&limit=10 +- url_query: html_url +- title_query: name +- content_query: description +- categories: [it, repos] +- shortcut: cb +- disabled: true +- about: +- website: https://codeberg.org/ +- wikidata_id: +- official_api_documentation: https://try.gitea.io/api/swagger +- use_official_api: false +- require_api_key: false +- results: JSON +- + - name: google + engine: google + shortcut: go +@@ -774,1372 +312,1837 @@ engines: + # result_container: + # - ['one_title_contains', 'Salvador'] + +- - name: google news +- engine: google_news +- shortcut: gon +- # additional_tests: +- # android: *test_android +- +- - name: google videos +- engine: google_videos +- shortcut: gov +- # additional_tests: +- # android: *test_android +- +- - name: google scholar +- engine: google_scholar +- shortcut: gos +- +- - name: google play apps +- engine: google_play +- categories: [files, apps] +- shortcut: gpa +- play_categ: apps +- disabled: true +- +- - name: google play movies +- engine: google_play +- categories: videos +- shortcut: gpm +- play_categ: movies +- disabled: true +- +- - name: material icons +- engine: material_icons +- categories: images +- shortcut: mi +- disabled: true +- +- - name: gpodder +- engine: json_engine +- shortcut: gpod +- timeout: 4.0 +- paging: false +- search_url: https://gpodder.net/search.json?q={query} +- url_query: url +- title_query: title +- content_query: description +- page_size: 19 +- categories: music +- disabled: true +- about: +- website: https://gpodder.net +- wikidata_id: Q3093354 +- official_api_documentation: https://gpoddernet.readthedocs.io/en/latest/api/ +- use_official_api: false +- requires_api_key: false +- results: JSON +- +- - name: habrahabr +- engine: xpath +- paging: true +- search_url: https://habr.com/en/search/page{pageno}/?q={query} +- results_xpath: //article[contains(@class, "tm-articles-list__item")] +- url_xpath: .//a[@class="tm-title__link"]/@href +- title_xpath: .//a[@class="tm-title__link"] +- content_xpath: .//div[contains(@class, "article-formatted-body")] +- categories: it +- timeout: 4.0 +- disabled: true +- shortcut: habr +- about: +- website: https://habr.com/ +- wikidata_id: Q4494434 +- official_api_documentation: https://habr.com/en/docs/help/api/ +- use_official_api: false +- require_api_key: false +- results: HTML +- +- - name: hoogle +- engine: xpath +- paging: true +- search_url: https://hoogle.haskell.org/?hoogle={query}&start={pageno} +- results_xpath: '//div[@class="result"]' +- title_xpath: './/div[@class="ans"]//a' +- url_xpath: './/div[@class="ans"]//a/@href' +- content_xpath: './/div[@class="from"]' +- page_size: 20 +- categories: [it, packages] +- shortcut: ho +- about: +- website: https://hoogle.haskell.org/ +- wikidata_id: Q34010 +- official_api_documentation: https://hackage.haskell.org/api +- use_official_api: false +- require_api_key: false +- results: JSON +- +- - name: imdb +- engine: imdb +- shortcut: imdb +- timeout: 6.0 +- disabled: true +- +- - name: ina +- engine: ina +- shortcut: in +- timeout: 6.0 +- disabled: true +- +- - name: invidious +- engine: invidious +- # Instanes will be selected randomly, see https://api.invidious.io/ for +- # instances that are stable (good uptime) and close to you. +- base_url: +- - https://invidious.io.lol +- - https://invidious.fdn.fr +- - https://yt.artemislena.eu +- - https://invidious.tiekoetter.com +- - https://invidious.flokinet.to +- - https://vid.puffyan.us +- - https://invidious.privacydev.net +- - https://inv.tux.pizza +- shortcut: iv +- timeout: 3.0 +- disabled: true +- +- - name: jisho +- engine: jisho +- shortcut: js +- timeout: 3.0 +- disabled: true +- +- - name: kickass +- engine: kickass +- shortcut: kc +- timeout: 4.0 +- disabled: true +- +- - name: lemmy communities +- engine: lemmy +- lemmy_type: Communities +- shortcut: leco +- +- - name: lemmy users +- engine: lemmy +- network: lemmy communities +- lemmy_type: Users +- shortcut: leus +- +- - name: lemmy posts +- engine: lemmy +- network: lemmy communities +- lemmy_type: Posts +- shortcut: lepo +- +- - name: lemmy comments +- engine: lemmy +- network: lemmy communities +- lemmy_type: Comments +- shortcut: lecom +- +- - name: library genesis +- engine: xpath +- search_url: https://libgen.fun/search.php?req={query} +- url_xpath: //a[contains(@href,"get.php?md5")]/@href +- title_xpath: //a[contains(@href,"book/")]/text()[1] +- content_xpath: //td/a[1][contains(@href,"=author")]/text() +- categories: files +- timeout: 7.0 +- disabled: true +- shortcut: lg +- about: +- website: https://libgen.fun/ +- wikidata_id: Q22017206 +- official_api_documentation: +- use_official_api: false +- require_api_key: false +- results: HTML +- +- - name: z-library +- engine: zlibrary +- shortcut: zlib +- categories: files +- timeout: 7.0 +- +- - name: library of congress +- engine: loc +- shortcut: loc +- categories: images +- +- - name: lingva +- engine: lingva +- shortcut: lv +- # set lingva instance in url, by default it will use the official instance +- # url: https://lingva.ml +- +- - name: lobste.rs +- engine: xpath +- search_url: https://lobste.rs/search?utf8=%E2%9C%93&q={query}&what=stories&order=relevance +- results_xpath: //li[contains(@class, "story")] +- url_xpath: .//a[@class="u-url"]/@href +- title_xpath: .//a[@class="u-url"] +- content_xpath: .//a[@class="domain"] +- categories: it +- shortcut: lo +- timeout: 5.0 +- disabled: true +- about: +- website: https://lobste.rs/ +- wikidata_id: Q60762874 +- official_api_documentation: +- use_official_api: false +- require_api_key: false +- results: HTML +- +- - name: azlyrics +- shortcut: lyrics +- engine: xpath +- timeout: 4.0 +- disabled: true +- categories: [music, lyrics] +- paging: true +- search_url: https://search.azlyrics.com/search.php?q={query}&w=lyrics&p={pageno} +- url_xpath: //td[@class="text-left visitedlyr"]/a/@href +- title_xpath: //span/b/text() +- content_xpath: //td[@class="text-left visitedlyr"]/a/small +- about: +- website: https://azlyrics.com +- wikidata_id: Q66372542 +- official_api_documentation: +- use_official_api: false +- require_api_key: false +- results: HTML +- +- - name: metacpan +- engine: metacpan +- shortcut: cpan +- disabled: true +- number_of_results: 20 +- +- # - name: meilisearch +- # engine: meilisearch +- # shortcut: mes +- # enable_http: true +- # base_url: http://localhost:7700 +- # index: my-index +- +- - name: mixcloud +- engine: mixcloud +- shortcut: mc +- +- # MongoDB engine +- # Required dependency: pymongo +- # - name: mymongo +- # engine: mongodb +- # shortcut: md +- # exact_match_only: false +- # host: '127.0.0.1' +- # port: 27017 +- # enable_http: true +- # results_per_page: 20 +- # database: 'business' +- # collection: 'reviews' # name of the db collection +- # key: 'name' # key in the collection to search for +- +- - name: mwmbl +- engine: mwmbl +- # api_url: https://api.mwmbl.org +- shortcut: mwm +- disabled: true +- +- - name: npm +- engine: json_engine +- paging: true +- first_page_num: 0 +- search_url: https://api.npms.io/v2/search?q={query}&size=25&from={pageno} +- results_query: results +- url_query: package/links/npm +- title_query: package/name +- content_query: package/description +- page_size: 25 +- categories: [it, packages] +- disabled: true +- timeout: 5.0 +- shortcut: npm +- about: +- website: https://npms.io/ +- wikidata_id: Q7067518 +- official_api_documentation: https://api-docs.npms.io/ +- use_official_api: false +- require_api_key: false +- results: JSON +- +- - name: nyaa +- engine: nyaa +- shortcut: nt +- disabled: true +- +- - name: mankier +- engine: json_engine +- search_url: https://www.mankier.com/api/v2/mans/?q={query} +- results_query: results +- url_query: url +- title_query: name +- content_query: description +- categories: it +- shortcut: man +- about: +- website: https://www.mankier.com/ +- official_api_documentation: https://www.mankier.com/api +- use_official_api: true +- require_api_key: false +- results: JSON +- +- - name: odysee +- engine: odysee +- shortcut: od +- disabled: true +- +- - name: openairedatasets +- engine: json_engine +- paging: true +- search_url: https://api.openaire.eu/search/datasets?format=json&page={pageno}&size=10&title={query} +- results_query: response/results/result +- url_query: metadata/oaf:entity/oaf:result/children/instance/webresource/url/$ +- title_query: metadata/oaf:entity/oaf:result/title/$ +- content_query: metadata/oaf:entity/oaf:result/description/$ +- content_html_to_text: true +- categories: "science" +- shortcut: oad +- timeout: 5.0 +- about: +- website: https://www.openaire.eu/ +- wikidata_id: Q25106053 +- official_api_documentation: https://api.openaire.eu/ +- use_official_api: false +- require_api_key: false +- results: JSON +- +- - name: openairepublications +- engine: json_engine +- paging: true +- search_url: https://api.openaire.eu/search/publications?format=json&page={pageno}&size=10&title={query} +- results_query: response/results/result +- url_query: metadata/oaf:entity/oaf:result/children/instance/webresource/url/$ +- title_query: metadata/oaf:entity/oaf:result/title/$ +- content_query: metadata/oaf:entity/oaf:result/description/$ +- content_html_to_text: true +- categories: science +- shortcut: oap +- timeout: 5.0 +- about: +- website: https://www.openaire.eu/ +- wikidata_id: Q25106053 +- official_api_documentation: https://api.openaire.eu/ +- use_official_api: false +- require_api_key: false +- results: JSON +- +- # - name: opensemanticsearch +- # engine: opensemantic +- # shortcut: oss +- # base_url: 'http://localhost:8983/solr/opensemanticsearch/' +- +- - name: openstreetmap +- engine: openstreetmap +- shortcut: osm +- +- - name: openrepos +- engine: xpath +- paging: true +- search_url: https://openrepos.net/search/node/{query}?page={pageno} +- url_xpath: //li[@class="search-result"]//h3[@class="title"]/a/@href +- title_xpath: //li[@class="search-result"]//h3[@class="title"]/a +- content_xpath: //li[@class="search-result"]//div[@class="search-snippet-info"]//p[@class="search-snippet"] +- categories: files +- timeout: 4.0 +- disabled: true +- shortcut: or +- about: +- website: https://openrepos.net/ +- wikidata_id: +- official_api_documentation: +- use_official_api: false +- require_api_key: false +- results: HTML +- +- - name: packagist +- engine: json_engine +- paging: true +- search_url: https://packagist.org/search.json?q={query}&page={pageno} +- results_query: results +- url_query: url +- title_query: name +- content_query: description +- categories: [it, packages] +- disabled: true +- timeout: 5.0 +- shortcut: pack +- about: +- website: https://packagist.org +- wikidata_id: Q108311377 +- official_api_documentation: https://packagist.org/apidoc +- use_official_api: true +- require_api_key: false +- results: JSON +- +- - name: pdbe +- engine: pdbe +- shortcut: pdb +- # Hide obsolete PDB entries. Default is not to hide obsolete structures +- # hide_obsolete: false +- +- - name: photon +- engine: photon +- shortcut: ph +- +- - name: piped +- engine: piped +- shortcut: ppd +- categories: videos +- piped_filter: videos +- timeout: 3.0 +- +- # URL to use as link and for embeds +- frontend_url: https://srv.piped.video +- # Instance will be selected randomly, for more see https://piped-instances.kavin.rocks/ +- backend_url: +- - https://pipedapi.kavin.rocks +- - https://pipedapi-libre.kavin.rocks +- - https://pipedapi.adminforge.de +- +- - name: piped.music +- engine: piped +- network: piped +- shortcut: ppdm +- categories: music +- piped_filter: music_songs +- timeout: 3.0 +- +- - name: piratebay +- engine: piratebay +- shortcut: tpb +- # You may need to change this URL to a proxy if piratebay is blocked in your +- # country +- url: https://thepiratebay.org/ +- timeout: 3.0 +- +- # Required dependency: psychopg2 +- # - name: postgresql +- # engine: postgresql +- # database: postgres +- # username: postgres +- # password: postgres +- # limit: 10 +- # query_str: 'SELECT * from my_table WHERE my_column = %(query)s' +- # shortcut : psql +- +- - name: pub.dev +- engine: xpath +- shortcut: pd +- search_url: https://pub.dev/packages?q={query}&page={pageno} +- paging: true +- results_xpath: //div[contains(@class,"packages-item")] +- url_xpath: ./div/h3/a/@href +- title_xpath: ./div/h3/a +- content_xpath: ./div/div/div[contains(@class,"packages-description")]/span +- categories: [packages, it] +- timeout: 3.0 +- disabled: true +- first_page_num: 1 +- about: +- website: https://pub.dev/ +- official_api_documentation: https://pub.dev/help/api +- use_official_api: false +- require_api_key: false +- results: HTML +- +- - name: pubmed +- engine: pubmed +- shortcut: pub +- timeout: 3.0 +- +- - name: pypi +- shortcut: pypi +- engine: xpath +- paging: true +- search_url: https://pypi.org/search/?q={query}&page={pageno} +- results_xpath: /html/body/main/div/div/div/form/div/ul/li/a[@class="package-snippet"] +- url_xpath: ./@href +- title_xpath: ./h3/span[@class="package-snippet__name"] +- content_xpath: ./p +- suggestion_xpath: /html/body/main/div/div/div/form/div/div[@class="callout-block"]/p/span/a[@class="link"] +- first_page_num: 1 +- categories: [it, packages] +- about: +- website: https://pypi.org +- wikidata_id: Q2984686 +- official_api_documentation: https://warehouse.readthedocs.io/api-reference/index.html +- use_official_api: false +- require_api_key: false +- results: HTML +- +- - name: qwant +- qwant_categ: web +- engine: qwant +- shortcut: qw +- categories: [general, web] +- additional_tests: +- rosebud: *test_rosebud +- +- - name: qwant news +- qwant_categ: news +- engine: qwant +- shortcut: qwn +- categories: news +- network: qwant +- +- - name: qwant images +- qwant_categ: images +- engine: qwant +- shortcut: qwi +- categories: [images, web] +- network: qwant +- +- - name: qwant videos +- qwant_categ: videos +- engine: qwant +- shortcut: qwv +- categories: [videos, web] +- network: qwant +- +- # - name: library +- # engine: recoll +- # shortcut: lib +- # base_url: 'https://recoll.example.org/' +- # search_dir: '' +- # mount_prefix: /export +- # dl_prefix: 'https://download.example.org' +- # timeout: 30.0 +- # categories: files +- # disabled: true +- +- # - name: recoll library reference +- # engine: recoll +- # base_url: 'https://recoll.example.org/' +- # search_dir: reference +- # mount_prefix: /export +- # dl_prefix: 'https://download.example.org' +- # shortcut: libr +- # timeout: 30.0 +- # categories: files +- # disabled: true +- +- - name: reddit +- engine: reddit +- shortcut: re +- page_size: 25 +- +- # Required dependency: redis +- # - name: myredis +- # shortcut : rds +- # engine: redis_server +- # exact_match_only: false +- # host: '127.0.0.1' +- # port: 6379 +- # enable_http: true +- # password: '' +- # db: 0 ++ - name: bing images ++ engine: bing_images ++ shortcut: bii + +- # tmp suspended: bad certificate +- # - name: scanr structures +- # shortcut: scs +- # engine: scanr_structures ++ # - name: 9gag ++ # engine: 9gag ++ # shortcut: 9g + # disabled: true +- +- - name: sepiasearch +- engine: sepiasearch +- shortcut: sep +- +- - name: soundcloud +- engine: soundcloud +- shortcut: sc +- +- - name: stackoverflow +- engine: stackexchange +- shortcut: st +- api_site: 'stackoverflow' +- categories: [it, q&a] +- +- - name: askubuntu +- engine: stackexchange +- shortcut: ubuntu +- api_site: 'askubuntu' +- categories: [it, q&a] +- +- - name: internetarchivescholar +- engine: internet_archive_scholar +- shortcut: ias +- timeout: 5.0 +- +- - name: superuser +- engine: stackexchange +- shortcut: su +- api_site: 'superuser' +- categories: [it, q&a] +- +- - name: searchcode code +- engine: searchcode_code +- shortcut: scc +- disabled: true +- +- - name: framalibre +- engine: framalibre +- shortcut: frl +- disabled: true +- +- # - name: searx +- # engine: searx_engine +- # shortcut: se +- # instance_urls : +- # - http://127.0.0.1:8888/ +- # - ... +- # disabled: true +- +- - name: semantic scholar +- engine: semantic_scholar +- disabled: true +- shortcut: se +- +- # Spotify needs API credentials +- # - name: spotify +- # engine: spotify +- # shortcut: stf +- # api_client_id: ******* +- # api_client_secret: ******* +- +- # - name: solr +- # engine: solr +- # shortcut: slr +- # base_url: http://localhost:8983 +- # collection: collection_name +- # sort: '' # sorting: asc or desc +- # field_list: '' # comma separated list of field names to display on the UI +- # default_fields: '' # default field to query +- # query_fields: '' # query fields +- # enable_http: true +- +- # - name: springer nature +- # engine: springer +- # # get your API key from: https://dev.springernature.com/signup +- # # working API key, for test & debug: "a69685087d07eca9f13db62f65b8f601" +- # api_key: 'unset' +- # shortcut: springer +- # timeout: 15.0 +- +- - name: startpage +- engine: startpage +- shortcut: sp +- timeout: 6.0 +- disabled: true +- additional_tests: +- rosebud: *test_rosebud +- +- - name: tokyotoshokan +- engine: tokyotoshokan +- shortcut: tt +- timeout: 6.0 +- disabled: true +- +- - name: solidtorrents +- engine: solidtorrents +- shortcut: solid +- timeout: 4.0 +- base_url: +- - https://solidtorrents.to +- - https://bitsearch.to +- +- # For this demo of the sqlite engine download: +- # https://liste.mediathekview.de/filmliste-v2.db.bz2 +- # and unpack into searx/data/filmliste-v2.db +- # Query to test: "!demo concert" +- # +- # - name: demo +- # engine: sqlite +- # shortcut: demo +- # categories: general +- # result_template: default.html +- # database: searx/data/filmliste-v2.db +- # query_str: >- +- # SELECT title || ' (' || time(duration, 'unixepoch') || ')' AS title, +- # COALESCE( NULLIF(url_video_hd,''), NULLIF(url_video_sd,''), url_video) AS url, +- # description AS content +- # FROM film +- # WHERE title LIKE :wildcard OR description LIKE :wildcard +- # ORDER BY duration DESC +- +- - name: tagesschau +- engine: tagesschau +- shortcut: ts +- disabled: true +- +- - name: tmdb +- engine: xpath +- paging: true +- search_url: https://www.themoviedb.org/search?page={pageno}&query={query} +- results_xpath: //div[contains(@class,"movie") or contains(@class,"tv")]//div[contains(@class,"card")] +- url_xpath: .//div[contains(@class,"poster")]/a/@href +- thumbnail_xpath: .//img/@src +- title_xpath: .//div[contains(@class,"title")]//h2 +- content_xpath: .//div[contains(@class,"overview")] +- shortcut: tm +- disabled: true +- +- # Requires Tor +- - name: torch +- engine: xpath +- paging: true +- search_url: +- http://xmh57jrknzkhv6y3ls3ubitzfqnkrwxhopf5aygthi7d6rplyvk3noyd.onion/cgi-bin/omega/omega?P={query}&DEFAULTOP=and +- results_xpath: //table//tr +- url_xpath: ./td[2]/a +- title_xpath: ./td[2]/b +- content_xpath: ./td[2]/small +- categories: onions +- enable_http: true +- shortcut: tch +- +- # torznab engine lets you query any torznab compatible indexer. Using this +- # engine in combination with Jackett opens the possibility to query a lot of +- # public and private indexers directly from SearXNG. More details at: +- # https://docs.searxng.org/dev/engines/online/torznab.html +- # +- # - name: Torznab EZTV +- # engine: torznab +- # shortcut: eztv +- # base_url: http://localhost:9117/api/v2.0/indexers/eztv/results/torznab +- # enable_http: true # if using localhost +- # api_key: xxxxxxxxxxxxxxx +- # show_magnet_links: true +- # show_torrent_files: false +- # # https://github.com/Jackett/Jackett/wiki/Jackett-Categories +- # torznab_categories: # optional +- # - 2000 +- # - 5000 +- +- - name: twitter +- shortcut: tw +- engine: twitter +- disabled: true +- +- # tmp suspended - too slow, too many errors +- # - name: urbandictionary +- # engine : xpath +- # search_url : https://www.urbandictionary.com/define.php?term={query} +- # url_xpath : //*[@class="word"]/@href +- # title_xpath : //*[@class="def-header"] +- # content_xpath: //*[@class="meaning"] +- # shortcut: ud +- +- - name: unsplash +- engine: unsplash +- shortcut: us +- +- - name: yahoo +- engine: yahoo +- shortcut: yh +- disabled: true +- +- - name: yahoo news +- engine: yahoo_news +- shortcut: yhn +- +- - name: youtube +- shortcut: yt +- # You can use the engine using the official stable API, but you need an API +- # key See: https://console.developers.google.com/project +- # +- # engine: youtube_api +- # api_key: 'apikey' # required! +- # +- # Or you can use the html non-stable engine, activated by default +- engine: youtube_noapi +- +- - name: dailymotion +- engine: dailymotion +- shortcut: dm +- +- - name: vimeo +- engine: vimeo +- shortcut: vm +- +- - name: wiby +- engine: json_engine +- paging: true +- search_url: https://wiby.me/json/?q={query}&p={pageno} +- url_query: URL +- title_query: Title +- content_query: Snippet +- categories: [general, web] +- shortcut: wib +- disabled: true +- about: +- website: https://wiby.me/ +- +- - name: alexandria +- engine: json_engine +- shortcut: alx +- categories: general +- paging: true +- search_url: https://api.alexandria.org/?a=1&q={query}&p={pageno} +- results_query: results +- title_query: title +- url_query: url +- content_query: snippet +- timeout: 1.5 +- disabled: true +- about: +- website: https://alexandria.org/ +- official_api_documentation: https://github.com/alexandria-org/alexandria-api/raw/master/README.md +- use_official_api: true +- require_api_key: false +- results: JSON +- +- - name: wikibooks +- engine: mediawiki +- weight: 0.5 +- shortcut: wb +- categories: [general, wikimedia] +- base_url: "https://{language}.wikibooks.org/" +- search_type: text +- disabled: true +- about: +- website: https://www.wikibooks.org/ +- wikidata_id: Q367 +- +- - name: wikinews +- engine: mediawiki +- shortcut: wn +- categories: [news, wikimedia] +- base_url: "https://{language}.wikinews.org/" +- search_type: text +- srsort: create_timestamp_desc +- about: +- website: https://www.wikinews.org/ +- wikidata_id: Q964 +- +- - name: wikiquote +- engine: mediawiki +- weight: 0.5 +- shortcut: wq +- categories: [general, wikimedia] +- base_url: "https://{language}.wikiquote.org/" +- search_type: text +- disabled: true +- additional_tests: +- rosebud: *test_rosebud +- about: +- website: https://www.wikiquote.org/ +- wikidata_id: Q369 +- +- - name: wikisource +- engine: mediawiki +- weight: 0.5 +- shortcut: ws +- categories: [general, wikimedia] +- base_url: "https://{language}.wikisource.org/" +- search_type: text +- disabled: true +- about: +- website: https://www.wikisource.org/ +- wikidata_id: Q263 +- +- - name: wikispecies +- engine: mediawiki +- shortcut: wsp +- categories: [general, science, wikimedia] +- base_url: "https://species.wikimedia.org/" +- search_type: text +- disabled: true +- about: +- website: https://species.wikimedia.org/ +- wikidata_id: Q13679 +- +- - name: wiktionary +- engine: mediawiki +- shortcut: wt +- categories: [dictionaries, wikimedia] +- base_url: "https://{language}.wiktionary.org/" +- search_type: text +- about: +- website: https://www.wiktionary.org/ +- wikidata_id: Q151 +- +- - name: wikiversity +- engine: mediawiki +- weight: 0.5 +- shortcut: wv +- categories: [general, wikimedia] +- base_url: "https://{language}.wikiversity.org/" +- search_type: text +- disabled: true +- about: +- website: https://www.wikiversity.org/ +- wikidata_id: Q370 +- +- - name: wikivoyage +- engine: mediawiki +- weight: 0.5 +- shortcut: wy +- categories: [general, wikimedia] +- base_url: "https://{language}.wikivoyage.org/" +- search_type: text +- disabled: true +- about: +- website: https://www.wikivoyage.org/ +- wikidata_id: Q373 +- +- - name: wikicommons.images +- engine: wikicommons +- shortcut: wc +- categories: images +- number_of_results: 10 +- +- - name: wolframalpha +- shortcut: wa +- # You can use the engine using the official stable API, but you need an API +- # key. See: https://products.wolframalpha.com/api/ +- # +- # engine: wolframalpha_api +- # api_key: '' +- # +- # Or you can use the html non-stable engine, activated by default +- engine: wolframalpha_noapi +- timeout: 6.0 +- categories: general +- disabled: true +- +- - name: dictzone +- engine: dictzone +- shortcut: dc +- +- - name: mymemory translated +- engine: translated +- shortcut: tl +- timeout: 5.0 +- # You can use without an API key, but you are limited to 1000 words/day +- # See: https://mymemory.translated.net/doc/usagelimits.php +- # api_key: '' +- +- # Required dependency: mysql-connector-python +- # - name: mysql +- # engine: mysql_server +- # database: mydatabase +- # username: user +- # password: pass +- # limit: 10 +- # query_str: 'SELECT * from mytable WHERE fieldname=%(query)s' +- # shortcut: mysql +- +- - name: 1337x +- engine: 1337x +- shortcut: 1337x +- disabled: true +- +- - name: duden +- engine: duden +- shortcut: du +- disabled: true +- +- - name: seznam +- shortcut: szn +- engine: seznam +- disabled: true +- +- # - name: deepl +- # engine: deepl +- # shortcut: dpl +- # # You can use the engine using the official stable API, but you need an API key +- # # See: https://www.deepl.com/pro-api?cta=header-pro-api +- # api_key: '' # required! +- # timeout: 5.0 +- # disabled: true +- +- - name: mojeek +- shortcut: mjk +- engine: xpath +- paging: true +- categories: [general, web] +- search_url: https://www.mojeek.com/search?q={query}&s={pageno}&lang={lang}&lb={lang} +- results_xpath: //ul[@class="results-standard"]/li/a[@class="ob"] +- url_xpath: ./@href +- title_xpath: ../h2/a +- content_xpath: ..//p[@class="s"] +- suggestion_xpath: //div[@class="top-info"]/p[@class="top-info spell"]/em/a +- first_page_num: 0 +- page_size: 10 +- disabled: true +- about: +- website: https://www.mojeek.com/ +- wikidata_id: Q60747299 +- official_api_documentation: https://www.mojeek.com/services/api.html/ +- use_official_api: false +- require_api_key: false +- results: HTML +- +- - name: moviepilot +- engine: moviepilot +- shortcut: mp +- disabled: true +- +- - name: naver +- shortcut: nvr +- categories: [general, web] +- engine: xpath +- paging: true +- search_url: https://search.naver.com/search.naver?where=webkr&sm=osp_hty&ie=UTF-8&query={query}&start={pageno} +- url_xpath: //a[@class="link_tit"]/@href +- title_xpath: //a[@class="link_tit"] +- content_xpath: //a[@class="total_dsc"]/div +- first_page_num: 1 +- page_size: 10 +- disabled: true +- about: +- website: https://www.naver.com/ +- wikidata_id: Q485639 +- official_api_documentation: https://developers.naver.com/docs/nmt/examples/ +- use_official_api: false +- require_api_key: false +- results: HTML +- language: ko +- +- - name: rubygems +- shortcut: rbg +- engine: xpath +- paging: true +- search_url: https://rubygems.org/search?page={pageno}&query={query} +- results_xpath: /html/body/main/div/a[@class="gems__gem"] +- url_xpath: ./@href +- title_xpath: ./span/h2 +- content_xpath: ./span/p +- suggestion_xpath: /html/body/main/div/div[@class="search__suggestions"]/p/a +- first_page_num: 1 +- categories: [it, packages] +- disabled: true +- about: +- website: https://rubygems.org/ +- wikidata_id: Q1853420 +- official_api_documentation: https://guides.rubygems.org/rubygems-org-api/ +- use_official_api: false +- require_api_key: false +- results: HTML +- +- - name: peertube +- engine: peertube +- shortcut: ptb +- paging: true +- # alternatives see: https://instances.joinpeertube.org/instances +- # base_url: https://tube.4aem.com +- categories: videos +- disabled: true +- timeout: 6.0 +- +- - name: mediathekviewweb +- engine: mediathekviewweb +- shortcut: mvw +- disabled: true +- +- # - name: yacy +- # engine: yacy +- # shortcut: ya +- # base_url: http://localhost:8090 +- # # required if you aren't using HTTPS for your local yacy instance' +- # enable_http: true +- # timeout: 3.0 +- # # Yacy search mode. 'global' or 'local'. +- # search_mode: 'global' +- +- - name: rumble +- engine: rumble +- shortcut: ru +- base_url: https://rumble.com/ +- paging: true +- categories: videos +- disabled: true +- +- - name: wordnik +- engine: wordnik +- shortcut: def +- base_url: https://www.wordnik.com/ +- categories: [dictionaries] +- timeout: 5.0 +- +- - name: woxikon.de synonyme +- engine: xpath +- shortcut: woxi +- categories: [dictionaries] +- timeout: 5.0 +- disabled: true +- search_url: https://synonyme.woxikon.de/synonyme/{query}.php +- url_xpath: //div[@class="upper-synonyms"]/a/@href +- content_xpath: //div[@class="synonyms-list-group"] +- title_xpath: //div[@class="upper-synonyms"]/a +- no_result_for_http_status: [404] +- about: +- website: https://www.woxikon.de/ +- wikidata_id: # No Wikidata ID +- use_official_api: false +- require_api_key: false +- results: HTML +- language: de +- +- - name: seekr news +- engine: seekr +- shortcut: senews +- categories: news +- seekr_category: news +- disabled: true +- +- - name: seekr images +- engine: seekr +- network: seekr news +- shortcut: seimg +- categories: images +- seekr_category: images +- disabled: true +- +- - name: seekr videos +- engine: seekr +- network: seekr news +- shortcut: sevid +- categories: videos +- seekr_category: videos +- disabled: true +- +- - name: sjp.pwn +- engine: sjp +- shortcut: sjp +- base_url: https://sjp.pwn.pl/ +- timeout: 5.0 +- disabled: true +- +- - name: svgrepo +- engine: svgrepo +- shortcut: svg +- timeout: 10.0 +- disabled: true +- +- - name: wallhaven +- engine: wallhaven +- # api_key: abcdefghijklmnopqrstuvwxyz +- shortcut: wh +- +- # wikimini: online encyclopedia for children +- # The fulltext and title parameter is necessary for Wikimini because +- # sometimes it will not show the results and redirect instead +- - name: wikimini +- engine: xpath +- shortcut: wkmn +- search_url: https://fr.wikimini.org/w/index.php?search={query}&title=Sp%C3%A9cial%3ASearch&fulltext=Search +- url_xpath: //li/div[@class="mw-search-result-heading"]/a/@href +- title_xpath: //li//div[@class="mw-search-result-heading"]/a +- content_xpath: //li/div[@class="searchresult"] +- categories: general +- disabled: true +- about: +- website: https://wikimini.org/ +- wikidata_id: Q3568032 +- use_official_api: false +- require_api_key: false +- results: HTML +- language: fr +- +- - name: wttr.in +- engine: wttr +- shortcut: wttr +- timeout: 9.0 +- +- - name: yummly +- engine: yummly +- shortcut: yum +- disabled: true +- +- - name: brave +- engine: brave +- shortcut: br +- time_range_support: true +- paging: true +- categories: [general, web] +- brave_category: search +- # brave_spellcheck: true +- +- - name: brave.images +- engine: brave +- network: brave +- shortcut: brimg +- categories: [images, web] +- brave_category: images +- +- - name: brave.videos +- engine: brave +- network: brave +- shortcut: brvid +- categories: [videos, web] +- brave_category: videos +- +- - name: brave.news +- engine: brave +- network: brave +- shortcut: brnews +- categories: news +- brave_category: news +- +- - name: lib.rs +- shortcut: lrs +- engine: xpath +- search_url: https://lib.rs/search?q={query} +- results_xpath: /html/body/main/div/ol/li/a +- url_xpath: ./@href +- title_xpath: ./div[@class="h"]/h4 +- content_xpath: ./div[@class="h"]/p +- categories: [it, packages] +- disabled: true +- about: +- website: https://lib.rs +- wikidata_id: Q113486010 +- use_official_api: false +- require_api_key: false +- results: HTML +- +- - name: sourcehut +- shortcut: srht +- engine: xpath +- paging: true +- search_url: https://sr.ht/projects?page={pageno}&search={query} +- results_xpath: (//div[@class="event-list"])[1]/div[@class="event"] +- url_xpath: ./h4/a[2]/@href +- title_xpath: ./h4/a[2] +- content_xpath: ./p +- first_page_num: 1 +- categories: [it, repos] +- disabled: true +- about: +- website: https://sr.ht +- wikidata_id: Q78514485 +- official_api_documentation: https://man.sr.ht/ +- use_official_api: false +- require_api_key: false +- results: HTML +- +- - name: goo +- shortcut: goo +- engine: xpath +- paging: true +- search_url: https://search.goo.ne.jp/web.jsp?MT={query}&FR={pageno}0 +- url_xpath: //div[@class="result"]/p[@class='title fsL1']/a/@href +- title_xpath: //div[@class="result"]/p[@class='title fsL1']/a +- content_xpath: //p[contains(@class,'url fsM')]/following-sibling::p +- first_page_num: 0 +- categories: [general, web] +- disabled: true +- timeout: 4.0 +- about: +- website: https://search.goo.ne.jp +- wikidata_id: Q249044 +- use_official_api: false +- require_api_key: false +- results: HTML +- language: ja +- +- - name: bt4g +- engine: bt4g +- shortcut: bt4g +- +- - name: pkg.go.dev +- engine: xpath +- shortcut: pgo +- search_url: https://pkg.go.dev/search?limit=100&m=package&q={query} +- results_xpath: /html/body/main/div[contains(@class,"SearchResults")]/div[not(@class)]/div[@class="SearchSnippet"] +- url_xpath: ./div[@class="SearchSnippet-headerContainer"]/h2/a/@href +- title_xpath: ./div[@class="SearchSnippet-headerContainer"]/h2/a +- content_xpath: ./p[@class="SearchSnippet-synopsis"] +- categories: [packages, it] +- timeout: 3.0 +- disabled: true +- about: +- website: https://pkg.go.dev/ +- use_official_api: false +- require_api_key: false +- results: HTML +- +-# Doku engine lets you access to any Doku wiki instance: +-# A public one or a privete/corporate one. +-# - name: ubuntuwiki +-# engine: doku +-# shortcut: uw +-# base_url: 'https://doc.ubuntu-fr.org' +- +-# Be careful when enabling this engine if you are +-# running a public instance. Do not expose any sensitive +-# information. You can restrict access by configuring a list +-# of access tokens under tokens. +-# - name: git grep +-# engine: command +-# command: ['git', 'grep', '{{QUERY}}'] +-# shortcut: gg +-# tokens: [] +-# disabled: true +-# delimiter: +-# chars: ':' +-# keys: ['filepath', 'code'] +- +-# Be careful when enabling this engine if you are +-# running a public instance. Do not expose any sensitive +-# information. You can restrict access by configuring a list +-# of access tokens under tokens. +-# - name: locate +-# engine: command +-# command: ['locate', '{{QUERY}}'] +-# shortcut: loc +-# tokens: [] +-# disabled: true +-# delimiter: +-# chars: ' ' +-# keys: ['line'] +- +-# Be careful when enabling this engine if you are +-# running a public instance. Do not expose any sensitive +-# information. You can restrict access by configuring a list +-# of access tokens under tokens. +-# - name: find +-# engine: command +-# command: ['find', '.', '-name', '{{QUERY}}'] +-# query_type: path +-# shortcut: fnd +-# tokens: [] +-# disabled: true +-# delimiter: +-# chars: ' ' +-# keys: ['line'] +- +-# Be careful when enabling this engine if you are +-# running a public instance. Do not expose any sensitive +-# information. You can restrict access by configuring a list +-# of access tokens under tokens. +-# - name: pattern search in files +-# engine: command +-# command: ['fgrep', '{{QUERY}}'] +-# shortcut: fgr +-# tokens: [] +-# disabled: true +-# delimiter: +-# chars: ' ' +-# keys: ['line'] +- +-# Be careful when enabling this engine if you are +-# running a public instance. Do not expose any sensitive +-# information. You can restrict access by configuring a list +-# of access tokens under tokens. +-# - name: regex search in files +-# engine: command +-# command: ['grep', '{{QUERY}}'] +-# shortcut: gr +-# tokens: [] +-# disabled: true +-# delimiter: +-# chars: ' ' +-# keys: ['line'] ++ # ++ # - name: annas archive ++ # engine: annas_archive ++ # disabled: true ++ # shortcut: aa ++ # ++ # # - name: annas articles ++ # # engine: annas_archive ++ # # shortcut: aaa ++ # # # https://docs.searxng.org/dev/engines/online/annas_archive.html ++ # # aa_content: 'journal_article' # book_any .. magazine, standards_document ++ # # aa_ext: 'pdf' # pdf, epub, .. ++ # # aa_sort: 'newest' # newest, oldest, largest, smallest ++ # ++ # - name: apk mirror ++ # engine: apkmirror ++ # timeout: 4.0 ++ # shortcut: apkm ++ # disabled: true ++ # ++ # - name: apple app store ++ # engine: apple_app_store ++ # shortcut: aps ++ # disabled: true ++ # ++ # # Requires Tor ++ # - name: ahmia ++ # engine: ahmia ++ # categories: onions ++ # enable_http: true ++ # shortcut: ah ++ # ++ # - name: anaconda ++ # engine: xpath ++ # paging: true ++ # first_page_num: 0 ++ # search_url: https://anaconda.org/search?q={query}&page={pageno} ++ # results_xpath: //tbody/tr ++ # url_xpath: ./td/h5/a[last()]/@href ++ # title_xpath: ./td/h5 ++ # content_xpath: ./td[h5]/text() ++ # categories: it ++ # timeout: 6.0 ++ # shortcut: conda ++ # disabled: true ++ # ++ # - name: arch linux wiki ++ # engine: archlinux ++ # shortcut: al ++ # ++ # - name: artic ++ # engine: artic ++ # shortcut: arc ++ # timeout: 4.0 ++ # ++ # - name: arxiv ++ # engine: arxiv ++ # shortcut: arx ++ # timeout: 4.0 ++ # ++ # # tmp suspended: dh key too small ++ # # - name: base ++ # # engine: base ++ # # shortcut: bs ++ # ++ # - name: bandcamp ++ # engine: bandcamp ++ # shortcut: bc ++ # categories: music ++ # ++ # - name: wikipedia ++ # engine: wikipedia ++ # shortcut: wp ++ # # add "list" to the array to get results in the results list ++ # display_type: ["infobox"] ++ # base_url: 'https://{language}.wikipedia.org/' ++ # categories: [general] ++ # ++ # - name: bilibili ++ # engine: bilibili ++ # shortcut: bil ++ # disabled: true ++ # ++ # - name: bing ++ # engine: bing ++ # shortcut: bi ++ # disabled: true ++ # ++ # ++ # - name: bing news ++ # engine: bing_news ++ # shortcut: bin ++ # ++ # - name: bing videos ++ # engine: bing_videos ++ # shortcut: biv ++ # ++ # - name: bitbucket ++ # engine: xpath ++ # paging: true ++ # search_url: https://bitbucket.org/repo/all/{pageno}?name={query} ++ # url_xpath: //article[@class="repo-summary"]//a[@class="repo-link"]/@href ++ # title_xpath: //article[@class="repo-summary"]//a[@class="repo-link"] ++ # content_xpath: //article[@class="repo-summary"]/p ++ # categories: [it, repos] ++ # timeout: 4.0 ++ # disabled: true ++ # shortcut: bb ++ # about: ++ # website: https://bitbucket.org/ ++ # wikidata_id: Q2493781 ++ # official_api_documentation: https://developer.atlassian.com/bitbucket ++ # use_official_api: false ++ # require_api_key: false ++ # results: HTML ++ # ++ # - name: btdigg ++ # engine: btdigg ++ # shortcut: bt ++ # disabled: true ++ # ++ # - name: ccc-tv ++ # engine: xpath ++ # paging: false ++ # search_url: https://media.ccc.de/search/?q={query} ++ # url_xpath: //div[@class="caption"]/h3/a/@href ++ # title_xpath: //div[@class="caption"]/h3/a/text() ++ # content_xpath: //div[@class="caption"]/h4/@title ++ # categories: videos ++ # disabled: true ++ # shortcut: c3tv ++ # about: ++ # website: https://media.ccc.de/ ++ # wikidata_id: Q80729951 ++ # official_api_documentation: https://github.com/voc/voctoweb ++ # use_official_api: false ++ # require_api_key: false ++ # results: HTML ++ # # We don't set language: de here because media.ccc.de is not just ++ # # for a German audience. It contains many English videos and many ++ # # German videos have English subtitles. ++ # ++ # - name: openverse ++ # engine: openverse ++ # categories: images ++ # shortcut: opv ++ # ++ # - name: chefkoch ++ # engine: chefkoch ++ # shortcut: chef ++ # # to show premium or plus results too: ++ # # skip_premium: false ++ # ++ # # - name: core.ac.uk ++ # # engine: core ++ # # categories: science ++ # # shortcut: cor ++ # # # get your API key from: https://core.ac.uk/api-keys/register/ ++ # # api_key: 'unset' ++ # ++ # - name: crossref ++ # engine: crossref ++ # shortcut: cr ++ # timeout: 30 ++ # disabled: true ++ # ++ # - name: crowdview ++ # engine: json_engine ++ # shortcut: cv ++ # categories: general ++ # paging: false ++ # search_url: https://crowdview-next-js.onrender.com/api/search-v3?query={query} ++ # results_query: results ++ # url_query: link ++ # title_query: title ++ # content_query: snippet ++ # disabled: true ++ # about: ++ # website: https://crowdview.ai/ ++ # ++ # - name: yep ++ # engine: json_engine ++ # shortcut: yep ++ # categories: general ++ # disabled: true ++ # paging: false ++ # content_html_to_text: true ++ # title_html_to_text: true ++ # search_url: https://api.yep.com/fs/1/?type=web&q={query}&no_correct=false&limit=100 ++ # results_query: 1/results ++ # title_query: title ++ # url_query: url ++ # content_query: snippet ++ # about: ++ # website: https://yep.com ++ # use_official_api: false ++ # require_api_key: false ++ # results: JSON ++ # ++ # - name: curlie ++ # engine: xpath ++ # shortcut: cl ++ # categories: general ++ # disabled: true ++ # paging: true ++ # lang_all: '' ++ # search_url: https://curlie.org/search?q={query}&lang={lang}&start={pageno}&stime=92452189 ++ # page_size: 20 ++ # results_xpath: //div[@id="site-list-content"]/div[@class="site-item"] ++ # url_xpath: ./div[@class="title-and-desc"]/a/@href ++ # title_xpath: ./div[@class="title-and-desc"]/a/div ++ # content_xpath: ./div[@class="title-and-desc"]/div[@class="site-descr"] ++ # about: ++ # website: https://curlie.org/ ++ # wikidata_id: Q60715723 ++ # use_official_api: false ++ # require_api_key: false ++ # results: HTML ++ # ++ # - name: currency ++ # engine: currency_convert ++ # categories: general ++ # shortcut: cc ++ # ++ # - name: deezer ++ # engine: deezer ++ # shortcut: dz ++ # disabled: true ++ # ++ # - name: deviantart ++ # engine: deviantart ++ # shortcut: da ++ # timeout: 3.0 ++ # ++ # - name: ddg definitions ++ # engine: duckduckgo_definitions ++ # shortcut: ddd ++ # weight: 2 ++ # disabled: true ++ # tests: *tests_infobox ++ # ++ # # cloudflare protected ++ # # - name: digbt ++ # # engine: digbt ++ # # shortcut: dbt ++ # # timeout: 6.0 ++ # # disabled: true ++ # ++ # - name: docker hub ++ # engine: docker_hub ++ # shortcut: dh ++ # categories: [it, packages] ++ # ++ # - name: erowid ++ # engine: xpath ++ # paging: true ++ # first_page_num: 0 ++ # page_size: 30 ++ # search_url: https://www.erowid.org/search.php?q={query}&s={pageno} ++ # url_xpath: //dl[@class="results-list"]/dt[@class="result-title"]/a/@href ++ # title_xpath: //dl[@class="results-list"]/dt[@class="result-title"]/a/text() ++ # content_xpath: //dl[@class="results-list"]/dd[@class="result-details"] ++ # categories: [] ++ # shortcut: ew ++ # disabled: true ++ # about: ++ # website: https://www.erowid.org/ ++ # wikidata_id: Q1430691 ++ # official_api_documentation: ++ # use_official_api: false ++ # require_api_key: false ++ # results: HTML ++ # ++ # # - name: elasticsearch ++ # # shortcut: es ++ # # engine: elasticsearch ++ # # base_url: http://localhost:9200 ++ # # username: elastic ++ # # password: changeme ++ # # index: my-index ++ # # # available options: match, simple_query_string, term, terms, custom ++ # # query_type: match ++ # # # if query_type is set to custom, provide your query here ++ # # #custom_query_json: {"query":{"match_all": {}}} ++ # # #show_metadata: false ++ # # disabled: true ++ # ++ # - name: wikidata ++ # engine: wikidata ++ # shortcut: wd ++ # timeout: 3.0 ++ # weight: 2 ++ # # add "list" to the array to get results in the results list ++ # display_type: ["infobox"] ++ # tests: *tests_infobox ++ # categories: [general] ++ # ++ # - name: duckduckgo ++ # engine: duckduckgo ++ # shortcut: ddg ++ # ++ # - name: duckduckgo images ++ # engine: duckduckgo_images ++ # shortcut: ddi ++ # timeout: 3.0 ++ # disabled: true ++ # ++ # - name: duckduckgo weather ++ # engine: duckduckgo_weather ++ # shortcut: ddw ++ # disabled: true ++ # ++ # - name: apple maps ++ # engine: apple_maps ++ # shortcut: apm ++ # disabled: true ++ # timeout: 5.0 ++ # ++ # - name: emojipedia ++ # engine: emojipedia ++ # timeout: 4.0 ++ # shortcut: em ++ # disabled: true ++ # ++ # - name: tineye ++ # engine: tineye ++ # shortcut: tin ++ # timeout: 9.0 ++ # disabled: true ++ # ++ # - name: etymonline ++ # engine: xpath ++ # paging: true ++ # search_url: https://etymonline.com/search?page={pageno}&q={query} ++ # url_xpath: //a[contains(@class, "word__name--")]/@href ++ # title_xpath: //a[contains(@class, "word__name--")] ++ # content_xpath: //section[contains(@class, "word__defination")] ++ # first_page_num: 1 ++ # shortcut: et ++ # categories: [dictionaries] ++ # about: ++ # website: https://www.etymonline.com/ ++ # wikidata_id: Q1188617 ++ # official_api_documentation: ++ # use_official_api: false ++ # require_api_key: false ++ # results: HTML ++ # ++ # # - name: ebay ++ # # engine: ebay ++ # # shortcut: eb ++ # # base_url: 'https://www.ebay.com' ++ # # disabled: true ++ # # timeout: 5 ++ # ++ # - name: 1x ++ # engine: www1x ++ # shortcut: 1x ++ # timeout: 3.0 ++ # disabled: true ++ # ++ # - name: fdroid ++ # engine: fdroid ++ # shortcut: fd ++ # disabled: true ++ # ++ # - name: flickr ++ # categories: images ++ # shortcut: fl ++ # # You can use the engine using the official stable API, but you need an API ++ # # key, see: https://www.flickr.com/services/apps/create/ ++ # # engine: flickr ++ # # api_key: 'apikey' # required! ++ # # Or you can use the html non-stable engine, activated by default ++ # engine: flickr_noapi ++ # ++ # - name: free software directory ++ # engine: mediawiki ++ # shortcut: fsd ++ # categories: [it, software wikis] ++ # base_url: https://directory.fsf.org/ ++ # search_type: title ++ # timeout: 5.0 ++ # disabled: true ++ # about: ++ # website: https://directory.fsf.org/ ++ # wikidata_id: Q2470288 ++ # ++ # # - name: freesound ++ # # engine: freesound ++ # # shortcut: fnd ++ # # disabled: true ++ # # timeout: 15.0 ++ # # API key required, see: https://freesound.org/docs/api/overview.html ++ # # api_key: MyAPIkey ++ # ++ # - name: frinkiac ++ # engine: frinkiac ++ # shortcut: frk ++ # disabled: true ++ # ++ # - name: genius ++ # engine: genius ++ # shortcut: gen ++ # ++ # - name: gentoo ++ # engine: gentoo ++ # shortcut: ge ++ # timeout: 10.0 ++ # ++ # - name: gitlab ++ # engine: json_engine ++ # paging: true ++ # search_url: https://gitlab.com/api/v4/projects?search={query}&page={pageno} ++ # url_query: web_url ++ # title_query: name_with_namespace ++ # content_query: description ++ # page_size: 20 ++ # categories: [it, repos] ++ # shortcut: gl ++ # timeout: 10.0 ++ # disabled: true ++ # about: ++ # website: https://about.gitlab.com/ ++ # wikidata_id: Q16639197 ++ # official_api_documentation: https://docs.gitlab.com/ee/api/ ++ # use_official_api: false ++ # require_api_key: false ++ # results: JSON ++ # ++ # - name: github ++ # engine: github ++ # shortcut: gh ++ # ++ # # This a Gitea service. If you would like to use a different instance, ++ # # change codeberg.org to URL of the desired Gitea host. Or you can create a ++ # # new engine by copying this and changing the name, shortcut and search_url. ++ # ++ # - name: codeberg ++ # engine: json_engine ++ # search_url: https://codeberg.org/api/v1/repos/search?q={query}&limit=10 ++ # url_query: html_url ++ # title_query: name ++ # content_query: description ++ # categories: [it, repos] ++ # shortcut: cb ++ # disabled: true ++ # about: ++ # website: https://codeberg.org/ ++ # wikidata_id: ++ # official_api_documentation: https://try.gitea.io/api/swagger ++ # use_official_api: false ++ # require_api_key: false ++ # results: JSON ++ # ++ # ++ # ++ # - name: google news ++ # engine: google_news ++ # shortcut: gon ++ # # additional_tests: ++ # # android: *test_android ++ # ++ # - name: google videos ++ # engine: google_videos ++ # shortcut: gov ++ # # additional_tests: ++ # # android: *test_android ++ # ++ # - name: google scholar ++ # engine: google_scholar ++ # shortcut: gos ++ # ++ # - name: google play apps ++ # engine: google_play ++ # categories: [files, apps] ++ # shortcut: gpa ++ # play_categ: apps ++ # disabled: true ++ # ++ # - name: google play movies ++ # engine: google_play ++ # categories: videos ++ # shortcut: gpm ++ # play_categ: movies ++ # disabled: true ++ # ++ # - name: material icons ++ # engine: material_icons ++ # categories: images ++ # shortcut: mi ++ # disabled: true ++ # ++ # - name: gpodder ++ # engine: json_engine ++ # shortcut: gpod ++ # timeout: 4.0 ++ # paging: false ++ # search_url: https://gpodder.net/search.json?q={query} ++ # url_query: url ++ # title_query: title ++ # content_query: description ++ # page_size: 19 ++ # categories: music ++ # disabled: true ++ # about: ++ # website: https://gpodder.net ++ # wikidata_id: Q3093354 ++ # official_api_documentation: https://gpoddernet.readthedocs.io/en/latest/api/ ++ # use_official_api: false ++ # requires_api_key: false ++ # results: JSON ++ # ++ # - name: habrahabr ++ # engine: xpath ++ # paging: true ++ # search_url: https://habr.com/en/search/page{pageno}/?q={query} ++ # results_xpath: //article[contains(@class, "tm-articles-list__item")] ++ # url_xpath: .//a[@class="tm-title__link"]/@href ++ # title_xpath: .//a[@class="tm-title__link"] ++ # content_xpath: .//div[contains(@class, "article-formatted-body")] ++ # categories: it ++ # timeout: 4.0 ++ # disabled: true ++ # shortcut: habr ++ # about: ++ # website: https://habr.com/ ++ # wikidata_id: Q4494434 ++ # official_api_documentation: https://habr.com/en/docs/help/api/ ++ # use_official_api: false ++ # require_api_key: false ++ # results: HTML ++ # ++ # - name: hoogle ++ # engine: xpath ++ # paging: true ++ # search_url: https://hoogle.haskell.org/?hoogle={query}&start={pageno} ++ # results_xpath: '//div[@class="result"]' ++ # title_xpath: './/div[@class="ans"]//a' ++ # url_xpath: './/div[@class="ans"]//a/@href' ++ # content_xpath: './/div[@class="from"]' ++ # page_size: 20 ++ # categories: [it, packages] ++ # shortcut: ho ++ # about: ++ # website: https://hoogle.haskell.org/ ++ # wikidata_id: Q34010 ++ # official_api_documentation: https://hackage.haskell.org/api ++ # use_official_api: false ++ # require_api_key: false ++ # results: JSON ++ # ++ # - name: imdb ++ # engine: imdb ++ # shortcut: imdb ++ # timeout: 6.0 ++ # disabled: true ++ # ++ # - name: ina ++ # engine: ina ++ # shortcut: in ++ # timeout: 6.0 ++ # disabled: true ++ # ++ # - name: invidious ++ # engine: invidious ++ # # Instanes will be selected randomly, see https://api.invidious.io/ for ++ # # instances that are stable (good uptime) and close to you. ++ # base_url: ++ # - https://invidious.io.lol ++ # - https://invidious.fdn.fr ++ # - https://yt.artemislena.eu ++ # - https://invidious.tiekoetter.com ++ # - https://invidious.flokinet.to ++ # - https://vid.puffyan.us ++ # - https://invidious.privacydev.net ++ # - https://inv.tux.pizza ++ # shortcut: iv ++ # timeout: 3.0 ++ # disabled: true ++ # ++ # - name: jisho ++ # engine: jisho ++ # shortcut: js ++ # timeout: 3.0 ++ # disabled: true ++ # ++ # - name: kickass ++ # engine: kickass ++ # shortcut: kc ++ # timeout: 4.0 ++ # disabled: true ++ # ++ # - name: lemmy communities ++ # engine: lemmy ++ # lemmy_type: Communities ++ # shortcut: leco ++ # ++ # - name: lemmy users ++ # engine: lemmy ++ # network: lemmy communities ++ # lemmy_type: Users ++ # shortcut: leus ++ # ++ # - name: lemmy posts ++ # engine: lemmy ++ # network: lemmy communities ++ # lemmy_type: Posts ++ # shortcut: lepo ++ # ++ # - name: lemmy comments ++ # engine: lemmy ++ # network: lemmy communities ++ # lemmy_type: Comments ++ # shortcut: lecom ++ # ++ # - name: library genesis ++ # engine: xpath ++ # search_url: https://libgen.fun/search.php?req={query} ++ # url_xpath: //a[contains(@href,"get.php?md5")]/@href ++ # title_xpath: //a[contains(@href,"book/")]/text()[1] ++ # content_xpath: //td/a[1][contains(@href,"=author")]/text() ++ # categories: files ++ # timeout: 7.0 ++ # disabled: true ++ # shortcut: lg ++ # about: ++ # website: https://libgen.fun/ ++ # wikidata_id: Q22017206 ++ # official_api_documentation: ++ # use_official_api: false ++ # require_api_key: false ++ # results: HTML ++ # ++ # - name: z-library ++ # engine: zlibrary ++ # shortcut: zlib ++ # categories: files ++ # timeout: 7.0 ++ # ++ # - name: library of congress ++ # engine: loc ++ # shortcut: loc ++ # categories: images ++ # ++ # - name: lingva ++ # engine: lingva ++ # shortcut: lv ++ # # set lingva instance in url, by default it will use the official instance ++ # # url: https://lingva.ml ++ # ++ # - name: lobste.rs ++ # engine: xpath ++ # search_url: https://lobste.rs/search?utf8=%E2%9C%93&q={query}&what=stories&order=relevance ++ # results_xpath: //li[contains(@class, "story")] ++ # url_xpath: .//a[@class="u-url"]/@href ++ # title_xpath: .//a[@class="u-url"] ++ # content_xpath: .//a[@class="domain"] ++ # categories: it ++ # shortcut: lo ++ # timeout: 5.0 ++ # disabled: true ++ # about: ++ # website: https://lobste.rs/ ++ # wikidata_id: Q60762874 ++ # official_api_documentation: ++ # use_official_api: false ++ # require_api_key: false ++ # results: HTML ++ # ++ # - name: azlyrics ++ # shortcut: lyrics ++ # engine: xpath ++ # timeout: 4.0 ++ # disabled: true ++ # categories: [music, lyrics] ++ # paging: true ++ # search_url: https://search.azlyrics.com/search.php?q={query}&w=lyrics&p={pageno} ++ # url_xpath: //td[@class="text-left visitedlyr"]/a/@href ++ # title_xpath: //span/b/text() ++ # content_xpath: //td[@class="text-left visitedlyr"]/a/small ++ # about: ++ # website: https://azlyrics.com ++ # wikidata_id: Q66372542 ++ # official_api_documentation: ++ # use_official_api: false ++ # require_api_key: false ++ # results: HTML ++ # ++ # - name: metacpan ++ # engine: metacpan ++ # shortcut: cpan ++ # disabled: true ++ # number_of_results: 20 ++ # ++ # # - name: meilisearch ++ # # engine: meilisearch ++ # # shortcut: mes ++ # # enable_http: true ++ # # base_url: http://localhost:7700 ++ # # index: my-index ++ # ++ # - name: mixcloud ++ # engine: mixcloud ++ # shortcut: mc ++ # ++ # # MongoDB engine ++ # # Required dependency: pymongo ++ # # - name: mymongo ++ # # engine: mongodb ++ # # shortcut: md ++ # # exact_match_only: false ++ # # host: '127.0.0.1' ++ # # port: 27017 ++ # # enable_http: true ++ # # results_per_page: 20 ++ # # database: 'business' ++ # # collection: 'reviews' # name of the db collection ++ # # key: 'name' # key in the collection to search for ++ # ++ # - name: mwmbl ++ # engine: mwmbl ++ # # api_url: https://api.mwmbl.org ++ # shortcut: mwm ++ # disabled: true ++ # ++ # - name: npm ++ # engine: json_engine ++ # paging: true ++ # first_page_num: 0 ++ # search_url: https://api.npms.io/v2/search?q={query}&size=25&from={pageno} ++ # results_query: results ++ # url_query: package/links/npm ++ # title_query: package/name ++ # content_query: package/description ++ # page_size: 25 ++ # categories: [it, packages] ++ # disabled: true ++ # timeout: 5.0 ++ # shortcut: npm ++ # about: ++ # website: https://npms.io/ ++ # wikidata_id: Q7067518 ++ # official_api_documentation: https://api-docs.npms.io/ ++ # use_official_api: false ++ # require_api_key: false ++ # results: JSON ++ # ++ # - name: nyaa ++ # engine: nyaa ++ # shortcut: nt ++ # disabled: true ++ # ++ # - name: mankier ++ # engine: json_engine ++ # search_url: https://www.mankier.com/api/v2/mans/?q={query} ++ # results_query: results ++ # url_query: url ++ # title_query: name ++ # content_query: description ++ # categories: it ++ # shortcut: man ++ # about: ++ # website: https://www.mankier.com/ ++ # official_api_documentation: https://www.mankier.com/api ++ # use_official_api: true ++ # require_api_key: false ++ # results: JSON ++ # ++ # - name: odysee ++ # engine: odysee ++ # shortcut: od ++ # disabled: true ++ # ++ # - name: openairedatasets ++ # engine: json_engine ++ # paging: true ++ # search_url: https://api.openaire.eu/search/datasets?format=json&page={pageno}&size=10&title={query} ++ # results_query: response/results/result ++ # url_query: metadata/oaf:entity/oaf:result/children/instance/webresource/url/$ ++ # title_query: metadata/oaf:entity/oaf:result/title/$ ++ # content_query: metadata/oaf:entity/oaf:result/description/$ ++ # content_html_to_text: true ++ # categories: "science" ++ # shortcut: oad ++ # timeout: 5.0 ++ # about: ++ # website: https://www.openaire.eu/ ++ # wikidata_id: Q25106053 ++ # official_api_documentation: https://api.openaire.eu/ ++ # use_official_api: false ++ # require_api_key: false ++ # results: JSON ++ # ++ # - name: openairepublications ++ # engine: json_engine ++ # paging: true ++ # search_url: https://api.openaire.eu/search/publications?format=json&page={pageno}&size=10&title={query} ++ # results_query: response/results/result ++ # url_query: metadata/oaf:entity/oaf:result/children/instance/webresource/url/$ ++ # title_query: metadata/oaf:entity/oaf:result/title/$ ++ # content_query: metadata/oaf:entity/oaf:result/description/$ ++ # content_html_to_text: true ++ # categories: science ++ # shortcut: oap ++ # timeout: 5.0 ++ # about: ++ # website: https://www.openaire.eu/ ++ # wikidata_id: Q25106053 ++ # official_api_documentation: https://api.openaire.eu/ ++ # use_official_api: false ++ # require_api_key: false ++ # results: JSON ++ # ++ # # - name: opensemanticsearch ++ # # engine: opensemantic ++ # # shortcut: oss ++ # # base_url: 'http://localhost:8983/solr/opensemanticsearch/' ++ # ++ # - name: openstreetmap ++ # engine: openstreetmap ++ # shortcut: osm ++ # ++ # - name: openrepos ++ # engine: xpath ++ # paging: true ++ # search_url: https://openrepos.net/search/node/{query}?page={pageno} ++ # url_xpath: //li[@class="search-result"]//h3[@class="title"]/a/@href ++ # title_xpath: //li[@class="search-result"]//h3[@class="title"]/a ++ # content_xpath: //li[@class="search-result"]//div[@class="search-snippet-info"]//p[@class="search-snippet"] ++ # categories: files ++ # timeout: 4.0 ++ # disabled: true ++ # shortcut: or ++ # about: ++ # website: https://openrepos.net/ ++ # wikidata_id: ++ # official_api_documentation: ++ # use_official_api: false ++ # require_api_key: false ++ # results: HTML ++ # ++ # - name: packagist ++ # engine: json_engine ++ # paging: true ++ # search_url: https://packagist.org/search.json?q={query}&page={pageno} ++ # results_query: results ++ # url_query: url ++ # title_query: name ++ # content_query: description ++ # categories: [it, packages] ++ # disabled: true ++ # timeout: 5.0 ++ # shortcut: pack ++ # about: ++ # website: https://packagist.org ++ # wikidata_id: Q108311377 ++ # official_api_documentation: https://packagist.org/apidoc ++ # use_official_api: true ++ # require_api_key: false ++ # results: JSON ++ # ++ # - name: pdbe ++ # engine: pdbe ++ # shortcut: pdb ++ # # Hide obsolete PDB entries. Default is not to hide obsolete structures ++ # # hide_obsolete: false ++ # ++ # - name: photon ++ # engine: photon ++ # shortcut: ph ++ # ++ # - name: piped ++ # engine: piped ++ # shortcut: ppd ++ # categories: videos ++ # piped_filter: videos ++ # timeout: 3.0 ++ # ++ # # URL to use as link and for embeds ++ # frontend_url: https://srv.piped.video ++ # # Instance will be selected randomly, for more see https://piped-instances.kavin.rocks/ ++ # backend_url: ++ # - https://pipedapi.kavin.rocks ++ # - https://pipedapi-libre.kavin.rocks ++ # - https://pipedapi.adminforge.de ++ # ++ # - name: piped.music ++ # engine: piped ++ # network: piped ++ # shortcut: ppdm ++ # categories: music ++ # piped_filter: music_songs ++ # timeout: 3.0 ++ # ++ # - name: piratebay ++ # engine: piratebay ++ # shortcut: tpb ++ # # You may need to change this URL to a proxy if piratebay is blocked in your ++ # # country ++ # url: https://thepiratebay.org/ ++ # timeout: 3.0 ++ # ++ # # Required dependency: psychopg2 ++ # # - name: postgresql ++ # # engine: postgresql ++ # # database: postgres ++ # # username: postgres ++ # # password: postgres ++ # # limit: 10 ++ # # query_str: 'SELECT * from my_table WHERE my_column = %(query)s' ++ # # shortcut : psql ++ # ++ # - name: pub.dev ++ # engine: xpath ++ # shortcut: pd ++ # search_url: https://pub.dev/packages?q={query}&page={pageno} ++ # paging: true ++ # results_xpath: //div[contains(@class,"packages-item")] ++ # url_xpath: ./div/h3/a/@href ++ # title_xpath: ./div/h3/a ++ # content_xpath: ./div/div/div[contains(@class,"packages-description")]/span ++ # categories: [packages, it] ++ # timeout: 3.0 ++ # disabled: true ++ # first_page_num: 1 ++ # about: ++ # website: https://pub.dev/ ++ # official_api_documentation: https://pub.dev/help/api ++ # use_official_api: false ++ # require_api_key: false ++ # results: HTML ++ # ++ # - name: pubmed ++ # engine: pubmed ++ # shortcut: pub ++ # timeout: 3.0 ++ # ++ # - name: pypi ++ # shortcut: pypi ++ # engine: xpath ++ # paging: true ++ # search_url: https://pypi.org/search/?q={query}&page={pageno} ++ # results_xpath: /html/body/main/div/div/div/form/div/ul/li/a[@class="package-snippet"] ++ # url_xpath: ./@href ++ # title_xpath: ./h3/span[@class="package-snippet__name"] ++ # content_xpath: ./p ++ # suggestion_xpath: /html/body/main/div/div/div/form/div/div[@class="callout-block"]/p/span/a[@class="link"] ++ # first_page_num: 1 ++ # categories: [it, packages] ++ # about: ++ # website: https://pypi.org ++ # wikidata_id: Q2984686 ++ # official_api_documentation: https://warehouse.readthedocs.io/api-reference/index.html ++ # use_official_api: false ++ # require_api_key: false ++ # results: HTML ++ # ++ # - name: qwant ++ # qwant_categ: web ++ # engine: qwant ++ # shortcut: qw ++ # categories: [general, web] ++ # additional_tests: ++ # rosebud: *test_rosebud ++ # ++ # - name: qwant news ++ # qwant_categ: news ++ # engine: qwant ++ # shortcut: qwn ++ # categories: news ++ # network: qwant ++ # ++ # - name: qwant images ++ # qwant_categ: images ++ # engine: qwant ++ # shortcut: qwi ++ # categories: [images, web] ++ # network: qwant ++ # ++ # - name: qwant videos ++ # qwant_categ: videos ++ # engine: qwant ++ # shortcut: qwv ++ # categories: [videos, web] ++ # network: qwant ++ # ++ # # - name: library ++ # # engine: recoll ++ # # shortcut: lib ++ # # base_url: 'https://recoll.example.org/' ++ # # search_dir: '' ++ # # mount_prefix: /export ++ # # dl_prefix: 'https://download.example.org' ++ # # timeout: 30.0 ++ # # categories: files ++ # # disabled: true ++ # ++ # # - name: recoll library reference ++ # # engine: recoll ++ # # base_url: 'https://recoll.example.org/' ++ # # search_dir: reference ++ # # mount_prefix: /export ++ # # dl_prefix: 'https://download.example.org' ++ # # shortcut: libr ++ # # timeout: 30.0 ++ # # categories: files ++ # # disabled: true ++ # ++ # - name: reddit ++ # engine: reddit ++ # shortcut: re ++ # page_size: 25 ++ # ++ # # Required dependency: redis ++ # # - name: myredis ++ # # shortcut : rds ++ # # engine: redis_server ++ # # exact_match_only: false ++ # # host: '127.0.0.1' ++ # # port: 6379 ++ # # enable_http: true ++ # # password: '' ++ # # db: 0 ++ # ++ # # tmp suspended: bad certificate ++ # # - name: scanr structures ++ # # shortcut: scs ++ # # engine: scanr_structures ++ # # disabled: true ++ # ++ # - name: sepiasearch ++ # engine: sepiasearch ++ # shortcut: sep ++ # ++ # - name: soundcloud ++ # engine: soundcloud ++ # shortcut: sc ++ # ++ # - name: stackoverflow ++ # engine: stackexchange ++ # shortcut: st ++ # api_site: 'stackoverflow' ++ # categories: [it, q&a] ++ # ++ # - name: askubuntu ++ # engine: stackexchange ++ # shortcut: ubuntu ++ # api_site: 'askubuntu' ++ # categories: [it, q&a] ++ # ++ # - name: internetarchivescholar ++ # engine: internet_archive_scholar ++ # shortcut: ias ++ # timeout: 5.0 ++ # ++ # - name: superuser ++ # engine: stackexchange ++ # shortcut: su ++ # api_site: 'superuser' ++ # categories: [it, q&a] ++ # ++ # - name: searchcode code ++ # engine: searchcode_code ++ # shortcut: scc ++ # disabled: true ++ # ++ # - name: framalibre ++ # engine: framalibre ++ # shortcut: frl ++ # disabled: true ++ # ++ # # - name: searx ++ # # engine: searx_engine ++ # # shortcut: se ++ # # instance_urls : ++ # # - http://127.0.0.1:8888/ ++ # # - ... ++ # # disabled: true ++ # ++ # - name: semantic scholar ++ # engine: semantic_scholar ++ # disabled: true ++ # shortcut: se ++ # ++ # # Spotify needs API credentials ++ # # - name: spotify ++ # # engine: spotify ++ # # shortcut: stf ++ # # api_client_id: ******* ++ # # api_client_secret: ******* ++ # ++ # # - name: solr ++ # # engine: solr ++ # # shortcut: slr ++ # # base_url: http://localhost:8983 ++ # # collection: collection_name ++ # # sort: '' # sorting: asc or desc ++ # # field_list: '' # comma separated list of field names to display on the UI ++ # # default_fields: '' # default field to query ++ # # query_fields: '' # query fields ++ # # enable_http: true ++ # ++ # # - name: springer nature ++ # # engine: springer ++ # # # get your API key from: https://dev.springernature.com/signup ++ # # # working API key, for test & debug: "a69685087d07eca9f13db62f65b8f601" ++ # # api_key: 'unset' ++ # # shortcut: springer ++ # # timeout: 15.0 ++ # ++ # - name: startpage ++ # engine: startpage ++ # shortcut: sp ++ # timeout: 6.0 ++ # disabled: true ++ # additional_tests: ++ # rosebud: *test_rosebud ++ # ++ # - name: tokyotoshokan ++ # engine: tokyotoshokan ++ # shortcut: tt ++ # timeout: 6.0 ++ # disabled: true ++ # ++ # - name: solidtorrents ++ # engine: solidtorrents ++ # shortcut: solid ++ # timeout: 4.0 ++ # base_url: ++ # - https://solidtorrents.to ++ # - https://bitsearch.to ++ # ++ # # For this demo of the sqlite engine download: ++ # # https://liste.mediathekview.de/filmliste-v2.db.bz2 ++ # # and unpack into searx/data/filmliste-v2.db ++ # # Query to test: "!demo concert" ++ # # ++ # # - name: demo ++ # # engine: sqlite ++ # # shortcut: demo ++ # # categories: general ++ # # result_template: default.html ++ # # database: searx/data/filmliste-v2.db ++ # # query_str: >- ++ # # SELECT title || ' (' || time(duration, 'unixepoch') || ')' AS title, ++ # # COALESCE( NULLIF(url_video_hd,''), NULLIF(url_video_sd,''), url_video) AS url, ++ # # description AS content ++ # # FROM film ++ # # WHERE title LIKE :wildcard OR description LIKE :wildcard ++ # # ORDER BY duration DESC ++ # ++ # - name: tagesschau ++ # engine: tagesschau ++ # shortcut: ts ++ # disabled: true ++ # ++ # - name: tmdb ++ # engine: xpath ++ # paging: true ++ # search_url: https://www.themoviedb.org/search?page={pageno}&query={query} ++ # results_xpath: //div[contains(@class,"movie") or contains(@class,"tv")]//div[contains(@class,"card")] ++ # url_xpath: .//div[contains(@class,"poster")]/a/@href ++ # thumbnail_xpath: .//img/@src ++ # title_xpath: .//div[contains(@class,"title")]//h2 ++ # content_xpath: .//div[contains(@class,"overview")] ++ # shortcut: tm ++ # disabled: true ++ # ++ # # Requires Tor ++ # - name: torch ++ # engine: xpath ++ # paging: true ++ # search_url: ++ # http://xmh57jrknzkhv6y3ls3ubitzfqnkrwxhopf5aygthi7d6rplyvk3noyd.onion/cgi-bin/omega/omega?P={query}&DEFAULTOP=and ++ # results_xpath: //table//tr ++ # url_xpath: ./td[2]/a ++ # title_xpath: ./td[2]/b ++ # content_xpath: ./td[2]/small ++ # categories: onions ++ # enable_http: true ++ # shortcut: tch ++ # ++ # # torznab engine lets you query any torznab compatible indexer. Using this ++ # # engine in combination with Jackett opens the possibility to query a lot of ++ # # public and private indexers directly from SearXNG. More details at: ++ # # https://docs.searxng.org/dev/engines/online/torznab.html ++ # # ++ # # - name: Torznab EZTV ++ # # engine: torznab ++ # # shortcut: eztv ++ # # base_url: http://localhost:9117/api/v2.0/indexers/eztv/results/torznab ++ # # enable_http: true # if using localhost ++ # # api_key: xxxxxxxxxxxxxxx ++ # # show_magnet_links: true ++ # # show_torrent_files: false ++ # # # https://github.com/Jackett/Jackett/wiki/Jackett-Categories ++ # # torznab_categories: # optional ++ # # - 2000 ++ # # - 5000 ++ # ++ # - name: twitter ++ # shortcut: tw ++ # engine: twitter ++ # disabled: true ++ # ++ # # tmp suspended - too slow, too many errors ++ # # - name: urbandictionary ++ # # engine : xpath ++ # # search_url : https://www.urbandictionary.com/define.php?term={query} ++ # # url_xpath : //*[@class="word"]/@href ++ # # title_xpath : //*[@class="def-header"] ++ # # content_xpath: //*[@class="meaning"] ++ # # shortcut: ud ++ # ++ # - name: unsplash ++ # engine: unsplash ++ # shortcut: us ++ # ++ # - name: yahoo ++ # engine: yahoo ++ # shortcut: yh ++ # disabled: true ++ # ++ # - name: yahoo news ++ # engine: yahoo_news ++ # shortcut: yhn ++ # ++ # - name: youtube ++ # shortcut: yt ++ # # You can use the engine using the official stable API, but you need an API ++ # # key See: https://console.developers.google.com/project ++ # # ++ # # engine: youtube_api ++ # # api_key: 'apikey' # required! ++ # # ++ # # Or you can use the html non-stable engine, activated by default ++ # engine: youtube_noapi ++ # ++ # - name: dailymotion ++ # engine: dailymotion ++ # shortcut: dm ++ # ++ # - name: vimeo ++ # engine: vimeo ++ # shortcut: vm ++ # ++ # - name: wiby ++ # engine: json_engine ++ # paging: true ++ # search_url: https://wiby.me/json/?q={query}&p={pageno} ++ # url_query: URL ++ # title_query: Title ++ # content_query: Snippet ++ # categories: [general, web] ++ # shortcut: wib ++ # disabled: true ++ # about: ++ # website: https://wiby.me/ ++ # ++ # - name: alexandria ++ # engine: json_engine ++ # shortcut: alx ++ # categories: general ++ # paging: true ++ # search_url: https://api.alexandria.org/?a=1&q={query}&p={pageno} ++ # results_query: results ++ # title_query: title ++ # url_query: url ++ # content_query: snippet ++ # timeout: 1.5 ++ # disabled: true ++ # about: ++ # website: https://alexandria.org/ ++ # official_api_documentation: https://github.com/alexandria-org/alexandria-api/raw/master/README.md ++ # use_official_api: true ++ # require_api_key: false ++ # results: JSON ++ # ++ # - name: wikibooks ++ # engine: mediawiki ++ # weight: 0.5 ++ # shortcut: wb ++ # categories: [general, wikimedia] ++ # base_url: "https://{language}.wikibooks.org/" ++ # search_type: text ++ # disabled: true ++ # about: ++ # website: https://www.wikibooks.org/ ++ # wikidata_id: Q367 ++ # ++ # - name: wikinews ++ # engine: mediawiki ++ # shortcut: wn ++ # categories: [news, wikimedia] ++ # base_url: "https://{language}.wikinews.org/" ++ # search_type: text ++ # srsort: create_timestamp_desc ++ # about: ++ # website: https://www.wikinews.org/ ++ # wikidata_id: Q964 ++ # ++ # - name: wikiquote ++ # engine: mediawiki ++ # weight: 0.5 ++ # shortcut: wq ++ # categories: [general, wikimedia] ++ # base_url: "https://{language}.wikiquote.org/" ++ # search_type: text ++ # disabled: true ++ # additional_tests: ++ # rosebud: *test_rosebud ++ # about: ++ # website: https://www.wikiquote.org/ ++ # wikidata_id: Q369 ++ # ++ # - name: wikisource ++ # engine: mediawiki ++ # weight: 0.5 ++ # shortcut: ws ++ # categories: [general, wikimedia] ++ # base_url: "https://{language}.wikisource.org/" ++ # search_type: text ++ # disabled: true ++ # about: ++ # website: https://www.wikisource.org/ ++ # wikidata_id: Q263 ++ # ++ # - name: wikispecies ++ # engine: mediawiki ++ # shortcut: wsp ++ # categories: [general, science, wikimedia] ++ # base_url: "https://species.wikimedia.org/" ++ # search_type: text ++ # disabled: true ++ # about: ++ # website: https://species.wikimedia.org/ ++ # wikidata_id: Q13679 ++ # ++ # - name: wiktionary ++ # engine: mediawiki ++ # shortcut: wt ++ # categories: [dictionaries, wikimedia] ++ # base_url: "https://{language}.wiktionary.org/" ++ # search_type: text ++ # about: ++ # website: https://www.wiktionary.org/ ++ # wikidata_id: Q151 ++ # ++ # - name: wikiversity ++ # engine: mediawiki ++ # weight: 0.5 ++ # shortcut: wv ++ # categories: [general, wikimedia] ++ # base_url: "https://{language}.wikiversity.org/" ++ # search_type: text ++ # disabled: true ++ # about: ++ # website: https://www.wikiversity.org/ ++ # wikidata_id: Q370 ++ # ++ # - name: wikivoyage ++ # engine: mediawiki ++ # weight: 0.5 ++ # shortcut: wy ++ # categories: [general, wikimedia] ++ # base_url: "https://{language}.wikivoyage.org/" ++ # search_type: text ++ # disabled: true ++ # about: ++ # website: https://www.wikivoyage.org/ ++ # wikidata_id: Q373 ++ # ++ # - name: wikicommons.images ++ # engine: wikicommons ++ # shortcut: wc ++ # categories: images ++ # number_of_results: 10 ++ # ++ # - name: wolframalpha ++ # shortcut: wa ++ # # You can use the engine using the official stable API, but you need an API ++ # # key. See: https://products.wolframalpha.com/api/ ++ # # ++ # # engine: wolframalpha_api ++ # # api_key: '' ++ # # ++ # # Or you can use the html non-stable engine, activated by default ++ # engine: wolframalpha_noapi ++ # timeout: 6.0 ++ # categories: general ++ # disabled: true ++ # ++ # - name: dictzone ++ # engine: dictzone ++ # shortcut: dc ++ # ++ # - name: mymemory translated ++ # engine: translated ++ # shortcut: tl ++ # timeout: 5.0 ++ # # You can use without an API key, but you are limited to 1000 words/day ++ # # See: https://mymemory.translated.net/doc/usagelimits.php ++ # # api_key: '' ++ # ++ # # Required dependency: mysql-connector-python ++ # # - name: mysql ++ # # engine: mysql_server ++ # # database: mydatabase ++ # # username: user ++ # # password: pass ++ # # limit: 10 ++ # # query_str: 'SELECT * from mytable WHERE fieldname=%(query)s' ++ # # shortcut: mysql ++ # ++ # - name: 1337x ++ # engine: 1337x ++ # shortcut: 1337x ++ # disabled: true ++ # ++ # - name: duden ++ # engine: duden ++ # shortcut: du ++ # disabled: true ++ # ++ # - name: seznam ++ # shortcut: szn ++ # engine: seznam ++ # disabled: true ++ # ++ # # - name: deepl ++ # # engine: deepl ++ # # shortcut: dpl ++ # # # You can use the engine using the official stable API, but you need an API key ++ # # # See: https://www.deepl.com/pro-api?cta=header-pro-api ++ # # api_key: '' # required! ++ # # timeout: 5.0 ++ # # disabled: true ++ # ++ # - name: mojeek ++ # shortcut: mjk ++ # engine: xpath ++ # paging: true ++ # categories: [general, web] ++ # search_url: https://www.mojeek.com/search?q={query}&s={pageno}&lang={lang}&lb={lang} ++ # results_xpath: //ul[@class="results-standard"]/li/a[@class="ob"] ++ # url_xpath: ./@href ++ # title_xpath: ../h2/a ++ # content_xpath: ..//p[@class="s"] ++ # suggestion_xpath: //div[@class="top-info"]/p[@class="top-info spell"]/em/a ++ # first_page_num: 0 ++ # page_size: 10 ++ # disabled: true ++ # about: ++ # website: https://www.mojeek.com/ ++ # wikidata_id: Q60747299 ++ # official_api_documentation: https://www.mojeek.com/services/api.html/ ++ # use_official_api: false ++ # require_api_key: false ++ # results: HTML ++ # ++ # - name: moviepilot ++ # engine: moviepilot ++ # shortcut: mp ++ # disabled: true ++ # ++ # - name: naver ++ # shortcut: nvr ++ # categories: [general, web] ++ # engine: xpath ++ # paging: true ++ # search_url: https://search.naver.com/search.naver?where=webkr&sm=osp_hty&ie=UTF-8&query={query}&start={pageno} ++ # url_xpath: //a[@class="link_tit"]/@href ++ # title_xpath: //a[@class="link_tit"] ++ # content_xpath: //a[@class="total_dsc"]/div ++ # first_page_num: 1 ++ # page_size: 10 ++ # disabled: true ++ # about: ++ # website: https://www.naver.com/ ++ # wikidata_id: Q485639 ++ # official_api_documentation: https://developers.naver.com/docs/nmt/examples/ ++ # use_official_api: false ++ # require_api_key: false ++ # results: HTML ++ # language: ko ++ # ++ # - name: rubygems ++ # shortcut: rbg ++ # engine: xpath ++ # paging: true ++ # search_url: https://rubygems.org/search?page={pageno}&query={query} ++ # results_xpath: /html/body/main/div/a[@class="gems__gem"] ++ # url_xpath: ./@href ++ # title_xpath: ./span/h2 ++ # content_xpath: ./span/p ++ # suggestion_xpath: /html/body/main/div/div[@class="search__suggestions"]/p/a ++ # first_page_num: 1 ++ # categories: [it, packages] ++ # disabled: true ++ # about: ++ # website: https://rubygems.org/ ++ # wikidata_id: Q1853420 ++ # official_api_documentation: https://guides.rubygems.org/rubygems-org-api/ ++ # use_official_api: false ++ # require_api_key: false ++ # results: HTML ++ # ++ # - name: peertube ++ # engine: peertube ++ # shortcut: ptb ++ # paging: true ++ # # alternatives see: https://instances.joinpeertube.org/instances ++ # # base_url: https://tube.4aem.com ++ # categories: videos ++ # disabled: true ++ # timeout: 6.0 ++ # ++ # - name: mediathekviewweb ++ # engine: mediathekviewweb ++ # shortcut: mvw ++ # disabled: true ++ # ++ # # - name: yacy ++ # # engine: yacy ++ # # shortcut: ya ++ # # base_url: http://localhost:8090 ++ # # # required if you aren't using HTTPS for your local yacy instance' ++ # # enable_http: true ++ # # timeout: 3.0 ++ # # # Yacy search mode. 'global' or 'local'. ++ # # search_mode: 'global' ++ # ++ # - name: rumble ++ # engine: rumble ++ # shortcut: ru ++ # base_url: https://rumble.com/ ++ # paging: true ++ # categories: videos ++ # disabled: true ++ # ++ # - name: wordnik ++ # engine: wordnik ++ # shortcut: def ++ # base_url: https://www.wordnik.com/ ++ # categories: [dictionaries] ++ # timeout: 5.0 ++ # ++ # - name: woxikon.de synonyme ++ # engine: xpath ++ # shortcut: woxi ++ # categories: [dictionaries] ++ # timeout: 5.0 ++ # disabled: true ++ # search_url: https://synonyme.woxikon.de/synonyme/{query}.php ++ # url_xpath: //div[@class="upper-synonyms"]/a/@href ++ # content_xpath: //div[@class="synonyms-list-group"] ++ # title_xpath: //div[@class="upper-synonyms"]/a ++ # no_result_for_http_status: [404] ++ # about: ++ # website: https://www.woxikon.de/ ++ # wikidata_id: # No Wikidata ID ++ # use_official_api: false ++ # require_api_key: false ++ # results: HTML ++ # language: de ++ # ++ # - name: seekr news ++ # engine: seekr ++ # shortcut: senews ++ # categories: news ++ # seekr_category: news ++ # disabled: true ++ # ++ # - name: seekr images ++ # engine: seekr ++ # network: seekr news ++ # shortcut: seimg ++ # categories: images ++ # seekr_category: images ++ # disabled: true ++ # ++ # - name: seekr videos ++ # engine: seekr ++ # network: seekr news ++ # shortcut: sevid ++ # categories: videos ++ # seekr_category: videos ++ # disabled: true ++ # ++ # - name: sjp.pwn ++ # engine: sjp ++ # shortcut: sjp ++ # base_url: https://sjp.pwn.pl/ ++ # timeout: 5.0 ++ # disabled: true ++ # ++ # - name: svgrepo ++ # engine: svgrepo ++ # shortcut: svg ++ # timeout: 10.0 ++ # disabled: true ++ # ++ # - name: wallhaven ++ # engine: wallhaven ++ # # api_key: abcdefghijklmnopqrstuvwxyz ++ # shortcut: wh ++ # ++ # # wikimini: online encyclopedia for children ++ # # The fulltext and title parameter is necessary for Wikimini because ++ # # sometimes it will not show the results and redirect instead ++ # - name: wikimini ++ # engine: xpath ++ # shortcut: wkmn ++ # search_url: https://fr.wikimini.org/w/index.php?search={query}&title=Sp%C3%A9cial%3ASearch&fulltext=Search ++ # url_xpath: //li/div[@class="mw-search-result-heading"]/a/@href ++ # title_xpath: //li//div[@class="mw-search-result-heading"]/a ++ # content_xpath: //li/div[@class="searchresult"] ++ # categories: general ++ # disabled: true ++ # about: ++ # website: https://wikimini.org/ ++ # wikidata_id: Q3568032 ++ # use_official_api: false ++ # require_api_key: false ++ # results: HTML ++ # language: fr ++ # ++ # - name: wttr.in ++ # engine: wttr ++ # shortcut: wttr ++ # timeout: 9.0 ++ # ++ # - name: yummly ++ # engine: yummly ++ # shortcut: yum ++ # disabled: true ++ # ++ # - name: brave ++ # engine: brave ++ # shortcut: br ++ # time_range_support: true ++ # paging: true ++ # categories: [general, web] ++ # brave_category: search ++ # # brave_spellcheck: true ++ # ++ # - name: brave.images ++ # engine: brave ++ # network: brave ++ # shortcut: brimg ++ # categories: [images, web] ++ # brave_category: images ++ # ++ # - name: brave.videos ++ # engine: brave ++ # network: brave ++ # shortcut: brvid ++ # categories: [videos, web] ++ # brave_category: videos ++ # ++ # - name: brave.news ++ # engine: brave ++ # network: brave ++ # shortcut: brnews ++ # categories: news ++ # brave_category: news ++ # ++ # - name: lib.rs ++ # shortcut: lrs ++ # engine: xpath ++ # search_url: https://lib.rs/search?q={query} ++ # results_xpath: /html/body/main/div/ol/li/a ++ # url_xpath: ./@href ++ # title_xpath: ./div[@class="h"]/h4 ++ # content_xpath: ./div[@class="h"]/p ++ # categories: [it, packages] ++ # disabled: true ++ # about: ++ # website: https://lib.rs ++ # wikidata_id: Q113486010 ++ # use_official_api: false ++ # require_api_key: false ++ # results: HTML ++ # ++ # - name: sourcehut ++ # shortcut: srht ++ # engine: xpath ++ # paging: true ++ # search_url: https://sr.ht/projects?page={pageno}&search={query} ++ # results_xpath: (//div[@class="event-list"])[1]/div[@class="event"] ++ # url_xpath: ./h4/a[2]/@href ++ # title_xpath: ./h4/a[2] ++ # content_xpath: ./p ++ # first_page_num: 1 ++ # categories: [it, repos] ++ # disabled: true ++ # about: ++ # website: https://sr.ht ++ # wikidata_id: Q78514485 ++ # official_api_documentation: https://man.sr.ht/ ++ # use_official_api: false ++ # require_api_key: false ++ # results: HTML ++ # ++ # - name: goo ++ # shortcut: goo ++ # engine: xpath ++ # paging: true ++ # search_url: https://search.goo.ne.jp/web.jsp?MT={query}&FR={pageno}0 ++ # url_xpath: //div[@class="result"]/p[@class='title fsL1']/a/@href ++ # title_xpath: //div[@class="result"]/p[@class='title fsL1']/a ++ # content_xpath: //p[contains(@class,'url fsM')]/following-sibling::p ++ # first_page_num: 0 ++ # categories: [general, web] ++ # disabled: true ++ # timeout: 4.0 ++ # about: ++ # website: https://search.goo.ne.jp ++ # wikidata_id: Q249044 ++ # use_official_api: false ++ # require_api_key: false ++ # results: HTML ++ # language: ja ++ # ++ # - name: bt4g ++ # engine: bt4g ++ # shortcut: bt4g ++ # ++ # - name: pkg.go.dev ++ # engine: xpath ++ # shortcut: pgo ++ # search_url: https://pkg.go.dev/search?limit=100&m=package&q={query} ++ # results_xpath: /html/body/main/div[contains(@class,"SearchResults")]/div[not(@class)]/div[@class="SearchSnippet"] ++ # url_xpath: ./div[@class="SearchSnippet-headerContainer"]/h2/a/@href ++ # title_xpath: ./div[@class="SearchSnippet-headerContainer"]/h2/a ++ # content_xpath: ./p[@class="SearchSnippet-synopsis"] ++ # categories: [packages, it] ++ # timeout: 3.0 ++ # disabled: true ++ # about: ++ # website: https://pkg.go.dev/ ++ # use_official_api: false ++ # require_api_key: false ++ # results: HTML ++ # ++ ## Doku engine lets you access to any Doku wiki instance: ++ ## A public one or a privete/corporate one. ++ ## - name: ubuntuwiki ++ ## engine: doku ++ ## shortcut: uw ++ ## base_url: 'https://doc.ubuntu-fr.org' ++ # ++ ## Be careful when enabling this engine if you are ++ ## running a public instance. Do not expose any sensitive ++ ## information. You can restrict access by configuring a list ++ ## of access tokens under tokens. ++ ## - name: git grep ++ ## engine: command ++ ## command: ['git', 'grep', '{{QUERY}}'] ++ ## shortcut: gg ++ ## tokens: [] ++ ## disabled: true ++ ## delimiter: ++ ## chars: ':' ++ ## keys: ['filepath', 'code'] ++ # ++ ## Be careful when enabling this engine if you are ++ ## running a public instance. Do not expose any sensitive ++ ## information. You can restrict access by configuring a list ++ ## of access tokens under tokens. ++ ## - name: locate ++ ## engine: command ++ ## command: ['locate', '{{QUERY}}'] ++ ## shortcut: loc ++ ## tokens: [] ++ ## disabled: true ++ ## delimiter: ++ ## chars: ' ' ++ ## keys: ['line'] ++ # ++ ## Be careful when enabling this engine if you are ++ ## running a public instance. Do not expose any sensitive ++ ## information. You can restrict access by configuring a list ++ ## of access tokens under tokens. ++ ## - name: find ++ ## engine: command ++ ## command: ['find', '.', '-name', '{{QUERY}}'] ++ ## query_type: path ++ ## shortcut: fnd ++ ## tokens: [] ++ ## disabled: true ++ ## delimiter: ++ ## chars: ' ' ++ ## keys: ['line'] ++ # ++ ## Be careful when enabling this engine if you are ++ ## running a public instance. Do not expose any sensitive ++ ## information. You can restrict access by configuring a list ++ ## of access tokens under tokens. ++ ## - name: pattern search in files ++ ## engine: command ++ ## command: ['fgrep', '{{QUERY}}'] ++ ## shortcut: fgr ++ ## tokens: [] ++ ## disabled: true ++ ## delimiter: ++ ## chars: ' ' ++ ## keys: ['line'] ++ # ++ ## Be careful when enabling this engine if you are ++ ## running a public instance. Do not expose any sensitive ++ ## information. You can restrict access by configuring a list ++ ## of access tokens under tokens. ++ ## - name: regex search in files ++ ## engine: command ++ ## command: ['grep', '{{QUERY}}'] ++ ## shortcut: gr ++ ## tokens: [] ++ ## disabled: true ++ ## delimiter: ++ ## chars: ' ' ++ ## keys: ['line'] + + doi_resolvers: + oadoi.org: 'https://oadoi.org/' diff --git a/src/portal/patches/snscrape_1036.diff b/src/portal/patches/snscrape_1036.diff new file mode 100644 index 0000000..0512f56 --- /dev/null +++ b/src/portal/patches/snscrape_1036.diff @@ -0,0 +1,28 @@ +diff --git a/.editorconfig b/.editorconfig +new file mode 100644 +index 0000000..e7c6935 +--- /dev/null ++++ b/.editorconfig +@@ -0,0 +1,3 @@ ++[*.py] ++indent_style = tab ++tab_width = 8 +diff --git a/snscrape/modules/__init__.py b/snscrape/modules/__init__.py +index e15a7b9..8963b93 100644 +--- a/snscrape/modules/__init__.py ++++ b/snscrape/modules/__init__.py +@@ -1,4 +1,5 @@ + import pkgutil ++import importlib + + + __all__ = [] +@@ -10,7 +11,7 @@ def _import_modules(): + assert not isPkg + moduleNameWithoutPrefix = moduleName[prefixLen:] + __all__.append(moduleNameWithoutPrefix) +- module = importer.find_module(moduleName).load_module(moduleName) ++ module = importlib.import_module(moduleName) + globals()[moduleNameWithoutPrefix] = module + + diff --git a/src/portal/patches/snscrape_auth.diff b/src/portal/patches/snscrape_auth.diff new file mode 100644 index 0000000..67962a9 --- /dev/null +++ b/src/portal/patches/snscrape_auth.diff @@ -0,0 +1,216 @@ +diff --git a/snscrape/modules/twitter.py b/snscrape/modules/twitter.py +index d1719a8..43d9076 100644 +--- a/snscrape/modules/twitter.py ++++ b/snscrape/modules/twitter.py +@@ -92,6 +92,7 @@ import typing + import urllib.parse + import urllib3.util.ssl_ + import warnings ++import hashlib + + + _logger = logging.getLogger(__name__) +@@ -99,10 +100,12 @@ _API_AUTHORIZATION_HEADER = 'Bearer AAAAAAAAAAAAAAAAAAAAANRILgAAAAAAnNwIzUejRCOu + _globalGuestTokenManager = None + _GUEST_TOKEN_VALIDITY = 10800 + _CIPHERS_CHROME = 'TLS_AES_128_GCM_SHA256:TLS_AES_256_GCM_SHA384:TLS_CHACHA20_POLY1305_SHA256:ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:ECDHE-RSA-AES128-SHA:ECDHE-RSA-AES256-SHA:AES128-GCM-SHA256:AES256-GCM-SHA384:AES128-SHA:AES256-SHA' ++_AGE_RESTRICTED_MSG = 'Age-restricted adult content. This content might not be appropriate for people under 18 years old. To view this media, you’ll need to log in to Twitter. Learn more' + + + @dataclasses.dataclass + class Tweet(snscrape.base.Item): ++ rawResponses: typing.Dict[str, str] + url: str + date: datetime.datetime + rawContent: str +@@ -793,17 +796,23 @@ class _TwitterAPIType(enum.Enum): + V2 = 0 # Introduced with the redesign + GRAPHQL = 1 + ++class _TwitterAPIAuthType(enum.Enum): ++ GUEST_TOKEN = 0 ++ OAUTH2 = 1 + + class _TwitterAPIScraper(snscrape.base.Scraper): +- def __init__(self, baseUrl, *, guestTokenManager = None, maxEmptyPages = 0, **kwargs): ++ def __init__(self, baseUrl, *, guestTokenManager = None, maxEmptyPages = 0, cookies = None, **kwargs): + super().__init__(**kwargs) + self._baseUrl = baseUrl +- if guestTokenManager is None: ++ self._syndicationUrl = 'https://cdn.syndication.twimg.com/tweet?id=' ++ self._authType = _TwitterAPIAuthType.OAUTH2 ++ if self._authType == _TwitterAPIAuthType.GUEST_TOKEN and guestTokenManager is None: + global _globalGuestTokenManager + if _globalGuestTokenManager is None: + _globalGuestTokenManager = GuestTokenManager() + guestTokenManager = _globalGuestTokenManager + self._guestTokenManager = guestTokenManager ++ self._cookies = cookies + self._maxEmptyPages = maxEmptyPages + self._apiHeaders = { + 'Authorization': _API_AUTHORIZATION_HEADER, +@@ -813,6 +822,11 @@ class _TwitterAPIScraper(snscrape.base.Scraper): + adapter = _TwitterTLSAdapter() + self._session.mount('https://twitter.com', adapter) + self._session.mount('https://api.twitter.com', adapter) ++ if self._authType == _TwitterAPIAuthType.OAUTH2: ++ self._ensure_oauth() ++ self._lastResponse = None ++ self._lastResponseHash = None ++ self._lastSentResponseHash = None + + def _check_guest_token_response(self, r): + if r.status_code != 200: +@@ -820,6 +834,8 @@ class _TwitterAPIScraper(snscrape.base.Scraper): + return True, None + + def _ensure_guest_token(self, url = None): ++ if not self._guestTokenManager: ++ return + if self._guestTokenManager.token is None: + _logger.info('Retrieving guest token') + r = self._get(self._baseUrl if url is None else url, responseOkCallback = self._check_guest_token_response) +@@ -843,18 +859,42 @@ class _TwitterAPIScraper(snscrape.base.Scraper): + self._apiHeaders['x-guest-token'] = self._guestTokenManager.token + + def _unset_guest_token(self, blockUntil): ++ if not self._guestTokenManager: ++ return + self._guestTokenManager.reset(blockUntil = blockUntil) + del self._session.cookies['gt'] + del self._apiHeaders['x-guest-token'] + ++ def _ensure_oauth(self): ++ if not self._cookies: ++ return ++ for k, v in self._cookies.items(): ++ self._session.cookies.set(k, v, domain = '.twitter.com', path = '/', secure = True) ++ self._apiHeaders['x-csrf-token'] = self._session.cookies.get('ct0') ++ self._apiHeaders['x-twitter-active-user'] = 'yes' ++ self._apiHeaders['x-twitter-auth-type'] = 'OAuth2Session' ++ self._apiHeaders['x-twitter-client-language'] = 'en' ++ ++ def _ensure_auth_updated(self, url = None): ++ if self._authType == _TwitterAPIAuthType.OAUTH2: ++ self._ensure_oauth() ++ else: ++ self._ensure_guest_token(url) ++ + def _check_api_response(self, r, apiType, instructionsPath): + if r.status_code in (403, 404, 429): ++ currentTime = time.time() + if r.status_code == 429 and r.headers.get('x-rate-limit-remaining', '') == '0' and 'x-rate-limit-reset' in r.headers: +- blockUntil = min(int(r.headers['x-rate-limit-reset']), int(time.time()) + 900) ++ blockUntil = min(int(r.headers['x-rate-limit-reset']), int(currentTime) + 900) ++ else: ++ blockUntil = int(currentTime) + 300 ++ if self._authType == _TwitterAPIAuthType.GUEST_TOKEN: ++ self._unset_guest_token(blockUntil) ++ self._ensure_guest_token() + else: +- blockUntil = int(time.time()) + 300 +- self._unset_guest_token(blockUntil) +- self._ensure_guest_token() ++ blockFor = (blockUntil - currentTime) * (int(r.headers.get('x-rate-limit-remaining', '0')) + 1) ++ _logger.warn(f'Got too many requests, blocking for {blockFor} seconds ({r.headers['x-rate-limit-reset']}) ({r.headers.get('x-rate-limit-remaining', '0')})') ++ time.sleep(blockFor) + return False, f'blocked ({r.status_code})' + if r.headers.get('content-type', '').replace(' ', '') != 'application/json;charset=utf-8': + return False, 'content type is not JSON' +@@ -868,6 +908,11 @@ class _TwitterAPIScraper(snscrape.base.Scraper): + r._snscrapeObj = obj + if apiType is _TwitterAPIType.GRAPHQL and 'errors' in obj: + msg = 'Twitter responded with an error: ' + ', '.join(f'{e["name"]}: {e["message"]}' for e in obj['errors']) ++ blockFor = 30.0 ++ _logger.warn(f'Got errors waiting for {blockFor} seconds') ++ time.sleep(blockFor) ++ return False, msg ++ ''' + instructions = obj + for k in instructionsPath: + instructions = instructions.get(k, {}) +@@ -877,13 +922,23 @@ class _TwitterAPIScraper(snscrape.base.Scraper): + return True, None + else: + return False, msg ++ ''' + return True, None + + def _get_api_data(self, endpoint, apiType, params, instructionsPath = None): +- self._ensure_guest_token() ++ self._ensure_auth_updated() + if apiType is _TwitterAPIType.GRAPHQL: + params = urllib.parse.urlencode({k: json.dumps(v, separators = (',', ':')) for k, v in params.items()}, quote_via = urllib.parse.quote) + r = self._get(endpoint, params = params, headers = self._apiHeaders, responseOkCallback = functools.partial(self._check_api_response, apiType = apiType, instructionsPath = instructionsPath)) ++ if self._authType == _TwitterAPIAuthType.OAUTH2: ++ csrf = r.cookies.get('ct0') ++ if csrf: ++ _logger.warn('Response contained updated csrf token') ++ self._apiHeaders['x-csrf-token'] = csrf ++ self._lastResponse = r._snscrapeObj ++ hasher = hashlib.new('sha256') ++ hasher.update(json.dumps(self._lastResponse).encode('utf-8')) ++ self._lastResponseHash = hasher.hexdigest() + return r._snscrapeObj + + def _iter_api_data(self, endpoint, apiType, params, paginationParams = None, cursor = None, direction = _ScrollDirection.BOTTOM, instructionsPath = None): +@@ -988,8 +1043,50 @@ class _TwitterAPIScraper(snscrape.base.Scraper): + def _get_tweet_id(self, tweet): + return tweet['id'] if 'id' in tweet else int(tweet['id_str']) + ++ def _entry_is_age_restricted(self, entry): ++ if entry['entryId'].startswith('tombstone-') and entry['content']['entryType'] == 'TimelineTimelineItem': ++ if entry['content']['itemContent']['tombstoneInfo']['richText']['text'] == _AGE_RESTRICTED_MSG: ++ return True ++ return False ++ ++ # Can we parse out more data here? ++ def _syndication_make_tweet(self, tweet): ++ tweetId = self._get_tweet_id(tweet) ++ kwargs = {} ++ kwargs['rawResponses'] = { ++ 'detached_api' : json.dumps(tweet) ++ } ++ kwargs['id'] = tweetId ++ kwargs['rawContent'] = tweet['text'] ++ kwargs['renderedContent'] = '' ++ username = tweet['user']['screen_name'] ++ kwargs['user'] = User(id = int(tweet['user']['id_str']), username = username, displayname = tweet['user']['name']) ++ kwargs['date'] = datetime.datetime.strptime(tweet['created_at'], '%Y-%m-%dT%H:%M:%S.%fZ') ++ kwargs['replyCount'] = tweet['conversation_count'] ++ kwargs['retweetCount'] = 0 ++ kwargs['quoteCount'] = 0 ++ kwargs['conversationId'] = 0 ++ kwargs['lang'] = tweet['lang'] ++ kwargs['source'] = '' ++ kwargs['url'] = f'https://twitter.com/{username}/status/{tweetId}' ++ kwargs['likeCount'] = tweet['favorite_count'] ++ media = [] ++ for p in tweet['photos']: ++ media.append(Photo(previewUrl = '', fullUrl = p['url'])) ++ kwargs['media'] = media ++ return Tweet(**kwargs) ++ ++ + def _make_tweet(self, tweet, user, retweetedTweet = None, quotedTweet = None, card = None, noteTweet = None, **kwargs): + tweetId = self._get_tweet_id(tweet) ++ kwargs['rawResponses'] = { ++ 'hash': self._lastResponseHash, ++ 'tweet': json.dumps(tweet), ++ 'user': user.json() ++ } ++ if not self._lastSentResponseHash or self._lastResponseHash != self._lastSentResponseHash: ++ kwargs['rawResponses']['detached_api'] = json.dumps(self._lastResponse) ++ self._lastSentResponseHash = self._lastResponseHash + kwargs['id'] = tweetId + if noteTweet and 'text' in noteTweet: + kwargs['rawContent'] = noteTweet['text'] +@@ -1789,7 +1886,7 @@ class TwitterUserScraper(TwitterSearchScraper): + self._baseUrl = f'https://twitter.com/{self._user}' if not self._isUserId else f'https://twitter.com/i/user/{self._user}' + + def _get_entity(self): +- self._ensure_guest_token() ++ self._ensure_auth_updated() + if not self._isUserId: + fieldName = 'screen_name' + endpoint = 'https://twitter.com/i/api/graphql/pVrmNaXcxPjisIvKtLDMEA/UserByScreenName' diff --git a/src/portal/py/__init__.py b/src/portal/py/__init__.py new file mode 100644 index 0000000..e69de29 --- /dev/null +++ b/src/portal/py/__init__.py diff --git a/src/portal/py/base.py b/src/portal/py/base.py new file mode 100644 index 0000000..c0a0b4f --- /dev/null +++ b/src/portal/py/base.py @@ -0,0 +1,116 @@ +import log +import time +import httpx +import threading +from enum import Enum +from collections import OrderedDict +from typing import Optional, Self, Any +from post import Post, User + +class Method(Enum): + HEAD = "HEAD" + GET = "GET" + POST = "POST" + +ParsedJson = Any + +class Search(): + def __init__(self): + self.pages: dict[int, list[str]] = {} + self.completed: bool = False + self.errored: bool = False + self.iter_page: int = 0 + self.iter_index: int = 0 + + def __iter__(self) -> Self: + self.iter_page = 0 + self.iter_index = 0 + return self + + def __next__(self) -> str: + page = self.get_page(self.iter_page) + if not page: + raise StopIteration + unique_id = page[self.iter_index] + self.iter_index += 1 + if self.iter_index >= len(page): + self.iter_page += 1 + self.iter_index = 0 + return unique_id + + def request_page(self, num: int) -> bool: + return False + + def get_page(self, num: int) -> Optional[list[str]]: + if self.errored: + return None + if num not in self.pages: + if self.completed: + return None + if not self.request_page(num): + if not self.completed: + self.errored = True + return None + return self.pages[num] + +class Module(): + def __init__(self): + self.headers: dict[str, str] = {} + self.cookies: dict[str, str] = {} + self.unique_id_map: OrderedDict = OrderedDict() + self.map_mutex: threading.Lock = threading.Lock() + self.session: httpx.Client = httpx.Client(follow_redirects=True, timeout=20.0, http2=True) + + def init(self) -> bool: + return True + + def do_request(self, method: Method, url: str, params: Optional[dict[str, str]] = None, retries: int = 2) -> Optional[httpx.Response]: + log.debug(f'HTTP {method.value} request {url} {params}.') + response: Optional[httpx.Response] = None + for _ in range(0, retries + 1): + try: + response = self.session.request(method.value, url, headers=self.headers, cookies=self.cookies, params=params) + except Exception as e: + log.info(f'Connection exception ({e}), retrying in 20 seconds...') + time.sleep(20) + continue + if not response: + continue + if response.status_code == 429: + log.info('Got too many requests, waiting for 3 minutes...') + time.sleep(60 * 3) + response = None + continue + if response.status_code == 404 or response.status_code == 403: # Shortcut 404, 403 + return None + if int(response.status_code / 100) != 2: # Non-200 response + log.error(f'Non-200 status code ({response.status_code}), retrying in 5 seconds...') + time.sleep(5) + response = None + continue + try: + response.read() + except Exception as e: + log.info(f'Read exception ({e}), retrying in 20 seconds...') + time.sleep(20) + response = None + continue + else: + break + return response + + def add_to_map(self, unique_id: str, item: Post | User): + with self.map_mutex: + self.unique_id_map[unique_id] = item + if len(self.unique_id_map) > 10240: + self.unique_id_map.popitem(last=False) + + def search(self, query: str, *extra_args: Any) -> Optional[Search]: + return None + + def get_item(self, unique_id: str) -> Optional[Post | User]: + with self.map_mutex: + return self.unique_id_map.get(unique_id, None) + + def get_download(self, unique_id: str, key: str) -> Optional[dict[str, Any]]: + return None diff --git a/src/portal/py/config.def.py b/src/portal/py/config.def.py new file mode 100644 index 0000000..c4e11c6 --- /dev/null +++ b/src/portal/py/config.def.py @@ -0,0 +1,20 @@ +from pathlib import Path + +BASE_DIR = '' + +TWITTER_ACCESS_TOKEN = '' +TWITTER_ACCESS_TOKEN_SECRET = '' +TWITTER_CONSUMER_TOKEN = '' +TWITTER_CONSUMER_TOKEN_SECRET = '' +TWITTER_COOKIES_PATH = '{}/cookies_twitter.txt'.format(BASE_DIR) +TWITTER_TIMEOUT = 5 # seconds + +FANBOX_SESSID = '' + +PIXIV_REFRESH_TOKEN = '' + +INSTAGRAM_USERNAME = '' +INSTAGRAM_PASSWORD = '' +INSTAGRAM_SESSION_ID = '' +INSTAGRAM_SETTINGS_PATH = Path('{}/ig_settings.json'.format(BASE_DIR)) +INSTAGRAM_USER_AGENT = '' diff --git a/src/portal/py/log.py b/src/portal/py/log.py new file mode 100644 index 0000000..b693a4c --- /dev/null +++ b/src/portal/py/log.py @@ -0,0 +1,34 @@ +class DefaultLogger(): + def debug(self, msg): + print(msg) + + def info(self, msg): + print(msg) + + def warn(self, msg): + print(msg) + + def error(self, msg): + print(msg) + +LOGGER = DefaultLogger() + +def set_logger(logger): + global LOGGER + LOGGER = logger + +def debug(msg): + global LOGGER + LOGGER.debug(msg) + +def info(msg): + global LOGGER + LOGGER.info(msg) + +def warn(msg): + global LOGGER + LOGGER.warn(msg) + +def error(msg): + global LOGGER + LOGGER.error(msg) diff --git a/src/portal/py/modules/__init__.py b/src/portal/py/modules/__init__.py new file mode 100644 index 0000000..480660c --- /dev/null +++ b/src/portal/py/modules/__init__.py @@ -0,0 +1,22 @@ +import config + +from modules.youtube import YoutubeModule +from modules.twitter import TwitterScrapeModule +from modules.pixiv_app import PixivAppModule +from modules.pixiv_web import PixivWebModule +from modules.fanbox import FanboxModule +from modules.instagram import InstagramModule + +ALL_MODULES = { + 'youtube': (YoutubeModule(), []), +# 'twitter': (TwitterModule( +# config.TWITTER_ACCESS_TOKEN, config.TWITTER_ACCESS_TOKEN_SECRET, +# config.TWITTER_CONSUMER_TOKEN, config.TWITTER_CONSUMER_TOKEN_SECRET), []), +# 'twitter': (TwitterScrapeModule(config.TWITTER_COOKIES_PATH), []), +# 'pixiv_web': (PixivWebModule(config.PIXIV_SESSID), []), +# 'pixiv_app': (PixivAppModule(config.PIXIV_REFRESH_TOKEN), []), +# 'fanbox': (FanboxModule(config.FANBOX_SESSID), []), +# 'instagram': (InstagramModule( +# config.INSTAGRAM_USER_AGENT, config.INSTAGRAM_SETTINGS_PATH, +# config.INSTAGRAM_SESSION_ID), []) +} diff --git a/src/portal/py/modules/common.py b/src/portal/py/modules/common.py new file mode 100644 index 0000000..62c3d5d --- /dev/null +++ b/src/portal/py/modules/common.py @@ -0,0 +1,9 @@ +from datetime import datetime, timezone + +def parse_pixiv_date(date_str: str) -> datetime: + rindex = date_str.rfind(':') + date_str = date_str[:rindex] + date_str[rindex + 1:] + return datetime.strptime(date_str, "%Y-%m-%dT%H:%M:%S%z") + +def get_current_utc_time() -> datetime: + return datetime.now(timezone.utc) diff --git a/src/portal/py/modules/fanbox.py b/src/portal/py/modules/fanbox.py new file mode 100644 index 0000000..01e66c3 --- /dev/null +++ b/src/portal/py/modules/fanbox.py @@ -0,0 +1,192 @@ +import json +import httpx +from typing import Optional, Any +from json.decoder import JSONDecodeError +from base import Search, Module, Method, ParsedJson +from post import MediaUrl, PostType, DateType, Post, User, Image, File, Tag +from query_parser import QueryParser +from modules.common import parse_pixiv_date, get_current_utc_time + +BASE_URL = "https://api.fanbox.cc" + +class FanboxBase(Search): + def __init__(self, userdata: Any): + super().__init__() + self.module: FanboxModule = userdata + + def check_api_response(self, response: Optional[httpx.Response]) -> Optional[ParsedJson]: + if not response: + return None + try: + obj = response.json() + except JSONDecodeError: + return None + return obj['body'] + + def create_media_url(self, url: str) -> MediaUrl: + return MediaUrl(url, url.split('.')[-1]) + + def create_post(self, data: ParsedJson) -> Optional[Post]: + kwargs = {} + post_id = data['id'] + unique_id = f'fanbox:p:{post_id}' + kwargs['unique_id'] = unique_id + kwargs['raw_responses'] = {} + kwargs['raw_responses']['api'] = json.dumps(data) + publish_date = parse_pixiv_date(data['publishedDatetime']).timestamp() + kwargs['dates'] = { + DateType.CREATED: publish_date, + DateType.RETRIEVED: get_current_utc_time().timestamp() + } + update_date = parse_pixiv_date(data['updatedDatetime']).timestamp() + if update_date != publish_date: + kwargs['dates'][DateType.EDITED] = update_date + user = User(unique_id=f'fanbox:u:{data['user']['userId']}', username=data['user']['name']) + user.profile_picture_url = self.create_media_url(data['user']['iconUrl']) + kwargs['author'] = user + kwargs['title'] = data['title'] + kwargs['likes'] = data['likeCount'] + kwargs['comments'] = data['commentCount'] + kwargs['tags'] = [Tag(name=tag) for tag in data['tags']] + if data['isRestricted']: + kwargs['type'] = PostType.PREVIEW + else: + kwargs['type'] = PostType.POST + text = '' + media = {} + if data['type'] == 'image': + text = data['body']['text'] + for i, value in enumerate(data['body']['images']): + media[str(i)] = Image(url=MediaUrl(value['originalUrl'], value['extension']), thumbnail_url=self.create_media_url(value['thumbnailUrl'])) + elif data['type'] == 'file': + text = data['body']['text'] + for i, value in enumerate(data['body']['files']): + media[str(i)] = File(url=MediaUrl(value['url'], value['extension']), name=value['name']) + elif data['type'] == 'article': + for block in data['body']['blocks']: + if block['type'] == 'p': + text += block['text'] + '\n' + elif block['type'] == 'image': + text += f'image::{block['imageId']}[]' + '\n' + elif block['type'] == 'url_embed': + text += f'embed::{block['urlEmbedId']}[]' + '\n' + for key, value in data['body']['imageMap'].items(): + media[key] = Image(url=MediaUrl(value['originalUrl'], value['extension']), thumbnail_url=self.create_media_url(value['thumbnailUrl'])) + for key, value in data['body']['fileMap'].items(): + media[key] = File(url=MediaUrl(value['url'], value['extension']), name=value['name']) + kwargs['text'] = text + kwargs['media'] = media + post = Post(**kwargs) + self.module.add_to_map(unique_id, post) + return post + + def request_post(self, post_id: int) -> Optional[Post]: + url = f'{BASE_URL}/post.info?postId={post_id}' + obj = self.check_api_response(self.module.do_request(Method.GET, url)) + if not obj: + return None + return self.create_post(obj) + + def create_user(self, data: ParsedJson) -> User: + unique_id = f'fanbox:u:{data['id']}' + user = User(unique_id=unique_id, username=data['creatorId'], display_name=data['user']['name']) + user.profile_picture_url = self.create_media_url(data['user']['iconUrl']) + self.module.add_to_map(unique_id, user) + return user + +class FanboxPost(FanboxBase): + def __init__(self, userdata: Any, arg: str): + super().__init__(userdata) + try: + self.id: int = int(arg) + except ValueError: + self.errored = True + + def request_page(self, num: int) -> bool: + post = self.request_post(self.id) + if not post: + self.errored = True + return False + self.pages[num] = [post.unique_id] + self.completed = True + return True + +class FanboxUser(FanboxBase): + def __init__(self, userdata: Any, arg: str): + super().__init__(userdata) + self.id = arg + self.pagination = [] + + def request_pagination(self) -> bool: + url = f'{BASE_URL}/post.paginateCreator?creatorId={self.id}' + obj = self.check_api_response(self.module.do_request(Method.GET, url)) + if not obj: + return False + for link in obj: + self.pagination.append(link) + return True + + def request_page(self, num: int) -> bool: + if len(self.pagination) == 0: + if not self.request_pagination(): + self.errored = True + return False + if num >= len(self.pagination): + return False + obj = self.check_api_response(self.module.do_request(Method.GET, self.pagination[num])) + if not obj: + self.errored = True + return False + self.pages[num] = [] + for item in obj['items']: + post = self.request_post(item['id']) + if post: + self.pages[num].append(post.unique_id) + if len(self.pages) == len(self.pagination): + self.completed = True + return True + +class FanboxSupporting(FanboxBase): + def __init__(self, userdata: Any, arg: str): + super().__init__(userdata) + + def request_page(self, num: int) -> bool: + url = f'{BASE_URL}/plan.listSupporting' + obj = self.check_api_response(self.module.do_request(Method.GET, url)) + if not obj: + self.errored = True + return False + self.pages[num] = [] + for user in obj: + self.pages[num].append(self.create_user(user).unique_id) + self.completed = True + return True + +class FanboxModule(Module): + def __init__(self, sessid: str): + super().__init__() + self.parser: QueryParser = QueryParser(self, 'post') + self.parser.add_command('post', FanboxPost) + self.parser.add_command('user', FanboxUser) + self.parser.add_command('supporting', FanboxSupporting) + self.headers['Accept-Encoding'] = 'gzip, deflate, br' + self.headers['Accept-Language'] = 'en-US,en;q=0.5' + self.headers['Origin'] = 'https://www.fanbox.cc' + self.headers['Referer'] = 'https://www.fanbox.cc/' + self.headers['User-Agent'] = 'Mozilla/5.0 (Windows NT 10.0; rv:122.0) Gecko/20100101 Firefox/122.0' + self.cookies['FANBOXSESSID'] = sessid + + def search(self, query: str, *extra_args: Any) -> Optional[Search]: + return self.parser.parse_query(query) + + def get_download(self, unique_id: str, key: str) -> Optional[dict[str, Any]]: + if unique_id not in self.unique_id_map: + return None + post = self.unique_id_map[unique_id] + if key not in post.media: + return None + return { + 'urls': [post.media[key].url], + 'headers': {}, + 'cookies': {} + } diff --git a/src/portal/py/modules/instagram.py b/src/portal/py/modules/instagram.py new file mode 100644 index 0000000..325caf6 --- /dev/null +++ b/src/portal/py/modules/instagram.py @@ -0,0 +1,154 @@ +import os +import log +import email.utils +from typing import Optional, Any +from pathlib import Path +from base import Module, Search +from post import MediaUrl, PostType, DateType, Post, User, Image, Video +from modules.common import get_current_utc_time +from instagrapi import Client +from instagrapi.exceptions import UserNotFound, LoginRequired, ChallengeRequired + +class InstagramSearch(Search): + REACH_LIMIT = 3 + + def __init__(self, module: Any, pk: str): + super().__init__() + self.module: InstagramModule = module + self.pk: str = pk + self.page: int = 0 + self.cursor: Any = None + + def create_media_url(self, url: str) -> MediaUrl: + question = url.find('?') + if question >= 0: + ext = url[:question].split('.')[-1] + else: + ext = url.split('.')[-1] + return MediaUrl(url, ext) + + def request_page(self, num: int) -> bool: + if num >= self.page + self.REACH_LIMIT: + return False + request_satisfied = False + for i in range(self.page, num + 1): + try: + media, self.cursor = self.module.cl.user_medias_paginated_v1(self.pk, 0, end_cursor=self.cursor) + except LoginRequired: + self.errored = True + return False + except ChallengeRequired: + self.errored = True + return False + if not self.cursor: + self.completed = True + self.pages[i] = [] + for m in media: + kwargs = {} + kwargs['type'] = PostType.POST + unique_id = 'instagram:p:{}'.format(m.id) + kwargs['unique_id'] = unique_id + kwargs['raw_responses'] = {} + kwargs['raw_responses']['tile'] = m.json() + kwargs['url'] = 'https://www.instagram.com/p/{}'.format(m.code) + kwargs['dates'] = { + DateType.CREATED: m.taken_at.timestamp(), + DateType.RETRIEVED: get_current_utc_time().timestamp() + } + user = User(unique_id='instagram:u:{}'.format(m.user.pk)) + if m.user.username: + user.username = m.user.username + if m.user.full_name: + user.display_name = m.user.full_name + if m.user.profile_pic_url: + user.profile_picture_url = self.create_media_url(str(m.user.profile_pic_url)) + kwargs['author'] = user + kwargs['title'] = m.title + kwargs['text'] = m.caption_text + kwargs['likes'] = m.like_count + kwargs['comments'] = m.comment_count + media = {} + if m.media_type == 8: # Album + for k, r in enumerate(m.resources): + if r.media_type == 1: # Photo + media[str(k)] = Image(url=self.create_media_url(str(r.thumbnail_url))) + elif r.media_type == 2: # Video + media[str(k)] = Video(url=self.create_media_url(str(r.video_url)), thumbnail_url=self.create_media_url(str(r.thumbnail_url))) + elif m.media_type == 2: # Video + kwargs['views'] = m.play_count + media[str(0)] = Video(url=self.create_media_url(str(m.video_url)), thumbnail_url=self.create_media_url(str(m.thumbnail_url))) + elif m.media_type == 1: # Photo + candidates = sorted(m.image_versions2['candidates'], key=lambda x: x['width'], reverse=True) + media[str(0)] = Image(url=self.create_media_url(candidates[0]['url']), thumbnail_url=self.create_media_url(candidates[1]['url'])) + kwargs['media'] = media + post = Post(**kwargs) + self.module.add_to_map(unique_id, post) + self.pages[i].append(unique_id) + self.page += 1 + if i == num: + request_satisfied = True + return request_satisfied + +class InstagramModule(Module): + def __init__(self, user_agent: str, settings: Path, session_id: str): + super().__init__() + self.cl: Client = Client() + # https://specdevice.com/showspec.php?id=c318-0c39-0033-c5870033c587 + device_set = { + 'app_version': '324.0.0.0.16', + 'android_version': 33, + 'android_release': '13.0.0', + 'dpi': '476dpi', + 'resolution': '1440x3120', + 'manufacturer': 'Google', + 'device': 'cheetah', + 'model': 'Pixel 7 Pro', + 'cpu': 'cheetah', + 'version_code': '9981770' + } + user_agent = f'Instagram {device_set['app_version']} Android ({device_set['android_version']}/{device_set['android_release']}; {device_set['dpi']}; {device_set['resolution']}; {device_set['manufacturer']}; {device_set['device']}; {device_set['model']}; {device_set['cpu']}; en_US; {device_set['version_code']})' + self.cl.set_country('US') + self.cl.set_country_code(1) # Phone code + self.cl.set_locale('en_US') + self.cl.set_timezone_offset(-14400) # New_York GMT-4 + self.cl.set_user_agent(user_agent) + self.cl.set_device(device = device_set) + self.settings: Path = settings + self.session_id: str = session_id + + def init(self) -> bool: + if os.path.exists(self.settings): + self.cl.load_settings(self.settings) + if not self.cl.login_by_sessionid(self.session_id): + return False + #if not self.cl.login(self.username, self.password): + # return False + self.cl.dump_settings(self.settings) + return True + + def search(self, query: str, *extra_args: Any) -> Optional[Search]: + try: + user = self.cl.user_info_by_username_v1(query) + except UserNotFound: + log.warn('User not found.') + return None + except LoginRequired: + log.error('Fetching user failed: Login required.') + return None + except ChallengeRequired: + log.error('Fetching user failed: Challenge required.') + return None + return InstagramSearch(self, user.pk) + + def get_download(self, unique_id: str, key: str) -> Optional[dict[str, Any]]: + if unique_id not in self.unique_id_map: + return None + post = self.unique_id_map[unique_id] + if key not in post.media: + return None + return { + 'urls': [post.media[key].url], + 'headers': { + 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; rv:122.0) Gecko/20100101 Firefox/122.0' + } + } diff --git a/src/portal/py/modules/pixiv_app.py b/src/portal/py/modules/pixiv_app.py new file mode 100644 index 0000000..8be3f9c --- /dev/null +++ b/src/portal/py/modules/pixiv_app.py @@ -0,0 +1,201 @@ +import json +from typing import Optional, Any +from base import Search, Module, ParsedJson +from post import MediaUrl, PostType, DateType, User, Post, Image, Tag, TagType +from query_parser import QueryParser +from pixivpy3 import * +from modules.common import get_current_utc_time, parse_pixiv_date + +class PixivAppBase(Search): + def __init__(self, userdata: Any): + super().__init__() + self.module: PixivAppModule = userdata + self.page: int = 0 + self.next_qs: Optional[dict[str, Any]] = None + + def api_request_page(self, num: int) -> tuple[bool, Optional[dict[str, Any]]]: + return False, None + + def request_page(self, num: int) -> bool: + request_satisfied = False + for i in range(self.page, num + 1): + if not self.next_qs: + break + if i in self.pages: + continue + self.pages[i] = [] + success, self.next_qs = self.api_request_page(i) + if not success: + self.errored = True + break + if i == num: + request_satisfied = True + self.page += 1 + return request_satisfied + + def create_media_url(self, url: str) -> MediaUrl: + return MediaUrl(url, url.split('.')[-1]) + + def create_tag(self, data: ParsedJson) -> Tag: + tag = Tag(type=TagType.GENERAL, name=data['name']) + if data['translated_name']: + tag.alts['en'] = data['translated_name'] + return tag + + def create_post(self, data: ParsedJson) -> Post: + if len(data['meta_pages']) == 0: + data['meta_pages'].append({ + 'image_urls': { + 'medium': data['image_urls']['medium'], + 'original': data['meta_single_page']['original_image_url'] + } + }) + kwargs = {} + kwargs['type'] = PostType.POST + unique_id = 'pixiv:i:{}'.format(data['id']) + kwargs['unique_id'] = unique_id + kwargs['raw_responses'] = {} + kwargs['raw_responses']['api'] = json.dumps(data) + kwargs['url'] = 'https://www.pixiv.net/en/artworks/{}'.format(data['id']) + kwargs['dates'] = { + DateType.CREATED: parse_pixiv_date(data['create_date']).timestamp(), + DateType.RETRIEVED: get_current_utc_time().timestamp() + } + user = User(unique_id=f'pixiv:u:{data['user']['id']}', username=data['user']['account'], display_name=data['user']['name']) + user.profile_picture_url = data['user']['profile_image_urls']['medium'] + kwargs['author'] = user + kwargs['title'] = data['title'] + kwargs['text'] = data['caption'] + kwargs['likes'] = data['total_bookmarks'] + if 'total_comments' in data: + kwargs['comments'] = data['total_comments'] + kwargs['views'] = data['total_view'] + kwargs['tags'] = [self.create_tag(tag) for tag in data['tags']] + media = {} + for i, page in enumerate(data['meta_pages']): + media[str(i)] = Image(url=self.create_media_url(page['image_urls']['original']), thumbnail_url=self.create_media_url(page['image_urls']['medium'])) + kwargs['media'] = media + post = Post(**kwargs) + self.module.add_to_map(unique_id, post) + return post + + def create_user(self, data: ParsedJson) -> User: + unique_id = f'pixiv:u:{data['id']}' + user = User(unique_id=unique_id, username=data['account'], display_name=data['name']) + self.module.add_to_map(unique_id, user) + return user + +class PixivAppSearch(PixivAppBase): + def __init__(self, userdata: Any, arg: str): + super().__init__(userdata) + self.next_qs: Optional[dict[str, Any]] = {} + self.next_qs['word'] = arg + self.next_qs['sort'] = 'popular_desc' + self.next_qs['search_ai_type'] = 1 + self.next_qs['start_date'] = '2019-01-01' + self.next_qs['end_date'] = '2019-12-31' + + def api_request_page(self, num: int) -> tuple[bool, Optional[dict[str, Any]]]: + if not self.next_qs: + return False, None + obj = self.module.api.search_illust(**self.next_qs) + if 'illusts' not in obj: + return False, None + for post in obj['illusts']: + self.pages[num].append(self.create_post(post).unique_id) + return True, self.module.api.parse_qs(obj['next_url']) + +class PixivAppIllust(PixivAppBase): + def __init__(self, userdata: Any, arg: str): + super().__init__(userdata) + self.next_qs: Optional[dict[str, Any]] = {} + self.next_qs['illust_id'] = arg + + def api_request_page(self, num: int) -> tuple[bool, Optional[dict[str, Any]]]: + if not self.next_qs: + return False, None + obj = self.module.api.illust_detail(**self.next_qs) + if 'illust' not in obj: + return False, None + self.pages[num].append(self.create_post(obj['illust']).unique_id) + return True, None + +class PixivAppUser(PixivAppBase): + def __init__(self, userdata: Any, arg: str): + super().__init__(userdata) + self.next_qs: Optional[dict[str, Any]] = {} + self.next_qs['user_id'] = arg + self.next_qs['type'] = 'illust' + + def api_request_page(self, num: int) -> tuple[bool, Optional[dict[str, Any]]]: + if not self.next_qs: + return False, None + obj = self.module.api.user_illusts(**self.next_qs) + if 'illusts' not in obj: + return False, None + for post in obj['illusts']: + self.pages[num].append(self.create_post(post).unique_id) + return True, self.module.api.parse_qs(obj['next_url']) + +class PixivAppBookmarks(PixivAppBase): + def __init__(self, userdata: Any, arg: str): + super().__init__(userdata) + self.next_qs: Optional[dict[str, Any]] = {} + self.next_qs['user_id'] = arg + self.next_qs['restrict'] = 'public' + + def api_request_page(self, num: int) -> tuple[bool, Optional[dict[str, Any]]]: + if not self.next_qs: + return False, None + obj = self.module.api.user_bookmarks_illust(**self.next_qs) + if 'illusts' not in obj: + return False, None + for post in obj['illusts']: + self.pages[num].append(self.create_post(post).unique_id) + return True, self.module.api.parse_qs(obj['next_url']) + +class PixivAppFollowing(PixivAppBase): + def __init__(self, userdata: Any, arg: str): + super().__init__(userdata) + self.next_qs: Optional[dict[str, Any]] = {} + self.next_qs['user_id'] = arg + self.next_qs['restrict'] = 'public' + + def api_request_page(self, num: int) -> tuple[bool, Optional[dict[str, Any]]]: + if not self.next_qs: + return False, None + obj = self.module.api.user_following(**self.next_qs) + if 'user_previews' not in obj: + return False, None + for user in obj['user_previews']: + self.pages[num].append(self.create_user(user['user']).unique_id) + return True, self.module.api.parse_qs(obj['next_url']) + +class PixivAppModule(Module): + def __init__(self, refresh_token: str): + super().__init__() + self.parser: QueryParser = QueryParser(self, 'search') + self.parser.add_command('search', PixivAppSearch) + self.parser.add_command('illust', PixivAppIllust) + self.parser.add_command('user', PixivAppUser) + self.parser.add_command('bookmarks', PixivAppBookmarks) + self.parser.add_command('following', PixivAppFollowing) + self.api: AppPixivAPI = AppPixivAPI() + self.api.auth(refresh_token=refresh_token) + + def search(self, query: str, *extra_args: Any) -> Optional[Search]: + return self.parser.parse_query(query) + + def get_download(self, unique_id: str, key: str) -> Optional[dict[str, Any]]: + if unique_id not in self.unique_id_map: + return None + post = self.unique_id_map[unique_id] + if key not in post.media: + return None + return { + 'urls': [post.media[key].url], + 'headers': { + 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; rv:122.0) Gecko/20100101 Firefox/122.0', + 'Referer': 'https://www.pixiv.net/' + } + } diff --git a/src/portal/py/modules/pixiv_web.py b/src/portal/py/modules/pixiv_web.py new file mode 100644 index 0000000..7bc043b --- /dev/null +++ b/src/portal/py/modules/pixiv_web.py @@ -0,0 +1,263 @@ +import log +import json +import httpx +import urllib.parse +from typing import Optional, Any +from json.decoder import JSONDecodeError +from base import Search, Module, Method, ParsedJson +from post import MediaUrl, PostType, DateType, Post, User, Media, Image, Animation, Tag, TagType +from query_parser import QueryParser +from modules.common import parse_pixiv_date, get_current_utc_time + +BASE_URL = 'https://www.pixiv.net/ajax' +LANG = 'en' +VERSION = '5f53980f6b59c9376220b6d86e8feb5ea9e22e10' + +class PixivWebBase(Search): + def __init__(self, userdata: Any): + super().__init__() + self.module: PixivWebModule = userdata + + def check_api_response(self, response: Optional[httpx.Response]) -> Optional[ParsedJson]: + if not response: + return None + try: + obj = response.json() + except JSONDecodeError: + return None + if obj['error']: + log.error(obj['message']) + return None + return obj['body'] + + def create_media_url(self, url: str) -> MediaUrl: + return MediaUrl(url, url.split('.')[-1]) + + def parse_pages(self, data: ParsedJson) -> dict[str, Media]: + media = {} + for i, m in enumerate(data): + media[str(i)] = Image(url=self.create_media_url(m['urls']['original']), thumbnail_url=self.create_media_url(m['urls']['small'])) + return media + + def parse_ugoira(self, data: ParsedJson, thumbnail_url: str) -> Animation: + animation = Animation(url=self.create_media_url(data['originalSrc']), thumbnail_url=self.create_media_url(thumbnail_url)) + for frame in data['frames']: + animation.frames.append((frame['file'], frame['delay'])) + return animation + + def seek_profile_picture_url(self, user_illusts: ParsedJson) -> Optional[MediaUrl]: + for illust in user_illusts.values(): + if illust is None: + continue + if 'profileImageUrl' in illust: + return self.create_media_url(illust['profileImageUrl']) + return None + + def create_tag(self, data: ParsedJson) -> Tag: + tag = Tag(type=TagType.GENERAL, name=data['tag']) + if 'romaji' in data: + tag.alts['romaji'] = data['romaji'] + if 'translation' in data: + if 'en' in data['translation']: + tag.alts['en'] = data['translation']['en'] + return tag + + def create_post(self, data: ParsedJson, pages: Optional[ParsedJson]=None, ugoira: Optional[ParsedJson]=None) -> Optional[Post]: + kwargs = {} + kwargs['type'] = PostType.POST + illust_id = data['illustId'] + unique_id = f'pixiv:i:{illust_id}' + kwargs['unique_id'] = unique_id + kwargs['raw_responses'] = {} + kwargs['raw_responses']['api'] = json.dumps(data) + kwargs['url'] = f'https://www.pixiv.net/{LANG}/artworks/{data['illustId']}' + create_date = parse_pixiv_date(data['createDate']).timestamp() + kwargs['dates'] = { + DateType.CREATED: create_date, + DateType.RETRIEVED: get_current_utc_time().timestamp() + } + upload_date = parse_pixiv_date(data['uploadDate']).timestamp() + if upload_date != create_date: + kwargs['dates'][DateType.EDITED] = upload_date + user = User(unique_id=f'pixiv:u:{data['userId']}', username=data['userAccount'], display_name=data['userName']) + profile_picture_url = self.seek_profile_picture_url(data['userIllusts']) + if profile_picture_url: + user.profile_picture_url = profile_picture_url + kwargs['author'] = user + kwargs['title'] = data['illustTitle'] + kwargs['text'] = data['illustComment'] + kwargs['likes'] = data['likeCount'] + kwargs['bookmarks'] = data['bookmarkCount'] + kwargs['comments'] = data['commentCount'] + kwargs['views'] = data['viewCount'] + kwargs['tags'] = [self.create_tag(tag) for tag in data['tags']['tags']] + if data['illustType'] == 2: # Ugoira + if not ugoira: + url = f'{BASE_URL}/illust/{illust_id}/ugoira_meta?lang={LANG}&version={VERSION}' + ugoira = self.check_api_response(self.module.do_request(Method.GET, url)) + if not ugoira: + return None + kwargs['raw_responses']['ugoira'] = json.dumps(ugoira) + kwargs['media'] = { '0': self.parse_ugoira(ugoira, data['urls']['original']) } + elif data['pageCount'] > 1: + if not pages: + url = f'{BASE_URL}/illust/{illust_id}/pages?lang={LANG}&version={VERSION}' + pages = self.check_api_response(self.module.do_request(Method.GET, url)) + if not pages: + return None + kwargs['raw_responses']['pages'] = json.dumps(pages) + kwargs['media'] = self.parse_pages(pages) + else: + kwargs['media'] = self.parse_pages([{ 'urls': data['urls'] }]) + post = Post(**kwargs) + self.module.add_to_map(unique_id, post) + return post + + def remake_post(self, raw_responses: ParsedJson, original_date: float) -> Optional[Post]: + api = json.loads(raw_responses['api']) + pages = raw_responses.get('pages', None) + if pages: + pages = json.loads(pages) + ugoira = raw_responses.get('ugoira', None) + if ugoira: + ugoira = json.loads(ugoira) + post = self.create_post(api, pages, ugoira) + if post: + post.dates[DateType.RETRIEVED] = original_date + return post + + def request_illust(self, illust_id: int) -> Optional[Post]: + url = f'{BASE_URL}/illust/{illust_id}?lang={LANG}&version={VERSION}' + obj = self.check_api_response(self.module.do_request(Method.GET, url)) + if not obj: + return None + return self.create_post(obj) + + def create_user(self, data: ParsedJson) -> User: + unique_id=f'pixiv:u:{data['userId']}' + user = User(unique_id=unique_id, display_name=data['userName']) + user.profile_picture_url = self.create_media_url(data['profileImageUrl']) + self.module.add_to_map(unique_id, user) + return user + +class PixivWebSearch(PixivWebBase): + def __init__(self, userdata: Any, arg: str): + super().__init__(userdata) + self.query: str = arg + self.last_page: Optional[int] = None + + def request_page(self, num: int) -> bool: + if self.last_page and num >= self.last_page: + return False + start = '2019-01-01' + end = '2019-12-31' + order = 'popular_d' + url = f'{BASE_URL}/search/artworks/{urllib.parse.quote(self.query)}?word={self.query}&order={order}&mode=all&scd={start}&ecd={end}&p={num + 1}&csw=0&s_mode=s_tag&type=all&lang={LANG}&version={VERSION}' + obj = self.check_api_response(self.module.do_request(Method.GET, url)) + if not obj: + self.errored = True + return False + self.last_page = obj['illustManga']['lastPage'] + self.pages[num] = [] + for partial_post in obj['illustManga']['data']: + post = self.request_illust(int(partial_post['id'])) + if not post: + self.errored = True + return False + self.pages[num].append(post.unique_id) + return True + +class PixivWebIllust(PixivWebBase): + def __init__(self, userdata: Any, arg: str): + super().__init__(userdata) + try: + self.id: int = int(arg) + except ValueError: + self.errored = True + + def request_page(self, num: int) -> bool: + post = self.request_illust(self.id) + if not post: + self.errored = True + return False + self.pages[num] = [post.unique_id] + self.completed = True + return True + +class PixivWebUser(PixivWebBase): + def __init__(self, userdata: Any, arg: str): + super().__init__(userdata) + try: + self.id: int = int(arg) + except ValueError: + self.errored = True + + def request_page(self, num: int) -> bool: + url = f'{BASE_URL}/user/{self.id}/profile/all?lang={LANG}&version={VERSION}' + obj = self.check_api_response(self.module.do_request(Method.GET, url)) + if not obj: + self.errored = True + return False + self.pages[num] = [] + if obj['manga']: + for illust in obj['manga'].keys(): + post = self.request_illust(int(illust)) + if not post: + self.errored = True + return False + self.pages[num].append(post.unique_id) + self.completed = True + return True + +class PixivWebFollowing(PixivWebBase): + LIMIT = 24 + + def __init__(self, userdata: Any, arg: str): + super().__init__(userdata) + try: + self.id: int = int(arg) + except ValueError: + self.errored = True + + def request_page(self, num: int) -> bool: + url = f'{BASE_URL}/user/{self.id}/following?offset={self.LIMIT * num}&limit={self.LIMIT}&rest=show&tag=&acceptingRequests=0&lang={LANG}&version={VERSION}' + obj = self.check_api_response(self.module.do_request(Method.GET, url)) + if not obj: + self.errored = True + return False + self.pages[num] = [] + for user in obj['users']: + self.pages[num].append(self.create_user(user).unique_id) + return True + +class PixivWebModule(Module): + def __init__(self, sessid: str): + super().__init__() + self.parser: QueryParser = QueryParser(self, 'search') + self.parser.add_command('search', PixivWebSearch) + self.parser.add_command('illust', PixivWebIllust) + self.parser.add_command('user', PixivWebUser) + self.parser.add_command('following', PixivWebFollowing) + self.headers['Accept-Encoding'] = 'gzip, deflate, br' + self.headers['Accept-Language'] = 'en-US,en;q=0.5' + self.headers['Referer'] = 'https://www.pixiv.net/' + self.headers['User-Agent'] = 'Mozilla/5.0 (Windows NT 10.0; rv:122.0) Gecko/20100101 Firefox/122.0' + self.headers['x-user-id'] = '22781328' + self.cookies['PHPSESSID'] = sessid + + def search(self, query: str, *extra_args: Any) -> Optional[Search]: + return self.parser.parse_query(query) + + def get_download(self, unique_id: str, key: str) -> Optional[dict[str, Any]]: + if unique_id not in self.unique_id_map: + return None + post = self.unique_id_map[unique_id] + if key not in post.media: + return None + return { + 'urls': [post.media[key].url], + 'headers': { + 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; rv:122.0) Gecko/20100101 Firefox/122.0', + 'Referer': 'https://www.pixiv.net/' + } + } diff --git a/src/portal/py/modules/searx.py b/src/portal/py/modules/searx.py new file mode 100644 index 0000000..4cc0b04 --- /dev/null +++ b/src/portal/py/modules/searx.py @@ -0,0 +1,95 @@ +import sys +import json + +from base import Module, Search +from post import Post, Image + +sys.path.append('../../vendor/searxng') +from searx import settings +from searx.utils import gen_useragent +from searx.engines import load_engine, register_engine + +# TODO: Put these on the Provider object. +ENGINES = {} + +ENGINE_INITIAL_INDEX = { + 'google images': 0, + 'google': 1, + 'bing images': 0 +} + +class SearxSearch(Search): + def __init__(self, plugin, query, engine): + super().__init__() + global ENGINES + self.query = query + self.plugin = plugin + self.key = engine.replace(' ', '_') + self.engine = ENGINES[engine] + self.index = ENGINE_INITIAL_INDEX[engine] + + def load_page(self, index): + self.pages[index] = [] + res = self.plugin.send_http_request(self.engine.request(self.query, { + 'language': 'en-US', + 'safesearch': 0, + 'time_range': None, + 'pageno': self.index + index, + 'cookies': self.plugin.cookies, + 'headers': self.plugin.headers, + 'data': None + })) + if not res: + return False + try: + ret = self.engine.response(res) + except: + return False + if len(ret) == 0: + # TODO: Test. + self.completed = True + return False + for i, p in enumerate(ret): + if 'url' not in p: + continue + # Don't add this to the global map because it's not actually unique. + unique_id = '{}:{}_{}:{}'.format(self.key, self.query, i, index) + media = [] + if 'img_src' in p: + media.append(Image(url=p['img_src'], thumbnail_url='')) + elif 'image_url' in p: + media.append(Image(url=p['image_url'], thumbnail_url='')) + kwargs = {} + kwargs['unique_id'] = unique_id + kwargs['raw_responses'] = {} + kwargs['raw_responses']['result'] = json.dumps(p) + kwargs['url'] = p['url'] + kwargs['title'] = p['title'] + kwargs['text'] = p['content'] + kwargs['media'] = media + self.pages[index].append(Post(**kwargs)) + return True + +class SearxModule(Module): + def __init__(self): + super().__init__() + global ENGINES + self.headers = { + 'User-Agent': gen_useragent(), + 'Accept-Language': 'en-US,en;q=0.5', + } + for engine_data in settings['engines']: + engine = load_engine(engine_data) + if engine: + register_engine(engine) + ENGINES[engine.name] = engine + + def send_http_request(self, params): + self.cookies.update(params['cookies']) + self.headers.update(params['headers']) + return self.get(params['url'], params=params['data']) + + def search(self, query, *extra_args): + if not extra_args: + return None + return SearxSearch(self, query, extra_args[0]) diff --git a/src/portal/py/modules/twitter.py b/src/portal/py/modules/twitter.py new file mode 100644 index 0000000..7e77253 --- /dev/null +++ b/src/portal/py/modules/twitter.py @@ -0,0 +1,190 @@ +import log +import http.cookiejar +from typing import Optional, Any, Iterator +from datetime import timezone +from base import Search, Module +from post import MediaUrl, PostType, DateType, PostRef, Post, User, Media, Image, Video +from query_parser import QueryParser +from modules.common import get_current_utc_time + +# GraphQL API +from snscrape.modules import twitter +from snscrape.base import ScraperException +from snscrape.modules.twitter import (Tweet, TweetRef, Tombstone, UserRef, + TwitterSearchScraper, TwitterProfileScraper, TwitterUserScraper, TwitterTweetScraper) + +class TwitterScrapeBase(Search): + # Posts per emulated page because snscrape returns an iterator. + POSTS_PER_PAGE = 8 + + # Don't allow requests for a page more than this amount past the current page. + REACH_LIMIT = 4 + + def __init__(self, userdata: Any): + super().__init__() + self.module: TwitterScrapeModule = userdata + self.iterator: Iterator[Tweet | TweetRef | Tombstone] + self.page: int = 0 + + def create_media_url(self, url: str) -> MediaUrl: + format = url.find('format=') + if format >= 0: + ext = url[format + 7:format + 10] + else: + question = url.rfind('?') + if question >= 0: + url = url[0:question] + ext = url.split('.')[-1] + return MediaUrl(url, ext) + + def parse_media(self, data: list[twitter.Medium]) -> dict[str, Media]: + media = {} + for i, m in enumerate(data): + if isinstance(m, twitter.Photo): + media[str(i)] = Image(url=self.create_media_url(m.fullUrl), thumbnail_url=self.create_media_url(m.previewUrl)) + elif isinstance(m, twitter.Video) or isinstance(m, twitter.Gif): + videos = sorted(m.variants, key=lambda x: x.bitrate if x.bitrate else 0, reverse=True) + media[str(i)] = Video(url=self.create_media_url(videos[0].url), thumbnail_url=self.create_media_url(m.thumbnailUrl)) + return media + + def create_post(self, tweet: Tweet | TweetRef | Tombstone) -> Post: + kwargs = {} + kwargs['unique_id'] = f'twitter:t:{tweet.id}' + if isinstance(tweet, TweetRef) or isinstance(tweet, Tombstone): + kwargs['type'] = PostType.TOMBSTONE + return Post(**kwargs) + kwargs['raw_responses'] = tweet.rawResponses + kwargs['url'] = tweet.url + # TODO: replace correct? + kwargs['dates'] = { + DateType.CREATED: tweet.date.replace(tzinfo=timezone.utc).timestamp(), + DateType.RETRIEVED: get_current_utc_time().timestamp() + } + author = User(unique_id=f'twitter:u:{tweet.user.id}') + if not isinstance(tweet.user, UserRef): + author.username = tweet.user.username + if tweet.user.displayname: + author.display_name = tweet.user.displayname + if tweet.user.profileImageUrl: + author.profile_picture_url = self.create_media_url(tweet.user.profileImageUrl) + kwargs['author'] = author + if tweet.retweetedTweet: + kwargs['type'] = PostType.REPOST + kwargs['post'] = PostRef(f'twitter:t:{tweet.retweetedTweet.id}') + return Post(**kwargs) + kwargs['type'] = PostType.POST + kwargs['text'] = tweet.rawContent + kwargs['likes'] = tweet.likeCount + kwargs['reposts'] = tweet.retweetCount + kwargs['quotes'] = tweet.quoteCount + kwargs['comments'] = tweet.replyCount + kwargs['views'] = tweet.viewCount + if tweet.media: + kwargs['media'] = self.parse_media(tweet.media) + if tweet.inReplyToTweetId: + kwargs['in_reply_to'] = PostRef(f'twitter:t:{tweet.inReplyToTweetId}') + if tweet.quotedTweet: + kwargs['quoted'] = PostRef(f'twitter:t:{tweet.quotedTweet.id}') + return Post(**kwargs) + + def add_post_to_page(self, num: int, item: Post | User) -> None: + self.module.add_to_map(item.unique_id, item) + self.pages[num].append(item.unique_id) + + def step_iterator_for_page(self, num: int) -> bool: + for _ in range(0, self.POSTS_PER_PAGE): + try: + tweet = next(self.iterator) + except StopIteration: + self.completed = True + break + except ScraperException as e: + log.error(repr(e)) + continue + self.add_post_to_page(num, self.create_post(tweet)) + if isinstance(tweet, TweetRef) or isinstance(tweet, Tombstone): + continue + if tweet.retweetedTweet: + retweet = self.create_post(tweet.retweetedTweet) + self.module.add_to_map(retweet.unique_id, retweet) + if tweet.quotedTweet: + quote = self.create_post(tweet.quotedTweet) + self.module.add_to_map(quote.unique_id, quote) + return len(self.pages[num]) > 0 + + def request_page(self, num: int) -> bool: + if num - self.page >= self.REACH_LIMIT: + return False + request_satisfied = False + for i in range(self.page, num + 1): + if self.completed: + break + if i in self.pages: + continue + self.pages[i] = [] + if not self.step_iterator_for_page(i): + break + self.page = i + if i == num: + request_satisfied = True + return request_satisfied + +class TwitterScrapeSearch(TwitterScrapeBase): + def __init__(self, userdata: Any, arg: str): + super().__init__(userdata) + self.iterator = TwitterSearchScraper(arg, top=True, cookies=self.module.cookies).get_items() + +class TwitterScrapeUser(TwitterScrapeBase): + def __init__(self, userdata: Any, arg: str): + super().__init__(userdata) + if arg.startswith('@'): + arg = arg[1:] + self.iterator = TwitterUserScraper(arg, cookies=self.module.cookies).get_items() + +class TwitterScrapeProfile(TwitterScrapeBase): + def __init__(self, userdata: Any, arg: str): + super().__init__(userdata) + if arg.startswith('@'): + arg = arg[1:] + self.iterator = TwitterProfileScraper(arg, cookies=self.module.cookies).get_items() + +class TwitterScrapeTweet(TwitterScrapeBase): + def __init__(self, userdata: Any, arg: str): + super().__init__(userdata) + if arg.startswith('https://'): + arg = arg[arg.rfind('/') + 1:] + question = arg.find('?') + if question >= 0: + arg = arg[:question] + self.iterator = TwitterTweetScraper(arg, cookies=self.module.cookies).get_items() + +class TwitterScrapeModule(Module): + def __init__(self, cookies_path: str): + super().__init__() + self.parser: QueryParser = QueryParser(self, 'search') + self.parser.add_command('search', TwitterScrapeSearch) + self.parser.add_command('user', TwitterScrapeUser) + self.parser.add_command('profile', TwitterScrapeProfile) + self.parser.add_command('tweet', TwitterScrapeTweet) + if cookies_path: + cookie_jar = http.cookiejar.MozillaCookieJar() + cookie_jar.load(filename=cookies_path, ignore_expires=True) + for c in cookie_jar: + if c.value: + self.cookies[c.name] = c.value + + def search(self, query: str, *extra_args: Any) -> Optional[Search]: + return self.parser.parse_query(query) + + def get_download(self, unique_id: str, key: str) -> Optional[dict[str, Any]]: + if unique_id not in self.unique_id_map: + return None + post = self.unique_id_map[unique_id] + if key not in post.media: + return None + return { + 'urls': [post.media[key].url], + 'headers': { + 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; rv:122.0) Gecko/20100101 Firefox/122.0' + } + } diff --git a/src/portal/py/modules/twitter_api.py b/src/portal/py/modules/twitter_api.py new file mode 100644 index 0000000..6d04855 --- /dev/null +++ b/src/portal/py/modules/twitter_api.py @@ -0,0 +1,269 @@ +from base import Search, Module +from query_parser import QueryParser + +# Official Twitter API +from twitter import Twitter2, TwitterError, OAuth + +# Quite incomplete API based backend. Should *not* be used for archiving. +# HOLD: I can't test this without paying $100 for the X API WTFFF. +''' +class TwitterBase(Search): + ALL_PARAMS = { + 'tweet.fields': 'attachments,author_id,context_annotations,conversation_id,created_at,entities,geo,id,in_reply_to_user_id,lang,public_metrics,possibly_sensitive,referenced_tweets,reply_settings,source,text,withheld', + 'user.fields': 'created_at,description,entities,id,location,name,pinned_tweet_id,profile_image_url,protected,public_metrics,url,username,verified,withheld', + 'media.fields': 'duration_ms,height,media_key,preview_image_url,type,url,width,public_metrics', + 'place.fields': 'contained_within,country,country_code,full_name,geo,id,name,place_type', + 'poll.fields': 'duration_minutes,end_datetime,id,options,voting_status', + } + ALL_EXPRESSIONS = 'author_id,referenced_tweets.id,referenced_tweets.id.author_id,entities.mentions.username,attachments.poll_ids,attachments.media_keys,in_reply_to_user_id,geo.place_id' + + SOME_PARAMS = { + 'tweet.fields': 'attachments,author_id,text,entities,referenced_tweets', + 'user.fields': 'id,name,profile_image_url,url,username', + 'media.fields': 'duration_ms,height,media_key,preview_image_url,type,url,width,public_metrics', + 'place.fields': '', + 'poll.fields': '', + } + SOME_EXPRESSIONS = 'author_id,referenced_tweets.id,referenced_tweets.id.author_id,entities.mentions.username,attachments.media_keys,in_reply_to_user_id' + + def __init__(self, userdata, arg): + super().__init__() + self.provider = userdata + self.params = self.SOME_PARAMS.copy() + self.media_map = {} + self.user_id = None + self.pagination_page = 0 + self.pagination_token = None + self.arg = arg + + def add_attachemnts(self, l, r, data): + if 'media_keys' not in data: + return + for m in data['media_keys']: + if m in r: + continue + r.append(m) + if m in self.media_map: + l.append(Image(url=self.media_map[m], thumbnail_url='')) + + def add_entity_urls(self, data): + if 'urls' in data: + for u in data['urls']: + if 'media_key' in u and u['media_key'] not in self.media_map: + self.media_map[u['media_key']] = u['expanded_url'] + + def make_post(self, tweet): + print(json.dumps(tweet, indent=4)) + media = [] + repeats = [] + if 'referenced_tweets' in t: + print(len(referenced_tweets)) + #for rt in t['referenced_tweets']: + # if rt['id'] in self.tweet_map: + # rrt = self.tweet_map[rt['id']] + # if 'entities' in rrt: + # self.add_entity_urls(rrt['entities']) + # if 'attachments' in rrt: + # self.add_attachemnts(media, repeats, rrt['attachments']) + if 'attachments' in t: + self.add_attachemnts(media, repeats, t['attachments']) + unique_id = 'twitter:t:{}'.format(t['id']) + url = 'https://twitter.com/{}/status/{}'.format(t['author_id'], t['id']) + kwargs = {} + kwargs['unique_id'] = unique_id + kwargs['raw_responses'] = { 'tweet' : json.dumps(t) } + kwargs['url'] = url + kwargs['author'] = User(unique_id='twitter:u:{}'.format(t['author_id'])) + kwargs['title'] = '' + kwargs['text'] = t['text'] + kwargs['media'] = media + return Post(**kwargs) + + def add_items_from_search(self, data): + if 'includes' in data: + includes = data['includes'] + if 'media' in includes: + for i in includes['media']: + if 'url' in i: + self.media_map[i['media_key']] = i['url'] + #if 'tweets' in includes: + # for t in includes['tweets']: + # self.tweet_map[t['id']] = t + if 'data' not in data or not data['data']: + return False + l = data['data'] if type(data['data']) == list else [data['data']] + for t in l: + if 'entities' in t: + self.add_entity_urls(t['entities']) + post = self.make_post(t) + self.pages[self.pagination_page].append(post) + return True + + def add_user_ids_from_search(self, data): + if not data['data']: + return False + for u in data['data']: + self.pages[self.pagination_page].append(User( + 'twitter:u:{}'.format(u['id']), username=u['username'], display_name=u['name'])) + return True + + def should_process_index(self, index): + if index - self.pagination_page >= REACH_LIMIT: + return False + if self.pagination_page > 0 and not self.pagination_token: + self.completed = True + return False + if self.pagination_token: + self.params['pagination_token'] = self.pagination_token + self.pages[self.pagination_page] = [] + return True + + def handle_data(self, data, user_ids=False): + if 'next_token' not in data['meta'].keys(): + self.pagination_token = None + else: + self.pagination_token = data['meta']['next_token'] + if user_ids: + if not self.add_user_ids_from_search(data): + return False + else: + if not self.add_items_from_search(data): + return False + self.pagination_page += 1 + return True + + def get_user_id(self, username): + if self.user_id: + return True + try: + data = self.provider.t.users.by.username._username( + _username=username, _timeout=config.TWITTER_TIMEOUT) + except TwitterError as e: + log.error(repr(e)) + return False + self.user_id = data['data']['id'] + return True + +class TwitterSearch(TwitterBase): + def load_page(self, index): + for _ in range(self.pagination_page, index + 1): + if not self.should_process_index(index): + return False + try: + data = self.provider.t.tweets.search.recent( + query=self.arg, expansions=self.SOME_EXPRESSIONS, params=self.params, + sort_order='relevancy', max_results=25, _timeout=config.TWITTER_TIMEOUT) + except TwitterError as e: + log.error(repr(e)) + return False + return self.handle_data(data) + +class TwitterUser(TwitterBase): + def __init__(self, userdata, arg): + if arg.startswith('@'): + arg = arg[1:] + super().__init__(userdata, arg) + + def load_page(self, index): + for _ in range(self.pagination_page, index + 1): + if not self.should_process_index(index): + return False + if not self.get_user_id(self.arg): + return False + try: + data = self.provider.t.users._id.tweets( + _id=self.user_id, expansions=self.SOME_EXPRESSIONS, params=self.params, + max_results=25, _timeout=config.TWITTER_TIMEOUT) + except TwitterError as e: + log.error(repr(e)) + return False + return self.handle_data(data) + +class TwitterTimeline(TwitterBase): + def __init__(self, userdata, arg): + if not arg: + arg = 'pizzabelly' + super().__init__(userdata, arg) + self.params['exclude'] = 'replies' + + def load_page(self, index): + for _ in range(self.pagination_page, index + 1): + if not self.should_process_index(index): + return False + if not self.get_user_id(self.arg): + return False + try: + data = self.provider.t.users._id.timelines.reverse_chronological( + _id=self.user_id, expansions=self.SOME_EXPRESSIONS, params=self.params, + max_results=25, _timeout=config.TWITTER_TIMEOUT) + except TwitterError as e: + log.error(repr(e)) + return False + return self.handle_data(data) + +class TwitterLikes(TwitterBase): + def __init__(self, userdata, arg): + if arg.startswith('@'): + arg = arg[1:] + super().__init__(userdata, arg) + + def load_page(self, index): + for _ in range(self.pagination_page, index + 1): + if not self.should_process_index(index): + return False + if not self.get_user_id(self.arg): + return False + try: + data = self.provider.t.users._id.liked_tweets( + _id=self.user_id, expansions=self.SOME_EXPRESSIONS, params=self.params, + max_results=25, _timeout=config.TWITTER_TIMEOUT) + except TwitterError as e: + log.error(repr(e)) + return False + return self.handle_data(data) + +class TwitterTweet(TwitterBase): + def load_page(self, index): + for _ in range(self.pagination_page, index + 1): + if not self.should_process_index(index): + return False + try: + data = self.provider.t.tweets( + ids=self.arg, expansions=self.SOME_EXPRESSIONS, params=self.params, + _timeout=config.TWITTER_TIMEOUT) + except TwitterError as e: + log.error(repr(e)) + return False + return self.handle_data(data) + +class TwitterFollowing(TwitterBase): + def __init__(self, userdata, arg): + if arg.startswith('@'): + arg = arg[1:] + super().__init__(userdata, arg) + del self.params['media.fields'] + del self.params['place.fields'] + del self.params['poll.fields'] + + def load_page(self, index): + for _ in range(self.pagination_page, index + 1): + if not self.should_process_index(index): + return False + if not self.get_user_id(self.arg): + return False + try: + data = self.provider.t.users._id.following( + _id=self.user_id, params=self.params, max_results=25, _timeout=config.TWITTER_TIMEOUT) + except TwitterError as e: + log.error(repr(e)) + return False + return self.handle_data(data, True) +''' + +class TwitterApiModule(Module): + def __init__(self, access_key: str, access_secret: str, consumer_key: str, consumer_secret: str): + self.t: Twitter2 = Twitter2(auth=OAuth(access_key, access_secret, consumer_key, consumer_secret), retry=True) + self.parser: QueryParser = QueryParser(self, 'timeline') + #self.parser.add_command('timeline', TwitterTimeline) + #self.parser.add_command('likes', TwitterLikes) + #self.parser.add_command('following', TwitterFollowing) diff --git a/src/portal/py/modules/youtube.py b/src/portal/py/modules/youtube.py new file mode 100644 index 0000000..d14dfe8 --- /dev/null +++ b/src/portal/py/modules/youtube.py @@ -0,0 +1,133 @@ +import log +from typing import Optional, Any +from base import Search, Module, ParsedJson +from post import MediaUrl, Post, PostType, Media, Video +from query_parser import QueryParser +from yt_dlp import YoutubeDL + +class YDLLogger(): + def debug(self, msg): + if msg.startswith('[debug] '): + log.debug(msg) + else: + log.info(msg) + + def info(self, msg): + log.info(msg) + + def warning(self, msg): + log.warn(msg) + + def error(self, msg): + log.error(msg) + +ydl_opts = { + 'quiet': False, + 'logger': YDLLogger(), + 'cachedir': False, +# 'cookiefile': '', +} + +ydl = YoutubeDL(ydl_opts) + +def get_playback_url(data, video=True): + if 'entries' in data: + if len(data['entries']) == 0: + return None + data = data['entries'][0] + + if 'formats' not in data: + if 'url' in data: + return data['url'] + return None + + # Filter out hls temporarily. + data['formats'] = list(filter(lambda f: not f['protocol'].startswith('m3u8'), data['formats'])) + + if len(data['formats']) == 0: + return None + + url = data['formats'][0]['url'] + + # audio_ext? + has_audio = list(filter(lambda f: 'acodec' not in f or f['acodec'] != 'none', data['formats'])) + + if video: + if len(has_audio) > 0: + data['formats'] = has_audio + try: + data['formats'] = list(filter(lambda f: 'quality' in f, data['formats'])) + url = max(data['formats'], key=lambda f: f['quality'])['url'] + except: + pass + else: + if len(has_audio) == 0: + return None + data['formats'] = has_audio + try: + audio_only = list(filter(lambda f: f['vcodec'] == 'none', data['formats'])) + if len(audio_only) > 0: + data['formats'] = audio_only + else: + data['formats'] = list(filter(lambda f: f['ext'] in ['mp4'], data['formats'])) + except: + pass + try: + data['formats'] = list(filter(lambda f: 'abr' in f, data['formats'])) + url = max(data['formats'], key=lambda f: f['abr'])['url'] + except: + pass + + return url + +class YoutubeBase(Search): + def __init__(self, userdata: Any): + super().__init__() + self.module: YoutubeModule = userdata + + def get_info(self) -> Optional[ParsedJson]: + return None + + def request_page(self, num: int) -> bool: + info = self.get_info() + if not info: + return False + url = get_playback_url(info) + if not url: + return False + media: dict[str, Media] = { '0': Video(url=MediaUrl(url=url)) } + unique_id = f'youtube:v:{info['id']}' + self.module.add_to_map(unique_id, Post(type=PostType.POST, unique_id=unique_id, url='', title=info['title'], text='', media=media)) + self.pages[num] = [unique_id] + return True + +class YoutubeSearch(YoutubeBase): + def __init__(self, userdata: Any, arg: str): + super().__init__(userdata) + self.query: str = arg + + def get_info(self) -> Optional[ParsedJson]: + return ydl.extract_info(f'ytsearch1:{self.query}', download=False) + +class YoutubeLink(YoutubeBase): + def __init__(self, userdata: Any, arg: str): + super().__init__(userdata) + self.link: str = arg + + def get_info(self) -> Optional[ParsedJson]: + return ydl.extract_info(self.link, download=False) + +class YoutubeModule(Module): + def __init__(self): + super().__init__() + self.parser: QueryParser = QueryParser(self, 'search') + self.parser.add_command('search', YoutubeSearch) + self.parser.add_command('link', YoutubeLink) + + def search(self, query: str, *extra_args: Any) -> Optional[Search]: + return self.parser.parse_query(query) + + def get_download(self, unique_id: str, key: str) -> Optional[dict[str, Any]]: + if unique_id not in self.unique_id_map: + return None + return None diff --git a/src/portal/py/post.py b/src/portal/py/post.py new file mode 100644 index 0000000..b064f97 --- /dev/null +++ b/src/portal/py/post.py @@ -0,0 +1,132 @@ +import dataclasses +from dataclasses import field +from typing import Optional +from enum import Enum +from json import JSONEncoder + +class PostType(int, Enum): + UNKNOWN = 0 + POST = 1 + REPOST = 2 + PREVIEW = 3 + TOMBSTONE = 4 + +class DateType(int, Enum): + CREATED = 0 + EDITED = 1 + RETRIEVED = 2 + +class MediaType(int, Enum): + UNKNOWN = 0 + FILE = 1 + AUDIO = 2 + IMAGE = 3 + VIDEO = 4 + VIDEO_SPLIT = 5 + ANIMATION = 6 + +class TagType(int, Enum): + UNKNOWN = 0 + GENERAL = 1 + ARTIST = 2 + CHARACTER = 3 + COPYRIGHT = 4 + META = 5 + DEPRECATED = 6 + +@dataclasses.dataclass +class MediaUrl(): + url: str = '' + ext: str = 'unknown' + +@dataclasses.dataclass +class Media(): + type: MediaType = MediaType.UNKNOWN + url: MediaUrl = field(default_factory=lambda: MediaUrl()) + +@dataclasses.dataclass +class File(Media): + type: MediaType = MediaType.FILE + name: str = '' + +@dataclasses.dataclass +class Audio(Media): + type: MediaType = MediaType.AUDIO + +@dataclasses.dataclass +class Image(Media): + type: MediaType = MediaType.IMAGE + thumbnail_url: MediaUrl = field(default_factory=lambda: MediaUrl()) + +@dataclasses.dataclass +class Video(Media): + type: MediaType = MediaType.VIDEO + thumbnail_url: MediaUrl = field(default_factory=lambda: MediaUrl()) + +@dataclasses.dataclass +class VideoSplit(Media): + type: MediaType = MediaType.VIDEO_SPLIT + audio_url: MediaUrl = field(default_factory=lambda: MediaUrl()) + subtitle_url: MediaUrl = field(default_factory=lambda: MediaUrl()) + thumbnail_url: MediaUrl = field(default_factory=lambda: MediaUrl()) + +@dataclasses.dataclass +class Animation(Media): + type: MediaType = MediaType.ANIMATION + thumbnail_url: MediaUrl = field(default_factory=lambda: MediaUrl()) + frames: list[tuple[str, int]] = field(default_factory=lambda: []) + +@dataclasses.dataclass +class Tag(): + type: TagType = TagType.UNKNOWN + name: str = '' + alts: dict[str, str] = field(default_factory=lambda: {}) + +@dataclasses.dataclass +class User(): + unique_id: str = '' + username: str = '' + display_name: str = '' + profile_picture_url: MediaUrl = field(default_factory=lambda: MediaUrl()) + +@dataclasses.dataclass +class PostRef(): + unique_id: str = '' + +# V4 Ideas: +# - Date estimate and range +# - Edited but unknown when +# - Generally an estimate/guess +# - Formated text/body + +@dataclasses.dataclass +class Post(): + version: int = 3 + type: PostType = PostType.UNKNOWN + unique_id: str = '' + raw_responses: dict[str, str] = field(default_factory=lambda: {}) + url: str = '' + dates: dict[DateType, float] = field(default_factory=lambda: {}) + author: User = field(default_factory=lambda: User()) + title: str = '' + text: str = '' + likes: Optional[int] = None + bookmarks: Optional[int] = None + reposts: Optional[int] = None + quotes: Optional[int] = None + comments: Optional[int] = None + views: Optional[int] = None + tags: list[Tag] = field(default_factory=lambda: []) + links: list[str] = field(default_factory=lambda: []) + media: dict[str, Media] = field(default_factory=lambda: {}) + post: PostRef = field(default_factory=lambda: PostRef()) + quoted: PostRef = field(default_factory=lambda: PostRef()) + in_reply_to: PostRef = field(default_factory=lambda: PostRef()) + +class PostEncoder(JSONEncoder): + def default(self, o): + if isinstance(o, Enum): + return o.value + if dataclasses.is_dataclass(o): + return dataclasses.asdict(o) + return o.__dict__ diff --git a/src/portal/py/query_parser.py b/src/portal/py/query_parser.py new file mode 100644 index 0000000..c16ded2 --- /dev/null +++ b/src/portal/py/query_parser.py @@ -0,0 +1,45 @@ +from typing import Optional, Any, Callable +from base import Search + +type QueryCommand = Callable[[Any, Optional[str]], Search] + +class QueryParser(): + def __init__(self, userdata: Any, default_command: str): + self.commands: dict[str, QueryCommand] = {} + self.escaped_commands: dict[str, str] = {} + self.userdata: Any = userdata + self.default_command: str = default_command + + def add_command(self, name, search) -> None: + self.commands[name] = search + self.escaped_commands['\\' + name] = name + + def parse_query(self, query: str) -> Search: + arg = query + search = self.commands[self.default_command] + + sp = query.split(' ') + + for i, word in enumerate(sp): + col = word.find(':') + if col == -1: + continue + command = word[0:col] + if command in self.escaped_commands: + query = query.replace(command, self.escaped_commands[command], 1) + continue + if command in self.commands: + if col + 1 < len(word): + arg = word[col+1:] + else: + arg = None + search = self.commands[command] + if i == 0: + if len(sp) > 1: + query = query.replace(word + ' ', '', 1) + else: + query = query.replace(word, '', 1) + else: + query = query.replace(' ' + word + ' ', '', 1) + + return search(self.userdata, arg) diff --git a/src/portal/requirements.txt b/src/portal/requirements.txt new file mode 100644 index 0000000..d241387 --- /dev/null +++ b/src/portal/requirements.txt @@ -0,0 +1,2 @@ +httpx[http2,brotli] +pillow diff --git a/src/portal/scripts/create_vendor.sh b/src/portal/scripts/create_vendor.sh new file mode 100755 index 0000000..f4d551e --- /dev/null +++ b/src/portal/scripts/create_vendor.sh @@ -0,0 +1,28 @@ +#! /usr/bin/env sh + +python3 -m venv ./venv +source ./venv/bin/activate + +cd vendor/ +rm -r vendor/ + +pip3 install -r ../requirements.txt --upgrade --target=./vendor + +cd yt-dlp/ +pip3 install . --upgrade --target=../vendor +cd ../ + +cd snscrape/ +pip3 install . --upgrade --target=../vendor +cd ../ + +cd pixivpy/ +pip3 install . --upgrade --target=../vendor +cd ../ + +cd instagrapi/ +pip3 install . --upgrade --target=../vendor +cd ../ + +cd ../ +rm -r venv diff --git a/src/portal/scripts/run_py.sh b/src/portal/scripts/run_py.sh new file mode 100755 index 0000000..ddbe869 --- /dev/null +++ b/src/portal/scripts/run_py.sh @@ -0,0 +1,4 @@ +#! /usr/bin/env sh +export PYTHONDONTWRITEBYTECODE=1 +export PYTHONPATH=$HOME/c/camu/src/portal/vendor/vendor +python3 $@ diff --git a/src/portal/src/packet_ext.c b/src/portal/src/packet_ext.c new file mode 100644 index 0000000..580c307 --- /dev/null +++ b/src/portal/src/packet_ext.c @@ -0,0 +1,98 @@ +#include "packet_ext.h" + +static void aki_packet_write_optional_int(struct aki_packet *packet, optional_int *o) +{ + AKI_PACKET_WRITE_TYPE(packet, s64, o->i); + AKI_PACKET_WRITE_TYPE(packet, bool, o->set); +} + +void aki_packet_write_camu_post(struct aki_packet *packet, struct camu_post *post) +{ + AKI_PACKET_WRITE_TYPE(packet, u16, post->version); + AKI_PACKET_WRITE_TYPE(packet, u8, post->type); + aki_packet_write_str(packet, &post->unique_id); + aki_packet_write_str(packet, &post->url); + AKI_PACKET_WRITE_TYPE(packet, u32, post->dates.size); + struct camu_post_date *date; + al_array_foreach_ptr(post->dates, i, date) { + AKI_PACKET_WRITE_TYPE(packet, u8, date->type); + AKI_PACKET_WRITE_TYPE(packet, f32, date->timestamp); + } + aki_packet_write_str(packet, &post->author.unique_id); + aki_packet_write_wstr(packet, &post->author.username); + aki_packet_write_wstr(packet, &post->author.display_name); + aki_packet_write_str(packet, &post->author.profile_picture_url); + aki_packet_write_wstr(packet, &post->title); + aki_packet_write_wstr(packet, &post->text); + aki_packet_write_optional_int(packet, &post->likes); + aki_packet_write_optional_int(packet, &post->bookmarks); + aki_packet_write_optional_int(packet, &post->reposts); + aki_packet_write_optional_int(packet, &post->quotes); + aki_packet_write_optional_int(packet, &post->comments); + aki_packet_write_optional_int(packet, &post->views); + AKI_PACKET_WRITE_TYPE(packet, u32, post->media.size); + struct camu_post_media *media; + al_array_foreach_ptr(post->media, i, media) { + AKI_PACKET_WRITE_TYPE(packet, u8, media->type); + aki_packet_write_str(packet, &media->key); + aki_packet_write_str(packet, &media->url); + aki_packet_write_str(packet, &media->ext); + aki_packet_write_str(packet, &media->thumbnail_url); + aki_packet_write_str(packet, &media->thumbnail_ext); + } + aki_packet_write_str(packet, &post->post.unique_id); + aki_packet_write_str(packet, &post->quoted.unique_id); + aki_packet_write_str(packet, &post->in_reply_to.unique_id); +} + +static void aki_packet_read_optional_int(struct aki_packet *packet, optional_int *o) +{ + AKI_PACKET_READ_TYPE(packet, s64, o->i); + AKI_PACKET_READ_TYPE(packet, bool, o->set); +} + +void aki_packet_read_camu_post(struct aki_packet *packet, struct camu_post *post) +{ + camu_post_reset(post); + AKI_PACKET_READ_TYPE(packet, u16, post->version); + AKI_PACKET_READ_TYPE(packet, u8, post->type); + aki_packet_read_str(packet, &post->unique_id); + aki_packet_read_str(packet, &post->url); + u32 dates_size; + AKI_PACKET_READ_TYPE(packet, u32, dates_size); + al_array_reserve(post->dates, dates_size); + for (u32 i = 0; i < dates_size; i++) { + struct camu_post_date date; + AKI_PACKET_READ_TYPE(packet, u8, date.type); + AKI_PACKET_READ_TYPE(packet, f32, date.timestamp); + al_array_push(post->dates, date); + } + aki_packet_read_str(packet, &post->author.unique_id); + aki_packet_read_wstr(packet, &post->author.username); + aki_packet_read_wstr(packet, &post->author.display_name); + aki_packet_read_str(packet, &post->author.profile_picture_url); + aki_packet_read_wstr(packet, &post->title); + aki_packet_read_wstr(packet, &post->text); + aki_packet_read_optional_int(packet, &post->likes); + aki_packet_read_optional_int(packet, &post->bookmarks); + aki_packet_read_optional_int(packet, &post->reposts); + aki_packet_read_optional_int(packet, &post->quotes); + aki_packet_read_optional_int(packet, &post->comments); + aki_packet_read_optional_int(packet, &post->views); + u32 media_size; + AKI_PACKET_READ_TYPE(packet, u32, media_size); + al_array_reserve(post->media, media_size); + for (u32 i = 0; i < media_size; i++) { + struct camu_post_media media; + AKI_PACKET_READ_TYPE(packet, u8, media.type); + aki_packet_read_str(packet, &media.key); + aki_packet_read_str(packet, &media.url); + aki_packet_read_str(packet, &media.ext); + aki_packet_read_str(packet, &media.thumbnail_url); + aki_packet_read_str(packet, &media.thumbnail_ext); + al_array_push(post->media, media); + } + aki_packet_read_str(packet, &post->post.unique_id); + aki_packet_read_str(packet, &post->quoted.unique_id); + aki_packet_read_str(packet, &post->in_reply_to.unique_id); +} diff --git a/src/portal/src/packet_ext.h b/src/portal/src/packet_ext.h new file mode 100644 index 0000000..eae1e07 --- /dev/null +++ b/src/portal/src/packet_ext.h @@ -0,0 +1,10 @@ +#pragma once + +#include <al/types.h> +#include <aki/common.h> +#include <aki/event_loop.h> + +#include "post.h" + +void aki_packet_write_post(struct aki_packet *packet, struct camu_post *post); +void aki_packet_read_post(struct aki_packet *packet, struct camu_post *post); diff --git a/src/portal/src/post.c b/src/portal/src/post.c new file mode 100644 index 0000000..c5eb4b5 --- /dev/null +++ b/src/portal/src/post.c @@ -0,0 +1,64 @@ +#include "post.h" + +void camu_post_reset(struct camu_post *post) +{ + al_bzero(post, sizeof(struct camu_post)); + al_array_init(post->dates); + al_array_init(post->media); +} + +void camu_post_clone(struct camu_post *dest, struct camu_post *src) +{ + camu_post_reset(dest); + dest->version = src->version; + dest->type = src->type; + al_str_clone(&dest->unique_id, &src->unique_id); + al_str_clone(&dest->url, &src->url); + al_array_clone(dest->dates, src->dates); + al_str_clone(&dest->author.unique_id, &src->author.unique_id); + al_wstr_clone(&dest->author.username, &src->author.username); + al_wstr_clone(&dest->author.display_name, &src->author.display_name); + al_str_clone(&dest->author.profile_picture_url, &src->author.profile_picture_url); + al_wstr_clone(&dest->title, &src->title); + al_wstr_clone(&dest->text, &src->text); + dest->likes = src->likes; + dest->bookmarks = src->bookmarks; + dest->reposts = src->reposts; + dest->quotes = src->quotes; + dest->comments = src->comments; + dest->views = src->views; + struct camu_post_media *media; + struct camu_post_media media_copy; + al_array_foreach_ptr(src->media, i, media) { + al_bzero(&media_copy, sizeof(struct camu_post_media)); + media_copy.type = media->type; + al_str_clone(&media_copy.key, &media->key); + al_str_clone(&media_copy.url, &media->url); + al_str_clone(&media_copy.ext, &media->ext); + al_str_clone(&media_copy.thumbnail_url, &media->thumbnail_url); + al_str_clone(&media_copy.thumbnail_ext, &media->thumbnail_ext); + al_array_push(dest->media, media_copy); + } + al_str_clone(&dest->post.unique_id, &src->post.unique_id); + al_str_clone(&dest->quoted.unique_id, &src->quoted.unique_id); + al_str_clone(&dest->in_reply_to.unique_id, &src->in_reply_to.unique_id); +} + +void camu_post_add_media(struct camu_post *post, u8 type, str *key, str *url, str *ext, str *thumbnail_url, str *thumbnail_ext) +{ + al_array_push(post->media, ((struct camu_post_media){ + .type = type, + .key = *key, + .url = *url, + .ext = *ext, + .thumbnail_url = *thumbnail_url, + .thumbnail_ext = *thumbnail_ext + })); +} + +void camu_post_add_date(struct camu_post *post, u8 type, f32 timestamp) +{ + al_array_push(post->dates, ((struct camu_post_date){ + .type = type, .timestamp = timestamp + })); +} diff --git a/src/portal/src/post.h b/src/portal/src/post.h new file mode 100644 index 0000000..11fa9b6 --- /dev/null +++ b/src/portal/src/post.h @@ -0,0 +1,102 @@ +#pragma once + +#include <al/types.h> +#include <al/str.h> +#include <al/wstr.h> +#include <al/array.h> + +enum { + CAMU_POST_UNKNOWN = 0, + CAMU_POST_POST, + CAMU_POST_REPOST, + CAMU_POST_PREVIEW, + CAMU_POST_TOMBSTONE +}; + +enum { + CAMU_DATE_CREATED = 0, + CAMU_DATE_EDITED, + CAMU_DATE_RETRIEVED +}; + +enum { + CAMU_MEDIA_UNKNOWN = 0, + CAMU_MEDIA_FILE, + CAMU_MEDIA_AUDIO, + CAMU_MEDIA_IMAGE, + CAMU_MEDIA_VIDEO, + CAMU_MEDIA_VIDEO_SPLIT, + CAMU_MEDIA_ANIMATION +}; + +enum { + CAMU_TAG_UNKNOWN = 0, + CAMU_TAG_GENERAL, + CAMU_TAG_ARTIST, + CAMU_TAG_CHARACTER, + CAMU_TAG_COPYRIGHT, + CAMU_TAG_META, + CAMU_TAG_DEPRECATED +}; + +typedef struct { + s64 i; + bool set; +} optional_int; + +struct camu_post_date { + u8 type; + f32 timestamp; +}; + +typedef array(struct camu_post_date) camu_dates_array; + +struct camu_post_media { + u8 type; + str key; + str url; + str ext; + str thumbnail_url; + str thumbnail_ext; +}; + +typedef array(struct camu_post_media) camu_media_array; + +struct camu_post_user { + str unique_id; + wstr username; + wstr display_name; + str profile_picture_url; +}; + +struct camu_post_ref { + str unique_id; +}; + +struct camu_post { + u16 version; + u8 type; + str unique_id; + str url; + camu_dates_array dates; + struct camu_post_user author; + wstr title; + wstr text; + optional_int likes; + optional_int bookmarks; + optional_int reposts; + optional_int quotes; + optional_int comments; + optional_int views; + camu_media_array media; + struct camu_post_ref post; // reposted post. + struct camu_post_ref quoted; + struct camu_post_ref in_reply_to; +}; + +void camu_post_reset(struct camu_post *post); +void camu_post_clone(struct camu_post *dest, struct camu_post *src); + +// Python internal. +void camu_post_add_date(struct camu_post *post, u8 type, f32 timestamp); +void camu_post_add_media(struct camu_post *post, u8 type, str *key, str *url, str *ext, str *thumbnail_url, str *thumbnail_ext); diff --git a/src/portal/src/post_cache.c b/src/portal/src/post_cache.c new file mode 100644 index 0000000..ad50404 --- /dev/null +++ b/src/portal/src/post_cache.c @@ -0,0 +1,30 @@ +#include "post_cache.h" + +void camu_post_cache_init(struct camu_post_cache *cache) +{ + al_array_init(cache->cache); +} + +void camu_post_cache_push(struct camu_post_cache *cache, struct camu_post *post) +{ + struct camu_post *check = camu_post_cache_get(cache, &post->unique_id); + if (!check) { + struct camu_post *npost = al_alloc_object(struct camu_post); + camu_post_clone(npost, post); + al_array_push(cache->cache, npost); + } +} + +struct camu_post *camu_post_cache_get(struct camu_post_cache *cache, str *unique_id) +{ + struct camu_post *post; + al_array_foreach(cache->cache, i, post) { + if (al_str_eq(&post->unique_id, unique_id)) { + return post; + } + if (al_str_eq(&post->author.unique_id, unique_id)) { + return post; + } + } + return NULL; +} diff --git a/src/portal/src/post_cache.h b/src/portal/src/post_cache.h new file mode 100644 index 0000000..a406c72 --- /dev/null +++ b/src/portal/src/post_cache.h @@ -0,0 +1,13 @@ +#pragma once + +#include <al/array.h> + +#include "post.h" + +struct camu_post_cache { + array(struct camu_post *) cache; +}; + +void camu_post_cache_init(struct camu_post_cache *cache); +void camu_post_cache_push(struct camu_post_cache *cache, struct camu_post *post); +struct camu_post *camu_post_cache_get(struct camu_post_cache *cache, str *unique_id); diff --git a/src/portal/src/search.c b/src/portal/src/search.c new file mode 100644 index 0000000..ec40d0e --- /dev/null +++ b/src/portal/src/search.c @@ -0,0 +1,141 @@ +#include <al/log.h> + +#include "../cpy/portal.c" + +#include "search.h" + +bool camu_python_init(void) +{ + PyPreConfig pre; + PyPreConfig_InitPythonConfig(&pre); + pre.utf8_mode = 1; + pre.dev_mode = 0; + Py_PreInitialize(&pre); + if (PyImport_AppendInittab("portal", PyInit_portal) == -1) { + al_log_error("portal", "Could not extend in-built modules table."); + camu_python_close(); + return false; + } + Py_Initialize(); + PyObject *module = PyImport_ImportModule("portal"); + if (!module) { + PyErr_Print(); + al_log_error("portal", "Could not import module."); + camu_python_close(); + return false; + } + return true; +} + +void camu_python_close(void) +{ + if (Py_IsInitialized()) Py_Finalize(); +} + +static struct camu_result_page *page_at_index(struct camu_search *search, u32 num) +{ + struct camu_result_page *page; + al_array_foreach_ptr(search->pages, i, page) { + if (page->num == num) return page; + } + al_array_push(search->pages, (struct camu_result_page){0}); + page = &al_array_last(search->pages); + page->num = num; + al_array_init(page->posts); + al_array_init(page->list); + return page; +} + + +void camu_portal_init(struct camu_portal *portal, struct camu_post_cache *cache) +{ + portal->cache = cache; + al_array_init(portal->searches); +} + +static void camu_search_init_internal(struct camu_search *search) +{ + search->page = 0; + al_array_init(search->pages); +} + +s32 camu_portal_create_search(struct camu_portal *portal, str *module, str *query) +{ + s32 id = portal_bridge_search(module, query); + if (id >= 0) { + struct camu_search *search = al_alloc_object(struct camu_search); + camu_search_init_internal(search); + search->id = id; + al_str_clone(&search->module, module); + al_str_clone(&search->query, query); + search->portal = portal; + al_array_push(portal->searches, search); + } + al_log_info("portal", "New search %i (%.*s).", id, AL_STR_PRINTF(query)); + return id; +} + +struct camu_search *camu_portal_get_search(struct camu_portal *portal, s32 id) +{ + struct camu_search *search; + al_array_foreach(portal->searches, i, search) { + if (search->id == id) return search; + } + return NULL; +} + +void camu_portal_discard_search(struct camu_portal *portal, s32 id) +{ + (void)portal; + (void)id; +} + +void camu_portal_close(struct camu_portal *portal) +{ + (void)portal; +} + +bool camu_search_get_page(struct camu_search *search, u32 num) +{ + struct camu_result_page *page; + al_array_foreach_ptr(search->pages, i, page) { + if (page->num == num) goto out; + } + al_log_info("portal", "Loading page %i (%.*s).", num, AL_STR_PRINTF(&search->query)); + if (portal_bridge_get_page(search, search->id, num) == -1) { + return false; + } + struct camu_portal *portal = search->portal; + if (portal->cache) { + struct camu_post *post; + al_array_foreach_ptr(al_array_at(search->pages, num).posts, i, post) { + camu_post_cache_push(portal->cache, post); + } + } +out: + search->page = num; + return true; +} + +void camu_search_free(struct camu_search *search) +{ + struct camu_result_page *page; + al_array_foreach_ptr(search->pages, i, page) { + // TODO: Free camu_post ? + al_array_free(page->posts); + al_array_free(page->list); + } + al_array_free(search->pages); +} + +void camu_search_add_post(struct camu_search *search, u32 num, struct camu_post *post) +{ + struct camu_result_page *page = page_at_index(search, num); + al_array_push(page->posts, *post); +} + +void camu_search_add_to_list(struct camu_search *search, u32 num, str *unique_id) +{ + struct camu_result_page *page = page_at_index(search, num); + al_array_push(page->list, *unique_id); +} diff --git a/src/portal/src/search.h b/src/portal/src/search.h new file mode 100644 index 0000000..4962170 --- /dev/null +++ b/src/portal/src/search.h @@ -0,0 +1,39 @@ +#pragma once + +#include "post.h" +#include "post_cache.h" + +struct camu_result_page { + u32 num; + array(struct camu_post) posts; + array(str) list; +}; + +struct camu_search { + s32 id; + str module; + str query; + u32 page; + array(struct camu_result_page) pages; + struct camu_portal *portal; +}; + +struct camu_portal { + struct camu_post_cache *cache; + array(struct camu_search *) searches; +}; + +bool camu_python_init(void); +void camu_python_close(void); + +void camu_portal_init(struct camu_portal *portal, struct camu_post_cache *cache); +s32 camu_portal_create_search(struct camu_portal *portal, str *module_str, str *search_str); +struct camu_search *camu_portal_get_search(struct camu_portal *portal, s32 id); +void camu_portal_discard_search(struct camu_portal *portal, s32 id); +void camu_portal_close(struct camu_portal *portal); + +bool camu_search_get_page(struct camu_search *search, u32 num); + +// Python internal. +void camu_search_add_post(struct camu_search *search, u32 num, struct camu_post *post); +void camu_search_add_to_list(struct camu_search *search, u32 num, str *unique_id); diff --git a/src/portal/tests/archive_query.py b/src/portal/tests/archive_query.py new file mode 100644 index 0000000..9f69c1f --- /dev/null +++ b/src/portal/tests/archive_query.py @@ -0,0 +1,262 @@ +import os +import sys +import signal +import json +import threading +import gzip +from datetime import datetime, timezone +sys.path.append('../py') +from base import Method +from post import PostEncoder, PostType, DateType, User +from modules import ALL_MODULES +from logger import Logger + +mode = 'instagram' +cmd = 'user' +module = ALL_MODULES[mode][0] +if not module.init(): + sys.exit(1) + +quit_mutex = threading.Lock() +queue_mutex = threading.Lock() +queue_cond = threading.Condition(queue_mutex) +is_running = True + +def default_media_download(obj, post, output_base): + for key in post.media.keys(): + download_params = module.get_download(post.unique_id, key) + if not download_params: + obj.log.write(f'Skipping media{key} for post {post.unique_id}.') + continue + for url in download_params['urls']: + media_path = f'{output_base}_media{key}.{url.ext}' + obj.log.write(f'Attempting to download file {media_path}.') + response = module.do_request(Method.GET, url.url) + if not response: + obj.log.write(f'ERROR: Downloading post media failed ({post.unique_id} #{key}).') + continue + if not obj.write_to_file(media_path, 'wb+', response.content, False): + return False + +class QueryDownloadThread(threading.Thread): + def __init__(self, log, complete, query, output_dir): + super().__init__() + self.log = log + self.complete = complete + self.query = query + self.output_dir = output_dir + self.media_download = default_media_download + + def make_path(self, post): + prefix: str + if post.type != PostType.TOMBSTONE: + prefix = post.author.unique_id.replace(':', '_') + else: + prefix = 'tombstones' + path = f'{self.output_dir}/{prefix}' + if not os.path.isdir(path): + os.mkdir(path) + return path + + def write_to_file(self, path, open_method, content, compress): + try: + if compress: + with gzip.open(path, open_method) as f: + f.write(content.encode('utf-8')) + else: + with open(path, open_method) as f: + f.write(content) + except: + self.log.write(f'ERROR: Writing file failed ({path}).') + return False + return True + + def write_post_to_disk(self, post): + if post.post.unique_id: + original = module.get_item(post.post.unique_id) + if original: + self.write_post_to_disk(original) + if post.quoted.unique_id: + quoted = module.get_item(post.quoted.unique_id) + if quoted: + self.write_post_to_disk(quoted) + message = f'Downloading post {post.unique_id}' + if 'detached_api' in post.raw_responses: + raw_response_path = f'{self.output_dir}/raw_responses' + if not os.path.isdir(raw_response_path): + os.mkdir(raw_response_path) + raw_response_path += f'/{post.raw_responses['hash']}.json.gz' + if not self.write_to_file(raw_response_path, 'wb+', json.dumps(json.loads(post.raw_responses['detached_api'])), True): + return False + del post.raw_responses['detached_api'] + if post.type != PostType.TOMBSTONE: + message += f':{datetime.fromtimestamp(post.dates[DateType.CREATED], tz=timezone.utc).isoformat()} from {post.author.username}({post.author.display_name}):{post.author.unique_id}.' + else: + message += '.' + output_path = self.make_path(post) + output_base = f'{output_path}/{post.unique_id.replace(':', '_')}' + output = output_base + '.json.gz' + ''' + if not os.path.isfile(output): + print('done') + sys.exit(1) + ''' + ''' + has_mp4 = False + for m in post.media.values(): + if m.url.ext == 'mp4': + self.log.write('Rewriting post with mp4.') + has_mp4 = True + break + if post.quoted.unique_id: + self.log.write('Rewriting post with quote.') + has_mp4 = True + ''' + if os.path.isfile(output): + skip_message = f'Skipping already downloaded post {post.unique_id}' + if post.type != PostType.TOMBSTONE: + skip_message += f' from {post.author.unique_id}.' + else: + skip_message += '.' + self.log.write(skip_message) + return True + self.log.write(message) + self.media_download(self, post, output_base) + if not self.write_to_file(output, 'wb+', json.dumps(post, cls=PostEncoder), True): + return False + return True + + def supply_post(self, post): + if not self.write_post_to_disk(post): + return False + return True + + def run(self): + global quit_mutex + global queue_cond + global is_running + search = module.search(self.query) + if not search: + return + completed = True + arg = self.query.split(':')[-1] + num = 0 + count = 0 + while completed: + self.log.write(f'Starting page {num} of {arg}.') + page = search.get_page(num) + if not page: + break + num += 1 + for unique_id in page: + if unique_id == 'twitter:t:1597637140833529856': + self.log.write('---------Hit target---------') + with quit_mutex: + if not is_running: + self.complete.write(f'{mode}:{cmd}:{arg}:i:{str(count)},{str(num)}') + completed = False + break + post = module.get_item(unique_id) + if not post: + self.log.write(f'Missing post with id {unique_id}.') + continue + if not self.write_post_to_disk(post): + continue + count += 1 + if completed: + self.complete.write(f'{mode}:{cmd}:{arg}:c:{str(count)},{str(num)}') + with queue_cond: + queue_cond.notify() + +RUNTIME_PATH = "/mnt/store/files/tmp/run" +LOG_FILE = f'{RUNTIME_PATH}/archive.log' +ARG_FILE = f'{RUNTIME_PATH}/completed_args.log' + +def read_completed_args(path): + args = [] + if os.path.isfile(path): + with open(path, 'r') as f: + for l in f.read().splitlines(): + if l[0] == '-': + continue + s = l.split(':') + if s[3] == 'c': + args.append(f'{s[0]}:{s[1]}:{s[2]}') + return args + +def download_args(): + global quit_mutex + global queue_cond + global compat_thread + + completed_args = read_completed_args(ARG_FILE) + + log = Logger(LOG_FILE) + complete = Logger(ARG_FILE) + + header = f'--------{datetime.now().isoformat()}---------' + log.write(header) + complete.write(header) + + output_dir = sys.argv[1] + if not os.path.isdir(output_dir): + os.mkdir(output_dir) + + #args = module.search(f'following:{22781328}') + args = ['byjoshuajamal'] + + threads = [] + start = True + + for arg in args: + #if mode == 'fanbox': + # user = module.get_item(arg) + # if not user or not isinstance(user, User): + # log.write(f'Failed to retrive information for user {arg}.') + # continue + # arg = user.username + with quit_mutex: + if not is_running: + break + if mode == 'pixiv_web' and not start: + if arg == '': + start = True + continue + if not start: + continue + if mode != 'instagram': + arg = f'{cmd}:{arg}' + if f'{mode}:{arg}' in completed_args: + log.write(f'Skipping already downloaded arg {arg}.') + continue + if len(threads) >= 1: + with queue_cond: + queue_cond.wait() + threads = [t for t in threads if t.is_alive()] + with quit_mutex: + if not is_running: + break + log.write(f'***********************\nStarting download of {arg}\n***********************\n') + arg_thread = QueryDownloadThread(log, complete, arg, output_dir) + threads.append(arg_thread) + arg_thread.start() + + for t in threads: + t.join() + + log.close() + complete.close() + +def signal_handler(sig, frame): + global quit_mutex + global is_running + print('Attempting to quit...') + with quit_mutex: + if not is_running: + sys.exit(1) + else: + is_running = False + +if __name__ == "__main__": + signal.signal(signal.SIGINT, signal_handler) + download_args() diff --git a/src/portal/tests/logger.py b/src/portal/tests/logger.py new file mode 100644 index 0000000..80e22da --- /dev/null +++ b/src/portal/tests/logger.py @@ -0,0 +1,15 @@ +import threading + +class Logger(): + def __init__(self, path): + self.file = open(path, 'a+') + self.mutex = threading.Lock() + + def write(self, msg): + with self.mutex: + print(msg) + self.file.write(msg + '\n') + self.file.flush() + + def close(self): + self.file.close() diff --git a/src/portal/tests/old_twitter_api.py b/src/portal/tests/old_twitter_api.py new file mode 100644 index 0000000..40d0346 --- /dev/null +++ b/src/portal/tests/old_twitter_api.py @@ -0,0 +1,271 @@ +import sys +import os +import shutil +import gzip +import json +import email.utils +sys.path.append('../py') +from base import Method +from post import PostEncoder, User, Post, PostRef, PostType, DateType, Image, Video, MediaUrl +from modules import ALL_MODULES +from archive_query import QueryDownloadThread +from logger import Logger + +module = ALL_MODULES['twitter'][0] +if not module.init(): + sys.exit(1) + +#origin_base = '/mnt/store/camu_db/twitter' +origin_base = '/mnt/store/files/camu_db/data/twitter_orig' + +#RUNTIME_PATH = '.' +RUNTIME_PATH = '/mnt/store/files/tmp/run' +LOG_FILE = f'{RUNTIME_PATH}/twitter_reparse3.log' + +log = Logger(LOG_FILE) + +compat_thread = QueryDownloadThread(log, None, None, sys.argv[1]) + +def compat_media_download(obj, post, output_base): + for i, key in enumerate(post.media.keys()): + media = post.media[key] + url = media.url + media_path = f'{output_base}_media{key}.{url.ext}' + if os.path.isfile(media_path): + obj.log.write(f'Skipping media {media_path}.') + continue + origin_path = f'{origin_base}/{post.author.unique_id.replace(':', '_')}/{post.unique_id.replace(':', '_')}_media{i}.{url.ext}' + if os.path.isfile(origin_path): + shutil.copyfile(origin_path, media_path) + #post.media[key].url.url = url.url.replace('name=orig', 'name=large') + obj.log.write(f'Used backup {origin_path}.') + continue + obj.log.write(f'Attempting to download file {media_path}.') + response = module.do_request(Method.GET, url.url, None, 1) + if not response: + return False + if not obj.write_to_file(media_path, 'wb+', response.content, False): + return False + +compat_thread.media_download = compat_media_download + +def process_post(post): + compat_thread.supply_post(post) + #print(json.dumps(post, indent=4, cls=PostEncoder)) + +def create_media_url(url, format=True): + question = url.rfind('?') + if question >= 0: + ext = url[0:question].rsplit('.')[-1] + else: + ext = url.rsplit('.')[-1] + if format: + return MediaUrl(f'{url}?format={ext}&name=orig', ext), MediaUrl(f'{url}?format={ext}&name=small', ext) + return MediaUrl(url, ext), None + +def create_media(m): + original, thumbnail = create_media_url(m['media_url_https']) + if m['type'] == 'photo': + return Image(url=original, thumbnail_url=thumbnail) + elif m['type'] == 'video' or m['type'] == 'animated_gif': + variants = sorted(m['video_info']['variants'], key=lambda x: x['bitrate'] if 'bitrate' in x else 0, reverse=True) + url = variants[0]['url'] + video_url, _ = create_media_url(url, False) + return Video(url=video_url, thumbnail_url=thumbnail) + return None + +def create_user(data): + unique_id = f'twitter:u:{data['id']}' + user = User(unique_id=unique_id, username=data['screen_name'], display_name=data['name']) + user.profile_picture_url = data['profile_image_url_https'] + return user + +def create_post_v1(data, user, mtime, hash): + kwargs = {} + kwargs['unique_id'] = f'twitter:t:{data['id']}' + kwargs['raw_responses'] = {} + kwargs['raw_responses']['hash'] = hash + kwargs['url'] = f'https://twitter.com/{user.username}/status/{data['id']}' + kwargs['dates'] = { + DateType.CREATED: email.utils.parsedate_to_datetime(data['created_at']).timestamp(), + DateType.RETRIEVED: mtime + } + kwargs['author'] = user + if 'retweeted_status_id' in data: + kwargs['type'] = PostType.REPOST + kwargs['post'] = PostRef(f'twitter:t:{data['retweeted_status_id']}') + process_post(Post(**kwargs)) + return + kwargs['type'] = PostType.POST + kwargs['text'] = data['full_text'] + kwargs['likes'] = data['favorite_count'] + kwargs['reposts'] = data['retweet_count'] + kwargs['quotes'] = data['quote_count'] + kwargs['comments'] = data['reply_count'] + media = {} + if 'media' in data['entities']: + for m in data['entities']['media']: + media_id = m['id_str'] if 'id_str' in m else str(m['id']) + media[media_id] = create_media(m) + if 'extended_entities' in data and 'media' in data['extended_entities']: + for m in data['extended_entities']['media']: + media_id = m['id_str'] if 'id_str' in m else str(m['id']) + media[media_id] = create_media(m) + kwargs['media'] = media + if data['in_reply_to_status_id']: + kwargs['in_reply_to'] = PostRef(f'twitter:t:{data['in_reply_to_status_id']}') + if 'quoted_status_id' in data: + kwargs['quoted'] = PostRef(f'twitter:t:{data['quoted_status_id']}') + process_post(Post(**kwargs)) + +def parse_v1_raw_response(obj, mtime, hash): + users = {} + for id, user in obj['globalObjects']['users'].items(): + users[id] = create_user(user) + for tweet in obj['globalObjects']['tweets'].values(): + create_post_v1(tweet, users[tweet['user_id_str']], mtime, hash) + +def create_post_graphql(data, users, tweet_id, mtime, hash): + if not tweet_id: + tweet_id = data.get('rest_id', None) + if data['__typename'] == 'TweetWithVisibilityResults': + data = data['tweet'] + if not tweet_id: + tweet_id = data.get('rest_id', None) + if not tweet_id: + log.write('no id') + return PostRef() + kwargs = {} + unique_id = f'twitter:t:{tweet_id}' + kwargs['unique_id'] = unique_id + kwargs['raw_responses'] = {} + kwargs['raw_responses']['hash'] = hash + if 'legacy' not in data or ('__typename' in data and (data['__typename'] == 'TweetTombstone' or data['__typename'] == 'TweetUnavailable')): + kwargs['type'] = PostType.TOMBSTONE + process_post(Post(**kwargs)) + return PostRef(unique_id) + tweet = data['legacy'] + user_id = tweet['user_id_str'] + if user_id not in users: + user_data = data['core']['user_results']['result']['legacy'] + user = User(unique_id=f'twitter:u:{user_id}', username=user_data['screen_name'], display_name=user_data['name']) + user.profile_picture_url, _ = create_media_url(user_data['profile_image_url_https'], False) + users[user_id] = user + else: + user = users[user_id] + kwargs['url'] = f'https://twitter.com/{user.username}/status/{tweet_id}' + kwargs['dates'] = { + DateType.CREATED: email.utils.parsedate_to_datetime(tweet['created_at']).timestamp(), + DateType.RETRIEVED: mtime + } + kwargs['author'] = user + if 'retweeted_status_result' in tweet: + if 'result' in tweet['retweeted_status_result']: + retweet_id = create_post_graphql(tweet['retweeted_status_result']['result'], users, None, mtime, hash) + elif 'retweeted_status_id' in tweet: + retweet_id = PostRef(f'twitter:t:{tweet['retweeted_status_id']}') + else: + retweet_id = PostRef() + kwargs['type'] = PostType.REPOST + kwargs['post'] = retweet_id + process_post(Post(**kwargs)) + return PostRef(unique_id) + kwargs['type'] = PostType.POST + kwargs['text'] = tweet['full_text'] + kwargs['likes'] = tweet['favorite_count'] + kwargs['reposts'] = tweet['retweet_count'] + kwargs['quotes'] = tweet['quote_count'] + kwargs['comments'] = tweet['reply_count'] + if 'views' in data and 'count' in data['views']: + kwargs['views'] = int(data['views']['count']) + if 'urls' in tweet['entities']: + urls = [] + for url in tweet['entities']['urls']: + urls.append(url['expanded_url']) + kwargs['links'] = urls + media = {} + if 'media' in tweet['entities']: + for m in tweet['entities']['media']: + media_id = m['id_str'] if 'id_str' in m else str(m['id']) + media[media_id] = create_media(m) + if 'extended_entities' in tweet and 'media' in tweet['extended_entities']: + for m in tweet['extended_entities']['media']: + media_id = m['id_str'] if 'id_str' in m else str(m['id']) + media[media_id] = create_media(m) + kwargs['media'] = media + if 'quoted_status_result' in data: + if 'result' not in data['quoted_status_result']: + quoted_id = PostRef(f'twitter:t:{tweet['quoted_status_id_str']}') + else: + quoted_id = create_post_graphql(data['quoted_status_result']['result'], users, None, mtime, hash) + kwargs['quoted'] = quoted_id + elif 'quoted_status_id_str' in tweet: + log.write('quote id no data') + quoted_id = PostRef(f'twitter:t:{tweet['quoted_status_id_str']}') + kwargs['quoted'] = quoted_id + elif data.get('quotedRefResult'): + log.write('ref quote') + quoted = data['quotedRefResult']['result'] + if quoted['__typename'] == 'TweetWithVisibilityResults': + quoted = quoted['tweet'] + quoted_id = PostRef(f'twitter:t:{quoted['rest_id']}') + kwargs['quoted'] = quoted_id + if 'in_reply_to_status_id_str' in tweet: + kwargs['in_reply_to'] = PostRef(f'twitter:t:{tweet['in_reply_to_status_id_str']}') + process_post(Post(**kwargs)) + return PostRef(unique_id) + +def parse_graphql_raw_response(obj, mtime, hash): + users = {} + if 'user' in obj['data']: + instructions = obj['data']['user']['result']['timeline_v2']['timeline']['instructions'] + elif 'search_by_raw_query' in obj['data']: + instructions = obj['data']['search_by_raw_query']['search_timeline']['timeline']['instructions'] + else: + instructions = [] + for inst in instructions: + if inst['type'] == 'TimelineAddEntries': + for entry in inst['entries']: + if entry['entryId'].startswith('tweet-'): + tweet_id = int(entry['entryId'].split('-', 1)[1]) + if entry['content']['entryType'] == 'TimelineTimelineItem' and entry['content']['itemContent']['itemType'] == 'TimelineTweet': + if 'result' not in entry['content']['itemContent']['tweet_results']: + continue + create_post_graphql(entry['content']['itemContent']['tweet_results']['result'], users, tweet_id, mtime, hash) + else: + log.write('Got unrecognised timeline tweet item(s)') # snscrape copy and paste + elif entry['entryId'].startswith(('homeConversation-', 'profile-conversation-')): + if entry['content']['entryType'] == 'TimelineTimelineModule': + for item in reversed(entry['content']['items']): + if not item['entryId'].startswith(entry['entryId'].split('ion-', 1)[0] + 'ion-') or '-tweet-' not in item['entryId']: + log.write(f'Unexpected conversation entry ID: {item['entryId']!r}') # snscrape copy and paste + continue + if item['item']['itemContent']['itemType'] == 'TimelineTweet': + tweet_id = int(item['entryId'].split('-tweet-', 1)[1]) + if 'result' in item['item']['itemContent']['tweet_results']: + create_post_graphql(item['item']['itemContent']['tweet_results']['result'], users, tweet_id, mtime, hash) + else: + kwargs = {} + unique_id = f'twitter:t:{tweet_id}' + kwargs['unique_id'] = unique_id + kwargs['raw_responses'] = {} + kwargs['raw_responses']['hash'] = hash + kwargs['type'] = PostType.TOMBSTONE + process_post(Post(**kwargs)) + +def parse_raw_response(path, hash): + mtime = os.path.getmtime(path) + with gzip.open(path, 'rb') as f: + obj = json.loads(f.read()) + if 'globalObjects' not in obj: + parse_graphql_raw_response(obj, mtime, hash) + else: + parse_v1_raw_response(obj, mtime, hash) + +#target = '/mnt/store/camu_db/twitter/raw_responses' +target = '/mnt/store/files/camu_db/data/twitter_orig/raw_responses' + +for file in os.listdir(target): + hash = file.split('.')[0] + print(hash) + parse_raw_response(f'{target}/{file}', hash) diff --git a/src/portal/tests/rewrite_post.py b/src/portal/tests/rewrite_post.py new file mode 100644 index 0000000..3bb152b --- /dev/null +++ b/src/portal/tests/rewrite_post.py @@ -0,0 +1,56 @@ +import sys +import os +import gzip +import json +import hashlib +sys.path.append('../py') +import config +#from post import PostEncoder, DateType +#from modules.pixiv_web import PixivWebBase, PixivWebModule +#from modules.twitter import TwitterScrapeBase, TwitterScrapeModule +from logger import Logger + +#module = PixivWebModule(config.PIXIV_SESSID) +#search = PixivWebBase(module) + +log = Logger('deleted_posts.log') + +''' +def rewrite_post(path): + print(f'Rewriting post @ {path}') + with gzip.open(path, 'rb') as f: + obj = json.loads(f.read()) + post = search.remake_post(obj['raw_responses'], obj['dates'][str(DateType.RETRIEVED.value)]) + if not post: + log.write(path) + return + #print(json.dumps(post, indent=4, cls=PostEncoder)) + with gzip.open(path, 'wb') as f: + f.write(json.dumps(post, cls=PostEncoder).encode('utf-8')) +''' + +def dump_raw_response(path, target): + print(f'Extracting raw_response from {path}') + with gzip.open(path, 'rb') as f: + obj = json.loads(f.read()) + if 'api' in obj['raw_responses']: + hasher = hashlib.new('sha256') + data = json.dumps(json.loads(obj['raw_responses']['api'])).encode('utf-8') + hasher.update(data) + output = f'{target}/{hasher.hexdigest()}.json.gz' + with gzip.open(output, 'wb+') as f: + f.write(data) + +target = sys.argv[1] +raw_responses = f'{target}/raw_responses' + +if not os.path.isdir(raw_responses): + os.mkdir(raw_responses) + +for user in os.listdir(target): + if user == 'raw_responses': + continue + for file in os.listdir(f'{target}/{user}'): + if file.split('.')[-1] == 'gz': + dump_raw_response(f'{target}/{user}/{file}', raw_responses) + #rewrite_post(f'{target}/{user}/{file}') diff --git a/src/portal/tests/test.py b/src/portal/tests/test.py new file mode 100644 index 0000000..cdf7178 --- /dev/null +++ b/src/portal/tests/test.py @@ -0,0 +1,47 @@ +import json +import httpx +import sys +sys.path.append('../py') +from base import Method +from post import DateType, PostEncoder +from modules import ALL_MODULES + +#ALL_MODULES['instagram'][0].init() +#search = ALL_MODULES['instagram'][0].search('opium_00pium') +#page = search.get_page(0) +#print(page) +#print(json.dumps(page, indent=4, cls=PostEncoder)) + +#module = ALL_MODULES['fanbox'][0] +#module = ALL_MODULES['pixiv_web'][0] +module = ALL_MODULES['instagram'][0] +#module = ALL_MODULES['twitter'][0] +module.init() + +search = module.search('opium_00pium') +page = search.get_page(1) +for unique_id in page: + post = module.get_item(unique_id) + print(json.dumps(post, indent=4, cls=PostEncoder)) + +#post = module.get_item(page[0]) +#for key in post.media.keys(): +# download_params = module.get_download(post.unique_id, key) +# for url in download_params['urls']: +# #response = module.do_request(Method.GET, url.url) +# r = httpx.get(url.url, headers=download_params['headers']) +# with open(f'test_{key}.{url.ext}', 'wb+') as f: +# f.write(r.content) + +#print(json.dumps(page, indent=4, cls=PostEncoder)) +#for i in range(0, len(post.media)): +# download_params = module.get_download(post.unique_id, i) +# r = httpx.get(download_params['url'], headers=download_params['headers']) +# if r.status_code != 200: +# print('error{}'.format(r.status_code)) +# else: +# with open('test{}.png'.format(i), 'wb+') as f: +# f.write(r.content) + +#print(json.dumps(page, indent=4, cls=PostEncoder)) +#print(page) diff --git a/src/portal/tests/unescape_json.py b/src/portal/tests/unescape_json.py new file mode 100644 index 0000000..eaf5194 --- /dev/null +++ b/src/portal/tests/unescape_json.py @@ -0,0 +1,10 @@ +import sys +import gzip +import json + +for path in sys.argv[1:]: + with gzip.open(path, 'rb') as f: + obj = json.loads(f.read().decode('utf-8')) + obj = json.loads(obj) + with gzip.open(path, 'wb') as f: + f.write(json.dumps(obj).encode('utf-8')) diff --git a/src/render/queue_libplacebo.c b/src/render/queue_libplacebo.c index de9df44..6316fe2 100644 --- a/src/render/queue_libplacebo.c +++ b/src/render/queue_libplacebo.c @@ -1,6 +1,6 @@ #include <al/lib.h> #include <al/log.h> -#ifdef HAVE_FFMPEG +#ifdef CAMU_HAVE_FFMPEG #define PL_LIBAV_IMPLEMENTATION 1 #include <libplacebo/utils/libav.h> #endif @@ -84,7 +84,7 @@ static void queue_lp_push(struct camu_frame_queue *queue, struct camu_frame *fra }); } -#ifdef HAVE_FFMPEG +#ifdef CAMU_HAVE_FFMPEG static bool map_av_frame(pl_gpu gpu, pl_tex *tex, const struct pl_source_frame *src, struct pl_frame *out_frame) { @@ -187,7 +187,7 @@ struct camu_frame_queue *camu_frame_queue_lp_create(void) { struct camu_frame_queue_lp *lq = al_alloc_object(struct camu_frame_queue_lp); lq->q.push = queue_lp_push; -#ifdef HAVE_FFMPEG +#ifdef CAMU_HAVE_FFMPEG lq->q.push_av_frame = queue_lp_push_av_frame; #endif lq->q.flush = queue_lp_flush; diff --git a/src/render/queue_tiger.c b/src/render/queue_tiger.c index 575b826..0e76502 100644 --- a/src/render/queue_tiger.c +++ b/src/render/queue_tiger.c @@ -33,7 +33,7 @@ static void queue_tiger_push(struct camu_frame_queue *queue, struct camu_frame * upload_texture(tq, frame->width, frame->height, frame->data, 0); } -#ifdef HAVE_FFMPEG +#ifdef CAMU_HAVE_FFMPEG static void queue_tiger_push_av_frame(struct camu_frame_queue *queue, AVFrame *frame, f64 pts) { struct camu_frame_queue_tiger *tq = (struct camu_frame_queue_tiger *)queue; @@ -80,7 +80,7 @@ struct camu_frame_queue *camu_frame_queue_tiger_create(void) { struct camu_frame_queue_tiger *tq = al_alloc_object(struct camu_frame_queue_tiger); tq->q.push = queue_tiger_push; -#ifdef HAVE_FFMPEG +#ifdef CAMU_HAVE_FFMPEG tq->q.push_av_frame = queue_tiger_push_av_frame; #endif tq->q.flush = queue_tiger_flush; diff --git a/src/render/renderer.h b/src/render/renderer.h index 64dc691..e60c3a5 100644 --- a/src/render/renderer.h +++ b/src/render/renderer.h @@ -2,7 +2,7 @@ #include <al/types.h> #include <skh/window.h> -#ifdef HAVE_FFMPEG +#ifdef CAMU_HAVE_FFMPEG #include <libavcodec/avcodec.h> #endif @@ -34,7 +34,7 @@ struct camu_renderer { struct camu_frame_queue *(*create_queue)(struct camu_renderer *); void (*resize)(struct camu_renderer *, s32 *, s32 *); void (*render)(struct camu_renderer *, struct camu_screen *); -#ifdef HAVE_FFMPEG +#ifdef CAMU_HAVE_FFMPEG s32 (*get_buffer2)(struct AVCodecContext *s, AVFrame *frame, s32 flags); void *opaque; #endif diff --git a/src/render/renderer_libplacebo.c b/src/render/renderer_libplacebo.c index 148574b..03f30a5 100644 --- a/src/render/renderer_libplacebo.c +++ b/src/render/renderer_libplacebo.c @@ -1,7 +1,7 @@ #include <al/lib.h> #include <al/log.h> #include <aki/file.h> -#ifdef HAVE_FFMPEG +#ifdef CAMU_HAVE_FFMPEG #include <libplacebo/utils/libav.h> #endif @@ -159,7 +159,7 @@ static bool renderer_lp_create_renderer(struct camu_renderer *renderer, s32 *wid } #endif -#ifdef HAVE_FFMPEG +#ifdef CAMU_HAVE_FFMPEG //lr->r.get_buffer2 = pl_get_buffer2; //lr->r.opaque = &lr->gpu; #endif diff --git a/src/render/renderer_tiger.c b/src/render/renderer_tiger.c index e4ae239..049a3b6 100644 --- a/src/render/renderer_tiger.c +++ b/src/render/renderer_tiger.c @@ -152,7 +152,7 @@ static bool renderer_tiger_create_renderer(struct camu_renderer *renderer, s32 * //glGenBuffers(1, &tr->texture_buffer); #endif -#ifdef HAVE_FFMPEG +#ifdef CAMU_HAVE_FFMPEG tr->r.get_buffer2 = NULL; tr->r.opaque = NULL; #endif diff --git a/src/screen/screen.c b/src/screen/screen.c index 604a357..b3ce0de 100644 --- a/src/screen/screen.c +++ b/src/screen/screen.c @@ -1,6 +1,7 @@ #include <aki/thread.h> #include <al/log.h> +#include "view.h" #include "screen.h" static struct camu_view *get_view_from_mouse_pos(struct camu_screen *scr) @@ -97,7 +98,18 @@ static bool mouse_button_callback(void *userdata, u8 state, u8 button) } break; case SEKIHI_MOUSE2: - break; // Move window. + switch (state) { + case SEKIHI_BUTTON_PRESSED: + scr->flags |= CAMU_SCREEN_MODIFIER; + break; + case SEKIHI_BUTTON_RELEASED: + scr->flags &= ~CAMU_SCREEN_MODIFIER; + break; + } + break; + case SEKIHI_MOUSE3: + scr->callback(scr->userdata, CAMU_SCREEN_CLOSE, 0.0); + break; default: break; } @@ -107,18 +119,24 @@ static bool mouse_button_callback(void *userdata, u8 state, u8 button) #if defined SEKIHI_WINDOW_WAYLAND #define SCROLL_MULTIPLIER 87.5 #else -#define SCROLL_MULTIPLIER 5.0 +#define SCROLL_MULTIPLIER 25.0 #endif static bool scroll_callback(void *userdata, f64 y) { struct camu_screen *scr = (struct camu_screen *)userdata; struct camu_view *view = get_view_from_mouse_pos(scr); - if (view && scr->flags & CAMU_SCREEN_ZOOM_PAN_SIMPLE) { - y = -y / SCROLL_MULTIPLIER; - if (camu_view_zoom_simple(view, scr->width, scr->height, scr->last_mouse_x, scr->last_mouse_y, y)) { - view->mode = CAMU_VIEW_DETACHED; + if (view) { + if (scr->flags & CAMU_SCREEN_MODIFIER) { + // Unimplemented. + view->rotation += y; return true; + } else if (scr->flags & CAMU_SCREEN_ZOOM_PAN_SIMPLE) { + y = -y / SCROLL_MULTIPLIER; + if (camu_view_zoom_simple(view, scr->width, scr->height, scr->last_mouse_x, scr->last_mouse_y, y)) { + view->mode = CAMU_VIEW_DETACHED; + return true; + } } } return false; @@ -163,6 +181,10 @@ static bool key_callback(void *userdata, u8 state, u8 button) } break; } + case 0x1f: + case 's': + scr->callback(scr->userdata, CAMU_SCREEN_RESEEK, 0.0); + break; case 0x1d: scr->flags |= CAMU_SCREEN_MODIFIER; break; @@ -229,7 +251,8 @@ bool camu_screen_init(struct camu_screen *scr) bool camu_screen_create_window(struct camu_screen *scr, const char *name) { s32 flags = SEKIHI_WINDOW_VSYNC; - if (!scr->window->create_window(scr->window, CAMU_SCREEN_WIDTH, CAMU_SCREEN_HEIGHT, NULL, name, flags)) { + //flags |= SEKIHI_WINDOW_WALLPAPER; + if (!scr->window->create_window(scr->window, CAMU_SCREEN_WIDTH, CAMU_SCREEN_HEIGHT, "DP-1", name, flags)) { return false; } scr->width = scr->window->width; @@ -355,7 +378,11 @@ bool camu_screen_poll(struct camu_screen *scr, bool block) #if defined SEKIHI_EVENT_BUFFER skh_window_read_events(scr->window); #endif +#if defined SEKIHI_PAUSE return scr->window->should_refresh(scr->window); +#else + return true; +#endif } bool camu_screen_tick(struct camu_screen *scr) @@ -366,7 +393,9 @@ bool camu_screen_tick(struct camu_screen *scr) void camu_screen_wake(struct camu_screen *scr) { +#if defined SEKIHI_POLL_INLINE && defined SEKIHI_PAUSE scr->window->wake(scr->window); +#endif } void camu_screen_close(struct camu_screen *scr) diff --git a/src/screen/screen.h b/src/screen/screen.h index 0ca5779..67cd7ed 100644 --- a/src/screen/screen.h +++ b/src/screen/screen.h @@ -35,6 +35,7 @@ enum { CAMU_SCREEN_PREVIOUS, CAMU_SCREEN_TOGGLE_PAUSE, CAMU_SCREEN_SEEK, + CAMU_SCREEN_RESEEK, CAMU_SCREEN_CLOSE }; diff --git a/src/screen/view.c b/src/screen/view.c index dddc11a..ab84277 100644 --- a/src/screen/view.c +++ b/src/screen/view.c @@ -12,6 +12,7 @@ void camu_view_calculate(struct camu_view *view, s32 window_width, s32 window_height) { + if (view->mode == CAMU_VIEW_DETACHED) return; f64 ratio = window_width / (f64)window_height; f64 frame_ratio = view->width / (f64)view->height; view->ratio = ratio / frame_ratio; diff --git a/src/screen/view.h b/src/screen/view.h index e9bec81..5f33742 100644 --- a/src/screen/view.h +++ b/src/screen/view.h @@ -19,6 +19,7 @@ struct camu_view { f64 zoom; f64 x_offset; f64 y_offset; + f64 rotation; s32 zindex; }; diff --git a/src/server/common.h b/src/server/common.h new file mode 100644 index 0000000..4d3a0ae --- /dev/null +++ b/src/server/common.h @@ -0,0 +1,42 @@ +#pragma once + +#include <al/str.h> + +#define CAMU_DB_PATH al_str_c("/mnt/store/files/camu_db") +//#define CAMU_DB_PATH al_str_c("/home/andrew/c/camu/data/camu_db_test") + +#define CAMU_PORT 14356 +#define CAMU_RESOURCE_PORT 14357 + +#define CAMU_SERVER_IP al_str_c("127.0.0.1") +//#define CAMU_SERVER_IP al_str_c("192.168.1.192") +//#define CAMU_SERVER_IP al_str_c("108.52.160.112") + + +enum { + CAMU_NODE = 0, + CAMU_CLIENT, + CAMU_SINK +}; + +enum { + CAMU_SRV_IDENTIFY = 0, + CAMU_SRV_LIST_ACTION, + CAMU_SRV_CREATE_SEARCH, + CAMU_SRV_GET_PAGE, + CAMU_SRV_CREATE_BROWSE, + CAMU_SRV_GET_PATH +}; + +enum { + CAMU_CONN_STATE +}; + +enum { + CAMU_LIST_ADD = 0, + CAMU_LIST_SKIP, + CAMU_LIST_SKIPTO, + CAMU_LIST_TOGGLE_PAUSE, + CAMU_LIST_SEEK, + CAMU_LIST_FINISHED +}; diff --git a/src/server/db.c b/src/server/db.c new file mode 100644 index 0000000..5042f2d --- /dev/null +++ b/src/server/db.c @@ -0,0 +1,56 @@ +#include <al/log.h> +#include <jansson.h> + +#include "server.h" + +static bool open_user(struct camu_server *srv, struct aki_dir_entry *dir) +{ + struct aki_file file; + if (!aki_file_open(&file, &dir->path, 0)) { + return false; + } + str s; + aki_file_read_as_str(&file, &s); + json_error_t error; + json_t *root = json_loadb(s.data, s.len, 0, &error); + if (!root) { + al_log_error("server", "Failed to parse json: %.*s:%d:%d (%s).", + AL_STR_PRINTF(&dir->path), error.line, error.column, error.text); + return false; + } + struct camu_user *user = al_alloc_object(struct camu_user); + al_str_from(&user->name, json_string_value(json_object_get(root, "username"))); + al_array_init(user->lists); + struct camu_list *default_list = al_alloc_object(struct camu_list); + camu_list_init(default_list, al_str_c("default")); + al_array_push(user->lists, default_list); + al_log_info("server", "Loaded user \"%.*s\"", AL_STR_PRINTF(&user->name)); + al_array_push(srv->users, user); + return true; +} + +bool camu_db_open(struct camu_server *srv, str *path) +{ + struct aki_dir camu_db; + if (!aki_dir_open(&camu_db, path)) { + return false; + } + struct aki_dir_entry entry; + while (aki_dir_read(&camu_db, &entry)) { + if (al_str_eq(&entry.name, al_str_c("users"))) { + struct aki_dir users; + if (aki_dir_open(&users, &entry.path)) { + struct aki_dir_entry user; + while (aki_dir_read(&users, &user)) { + if (user.type == AKI_ENTRY_FILE) { + open_user(srv, &user); + } + } + aki_dir_close(&users); + } + } + aki_dir_entry_free(&entry); + } + aki_dir_close(&camu_db); + return true; +} diff --git a/src/server/db.h b/src/server/db.h new file mode 100644 index 0000000..8e8b069 --- /dev/null +++ b/src/server/db.h @@ -0,0 +1,9 @@ +#pragma once + +#include <al/str.h> + +struct camu_server; +struct camu_db { +}; + +bool camu_db_open(struct camu_server *srv, str *path); diff --git a/src/server/local_compat.h b/src/server/local_compat.h new file mode 100644 index 0000000..cdb88ce --- /dev/null +++ b/src/server/local_compat.h @@ -0,0 +1,68 @@ +#pragma once + +#include "../portal/src/search.h" + +#ifdef AKIYO_HAS_CURL +#include "../cache/handlers/http.h" +#endif +#include "../cache/handlers/file.h" + +static struct cch_entry *entry_for_external_path(struct camu_portal *portal, str *path) +{ + struct cch_entry *entry = NULL; +#ifdef AKIYO_HAS_CURL + bool is_search = al_str_at(path, 0) == ';'; + if (is_search || al_str_cmp(path, al_str_c("https://"), 0, 8) == 0 || + al_str_cmp(path, al_str_c("http://"), 0, 7) == 0) { + str module; + str query; + al_str_from(&module, ""); + al_str_from(&query, ""); + if (al_str_cmp(path, al_str_c("https://twitter.com"), 0, 19) == 0 || + al_str_cmp(path, al_str_c("https://x.com"), 0, 13) == 0) { + al_str_cat(&query, al_str_c("tweet:")); + al_str_cat(&query, path); + al_str_cat(&module, al_str_c("twitter")); + } else if (al_str_cmp(path, al_str_c("https://instagram.com"), 0, 21) == 0) { + s32 slash = al_str_rfind(path, '/'); + if (slash >= 0) { + al_str_cat(&query, al_str_substr(path, slash + 1, path->len)); + } + al_str_cat(&module, al_str_c("instagram")); + } else { + if (is_search) { + al_str_cat(&query, al_str_substr(path, 1, path->len)); + } else { + al_str_cat(&query, al_str_c("link:")); + al_str_cat(&query, path); + } + al_str_cat(&module, al_str_c("youtube")); + } + s32 id = camu_portal_create_search(portal, &module, &query); + al_str_free(&module); + al_str_free(&query); + if (id < 0) return NULL; + struct camu_search *search = camu_portal_get_search(portal, id); + if (!search || !camu_search_get_page(search, 0)) { + return NULL; + } + struct camu_result_page *page = &al_array_at(search->pages, 0); + struct camu_post *post; + al_array_foreach_ptr(page->posts, i, post) { + struct camu_post_media *media; + al_array_foreach_ptr(post->media, j, media) { + if (media->url.len > 0) { + entry = cch_handler_http_create(&media->url); + break; + } + } + if (entry) break; + } + camu_portal_discard_search(portal, id); + if (entry) entry->handler->maybe_spawn_worker(entry->handler, 0); + } else // { +#endif + entry = cch_handler_file_create(path); + // } + return entry; +} diff --git a/src/server/meson.build b/src/server/meson.build new file mode 100644 index 0000000..d10b131 --- /dev/null +++ b/src/server/meson.build @@ -0,0 +1,6 @@ +server_src = [ + 'server.c', + 'db.c' +] +server_deps = [common_deps, portal, cache, shrub_server, list] +executable('server', sources: server_src, dependencies: server_deps) diff --git a/src/server/server.c b/src/server/server.c new file mode 100644 index 0000000..0fd1d71 --- /dev/null +++ b/src/server/server.c @@ -0,0 +1,308 @@ +#include <al/log.h> + +#include "../libsink/common.h" +#include "../shrub/common.h" + +#include "server.h" +#include "common.h" +#include "db.h" +#ifdef CAMU_LOCAL_SOCKET +#include "local_compat.h" +#endif + +static struct camu_user *get_user_by_username(struct camu_server *tree, str *username) +{ + struct camu_user *user; + al_array_foreach(tree->users, i, user) { + if (al_str_eq(&user->name, username)) { + return user; + } + } + return NULL; +} + +static void list_callback(void *userdata, u8 op, void *opaque, s32 sequence) +{ + struct camu_srv_sink *sink = (struct camu_srv_sink *)userdata; + struct camu_srv_resource *resource = (struct camu_srv_resource *)opaque; + struct aki_packet *packet = aki_rpc_get_packet(sink->conn->rpc, op); + aki_packet_write_str(packet, CAMU_SERVER_IP); + aki_packet_write_s32(packet, CAMU_RESOURCE_PORT); + aki_packet_write_u16(packet, resource->node->id); + aki_packet_write_s32(packet, sequence); + aki_packet_write_u64(packet, resource->node->start); + aki_rpc_connection_command(sink->conn, packet, NULL, NULL); + if (op == CAMU_SINK_SET) { + al_printf("Now Playing: %.*s\n", AL_STR_PRINTF(&resource->unique_id)); + } +} + +static bool identify_command_callback(void *userdata, struct aki_rpc_connection *conn, + struct aki_packet *packet, struct aki_packet *rpacket) +{ + struct camu_server *srv = (struct camu_server *)userdata; + (void)rpacket; + switch (aki_packet_read_u8(packet)) { + case CAMU_NODE: { + struct camu_srv_node *node = al_alloc_object(struct camu_srv_node); + node->conn = conn; + al_array_push(srv->nodes, node); + al_log_info("server", "New node."); + break; + } + case CAMU_CLIENT: { + str username; + aki_packet_read_str(packet, &username); + struct camu_user *user = get_user_by_username(srv, &username); + if (user) { + struct camu_srv_client *client = al_alloc_object(struct camu_srv_client); + client->conn = conn; + client->user = user; + al_array_push(srv->clients, client); + //send_current_state(tree, user, conn); + al_log_info("server", "User \"%.*s\" logged in.", AL_STR_PRINTF(&user->name)); + } else { + al_log_info("server", "User \"%.*s\" not found.", AL_STR_PRINTF(&user->name)); + } + break; + } + case CAMU_SINK: { + struct camu_srv_sink *sink = al_alloc_object(struct camu_srv_sink); + sink->conn = conn; + al_array_push(srv->sinks, sink); + struct camu_user *user = al_array_at(srv->users, 0); + struct camu_list *list = al_array_at(user->lists, 0); + camu_list_add_sink(list, list_callback, sink); + al_log_info("server", "New sink."); + break; + } + } + aki_packet_free(packet); + return true; +} + +static bool list_command_callback(void *userdata, struct aki_rpc_connection *conn, + struct aki_packet *packet, struct aki_packet *rpacket) +{ + struct camu_server *srv = (struct camu_server *)userdata; + (void)conn; + (void)rpacket; + + struct camu_user *user = al_array_at(srv->users, 0); + struct camu_list *list = al_array_at(user->lists, 0); + + switch (aki_packet_read_u8(packet)) { + case CAMU_LIST_ADD: { + break; + } + case CAMU_LIST_SKIP: { + s32 sequence = aki_packet_read_s32(packet); + s32 n = aki_packet_read_s32(packet); + camu_list_skip(list, sequence, n); + break; + } + case CAMU_LIST_SKIPTO: { + s32 i = aki_packet_read_s32(packet); + camu_list_skipto(list, i); + break; + } + case CAMU_LIST_TOGGLE_PAUSE: { + s32 sequence = aki_packet_read_s32(packet); + u64 pos = aki_packet_read_u64(packet); + camu_list_toggle_pause(list, sequence, pos); + break; + } + case CAMU_LIST_SEEK: { + s32 sequence = aki_packet_read_s32(packet); + f64 percent = aki_packet_read_f64(packet); + camu_list_seek(list, sequence, percent); + break; + } + case CAMU_LIST_FINISHED: { + s32 sequence = aki_packet_read_s32(packet); + camu_list_finished(list, sequence); + break; + } + } + + aki_packet_free(packet); + return false; +} + +static struct aki_rpc_command commands[] = { + { .op = CAMU_SRV_IDENTIFY, .callback = identify_command_callback, .userdata = NULL }, + { .op = CAMU_SRV_LIST_ACTION, .callback = list_command_callback, .userdata = NULL }, + //{ .op = CAMU_SRV_CREATE_SEARCH, .callback = create_search_command_callback, .userdata = NULL }, + //{ .op = CAMU_SRV_GET_PAGE, .callback = get_page_command_callback, .userdata = NULL }, +}; + +static void connection_callback(void *userdata, struct aki_rpc_connection *conn) +{ + (void)userdata; + (void)conn; +} + +static void cleanup_node(struct camu_srv_node *node) +{ + al_free(node); +} + +static void cleanup_client(struct camu_srv_client *client) +{ + al_free(client); +} + +static void cleanup_sink(struct camu_srv_sink *sink) +{ + al_free(sink); +} + +static void connection_closed_callback(void *userdata, struct aki_rpc_connection *conn) +{ + struct camu_server *srv = (struct camu_server *)userdata; + + struct camu_srv_node *node; + al_array_foreach(srv->nodes, i, node) { + if (node->conn == conn) { + al_log_info("server", "Node removed."); + cleanup_node(node); + al_array_remove_at_iter(srv->nodes, i); + break; + } + } + + struct camu_srv_client *client; + al_array_foreach(srv->clients, i, client) { + if (client->conn == conn) { + al_log_info("server", "User \"%.*s\" logged out.", AL_STR_PRINTF(&client->user->name)); + cleanup_client(client); + al_array_remove_at_iter(srv->clients, i); + break; + } + } + + struct camu_srv_sink *sink; + al_array_foreach(srv->sinks, i, sink) { + if (sink->conn == conn) { + al_log_info("server", "Sink removed."); + al_array_remove_at_iter(srv->sinks, i); + struct camu_user *user = al_array_at(srv->users, 0); + struct camu_list *list = al_array_at(user->lists, 0); + camu_list_remove_sink(list, sink); + cleanup_sink(sink); + break; + } + } +} + +static void list_entry_callback(void *entry, u8 op, void *opaque) +{ + struct camu_srv_resource *resource = (struct camu_srv_resource *)entry; + switch (op) { + case CAMU_LIST_ENTRY_IMPULSE: { + struct camu_list_timing *timing = (struct camu_list_timing *)opaque; + shrb_node_set_start(resource->node, timing->start); + break; + } + case CAMU_LIST_ENTRY_TOGGLE_PAUSE: + shrb_node_toggle_pause(resource->node, *(u64 *)opaque); + break; + case CAMU_LIST_ENTRY_SEEK: + shrb_node_seek(resource->node, *(u64 *)opaque); + break; + } +} + +#ifdef CAMU_LOCAL_SOCKET +static u8 server_line_callback(void *userdata, str *line) +{ + struct camu_server *srv = (struct camu_server *)userdata; + struct camu_user *user = al_array_at(srv->users, 0); + struct camu_list *list = al_array_at(user->lists, 0); + if (al_str_eq(line, al_str_c(";NEXT"))) { + camu_list_skip(list, CAMU_SEQUENCE_INVALID, 1); + } else if (al_str_eq(line, al_str_c(";PREV"))) { + camu_list_skip(list, CAMU_SEQUENCE_INVALID, -1); + } else if (al_str_eq(line, al_str_c(";SHUFFLE"))) { + camu_list_shuffle(list); + } else if (al_str_eq(line, al_str_c(";SORT"))) { + } else if (al_str_eq(line, al_str_c(";CLEAR"))) { + } else { + struct cch_entry *entry = entry_for_external_path(&srv->portal, line); + if (entry) { + struct shrb_node *node = shrb_server_create_node(&srv->resource, 0, entry); + if (node) { + struct camu_srv_resource *resource = al_alloc_object(struct camu_srv_resource); + al_array_push(srv->resources, resource); + al_str_clone(&resource->unique_id, line); + resource->entry = entry; + resource->node = node; + camu_list_add(list, resource, list_entry_callback, resource->node->duration, false); + } + } + } + return AKI_LINE_PROCESSOR_CONTINUE; +} +#endif + +static void sigint_handler(s32 signum) +{ + (void)signum; + // explode. + exit(EXIT_FAILURE); +} + +static struct camu_server srv = { 0 }; + +s32 main(void) +{ + aki_common_init(); + + signal(SIGINT, sigint_handler); + + al_array_init(srv.nodes); + al_array_init(srv.clients); + al_array_init(srv.sinks); + al_array_init(srv.users); + + if (!camu_db_open(&srv, CAMU_DB_PATH)) return EXIT_FAILURE; + + camu_post_cache_init(&srv.cache); + + bool py_init = camu_python_init(); + + camu_portal_init(&srv.portal, &srv.cache); + + aki_event_loop_init(&srv.loop); + + aki_rpc_init(&srv.server, AKI_SOCKET_TCP, connection_callback, connection_closed_callback, &srv); + for (u32 i = 0; i < AL_ARRAY_SIZE(commands); i++) { + commands[i].userdata = &srv; + aki_rpc_add_command(&srv.server, &commands[i]); + } + aki_rpc_listen(&srv.server, &srv.loop, al_str_c("0.0.0.0"), CAMU_PORT); + + shrb_server_init(&srv.resource); + shrb_server_listen(&srv.resource, &srv.loop, al_str_c("0.0.0.0"), CAMU_RESOURCE_PORT); + +#ifdef CAMU_LOCAL_SOCKET + srv.socket.type = AKI_SOCKET_UNIX; + aki_socket_init(&srv.socket); + aki_socket_set_blocking(&srv.socket, false); + srv.pro.callback = server_line_callback; + srv.pro.userdata = &srv; + aki_line_processor_init(&srv.pro, al_str_c("\n")); + aki_line_processor_open_socket(&srv.pro, &srv.socket); + if (aki_socket_listen(&srv.socket, al_str_c("/tmp/camu_sock"), 0)) { + aki_line_processor_run(&srv.pro, &srv.loop); + } +#endif + + aki_event_loop_run(&srv.loop); + + if (py_init) camu_python_close(); + + aki_common_close(); + + return EXIT_SUCCESS; +} diff --git a/src/server/server.h b/src/server/server.h new file mode 100644 index 0000000..83e8523 --- /dev/null +++ b/src/server/server.h @@ -0,0 +1,54 @@ +#pragma once + +#define CAMU_LOCAL_SOCKET + +#include <aki/rpc2.h> +#ifdef CAMU_LOCAL_SOCKET +#include <aki/line_processor.h> +#endif + +#include "../portal/src/search.h" +#include "../cache/entry.h" +#include "../shrub/server.h" +#include "../list/list.h" + +struct camu_srv_node { + struct aki_rpc_connection *conn; +}; + +struct camu_user { + str name; + array(struct camu_list *) lists; +}; + +struct camu_srv_client { + struct aki_rpc_connection *conn; + struct camu_user *user; +}; + +struct camu_srv_sink { + struct aki_rpc_connection *conn; +}; + +struct camu_srv_resource { + str unique_id; + struct cch_entry *entry; + struct shrb_node *node; +}; + +struct camu_server { + struct aki_event_loop loop; + struct aki_rpc server; + array(struct camu_srv_node *) nodes; + array(struct camu_srv_client *) clients; + array(struct camu_srv_sink *) sinks; + array(struct camu_user *) users; + struct shrb_server resource; + array(struct camu_srv_resource *) resources; + struct camu_portal portal; + struct camu_post_cache cache; +#ifdef CAMU_LOCAL_SOCKET + struct aki_socket socket; + struct aki_line_processor pro; +#endif +}; diff --git a/src/bimu/client.c b/src/shrub/client.c index 03910ee..015f47f 100644 --- a/src/bimu/client.c +++ b/src/shrub/client.c @@ -1,57 +1,44 @@ #include <al/log.h> -#include "../codec/libav/packet_ext.h" +#include "../codec/ffmpeg/packet_ext.h" #include "handlers/codec.h" #include "client.h" +#include "handler.h" #include "common.h" static void data_connection_callback(void *userdata, struct aki_packet_stream *stream) { - struct bmu_client *client = (struct bmu_client *)userdata; + struct shrb_client *client = (struct shrb_client *)userdata; (void)stream; struct aki_packet *packet = aki_packet_create(); - aki_packet_write_u8(packet, BIMU_CONNECTION_DATA); + aki_packet_write_u8(packet, SHRUB_CONNECTION_DATA); aki_packet_write_u16(packet, client->connection_id); aki_packet_write_u16(packet, client->node_id); aki_packet_write_u64(packet, client->seek_pos); aki_packet_stream_send_packet(&client->data, packet); } -static void send_reconnect_packet(struct bmu_client *client) +static void send_reconnect_packet(struct shrb_client *client) { struct aki_packet *packet = aki_packet_create(); - aki_packet_write_u8(packet, BIMU_CONTROL_RECONNECT); + aki_packet_write_u8(packet, SHRUB_CONTROL_RECONNECT); aki_packet_write_u32(packet, client->level); aki_packet_stream_send_packet(&client->control, packet); } -static void cleanup_connection_internal(struct bmu_client *client) +static void cleanup_connection_internal(struct shrb_client *client) { - client->callback(client->userdata, BIMU_CLIENT_CLOSED, NULL, NULL); -} - -static void data_connection_closed_callback(void *userdata, struct aki_packet_stream *stream) -{ - struct bmu_client *client = (struct bmu_client *)userdata; - (void)stream; - if (client->closed) { - if (!client->control.connected) cleanup_connection_internal(client); - return; - } - aki_packet_stream_free(&client->data); - client->callback(client->userdata, BIMU_CLIENT_REMOVE_BUFFERS, NULL, NULL); - bmu_vcr_flush(&client->vcr); - send_reconnect_packet(client); + client->callback(client->userdata, SHRUB_CLIENT_CLOSED, NULL, NULL); } static void data_packet_callback(void *userdata, struct aki_packet_stream *stream, struct aki_packet *packet) { - struct bmu_client *client = (struct bmu_client *)userdata; + struct shrb_client *client = (struct shrb_client *)userdata; (void)stream; - bmu_vcr_push_packet(&client->vcr, packet); + shrb_vcr_push_packet(&client->vcr, packet); } static void packet_sent_callback(void *userdata, struct aki_packet *packet) @@ -60,19 +47,57 @@ static void packet_sent_callback(void *userdata, struct aki_packet *packet) aki_packet_free(packet); } -void connect_data(struct bmu_client *client) +static void data_connection_closed_callback(void *userdata, struct aki_packet_stream *stream); + +void connect_data(struct shrb_client *client) { + if (!client->reconnect) { + client->reconnect = true; + } else { + aki_packet_stream_free(&client->data); + } aki_packet_stream_init(&client->data, AKI_SOCKET_TCP, data_connection_callback, data_connection_closed_callback, data_packet_callback, packet_sent_callback, client); aki_packet_stream_connect(&client->data, client->loop, &client->addr, client->port); } +static void parse_pause_packet(struct shrb_client *client, struct aki_packet *packet) +{ + u64 delay = aki_packet_read_u64(packet); + u64 seek_pos = aki_packet_read_u64(packet); + struct shrb_seek_req req = { .base = seek_pos, .ts = UINT64_MAX, .delay = delay }; + // TODO: separare function. + client->callback(client->userdata, SHRUB_CLIENT_SET, NULL, &req); + client->seek_pos = req.base; +} + +void data_connection_closed_callback(void *userdata, struct aki_packet_stream *stream) +{ + struct shrb_client *client = (struct shrb_client *)userdata; + (void)stream; + if (client->closed) { + if (!client->control.connected) cleanup_connection_internal(client); + return; + } + client->callback(client->userdata, SHRUB_CLIENT_REMOVE_BUFFERS, NULL, NULL); + shrb_vcr_flush(&client->vcr); + if (!client->paused) { + send_reconnect_packet(client); + } else { + parse_pause_packet(client, client->paused); + aki_packet_free(client->paused); + client->paused = NULL; + client->callback(client->userdata, SHRUB_CLIENT_PAUSE, NULL, NULL); + connect_data(client); + } +} + static void control_connection_callback(void *userdata, struct aki_packet_stream *stream) { - struct bmu_client *client = (struct bmu_client *)userdata; + struct shrb_client *client = (struct shrb_client *)userdata; (void)stream; struct aki_packet *packet = aki_packet_create(); - aki_packet_write_u8(packet, BIMU_CONNECTION_CONTROL); + aki_packet_write_u8(packet, SHRUB_CONNECTION_CONTROL); aki_packet_write_u16(packet, 0); aki_packet_write_u16(packet, client->node_id); aki_packet_stream_send_packet(&client->control, packet); @@ -80,7 +105,7 @@ static void control_connection_callback(void *userdata, struct aki_packet_stream static void control_connection_closed_callback(void *userdata, struct aki_packet_stream *stream) { - struct bmu_client *client = (struct bmu_client *)userdata; + struct shrb_client *client = (struct shrb_client *)userdata; (void)stream; if (client->closed) { if (!client->data.connected) cleanup_connection_internal(client); @@ -90,24 +115,24 @@ static void control_connection_closed_callback(void *userdata, struct aki_packet // TODO: Reconnect. } -static void parse_info_packet(struct bmu_client *client, struct aki_packet *packet) +static void parse_info_packet(struct shrb_client *client, struct aki_packet *packet) { client->duration = aki_packet_read_u64(packet); u32 count = aki_packet_read_u32(packet); for (u32 i = 0; i < count; i++) { - struct bmu_vcr_stream *stream = al_alloc_object(struct bmu_vcr_stream); - stream->client = bmu_codec_client_create(); + struct shrb_vcr_stream *stream = al_alloc_object(struct shrb_vcr_stream); + stream->client = shrb_codec_client_create(); stream->client->callback = client->callback; stream->client->userdata = client->userdata; stream->stream.type = aki_packet_read_u8(packet); switch (stream->stream.type) { case CAMU_NORMAL: - stream->type = BIMU_STREAM_VIDEO; + stream->type = SHRUB_STREAM_VIDEO; stream->index = 0; stream->stream.video.width = aki_packet_read_s32(packet); stream->stream.video.height = aki_packet_read_s32(packet); break; -#ifdef HAVE_FFMPEG +#ifdef CAMU_HAVE_FFMPEG case CAMU_FFMPEG_COMPAT: { stream->stream.type = CAMU_FFMPEG_COMPAT; stream->stream.av.format_context = avformat_alloc_context(); @@ -116,16 +141,16 @@ static void parse_info_packet(struct bmu_client *client, struct aki_packet *pack stream->index = stream->stream.av.stream->index; switch (stream->stream.av.stream->codecpar->codec_type) { case AVMEDIA_TYPE_AUDIO: - stream->type = BIMU_STREAM_AUDIO; + stream->type = SHRUB_STREAM_AUDIO; al_array_push(client->subscribed, stream->index); break; case AVMEDIA_TYPE_VIDEO: - stream->type = BIMU_STREAM_VIDEO; + stream->type = SHRUB_STREAM_VIDEO; al_array_push(client->subscribed, stream->index); break; case AVMEDIA_TYPE_SUBTITLE: default: - stream->type = BIMU_STREAM_UNKNOWN; + stream->type = SHRUB_STREAM_UNKNOWN; break; } break; @@ -133,15 +158,22 @@ static void parse_info_packet(struct bmu_client *client, struct aki_packet *pack #endif } if (!stream->client->init(stream->client, NULL, stream)) { + // TODO: Handle this. } - bmu_vcr_add_stream(&client->vcr, stream); + shrb_vcr_add_stream(&client->vcr, stream); } } -static void send_subscribe_packet(struct bmu_client *client) +static void disconnect_data_internal(struct shrb_client *client) +{ + client->corked = false; + aki_packet_stream_disconnect(&client->data); +} + +static void send_subscribe_packet(struct shrb_client *client) { struct aki_packet *packet = aki_packet_create(); - aki_packet_write_u8(packet, BIMU_CONTROL_SUBSCRIBE); + aki_packet_write_u8(packet, SHRUB_CONTROL_SUBSCRIBE); aki_packet_write_u32(packet, client->subscribed.size); s32 index; al_array_foreach(client->subscribed, i, index) { @@ -150,40 +182,61 @@ static void send_subscribe_packet(struct bmu_client *client) aki_packet_stream_send_packet(&client->control, packet); } -static void parse_start_packet(struct bmu_client *client, struct aki_packet *packet) +static void parse_start_packet(struct shrb_client *client, struct aki_packet *packet) { u64 ts = aki_packet_read_u64(packet); u64 delay = aki_packet_read_u64(packet); u64 seek_pos = aki_packet_read_u64(packet); - struct bmu_seek_req req = { .base = seek_pos, .ts = ts, .delay = delay }; - client->callback(client->userdata, BIMU_CLIENT_SET, NULL, &req); + u8 paused = aki_packet_read_u8(packet); + (void)paused; + struct shrb_seek_req req = { .base = seek_pos, .ts = ts, .delay = delay }; + client->callback(client->userdata, SHRUB_CLIENT_SET, NULL, &req); + client->seek_pos = req.base; +} + +static void parse_resume_packet(struct shrb_client *client, struct aki_packet *packet) +{ + u64 ts = aki_packet_read_u64(packet); + s64 delay = aki_packet_read_s64(packet); + u64 seek_pos = aki_packet_read_u64(packet); + struct shrb_seek_req req = { .base = seek_pos, .ts = ts, .delay = delay }; + client->callback(client->userdata, SHRUB_CLIENT_RESUME, NULL, &req); client->seek_pos = req.base; } static void control_packet_callback(void *userdata, struct aki_packet_stream *stream, struct aki_packet *packet) { - struct bmu_client *client = (struct bmu_client *)userdata; + struct shrb_client *client = (struct shrb_client *)userdata; (void)stream; switch (aki_packet_read_u8(packet)) { - case BIMU_CONTROL_INFO: + case SHRUB_CONTROL_INFO: client->connection_id = aki_packet_read_u16(packet); parse_info_packet(client, packet); send_subscribe_packet(client); break; - case BIMU_CONTROL_START: + case SHRUB_CONTROL_START: parse_start_packet(client, packet); connect_data(client); break; - case BIMU_CONTROL_PAUSE: + case SHRUB_CONTROL_PAUSE: { + al_assert(!client->paused); + f64 pts = aki_packet_read_u64(packet) / 1000000.f; + client->callback(client->userdata, SHRUB_CLIENT_PAUSE, NULL, &pts); + //client->paused = packet; + //disconnect_data_internal(client); break; - case BIMU_CONTROL_RESUME: + } + case SHRUB_CONTROL_RESUME: { + u64 ts = aki_packet_read_u64(packet); + client->callback(client->userdata, SHRUB_CLIENT_RESUME, NULL, &ts); + //parse_resume_packet(client, packet); break; - case BIMU_CONTROL_SEEK: + } + case SHRUB_CONTROL_SEEK: client->level = 0; - client->corked = false; - aki_packet_stream_disconnect(&client->data); + disconnect_data_internal(client); break; default: break; @@ -192,55 +245,65 @@ static void control_packet_callback(void *userdata, struct aki_packet_stream *st aki_packet_free(packet); } -void bmu_client_connect(struct bmu_client *client, struct aki_event_loop *loop, str *addr, s32 port, u16 node_id) +void shrb_client_connect(struct shrb_client *client, struct aki_event_loop *loop, str *addr, s32 port, u16 node_id) { client->loop = loop; client->node_id = node_id; al_str_clone(&client->addr, addr); client->port = port; + client->closed = false; + client->reconnect = false; client->level = 0; client->corked = false; + client->paused = NULL; al_array_init(client->subscribed); - bmu_vcr_init(&client->vcr, client->loop, &client->data); + shrb_vcr_init(&client->vcr, client->loop, &client->data); aki_packet_stream_init(&client->control, AKI_SOCKET_TCP, control_connection_callback, control_connection_closed_callback, control_packet_callback, packet_sent_callback, client); aki_packet_stream_set_no_delay(&client->control, 1); aki_packet_stream_connect(&client->control, client->loop, &client->addr, client->port); } -void bmu_client_reseek(struct bmu_client *client) +void shrb_client_toggle_pause(struct shrb_client *client, f64 pts) +{ + struct aki_packet *packet = aki_packet_create(); + aki_packet_write_u8(packet, SHRUB_CONTROL_TOGGLE_PAUSE); + aki_packet_write_u64(packet, pts * 1000000L); + aki_packet_stream_send_packet(&client->control, packet); +} + +void shrb_client_reseek(struct shrb_client *client) { client->level++; al_log_debug("bimu", "Attempting re-seek at level %i.", client->level); - client->corked = false; - aki_packet_stream_disconnect(&client->data); + disconnect_data_internal(client); } -void bmu_client_seek(struct bmu_client *client, f64 percent) +void shrb_client_seek(struct shrb_client *client, f64 percent) { u64 pos = client->duration * percent; struct aki_packet *packet = aki_packet_create(); - aki_packet_write_u8(packet, BIMU_CONTROL_SEEK); + aki_packet_write_u8(packet, SHRUB_CONTROL_SEEK); aki_packet_write_u64(packet, pos); aki_packet_stream_send_packet(&client->control, packet); } -void bmu_client_close(struct bmu_client *client) +void shrb_client_close(struct shrb_client *client) { - struct bmu_vcr_stream *stream; + struct shrb_vcr_stream *stream; al_array_foreach(client->vcr.streams, i, stream) { - bmu_vcr_stream_close(stream); + shrb_vcr_stream_close(stream); } client->closed = true; aki_packet_stream_disconnect(&client->control); aki_packet_stream_disconnect(&client->data); } -void bmu_client_free(struct bmu_client *client) +void shrb_client_free(struct shrb_client *client) { aki_packet_stream_free(&client->control); aki_packet_stream_free(&client->data); al_str_free(&client->addr); al_array_free(client->subscribed); - bmu_vcr_free(&client->vcr); + shrb_vcr_free(&client->vcr); } diff --git a/src/shrub/client.h b/src/shrub/client.h new file mode 100644 index 0000000..cb3250d --- /dev/null +++ b/src/shrub/client.h @@ -0,0 +1,34 @@ +#pragma once + +#include <al/types.h> +#include <aki/packet_stream.h> + +#include "vcr.h" + +struct shrb_client { + struct aki_event_loop *loop; + u16 node_id; + u16 connection_id; + str addr; + s32 port; + struct aki_packet_stream control; + struct aki_packet_stream data; + bool closed; + bool reconnect; + u32 level; + bool corked; + u64 duration; + u64 seek_pos; + struct aki_packet *paused; + array(s32) subscribed; + struct shrb_vcr vcr; + void (*callback)(void *, u8, struct shrb_vcr_stream *stream, void *); + void *userdata; +}; + +void shrb_client_connect(struct shrb_client *client, struct aki_event_loop *loop, str *addr, s32 port, u16 node_id); +void shrb_client_toggle_pause(struct shrb_client *client, f64 pts); +void shrb_client_reseek(struct shrb_client *client); +void shrb_client_seek(struct shrb_client *client, f64 percent); +void shrb_client_close(struct shrb_client *client); +void shrb_client_free(struct shrb_client *client); diff --git a/src/shrub/common.h b/src/shrub/common.h new file mode 100644 index 0000000..0be00e4 --- /dev/null +++ b/src/shrub/common.h @@ -0,0 +1,27 @@ +#pragma once + +enum { + SHRUB_STREAM_AUDIO = 0, + SHRUB_STREAM_VIDEO, + SHRUB_STREAM_UNKNOWN, +}; + +enum { + SHRUB_CONNECTION_CONTROL = 0, + SHRUB_CONNECTION_DATA +}; + +enum { + SHRUB_CONTROL_INFO = 0, + SHRUB_CONTROL_SUBSCRIBE, + SHRUB_CONTROL_RECONNECT, + SHRUB_CONTROL_START, + SHRUB_CONTROL_TOGGLE_PAUSE, + SHRUB_CONTROL_PAUSE, + SHRUB_CONTROL_RESUME, + SHRUB_CONTROL_SEEK +}; + +#define SHRUB_BASE_DELAY 500000Lu +#define SHRUB_PAUSE_DELAY 60000Lu +#define SHRUB_DELAY_IGNORE 0Lu diff --git a/src/shrub/handler.h b/src/shrub/handler.h new file mode 100644 index 0000000..4ff2f2d --- /dev/null +++ b/src/shrub/handler.h @@ -0,0 +1,52 @@ +#pragma once + +#include <aki/packet_pool.h> + +#include "../codec/codec.h" +#include "../cache/handle.h" + +#include "vcr.h" + +struct shrb_server_handler { + bool (*init)(struct shrb_server_handler *, struct cch_handle *, struct aki_packet_pool *); + void (*write_info)(struct shrb_server_handler *, struct aki_packet *); + void (*subscribe)(struct shrb_server_handler *, s32, bool); + u64 (*get_duration)(struct shrb_server_handler *); + bool (*seek)(struct shrb_server_handler *, u64); + bool (*step)(struct shrb_server_handler *); + void (*free)(struct shrb_server_handler **); +}; + +enum { + SHRUB_PACKET_DATA = 0, + SHRUB_PACKET_EOF, + SHRUB_PACKET_ERROR +}; + +enum { + SHRUB_CLIENT_CONFIGURE = 0, + SHRUB_CLIENT_SET, + SHRUB_CLIENT_PAUSE, + SHRUB_CLIENT_RESUME, + SHRUB_CLIENT_DATA, + SHRUB_CLIENT_REMOVE_BUFFERS, + SHRUB_CLIENT_EOF, + SHRUB_CLIENT_CLOSED +}; + +struct shrb_seek_req { + u64 base; + u64 ts; + u64 delay; +}; + +struct shrb_client_stream; +struct shrb_client_handler { + bool (*init)(struct shrb_client_handler *, struct camu_renderer *, struct shrb_vcr_stream *); + void (*handle_packet)(struct shrb_client_handler *, struct aki_packet *); + void (*flush)(struct shrb_client_handler *); + void (*free)(struct shrb_client_handler **); + struct shrb_vcr_stream *stream; + void (*callback)(void *, u8, struct shrb_vcr_stream *stream, void *); + void *userdata; +}; diff --git a/src/bimu/handlers/cdio.h b/src/shrub/handlers/cdio.h index 0f614af..0f614af 100644 --- a/src/bimu/handlers/cdio.h +++ b/src/shrub/handlers/cdio.h diff --git a/src/bimu/handlers/cdio_server.c b/src/shrub/handlers/cdio_server.c index 151b94f..151b94f 100644 --- a/src/bimu/handlers/cdio_server.c +++ b/src/shrub/handlers/cdio_server.c diff --git a/src/shrub/handlers/codec.h b/src/shrub/handlers/codec.h new file mode 100644 index 0000000..f17b17d --- /dev/null +++ b/src/shrub/handlers/codec.h @@ -0,0 +1,20 @@ +#pragma once + +#include "../handler.h" + +#include "../../codec/codec.h" + +struct shrb_codec_server { + struct shrb_server_handler handler; + struct camu_demuxer *demux; + struct camu_packet packet; + struct aki_packet_pool *pool; +}; + +struct shrb_codec_client { + struct shrb_client_handler handler; + struct camu_decoder *dec; +}; + +struct shrb_server_handler *shrb_codec_server_create(void); +struct shrb_client_handler *shrb_codec_client_create(void); diff --git a/src/shrub/handlers/codec_client.c b/src/shrub/handlers/codec_client.c new file mode 100644 index 0000000..f55a740 --- /dev/null +++ b/src/shrub/handlers/codec_client.c @@ -0,0 +1,102 @@ +#include "../../codec/codec.h" +#include "../../codec/ffmpeg/decoder.h" +#include "../../codec/ffmpeg/packet_ext.h" + +#include "../vcr.h" + +#include "codec.h" + +static void data_callback(void *userdata, struct camu_frame *frame) +{ + struct shrb_codec_client *codec = (struct shrb_codec_client *)userdata; + codec->handler.callback(codec->handler.userdata, SHRUB_CLIENT_DATA, codec->handler.stream, frame); +} + +static bool codec_client_init(struct shrb_client_handler *handler, struct camu_renderer *renderer, + struct shrb_vcr_stream *stream) +{ + struct shrb_codec_client *codec = (struct shrb_codec_client *)handler; + codec->dec = camu_ff_decoder_create(); + codec->handler.stream = stream; + if (!codec->dec->init(codec->dec, renderer, &stream->stream)) { + return false; + } + codec->dec->stream = &stream->stream; + codec->handler.callback(codec->handler.userdata, SHRUB_CLIENT_CONFIGURE, codec->handler.stream, NULL); + codec->dec->set_callback(codec->dec, data_callback, codec); + return true; +} + +#ifdef CAMU_HAVE_FFMPEG +static void push_av_packet(struct shrb_codec_client *codec, AVPacket *pkt) +{ + struct camu_packet packet; + packet.av.pkt = pkt; + s32 ret = codec->dec->push(codec->dec, &packet); + av_packet_unref(pkt); + av_packet_free(&pkt); + al_assert(ret == CAMU_OK); +} +#endif + +static void push_packet(struct shrb_codec_client *codec, struct aki_buffer *buffer) +{ + struct camu_packet packet; + packet.buffer = buffer; + s32 ret = codec->dec->push(codec->dec, &packet); + al_assert(ret == CAMU_OK); +} + +static void codec_client_handle_packet(struct shrb_client_handler *handler, struct aki_packet *packet) +{ + struct shrb_codec_client *codec = (struct shrb_codec_client *)handler; + if (!packet) { + struct shrb_codec_client *codec = (struct shrb_codec_client *)handler; + s32 ret = codec->dec->push(codec->dec, NULL); + // Flush returns success. + ret = codec->dec->process(codec->dec); + al_assert(ret == CAMU_ERR_EOF); + codec->handler.callback(codec->handler.userdata, SHRUB_CLIENT_EOF, codec->handler.stream, NULL); + return; + } + switch (aki_packet_read_u8(packet)) { + case CAMU_NORMAL: { + struct aki_buffer buffer; + aki_packet_read_buffer(packet, &buffer); + push_packet(codec, &buffer); + break; + } +#ifdef CAMU_HAVE_FFMPEG + case CAMU_FFMPEG_COMPAT: { + push_av_packet(codec, aki_packet_read_av_packet(packet)); + break; + } +#endif + } + s32 ret = codec->dec->process(codec->dec); + al_assert(ret == CAMU_ERR_AGAIN || ret == CAMU_ERR_EOF); +} + +static void codec_client_flush(struct shrb_client_handler *handler) +{ + struct shrb_codec_client *codec = (struct shrb_codec_client *)handler; + codec->dec->flush(codec->dec); +} + +static void codec_client_free(struct shrb_client_handler **handler) +{ + struct shrb_codec_client *codec = (struct shrb_codec_client *)*handler; + codec->dec->free(&codec->dec); + al_free(codec); + *handler = NULL; +} + +struct shrb_client_handler *shrb_codec_client_create(void) +{ + struct shrb_codec_client *codec = al_alloc_object(struct shrb_codec_client); + codec->handler.init = codec_client_init; + codec->handler.handle_packet = codec_client_handle_packet; + codec->handler.flush = codec_client_flush; + codec->handler.free = codec_client_free; + return (struct shrb_client_handler *)codec; +} diff --git a/src/bimu/handlers/codec_server.c b/src/shrub/handlers/codec_server.c index a30ce68..cab615a 100644 --- a/src/bimu/handlers/codec_server.c +++ b/src/shrub/handlers/codec_server.c @@ -1,23 +1,23 @@ #include "../../codec/codec.h" -#include "../../codec/libav/demuxer.h" -#include "../../codec/libav/packet_ext.h" +#include "../../codec/ffmpeg/demuxer.h" +#include "../../codec/ffmpeg/packet_ext.h" #include "../handler.h" #include "codec.h" -static bool codec_server_init(struct bmu_server_handler *handler, struct cch_handle *handle, +static bool codec_server_init(struct shrb_server_handler *handler, struct cch_handle *handle, struct aki_packet_pool *pool) { - struct bmu_codec_server *codec = (struct bmu_codec_server *)handler; - codec->demux = camu_lav_demuxer_create(); + struct shrb_codec_server *codec = (struct shrb_codec_server *)handler; + codec->demux = camu_ff_demuxer_create(); if (!codec->demux->init(codec->demux, handle)) { return false; } switch (codec->demux->type) { case CAMU_NORMAL: break; -#ifdef HAVE_FFMPEG +#ifdef CAMU_HAVE_FFMPEG case CAMU_FFMPEG_COMPAT: codec->packet.av.pkt = av_packet_alloc(); break; @@ -27,9 +27,9 @@ static bool codec_server_init(struct bmu_server_handler *handler, struct cch_han return true; } -static void codec_server_write_info(struct bmu_server_handler *handler, struct aki_packet *packet) +static void codec_server_write_info(struct shrb_server_handler *handler, struct aki_packet *packet) { - struct bmu_codec_server *codec = (struct bmu_codec_server *)handler; + struct shrb_codec_server *codec = (struct shrb_codec_server *)handler; aki_packet_write_u64(packet, codec->demux->get_duration(codec->demux)); aki_packet_write_u32(packet, codec->demux->streams.size); struct camu_stream *stream; @@ -40,7 +40,7 @@ static void codec_server_write_info(struct bmu_server_handler *handler, struct a aki_packet_write_s32(packet, stream->video.width); aki_packet_write_s32(packet, stream->video.height); break; -#ifdef HAVE_FFMPEG +#ifdef CAMU_HAVE_FFMPEG case CAMU_FFMPEG_COMPAT: aki_packet_write_av_codec_id(packet, stream->av.stream->codecpar->codec_id); aki_packet_write_av_stream(packet, stream->av.stream); @@ -50,29 +50,29 @@ static void codec_server_write_info(struct bmu_server_handler *handler, struct a } } -static void codec_server_subscribe(struct bmu_server_handler *handler, s32 index, bool subscribe) +static void codec_server_subscribe(struct shrb_server_handler *handler, s32 index, bool subscribe) { - struct bmu_codec_server *codec = (struct bmu_codec_server *)handler; + struct shrb_codec_server *codec = (struct shrb_codec_server *)handler; if (subscribe) { al_array_push(codec->demux->subscribed, index); } } -static u64 codec_server_get_duration(struct bmu_server_handler *handler) +static u64 codec_server_get_duration(struct shrb_server_handler *handler) { - struct bmu_codec_server *codec = (struct bmu_codec_server *)handler; + struct shrb_codec_server *codec = (struct shrb_codec_server *)handler; return codec->demux->get_duration(codec->demux); } -static bool codec_server_seek(struct bmu_server_handler *handler, u64 pos) +static bool codec_server_seek(struct shrb_server_handler *handler, u64 pos) { - struct bmu_codec_server *codec = (struct bmu_codec_server *)handler; + struct shrb_codec_server *codec = (struct shrb_codec_server *)handler; return codec->demux->seek(codec->demux, pos); } -static bool codec_server_step(struct bmu_server_handler *handler) +static bool codec_server_step(struct shrb_server_handler *handler) { - struct bmu_codec_server *codec = (struct bmu_codec_server *)handler; + struct shrb_codec_server *codec = (struct shrb_codec_server *)handler; struct aki_packet *packet = aki_packet_pool_get(codec->pool); if (!packet) return false; s32 ret = codec->demux->get_packet(codec->demux, &codec->packet); @@ -80,12 +80,12 @@ static bool codec_server_step(struct bmu_server_handler *handler) aki_packet_write_u8(packet, 0); switch (codec->packet.type) { case CAMU_NORMAL: { - aki_packet_write_s32(packet, BIMU_PACKET_DATA); + aki_packet_write_s32(packet, SHRUB_PACKET_DATA); aki_packet_write_u8(packet, codec->packet.type); aki_packet_write_buffer(packet, codec->packet.buffer); break; } -#ifdef HAVE_FFMPEG +#ifdef CAMU_HAVE_FFMPEG case CAMU_FFMPEG_COMPAT: { AVPacket *pkt = codec->packet.av.pkt; aki_packet_write_s32(packet, pkt->stream_index); @@ -98,24 +98,24 @@ static bool codec_server_step(struct bmu_server_handler *handler) } aki_packet_pool_submit(codec->pool, packet); } else if (ret == CAMU_ERR_EOF) { - aki_packet_write_u8(packet, BIMU_PACKET_EOF); + aki_packet_write_u8(packet, SHRUB_PACKET_EOF); aki_packet_pool_submit(codec->pool, packet); return false; } else { - aki_packet_write_u8(packet, BIMU_PACKET_ERROR); + aki_packet_write_u8(packet, SHRUB_PACKET_ERROR); aki_packet_pool_submit(codec->pool, packet); return false; } return true; } -static void codec_server_free(struct bmu_server_handler **handler) +static void codec_server_free(struct shrb_server_handler **handler) { - struct bmu_codec_server *codec = (struct bmu_codec_server *)*handler; + struct shrb_codec_server *codec = (struct shrb_codec_server *)*handler; switch (codec->demux->type) { case CAMU_NORMAL: break; -#ifdef HAVE_FFMPEG +#ifdef CAMU_HAVE_FFMPEG case CAMU_FFMPEG_COMPAT: av_packet_free(&codec->packet.av.pkt); break; @@ -126,9 +126,9 @@ static void codec_server_free(struct bmu_server_handler **handler) *handler = NULL; } -struct bmu_server_handler *bmu_codec_server_create(void) +struct shrb_server_handler *shrb_codec_server_create(void) { - struct bmu_codec_server *codec = al_alloc_object(struct bmu_codec_server); + struct shrb_codec_server *codec = al_alloc_object(struct shrb_codec_server); codec->handler.init = codec_server_init; codec->handler.write_info = codec_server_write_info; codec->handler.subscribe = codec_server_subscribe; @@ -136,5 +136,5 @@ struct bmu_server_handler *bmu_codec_server_create(void) codec->handler.seek = codec_server_seek; codec->handler.step = codec_server_step; codec->handler.free = codec_server_free; - return (struct bmu_server_handler *)codec; + return (struct shrb_server_handler *)codec; } diff --git a/src/bimu/handlers/dvd.h b/src/shrub/handlers/dvd.h index db9bfe9..db9bfe9 100644 --- a/src/bimu/handlers/dvd.h +++ b/src/shrub/handlers/dvd.h diff --git a/src/bimu/handlers/dvd_server.c b/src/shrub/handlers/dvd_server.c index ea3a9d9..ea3a9d9 100644 --- a/src/bimu/handlers/dvd_server.c +++ b/src/shrub/handlers/dvd_server.c diff --git a/src/bimu/meson.build b/src/shrub/meson.build index cc1ff43..a3fc77b 100644 --- a/src/bimu/meson.build +++ b/src/shrub/meson.build @@ -1,29 +1,29 @@ -bimu_server_src = [ +shrub_server_src = [ 'server.c', 'handlers/codec_server.c' ] -bimu_client_src = [ +shrub_client_src = [ 'client.c', 'vcr.c', 'handlers/codec_client.c' ] -bimu_server_deps = [av_server] -bimu_client_deps = [av_client] +shrub_server_deps = [ffmpeg_server] +shrub_client_deps = [ffmpeg_client] #libcdio_paranoia = dependency('libcdio_paranoia', required: false, allow_fallback: true) #libcdio_cdda = dependency('libcdio_cdda', required: false, allow_fallback: true) #if libcdio_paranoia.found() and libcdio_cdda.found() -# bimu_server_src += ['handlers/cdio_server.c'] -# bimu_server_deps += [libcdio_paranoia, libcdio_cdda] +# shrub_server_src += ['handlers/cdio_server.c'] +# shrub_server_deps += [libcdio_paranoia, libcdio_cdda] #endif # #libdvdcss = dependency('libdvdcss', required: false, allow_fallback: true) #libdvdread = dependency('libdvdread', required: false, allow_fallback: true) #libdvdnav = dependency('libdvdnav', required: false, allow_fallback: true) #if libdvdcss.found() and libdvdread.found() and libdvdnav.found() -# bimu_server_src += ['handlers/dvd_server.c'] -# bimu_server_deps += [libdvdcss] +# shrub_server_src += ['handlers/dvd_server.c'] +# shrub_server_deps += [libdvdcss] #endif -bimu_server = declare_dependency(sources: bimu_server_src, dependencies: bimu_server_deps) -bimu_client = declare_dependency(sources: bimu_client_src, dependencies: bimu_client_deps) +shrub_server = declare_dependency(sources: shrub_server_src, dependencies: shrub_server_deps) +shrub_client = declare_dependency(sources: shrub_client_src, dependencies: shrub_client_deps) diff --git a/src/bimu/server.c b/src/shrub/server.c index 6db0dd2..ba7259c 100644 --- a/src/bimu/server.c +++ b/src/shrub/server.c @@ -8,9 +8,9 @@ static void control_connection_closed_callback(void *userdata, struct aki_packet_stream *stream) { - struct bmu_node_connection *connection = (struct bmu_node_connection *)userdata; + struct shrb_node_connection *connection = (struct shrb_node_connection *)userdata; (void)stream; - struct bmu_node_connection *rconnection; + struct shrb_node_connection *rconnection; al_array_foreach(connection->node->connections, i, rconnection) { if (rconnection == connection) { al_array_remove_at_iter(connection->node->connections, i); @@ -26,7 +26,7 @@ static void control_connection_closed_callback(void *userdata, struct aki_packet static void data_connection_closed_callback(void *userdata, struct aki_packet_stream *stream) { - struct bmu_node_connection *connection = (struct bmu_node_connection *)userdata; + struct shrb_node_connection *connection = (struct shrb_node_connection *)userdata; (void)stream; connection->running = 0; aki_packet_pool_disable(&connection->pool); @@ -44,7 +44,7 @@ static void default_packet_sent_callback(void *userdata, struct aki_packet *pack static void packet_sent_callback(void *userdata, struct aki_packet *packet) { - struct bmu_node_connection *connection = (struct bmu_node_connection *)userdata; + struct shrb_node_connection *connection = (struct shrb_node_connection *)userdata; aki_packet_pool_return(&connection->pool, packet); } @@ -57,7 +57,7 @@ static void data_packet_callback(void *userdata, struct aki_packet_stream *strea static u8 packet_pool_callback(void *userdata, struct aki_packet *packet) { - struct bmu_node_connection *connection = (struct bmu_node_connection *)userdata; + struct shrb_node_connection *connection = (struct shrb_node_connection *)userdata; if (connection->data->connected) { aki_packet_stream_send_packet(connection->data, packet); return AKI_PACKET_POOL_KEEP; @@ -65,187 +65,189 @@ static u8 packet_pool_callback(void *userdata, struct aki_packet *packet) return AKI_PACKET_POOL_RETURN; } -static void send_resume_packet(struct bmu_node_connection *connection, u64 seek_pos, u64 ts) +static void send_resume_packet(struct shrb_node_connection *connection, u64 ts) { struct aki_packet *packet = aki_packet_create(); - aki_packet_write_u8(packet, BIMU_CONTROL_RESUME); - aki_packet_write_u64(packet, seek_pos); + aki_packet_write_u8(packet, SHRUB_CONTROL_RESUME); aki_packet_write_u64(packet, ts); aki_packet_stream_send_packet(connection->control, packet); } -static void send_pause_packet(struct bmu_node_connection *connection, u64 seek_pos) +static void send_pause_packet(struct shrb_node_connection *connection, u64 seek_pos) { struct aki_packet *packet = aki_packet_create(); - aki_packet_write_u8(packet, BIMU_CONTROL_PAUSE); + aki_packet_write_u8(packet, SHRUB_CONTROL_PAUSE); aki_packet_write_u64(packet, seek_pos); aki_packet_stream_send_packet(connection->control, packet); } -#define DELAY 275000Lu // 275ms -#define DELAY_GIVE 30000Lu // 30ms +static struct shrb_node *get_node_by_id(struct shrb_server *server, u16 id) +{ + struct shrb_node *node; + al_array_foreach(server->nodes, i, node) { + if (node->id == id) return node; + } + return NULL; +} -static void send_start_packet(struct bmu_node_connection *connection, u32 level) +// Possible connection senarios: +// - initial connection and start +// - connected while playing +// - connected while paused +// - connected while queued +// - connected while queued after a paused stream +// Questions: +// - can the server estimate the position of the stream? I think yes + +static void send_start_packet(struct shrb_node_connection *connection) { + struct shrb_node *node = connection->node; struct aki_packet *packet = aki_packet_create(); - aki_packet_write_u8(packet, BIMU_CONTROL_START); - u64 ts = aki_get_timestamp(); - u64 delay = DELAY * powl(3, level); - u64 seek_pos = connection->node->seek_pos; - if (!(connection->node->flags & BIMU_NODE_PAUSED)) { - if (connection->node->flags & BIMU_NODE_STARTED) { - if ((ts - DELAY_GIVE) < (u64)connection->node->start) { - // Current time is close enough to node start. - ts = connection->node->start; - } else { - seek_pos += (ts - connection->node->start) + (delay - DELAY); - } - } else { - // Must be set at level 0, this applies for seeks too. - connection->node->start = ts; - connection->node->flags |= BIMU_NODE_STARTED; - } + aki_packet_write_u8(packet, SHRUB_CONTROL_START); + u64 seek_pos = node->seek_pos; + u64 delay = SHRUB_BASE_DELAY * powl(3, connection->level); + u64 start = node->start; + s64 diff = aki_get_timestamp() - start; + if (diff > (s64)SHRUB_BASE_DELAY) { + start += diff; + seek_pos += diff; } - aki_packet_write_u64(packet, ts); + seek_pos += delay - SHRUB_BASE_DELAY; + aki_packet_write_u64(packet, start); aki_packet_write_u64(packet, delay); aki_packet_write_u64(packet, seek_pos); + aki_packet_write_u8(packet, node->paused ? 1 : 0); aki_packet_stream_send_packet(connection->control, packet); } -static void send_seek_packet(struct bmu_node *node) +static void send_seek_packet(struct shrb_node_connection *connection, u64 pos) { - struct aki_packet *packet; - struct bmu_node_connection *connection; - al_array_foreach(node->connections, i, connection) { - packet = aki_packet_create(); - aki_packet_write_u8(packet, BIMU_CONTROL_SEEK); - aki_packet_stream_send_packet(connection->control, packet); - } + connection->seek_pos = pos; + struct aki_packet *packet = aki_packet_create(); + aki_packet_write_u8(packet, SHRUB_CONTROL_SEEK); + aki_packet_stream_send_packet(connection->control, packet); } static void control_packet_callback(void *userdata, struct aki_packet_stream *stream, struct aki_packet *packet) { - struct bmu_node_connection *connection = (struct bmu_node_connection *)userdata; + struct shrb_node_connection *connection = (struct shrb_node_connection *)userdata; (void)stream; switch (aki_packet_read_u8(packet)) { - case BIMU_CONTROL_SUBSCRIBE: { + case SHRUB_CONTROL_SUBSCRIBE: { u32 count = aki_packet_read_u32(packet); for (u32 i = 0; i < count; i++) { connection->handler->subscribe(connection->handler, aki_packet_read_s32(packet), true); } - send_start_packet(connection, 0); + send_start_packet(connection); break; } - case BIMU_CONTROL_RECONNECT: { - send_start_packet(connection, aki_packet_read_u32(packet)); + case SHRUB_CONTROL_RECONNECT: { + connection->level = aki_packet_read_u32(packet); + send_start_packet(connection); break; } - case BIMU_CONTROL_TOGGLE_PAUSE: { - struct bmu_node *node = connection->node; + case SHRUB_CONTROL_TOGGLE_PAUSE: { + struct shrb_node *node = connection->node; + /* u64 pos = aki_packet_read_u64(packet); u8 paused = connection->paused; - if (paused == BIMU_PLAYING) { + if (paused == SHRUB_PLAYING) { node->seek_pos = pos; - node->flags |= BIMU_NODE_PAUSED; - // Update node start? - } else if (paused == BIMU_PAUSED_AND_RUNNING) { - node->flags &= ~BIMU_NODE_PAUSED; + node->flags |= SHRUB_NODE_PAUSED; + // node->start won't be touched while SHRUB_NODE_PAUSED is set. + } else if (paused == SHRUB_PAUSED_AND_RUNNING) { + node->flags &= ~SHRUB_NODE_PAUSED; + // TODO: + node->start = aki_get_timestamp() + DELAY; } - struct bmu_node_connection *rconnection; + struct shrb_node_connection *rconnection; al_array_foreach(node->connections, i, rconnection) { switch (rconnection->paused) { - case BIMU_PLAYING: - al_assert(paused == BIMU_PLAYING || paused == BIMU_NOT_PAUSED); - rconnection->paused = BIMU_PAUSED; + case SHRUB_PLAYING: + al_assert(paused == SHRUB_PLAYING || paused == SHRUB_NOT_PAUSED); + rconnection->paused = SHRUB_PAUSED; send_pause_packet(rconnection, node->seek_pos); break; - case BIMU_NOT_PAUSED: - al_assert(paused != BIMU_PAUSED); - rconnection->paused = BIMU_PAUSED; + case SHRUB_NOT_PAUSED: + al_assert(paused != SHRUB_PAUSED); + rconnection->paused = SHRUB_PAUSED; break; - case BIMU_PAUSED: - al_assert(paused != BIMU_NOT_PAUSED); - rconnection->paused = BIMU_NOT_PAUSED; + case SHRUB_PAUSED: + al_assert(paused != SHRUB_NOT_PAUSED); + rconnection->paused = SHRUB_NOT_PAUSED; break; - case BIMU_PAUSED_AND_RUNNING: - al_assert(paused != BIMU_NOT_PAUSED); + case SHRUB_PAUSED_AND_RUNNING: + al_assert(paused != SHRUB_NOT_PAUSED); send_resume_packet(rconnection, node->seek_pos, node->start); - rconnection->paused = BIMU_PLAYING; + rconnection->paused = SHRUB_PLAYING; break; } } + */ break; } - case BIMU_CONTROL_SEEK: { + case SHRUB_CONTROL_SEEK: { u64 pos = aki_packet_read_u64(packet); connection->node->seek_pos = pos; - connection->node->flags |= BIMU_NODE_SEEKED; - connection->node->flags &= ~BIMU_NODE_STARTED; - send_seek_packet(connection->node); + connection->node->seeked = true; + //send_seek_packet(connection->node); break; } } aki_packet_free(packet); } -static struct bmu_node *get_node_by_id(struct bmu_server *server, u16 id) -{ - struct bmu_node *node; - al_array_foreach(server->nodes, i, node) { - if (node->id == id) return node; - } - return NULL; -} - -static struct bmu_node_connection *get_connection_by_id(struct bmu_node *node, u16 id) +static struct shrb_node_connection *get_connection_by_id(struct shrb_node *node, u16 id) { - struct bmu_node_connection *connection; + struct shrb_node_connection *connection; al_array_foreach(node->connections, i, connection) { if (connection->id == id) return connection; } return NULL; } -static void send_control_info_packet(struct bmu_node_connection *connection) +static void send_control_info_packet(struct shrb_node_connection *connection) { struct aki_packet *packet = aki_packet_create(); - aki_packet_write_u8(packet, BIMU_CONTROL_INFO); + aki_packet_write_u8(packet, SHRUB_CONTROL_INFO); aki_packet_write_u16(packet, connection->id); - connection->handler = bmu_codec_server_create(); - connection->handler->init(connection->handler, &connection->handle, &connection->pool); + connection->handler = shrb_codec_server_create(); + if (!connection->handler->init(connection->handler, &connection->handle, &connection->pool)) { + al_assert(false); + } connection->handler->write_info(connection->handler, packet); aki_packet_stream_send_packet(connection->control, packet); } static aki_thread_result AKI_THREADCALL handler_thread(void *userdata) { - struct bmu_node_connection *connection = (struct bmu_node_connection *)userdata; + struct shrb_node_connection *connection = (struct shrb_node_connection *)userdata; while (connection->running && connection->handler->step(connection->handler)) {} return 0; } -static void maybe_start_handler(struct bmu_node_connection *connection) +static void maybe_start_handler(struct shrb_node_connection *connection) { al_assert(!connection->running); // Don't seek to 0 unless NODE_SEEKED is set. - if (connection->seek_pos > 0 || (connection->node->flags & BIMU_NODE_SEEKED)) { + if (connection->seek_pos > 0 || connection->node->seeked) { connection->handler->seek(connection->handler, connection->seek_pos); } connection->running = 1; aki_thread_create(&connection->thread, handler_thread, connection); - struct bmu_node *node = connection->node; - if (connection->paused == BIMU_NOT_PAUSED) { - send_resume_packet(connection, connection->seek_pos, node->start); - connection->paused = BIMU_PLAYING; - } else if (connection->paused == BIMU_PAUSED) { - connection->paused = BIMU_PAUSED_AND_RUNNING; - } + //struct shrb_node *node = connection->node; + //if (connection->paused == SHRUB_NOT_PAUSED) { + // send_resume_packet(connection, connection->seek_pos, node->start); + // connection->paused = SHRUB_PLAYING; + //} else if (connection->paused == SHRUB_PAUSED) { + // connection->paused = SHRUB_PAUSED_AND_RUNNING; + //} } static void identify_packet_callback(void *userdata, struct aki_packet_stream *stream, struct aki_packet *packet) { - struct bmu_server *server = (struct bmu_server *)userdata; + struct shrb_server *server = (struct shrb_server *)userdata; u8 type = aki_packet_read_u8(packet); @@ -260,13 +262,13 @@ static void identify_packet_callback(void *userdata, struct aki_packet_stream *s // - Data connected before control. // - Data connected more than once. - struct bmu_node *node = get_node_by_id(server, node_id); + struct shrb_node *node = get_node_by_id(server, node_id); al_assert(node); - struct bmu_node_connection *connection = get_connection_by_id(node, connection_id); + struct shrb_node_connection *connection = get_connection_by_id(node, connection_id); if (!connection) { - al_assert(type == BIMU_CONNECTION_CONTROL); - connection = al_alloc_object(struct bmu_node_connection); + al_assert(type == SHRUB_CONNECTION_CONTROL); + connection = al_alloc_object(struct shrb_node_connection); al_array_push(node->connections, connection); connection->id = node->connection_id++; connection->node = node; @@ -274,12 +276,13 @@ static void identify_packet_callback(void *userdata, struct aki_packet_stream *s connection->data = NULL; aki_packet_pool_init(&connection->pool, 16, server->loop, packet_pool_callback, connection); cch_entry_get_handle(connection->node->entry, &connection->handle); - connection->paused = BIMU_NOT_PAUSED; + //connection->paused = SHRUB_PLAYING; + connection->level = 0; connection->running = 0; } switch (type) { - case BIMU_CONNECTION_CONTROL: + case SHRUB_CONNECTION_CONTROL: aki_packet_stream_set_no_delay(stream, 1); connection->control = stream; stream->packet_callback = control_packet_callback; @@ -288,7 +291,7 @@ static void identify_packet_callback(void *userdata, struct aki_packet_stream *s stream->userdata = connection; send_control_info_packet(connection); break; - case BIMU_CONNECTION_DATA: + case SHRUB_CONNECTION_DATA: // TODO: Account for control connection being in a reconnect state. // TODO: is it possible for data->data to be non-NULL during normal operation // (disconnect and reconnect very fast?). @@ -315,7 +318,7 @@ static void connection_closed_callback(void *userdata, struct aki_packet_stream static void connection_callback(void *userdata, struct aki_packet_stream *stream) { - struct bmu_server *server = (struct bmu_server *)userdata; + struct shrb_server *server = (struct shrb_server *)userdata; stream->connection_closed_callback = connection_closed_callback; stream->packet_callback = identify_packet_callback; stream->packet_sent_callback = default_packet_sent_callback; @@ -323,34 +326,94 @@ static void connection_callback(void *userdata, struct aki_packet_stream *stream stream->userdata = server; } -bool bmu_server_init(struct bmu_server *server) +bool shrb_server_init(struct shrb_server *server) { al_array_init(server->nodes); return aki_packet_stream_init(&server->server, AKI_SOCKET_TCP, connection_callback, NULL, NULL, NULL, server); } -void bmu_server_listen(struct bmu_server *server, struct aki_event_loop *loop, str *addr, s32 port) +void shrb_server_listen(struct shrb_server *server, struct aki_event_loop *loop, str *addr, s32 port) { server->loop = loop; aki_packet_stream_listen(&server->server, loop, addr, port); } -u16 bmu_server_create_node(struct bmu_server *server, struct cch_entry *entry) +struct shrb_node *shrb_server_create_node(struct shrb_server *server, u16 prev, struct cch_entry *entry) { - struct bmu_node *node = al_alloc_object(struct bmu_node); - al_array_push(server->nodes, node); + struct shrb_node *node = al_alloc_object(struct shrb_node); node->id = al_rand_u16(); + node->prev = prev; node->entry = entry; - node->flags = 0; + node->paused = false; + node->seeked = false; node->start = 0; node->seek_pos = 0; + node->offset = 0; node->connection_id = 1; al_array_init(node->connections); node->server = server; - return node->id; + struct cch_handle handle; + if (!cch_entry_get_handle(node->entry, &handle)) { + al_free(node); + return NULL; + } + struct shrb_server_handler *handler = shrb_codec_server_create(); + if (!handler->init(handler, &handle, NULL)) { + al_free(node); + return NULL; + } + node->duration = handler->get_duration(handler); + handler->free(&handler); + cch_entry_return_handle(node->entry, &handle); + al_array_push(server->nodes, node); + return node; +} + +void shrb_node_set_start(struct shrb_node *node, u64 start) +{ + node->start = start; +} + +void shrb_node_set_offset(struct shrb_node *node, u64 offset) +{ + node->offset = offset; +} + +void shrb_node_toggle_pause(struct shrb_node *node, u64 pos) +{ + if (!node->paused) { + node->seek_pos = pos; + node->seek_pos += SHRUB_PAUSE_DELAY; + } else { + node->start = aki_get_timestamp() + SHRUB_PAUSE_DELAY; + } + struct shrb_node_connection *connection; + al_array_foreach(node->connections, i, connection) { + if (!node->paused) { + send_pause_packet(connection, node->seek_pos); + } else { + send_resume_packet(connection, node->start); + } + } + if (node->paused) { + node->start -= SHRUB_BASE_DELAY; + node->offset = 0; + } + node->paused = !node->paused; +} + +void shrb_node_seek(struct shrb_node *node, u64 pos) +{ + node->seek_pos = pos; + node->start = aki_get_timestamp(); + node->seeked = true; + struct shrb_node_connection *connection; + al_array_foreach(node->connections, i, connection) { + send_seek_packet(connection, node->seek_pos); + } } -void bmu_server_close(struct bmu_server *server) +void shrb_server_close(struct shrb_server *server) { (void)server; } diff --git a/src/shrub/server.h b/src/shrub/server.h new file mode 100644 index 0000000..2bce369 --- /dev/null +++ b/src/shrub/server.h @@ -0,0 +1,62 @@ +#pragma once + +#include <al/array.h> +#include <aki/packet_stream.h> +#include <aki/packet_pool.h> + +#include "../cache/entry.h" +#include "../cache/handle.h" + +#include "handler.h" + +enum { + SHRUB_PLAYING = 0, + SHRUB_NOT_PAUSED, + SHRUB_PAUSED, + SHRUB_PAUSED_AND_RUNNING +}; + +struct shrb_node_connection { + u16 id; + struct shrb_node *node; + struct aki_packet_stream *control; + struct aki_packet_stream *data; + struct aki_packet_pool pool; + s32 running; + //u8 paused; + u32 level; + u64 seek_pos; + struct cch_handle handle; + struct shrb_server_handler *handler; + struct aki_thread thread; +}; + +struct shrb_node { + u16 id; + u16 prev; + struct cch_entry *entry; + bool paused; + bool seeked; + u64 start; + u64 seek_pos; + u64 offset; + u64 duration; + u16 connection_id; + array(struct shrb_node_connection *) connections; + struct shrb_server *server; +}; + +struct shrb_server { + struct aki_event_loop *loop; + struct aki_packet_stream server; + array(struct shrb_node *) nodes; +}; + +bool shrb_server_init(struct shrb_server *server); +void shrb_server_listen(struct shrb_server *server, struct aki_event_loop *loop, str *addr, s32 port); +struct shrb_node *shrb_server_create_node(struct shrb_server *server, u16 prev, struct cch_entry *entry); +void shrb_node_set_start(struct shrb_node *node, u64 start); +void shrb_node_set_offset(struct shrb_node *node, u64 offset); +void shrb_node_toggle_pause(struct shrb_node *node, u64 pos); +void shrb_node_seek(struct shrb_node *node, u64 pos); +void shrb_server_close(struct shrb_server *server); diff --git a/src/bimu/vcr.c b/src/shrub/vcr.c index 109edd7..1816142 100644 --- a/src/bimu/vcr.c +++ b/src/shrub/vcr.c @@ -9,11 +9,11 @@ static void signal_callback(void *userdata) { - struct bmu_vcr *vcr = (struct bmu_vcr *)userdata; + struct shrb_vcr *vcr = (struct shrb_vcr *)userdata; aki_packet_stream_cork(vcr->data, false); } -void bmu_vcr_init(struct bmu_vcr *vcr, struct aki_event_loop *loop, struct aki_packet_stream *data) +void shrb_vcr_init(struct shrb_vcr *vcr, struct aki_event_loop *loop, struct aki_packet_stream *data) { al_array_init(vcr->streams); al_atomic_u64_store(&vcr->count, 0, AL_ATOMIC_RELAXED); @@ -24,33 +24,33 @@ void bmu_vcr_init(struct bmu_vcr *vcr, struct aki_event_loop *loop, struct aki_p static aki_thread_result AKI_THREADCALL vcr_stream_thread(void *userdata) { - struct bmu_vcr_stream *stream = (struct bmu_vcr_stream *)userdata; - struct bmu_vcr *vcr = stream->vcr; + struct shrb_vcr_stream *stream = (struct shrb_vcr_stream *)userdata; + struct shrb_vcr *vcr = stream->vcr; while (aki_packet_cache_wait(&stream->cache)) { struct aki_packet *packet = aki_packet_cache_pop(&stream->cache); if (!packet) { - stream->client->handle_eof(stream->client); + stream->client->handle_packet(stream->client, NULL); break; } if (al_atomic_u64_sub(&vcr->count, 1, AL_ATOMIC_RELAXED) < VCR_BUFFER_LOW && stream->buffered) { aki_signal_send(&vcr->signal); } - if (al_atomic_s32_load(&stream->state, AL_ATOMIC_RELAXED) == BIMU_STREAM_STOPPED) { + if (al_atomic_s32_load(&stream->state, AL_ATOMIC_RELAXED) == SHRUB_STREAM_STOPPED) { aki_mutex_lock(&stream->mutex); aki_cond_wait(&stream->cond, &stream->mutex); aki_mutex_unlock(&stream->mutex); } - if (al_atomic_s32_load(&stream->state, AL_ATOMIC_RELAXED) == BIMU_STREAM_CLOSED) { + if (al_atomic_s32_load(&stream->state, AL_ATOMIC_RELAXED) == SHRUB_STREAM_CLOSED) { aki_packet_free(packet); break; } - stream->client->handle_data_packet(stream->client, packet); + stream->client->handle_packet(stream->client, packet); aki_packet_free(packet); } return 0; } -void bmu_vcr_add_stream(struct bmu_vcr *vcr, struct bmu_vcr_stream *stream) +void shrb_vcr_add_stream(struct shrb_vcr *vcr, struct shrb_vcr_stream *stream) { stream->vcr = vcr; aki_cond_init(&stream->cond); @@ -58,23 +58,23 @@ void bmu_vcr_add_stream(struct bmu_vcr *vcr, struct bmu_vcr_stream *stream) aki_packet_cache_init(&stream->cache, VCR_BUFFER_HIGH); stream->buffered = 0; al_array_push(vcr->streams, stream); - al_atomic_s32_store(&stream->state, BIMU_STREAM_RUNNING, AL_ATOMIC_RELAXED); + al_atomic_s32_store(&stream->state, SHRUB_STREAM_RUNNING, AL_ATOMIC_RELAXED); } -static struct bmu_vcr_stream *get_stream_from_index(struct bmu_vcr *vcr, s32 index) +static struct shrb_vcr_stream *get_stream_from_index(struct shrb_vcr *vcr, s32 index) { - struct bmu_vcr_stream *stream; + struct shrb_vcr_stream *stream; al_array_foreach(vcr->streams, i, stream) { if (stream->index == index) return stream; } return NULL; } -bool bmu_vcr_push_packet(struct bmu_vcr *vcr, struct aki_packet *packet) +bool shrb_vcr_push_packet(struct shrb_vcr *vcr, struct aki_packet *packet) { - struct bmu_vcr_stream *stream; + struct shrb_vcr_stream *stream; switch (aki_packet_read_u8(packet)) { - case BIMU_PACKET_DATA: + case SHRUB_PACKET_DATA: stream = get_stream_from_index(vcr, aki_packet_read_s32(packet)); al_assert(stream); if (!stream->running) { @@ -90,15 +90,15 @@ bool bmu_vcr_push_packet(struct bmu_vcr *vcr, struct aki_packet *packet) } } break; - case BIMU_PACKET_EOF: + case SHRUB_PACKET_EOF: al_array_foreach(vcr->streams, i, stream) { aki_packet_cache_send_packet(&stream->cache, NULL); } aki_packet_free(packet); break; - case BIMU_PACKET_ERROR: - aki_packet_free(packet); + case SHRUB_PACKET_ERROR: al_log_warn("bimu", "Unhandled error packet."); + aki_packet_free(packet); break; default: al_assert(false); @@ -106,9 +106,9 @@ bool bmu_vcr_push_packet(struct bmu_vcr *vcr, struct aki_packet *packet) return true; } -static void vcr_stream_close_internal(struct bmu_vcr_stream *stream) +static void vcr_stream_close_internal(struct shrb_vcr_stream *stream) { - al_atomic_s32_store(&stream->state, BIMU_STREAM_CLOSED, AL_ATOMIC_RELAXED); + al_atomic_s32_store(&stream->state, SHRUB_STREAM_CLOSED, AL_ATOMIC_RELAXED); aki_packet_cache_disable(&stream->cache); aki_mutex_lock(&stream->mutex); if (aki_cond_is_waiting(&stream->cond)) { @@ -124,32 +124,32 @@ static void vcr_stream_close_internal(struct bmu_vcr_stream *stream) } } -void bmu_vcr_stream_close(struct bmu_vcr_stream *stream) +void shrb_vcr_stream_close(struct shrb_vcr_stream *stream) { vcr_stream_close_internal(stream); } -void bmu_vcr_flush(struct bmu_vcr *vcr) +void shrb_vcr_flush(struct shrb_vcr *vcr) { - struct bmu_vcr_stream *stream; + struct shrb_vcr_stream *stream; al_array_foreach(vcr->streams, i, stream) { vcr_stream_close_internal(stream); stream->client->flush(stream->client); aki_packet_cache_enable(&stream->cache); stream->running = false; - al_atomic_s32_store(&stream->state, BIMU_STREAM_RUNNING, AL_ATOMIC_RELAXED); + al_atomic_s32_store(&stream->state, SHRUB_STREAM_RUNNING, AL_ATOMIC_RELAXED); } al_atomic_u64_store(&vcr->count, 0, AL_ATOMIC_RELAXED); } -void bmu_vcr_stream_cork(struct bmu_vcr_stream *stream) +void shrb_vcr_stream_cork(struct shrb_vcr_stream *stream) { - al_atomic_s32_store(&stream->state, BIMU_STREAM_STOPPED, AL_ATOMIC_RELAXED); + al_atomic_s32_store(&stream->state, SHRUB_STREAM_STOPPED, AL_ATOMIC_RELAXED); } -void bmu_vcr_stream_uncork(struct bmu_vcr_stream *stream) +void shrb_vcr_stream_uncork(struct shrb_vcr_stream *stream) { - al_atomic_s32_store(&stream->state, BIMU_STREAM_RUNNING, AL_ATOMIC_RELAXED); + al_atomic_s32_store(&stream->state, SHRUB_STREAM_RUNNING, AL_ATOMIC_RELAXED); aki_mutex_lock(&stream->mutex); if (aki_cond_is_waiting(&stream->cond)) { aki_cond_signal(&stream->cond); @@ -157,10 +157,10 @@ void bmu_vcr_stream_uncork(struct bmu_vcr_stream *stream) aki_mutex_unlock(&stream->mutex); } -void bmu_vcr_free(struct bmu_vcr *vcr) +void shrb_vcr_free(struct shrb_vcr *vcr) { aki_signal_stop(&vcr->signal); - struct bmu_vcr_stream *stream; + struct shrb_vcr_stream *stream; al_array_foreach(vcr->streams, i, stream) { aki_packet_cache_free(&stream->cache); stream->client->free(&stream->client); diff --git a/src/shrub/vcr.h b/src/shrub/vcr.h new file mode 100644 index 0000000..4fae2c0 --- /dev/null +++ b/src/shrub/vcr.h @@ -0,0 +1,45 @@ +#pragma once + +#include <aki/packet_cache.h> +#include <al/atomic.h> +#include <aki/packet_stream.h> +#include <aki/signal.h> + +#include "../codec/codec.h" + +enum { + SHRUB_STREAM_RUNNING = 0, + SHRUB_STREAM_STOPPED, + SHRUB_STREAM_CLOSED +}; + +struct shrb_vcr_stream { + u8 type; + s32 index; + struct camu_stream stream; + struct shrb_client_handler *client; + atomic_s32 state; + struct aki_packet_cache cache; + s32 buffered; + struct aki_cond cond; + struct aki_mutex mutex; + bool running; + struct aki_thread thread; + struct shrb_vcr *vcr; +}; + +struct shrb_vcr { + array(struct shrb_vcr_stream *) streams; + atomic_u64 count; + struct aki_packet_stream *data; + struct aki_signal signal; +}; + +void shrb_vcr_init(struct shrb_vcr *vcr, struct aki_event_loop *loop, struct aki_packet_stream *data); +void shrb_vcr_add_stream(struct shrb_vcr *vcr, struct shrb_vcr_stream *stream); +bool shrb_vcr_push_packet(struct shrb_vcr *vcr, struct aki_packet *packet); +void shrb_vcr_stream_close(struct shrb_vcr_stream *stream); +void shrb_vcr_flush(struct shrb_vcr *vcr); +void shrb_vcr_stream_cork(struct shrb_vcr_stream *stream); +void shrb_vcr_stream_uncork(struct shrb_vcr_stream *stream); +void shrb_vcr_free(struct shrb_vcr *vcr); diff --git a/src/tree/common.h b/src/tree/common.h deleted file mode 100644 index f740a4e..0000000 --- a/src/tree/common.h +++ /dev/null @@ -1,27 +0,0 @@ -#pragma once - -#include <al/str.h> - -#define TREE_PORT 14356 -#define TREE_RESOURCE_PORT 14357 -#define TREE_STREAM_PORT 14358 - -#define TREE_SERVER_IP al_str_c("127.0.0.1") -//#define TREE_SERVER_IP al_str_c("108.52.160.112") - -#define CAMU_DB_PATH al_str_c("/mnt/store/files/camu_db") - -enum { - TREE_NODE = 0, - TREE_CLIENT, - TREE_SINK -}; - -enum { - TREE_CMD_IDENTIFY = 0, - TREE_CMD_UPDATE_STATE, - TREE_CMD_CREATE_SEARCH, - TREE_CMD_GET_PAGE, - TREE_CMD_ADD, - TREE_CMD_SKIP -}; diff --git a/src/tree/list.c b/src/tree/list.c deleted file mode 100644 index 948d3cc..0000000 --- a/src/tree/list.c +++ /dev/null @@ -1,247 +0,0 @@ -#include <al/random.h> - -#ifdef AKIYO_HAS_CURL -#include "../cache/handlers/http.h" -#endif -#include "../cache/handlers/file.h" - -#include "../libsink/common.h" - -#include "list.h" -#include "tree.h" -#include "common.h" - -void tree_list_init(struct tree_list *list, struct tree_server *tree, str *name) -{ - al_str_clone(&list->name, name); - list->set = -1; - list->current = 0; - list->backwards = false; - al_array_init(list->entries); - al_array_init(list->sinks); - list->tree = tree; -} - -/* -static void send_buffer_cmd(struct tree_list *list, struct tree_sink *sink, struct tree_list_entry *entry) -{ - struct aki_packet *packet = aki_rpc_get_packet(&list->tree->server, CAMU_SINK_CMD_BUFFER); - aki_packet_write_str(packet, TREE_SERVER_IP); - aki_packet_write_s32(packet, TREE_STREAM_PORT); - aki_packet_write_u16(packet, entry->node_id); - aki_rpc_connection_command(sink->conn, packet, NULL, NULL); -} -*/ - -static void send_set_cmd(struct tree_list *list, struct tree_sink *sink, struct tree_list_entry *entry) -{ - struct aki_packet *packet = aki_rpc_get_packet(&list->tree->server, CAMU_SINK_CMD_SET); - aki_packet_write_str(packet, TREE_SERVER_IP); - aki_packet_write_s32(packet, TREE_STREAM_PORT); - aki_packet_write_u16(packet, entry->node_id); - aki_rpc_connection_command(sink->conn, packet, NULL, NULL); -} - -/* -static void send_queue_cmd(struct tree_list *list, struct tree_sink *sink, struct tree_list_entry *entry) -{ - struct aki_packet *packet = aki_rpc_get_packet(&list->tree->server, CAMU_SINK_CMD_QUEUE); - aki_packet_write_str(packet, TREE_SERVER_IP); - aki_packet_write_s32(packet, TREE_STREAM_PORT); - aki_packet_write_u16(packet, entry->node_id); - aki_rpc_connection_command(sink->conn, packet, NULL, NULL); -} -*/ - -void tree_list_add_sink(struct tree_list *list, struct tree_sink *sink) -{ - al_array_push(list->sinks, sink); - if (list->set == list->current) { - struct tree_list_entry *entry = &al_array_at(list->entries, list->current); - send_set_cmd(list, sink, entry); - } -} - -void tree_list_remove_sink(struct tree_list *list, struct tree_sink *sink) -{ - struct tree_sink *rsink; - al_array_foreach(list->sinks, i, rsink) { - if (rsink == sink) { - al_array_remove_at_iter(list->sinks, i); - break; - } - } -} - -bool tree_list_add(struct tree_list *list, str *unique_id, u32 index) -{ - struct tree_server *tree = list->tree; - struct sho_post *post = sho_post_cache_get(&tree->resources.cache, unique_id); - if (!post) return false; - - struct tree_list_entry entry; - entry.index = index; - al_str_clone(&entry.unique_id, unique_id); - entry.post = post; - - entry.entry = cch_handler_http_create(&al_array_at(entry.post->media, entry.index).url); - if (!entry.entry) return false; - entry.entry->handler->maybe_spawn_worker(entry.entry->handler, 0); - - entry.node_id = bmu_server_create_node(&tree->streams.server, entry.entry); - - al_array_push(list->entries, entry); - - tree_list_pump(list); - - return true; -} - -static struct cch_entry *entry_for_external_path(struct sho_client *client, str *path) -{ - struct cch_entry *entry = NULL; -#ifdef AKIYO_HAS_CURL - bool is_search = al_str_at(path, 0) == ';'; - if (is_search || al_str_cmp(path, al_str_c("https://"), 0, 8) == 0 || - al_str_cmp(path, al_str_c("http://"), 0, 7) == 0) { - str module; - str query; - al_str_from(&module, ""); - al_str_from(&query, ""); - if (al_str_cmp(path, al_str_c("https://twitter.com"), 0, 19) == 0 || - al_str_cmp(path, al_str_c("https://x.com"), 0, 13) == 0) { - al_str_cat(&query, al_str_c("tweet:")); - al_str_cat(&query, path); - al_str_cat(&module, al_str_c("twitter")); - } else if (al_str_cmp(path, al_str_c("https://instagram.com"), 0, 21) == 0) { - s32 slash = al_str_rfind(path, '/'); - if (slash >= 0) { - al_str_cat(&query, al_str_substr(path, slash + 1, path->len)); - } - al_str_cat(&module, al_str_c("instagram")); - } else { - if (is_search) { - al_str_cat(&query, al_str_substr(path, 1, path->len)); - } else { - al_str_cat(&query, al_str_c("link:")); - al_str_cat(&query, path); - } - al_str_cat(&module, al_str_c("youtube")); - } - s32 id = sho_client_create_search(client, &module, &query); - al_str_free(&module); - al_str_free(&query); - if (id < 0) return NULL; - struct sho_search *search = sho_client_get_search(client, id); - if (!search || !sho_search_get_page(search, 0)) return NULL; - struct sho_result_page *page = &al_array_at(search->pages, 0); - struct sho_post *post; - al_array_foreach_ptr(page->posts, i, post) { - struct sho_post_media *media; - al_array_foreach_ptr(post->media, j, media) { - if (media->url.len > 0) { - entry = cch_handler_http_create(&media->url); - break; - } - } - if (entry) break; - } - sho_client_discard_search(client, id); - if (entry) entry->handler->maybe_spawn_worker(entry->handler, 0); - } else // { -#endif - entry = cch_handler_file_create(path); - // } - return entry; -} - -bool tree_list_add_external(struct tree_list *list, str *path) -{ - struct tree_server *tree = list->tree; - - struct tree_list_entry entry; - entry.index = 0; - al_str_clone(&entry.unique_id, path); - entry.post = NULL; - - struct tree_user *user = al_array_at(tree->users, 0); - - entry.entry = entry_for_external_path(&user->search, path); - if (!entry.entry) return false; - - entry.node_id = bmu_server_create_node(&tree->streams.server, entry.entry); - - al_array_push(list->entries, entry); - - tree_list_pump(list); - - return true; -} - -void tree_list_skip(struct tree_list *list, s32 n) -{ - s32 size = (s32)list->entries.size; - if (list->current + n < 0 || list->current + n >= size) { - return; - } - list->current += n; - list->backwards = n < 0; - tree_list_pump(list); -} - -static s32 sort_list_func(void *_a, void *_b) -{ - struct tree_list_entry *a = (struct tree_list_entry *)_a; - struct tree_list_entry *b = (struct tree_list_entry *)_b; - return al_str_cmp(&a->unique_id, &b->unique_id, 0, a->unique_id.len); -} - -void tree_list_sort(struct tree_list *list) -{ - al_array_sort(list->entries, struct tree_list_entry, sort_list_func); -} - -void tree_list_shuffle(struct tree_list *list) -{ - u32 size = list->entries.size; - if (size == 0) return; - for (u32 i = 0; i < size - 1; i++) { - u32 j = i + al_rand() / (AL_RAND_MAX / (size - i) + 1); - struct tree_list_entry tmp = al_array_at(list->entries, j); - al_array_at(list->entries, j) = al_array_at(list->entries, i); - al_array_at(list->entries, i) = tmp; - } - list->set = -1; - tree_list_pump(list); -} - -void tree_list_clear(struct tree_list *list) -{ - //list->entries.size = 0; -} - -void tree_list_pump(struct tree_list *list) -{ - s32 size = (s32)list->entries.size; - if (size <= list->current) return; - struct tree_sink *sink; - if (list->current + 1 < size) { - // This needs a massive rethinking on how to sync the lists. - // - Query sinks on server side skip request? - //struct tree_list_entry *upcoming = &al_array_at(list->entries, list->current + 1); - //al_array_foreach(list->sinks, i, sink) { - // send_queue_cmd(list, sink, upcoming); - //} - } - if (list->set == list->current) return; - struct tree_list_entry *entry = &al_array_at(list->entries, list->current); - al_array_foreach(list->sinks, i, sink) { - send_set_cmd(list, sink, entry); - } - list->set = list->current; -} - -void tree_list_free(struct tree_list *list) -{ - al_str_free(&list->name); -} diff --git a/src/tree/list.h b/src/tree/list.h deleted file mode 100644 index 3702983..0000000 --- a/src/tree/list.h +++ /dev/null @@ -1,37 +0,0 @@ -#pragma once - -#include <al/str.h> -#include <al/array.h> - -#include "../cache/entry.h" - -struct tree_list_entry { - u32 index; - str unique_id; - struct sho_post *post; - struct cch_entry *entry; - u16 node_id; -}; - -struct tree_sink; -struct tree_list { - str name; - s32 set; - s32 current; - bool backwards; - array(struct tree_list_entry) entries; - array(struct tree_sink *) sinks; - struct tree_server *tree; -}; - -void tree_list_init(struct tree_list *list, struct tree_server *tree, str *name); -void tree_list_add_sink(struct tree_list *list, struct tree_sink *sink); -void tree_list_remove_sink(struct tree_list *list, struct tree_sink *sink); -bool tree_list_add(struct tree_list *list, str *unique_id, u32 index); -bool tree_list_add_external(struct tree_list *list, str *path); -void tree_list_skip(struct tree_list *list, s32 n); -void tree_list_sort(struct tree_list *list); -void tree_list_shuffle(struct tree_list *list); -void tree_list_clear(struct tree_list *list); -void tree_list_pump(struct tree_list *list); -void tree_list_free(struct tree_list *list); diff --git a/src/tree/meson.build b/src/tree/meson.build deleted file mode 100644 index 3aba0a6..0000000 --- a/src/tree/meson.build +++ /dev/null @@ -1,7 +0,0 @@ -tree_src = [ - 'tree.c', - 'resource_manager.c', - 'list.c' -] -tree_deps = [common_deps, shoki, cache, bimu_server] -executable('tree', sources: tree_src, dependencies: tree_deps) diff --git a/src/tree/resource_manager.c b/src/tree/resource_manager.c deleted file mode 100644 index dd24310..0000000 --- a/src/tree/resource_manager.c +++ /dev/null @@ -1,75 +0,0 @@ -#include <al/random.h> -#include <aki/http.h> - -#include "resource_manager.h" - -static u8 packet_pool_callback(void *userdata, struct aki_packet *packet) -{ - (void)userdata; - struct tree_resource_request *request = (struct tree_resource_request *)packet->userdata; - aki_packet_stream_send_packet(request->stream, packet); - return AKI_PACKET_POOL_KEEP; -} - -static void packet_callback(void *userdata, struct aki_packet_stream *stream, struct aki_packet *packet) -{ - struct tree_resource_server *server = (struct tree_resource_server *)userdata; - struct tree_resource_request *request = al_alloc_object(struct tree_resource_request); - request->stream = stream; - request->id = aki_packet_read_u16(packet); - str unique_id; - aki_packet_read_str(packet, &unique_id); - al_str_clone(&request->unique_id, &unique_id); - request->index = aki_packet_read_u32(packet); - request->packet = aki_packet_pool_get(&server->pool); - request->packet->userdata = request; - request->server = server; - if (!server->request_resource(server->userdata, request)) { - aki_packet_pool_return(&server->pool, request->packet); - } - aki_packet_free(packet); -} - -static void connection_closed_callback(void *userdata, struct aki_packet_stream *stream) -{ - (void)userdata; - (void)stream; -} - -static void packet_sent_callback(void *userdata, struct aki_packet *packet) -{ - struct tree_resource_server *server = (struct tree_resource_server *)userdata; - struct tree_resource_request *request = (struct tree_resource_request *)packet->userdata; - aki_http_request_close(&request->request); - aki_packet_pool_return(&server->pool, packet); -} - -static void flushed_callback(void *userdata) -{ - (void)userdata; -} - -static void connection_callback(void *userdata, struct aki_packet_stream *stream) -{ - stream->userdata = userdata; - stream->packet_callback = packet_callback; - stream->packet_sent_callback = packet_sent_callback; - stream->connection_closed_callback = connection_closed_callback; - stream->flushed_callback = flushed_callback; -} - -bool tree_resource_server_init(struct tree_resource_server *server, - bool (*request_resource)(void *, struct tree_resource_request *), void *userdata) -{ - server->request_resource = request_resource; - server->userdata = userdata; - return aki_packet_stream_init(&server->server, AKI_SOCKET_TCP, connection_callback, - NULL, NULL, NULL, server); -} - -void tree_resource_server_listen(struct tree_resource_server *server, - struct aki_event_loop *loop, str *addr, s32 port) -{ - aki_packet_pool_init(&server->pool, 50, loop, packet_pool_callback, server); - aki_packet_stream_listen(&server->server, loop, addr, port); -} diff --git a/src/tree/resource_manager.h b/src/tree/resource_manager.h deleted file mode 100644 index a81e3a9..0000000 --- a/src/tree/resource_manager.h +++ /dev/null @@ -1,27 +0,0 @@ -#pragma once - -#include <aki/packet_stream.h> -#include <aki/packet_pool.h> -#include <aki/http.h> - -struct tree_resource_request { - u16 id; - str unique_id; - u32 index; - struct aki_packet *packet; - struct aki_packet_stream *stream; - struct aki_http_request request; - struct tree_resource_server *server; -}; - -struct tree_resource_server { - struct aki_packet_stream server; - struct aki_packet_pool pool; - bool (*request_resource)(void *, struct tree_resource_request *); - void *userdata; -}; - -bool tree_resource_server_init(struct tree_resource_server *server, - bool (*request_resource)(void *, struct tree_resource_request *), void *userdata); -void tree_resource_server_listen(struct tree_resource_server *server, - struct aki_event_loop *loop, str *addr, s32 port); diff --git a/src/tree/tree.c b/src/tree/tree.c deleted file mode 100644 index c57d56c..0000000 --- a/src/tree/tree.c +++ /dev/null @@ -1,460 +0,0 @@ -#include <al/log.h> -#include <aki/file.h> -#include <blake3.h> -#include <jansson.h> - -#include "tree.h" -#include "common.h" - -#define USER_AGENT al_str_c("Mozilla/5.0 (Windows NT 10.0; rv:122.0) Gecko/20100101 Firefox/122.0") - -static void http_callback(void *userdata, struct aki_http_request *response, bool success) -{ - struct tree_resource_request *request = (struct tree_resource_request *)userdata; - struct aki_packet *packet = request->packet; - if (!success) { - aki_packet_pool_return(&request->server->pool, packet); - return; - } - aki_packet_write_u16(packet, request->id); - aki_packet_write_buffer(packet, &response->response); - aki_packet_pool_submit(&request->server->pool, packet); -} - -static bool request_resource(void *userdata, struct tree_resource_request *request) -{ - struct tree_server *tree = (struct tree_server *)userdata; - struct sho_post *post = sho_post_cache_get(&tree->resources.cache, &request->unique_id); - struct aki_packet *packet = request->packet; - if (!post) { - aki_packet_pool_return(&request->server->pool, packet); - return false; - } - str *url = NULL; - if (al_str_eq(&post->author.unique_id, &request->unique_id)) { - url = &post->author.profile_image_url; - } else { - if (post->media.size <= request->index) return false; - struct sho_post_media *media = &al_array_at(post->media, request->index); - url = &media->thumbnail_url; - } - aki_http_request_init(&request->request); - aki_http_set_url(&request->request.http, url); - aki_http_set_user_agent(&request->request.http, USER_AGENT); - if (!aki_http_request(&request->request, AKI_HTTP_GET, &tree->loop, http_callback, request)) { - aki_packet_pool_return(&request->server->pool, packet); - return false; - } - return true; -} - -static struct tree_user *get_user_by_username(struct tree_server *tree, str *username) -{ - struct tree_user *user; - al_array_foreach(tree->users, i, user) { - if (al_str_eq(&user->username, username)) { - return user; - } - } - return NULL; -} - -static struct tree_user *get_user_by_connection(struct tree_server *tree, struct aki_rpc_connection *conn) -{ - struct tree_user *user; - al_array_foreach(tree->users, i, user) { - struct tree_client *client; - al_array_foreach(user->clients, j, client) { - if (client->conn == conn) return user; - } - struct tree_list *list; - al_array_foreach(user->lists, j, list) { - struct tree_sink *sink; - al_array_foreach(list->sinks, k, sink) { - if (sink->conn == conn) return user; - } - } - } - return NULL; -} - -static void send_current_state(struct tree_server *server, struct tree_user *user, struct aki_rpc_connection *conn) -{ - struct aki_packet *packet = aki_rpc_get_packet(&server->server, TREE_CMD_UPDATE_STATE); - aki_packet_write_u32(packet, user->lists.size); - struct tree_list *list; - al_array_foreach(user->lists, i, list) { - aki_packet_write_str(packet, &list->name); - aki_packet_write_u32(packet, list->entries.size); - struct tree_list_entry *entry; - al_array_foreach_ptr(list->entries, i, entry) { - aki_packet_write_str(packet, &entry->unique_id); - } - aki_packet_write_s32(packet, list->current); - } - aki_packet_write_u32(packet, user->search.searches.size); - struct sho_search *search; - al_array_foreach(user->search.searches, i, search) { - aki_packet_write_s32(packet, search->id); - aki_packet_write_str(packet, &search->module); - aki_packet_write_str(packet, &search->query); - aki_packet_write_u32(packet, search->page); - } - aki_rpc_connection_command(conn, packet, NULL, NULL); -} - -static bool identify_command_callback(void *userdata, struct aki_rpc_connection *conn, - struct aki_packet *packet, struct aki_packet *rpacket) -{ - struct tree_server *tree = (struct tree_server *)userdata; - (void)rpacket; - switch (aki_packet_read_u8(packet)) { - case TREE_NODE: { - struct tree_node *node = al_alloc_object(struct tree_node); - node->conn = conn; - al_array_push(tree->nodes, node); - al_log_info("tree", "New node."); - break; - } - case TREE_CLIENT: { - str username; - aki_packet_read_str(packet, &username); - struct tree_user *user = get_user_by_username(tree, &username); - if (user) { - struct tree_client *client = al_alloc_object(struct tree_client); - client->conn = conn; - client->user = user; - al_array_push(tree->clients, client); - al_array_push(user->clients, client); - send_current_state(tree, user, conn); - al_log_info("tree", "User \"%.*s\" logged in.", AL_STR_PRINTF(&user->username)); - } - break; - } - case TREE_SINK: { - struct tree_sink *sink = al_alloc_object(struct tree_sink); - sink->conn = conn; - al_array_push(tree->sinks, sink); - struct tree_user *user = al_array_at(tree->users, 0); - struct tree_list *list = al_array_at(user->lists, 0); - tree_list_add_sink(list, sink); - al_log_info("tree", "New sink."); - break; - } - } - aki_packet_free(packet); - return true; -} - -static bool create_search_command_callback(void *userdata, struct aki_rpc_connection *conn, - struct aki_packet *packet, struct aki_packet *rpacket) -{ - struct tree_server *tree = (struct tree_server *)userdata; - struct tree_user *user = get_user_by_connection(tree, conn); - if (!user) { - aki_packet_write_s32(rpacket, -1); - goto out; - } - - str module, query; - aki_packet_read_str(packet, &module); - aki_packet_read_str(packet, &query); - aki_packet_write_s32(rpacket, sho_client_create_search(&user->search, &module, &query)); - -out: - aki_packet_free(packet); - return true; -} - -static bool get_page_command_callback(void *userdata, struct aki_rpc_connection *conn, - struct aki_packet *packet, struct aki_packet *rpacket) -{ - s32 id = aki_packet_read_s32(packet); - - // Always reciprocate the requested id, even if it turns out to be invalid. - aki_packet_write_s32(rpacket, id); - - struct tree_server *tree = (struct tree_server *)userdata; - struct tree_user *user = get_user_by_connection(tree, conn); - if (!user) { - aki_packet_write_s32(rpacket, -1); - goto out; - } - - u32 page_request = aki_packet_read_u32(packet); - - struct sho_search *search = sho_client_get_search(&user->search, id); - if (!search) { - aki_packet_write_s32(rpacket, -1); - goto out; - } - - if (!sho_search_get_page(search, page_request)) { - aki_packet_write_s32(rpacket, -1); - goto out; - } - - aki_packet_write_s32(rpacket, 0); - aki_packet_write_u32(rpacket, page_request); - struct sho_result_page *page = &al_array_at(search->pages, page_request); - aki_packet_write_u32(rpacket, page->posts.size); - struct sho_post *post; - al_array_foreach_ptr(page->posts, i, post) { - aki_packet_write_sho_post(rpacket, post); - } - aki_packet_write_u32(rpacket, page->list.size); - str *unique_id; - al_array_foreach_ptr(page->list, i, unique_id) { - aki_packet_write_str(rpacket, unique_id); - } - -out: - aki_packet_free(packet); - return true; -} - -static bool add_command_callback(void *userdata, struct aki_rpc_connection *conn, - struct aki_packet *packet, struct aki_packet *rpacket) -{ - struct tree_server *tree = (struct tree_server *)userdata; - (void)rpacket; - struct tree_user *user = get_user_by_connection(tree, conn); - if (!user) goto out; - - str unique_id; - aki_packet_read_str(packet, &unique_id); - u32 index = aki_packet_read_u32(packet); - - struct tree_list *list = al_array_at(user->lists, 0); - tree_list_add(list, &unique_id, index); - -out: - aki_packet_free(packet); - return false; -} - -static bool skip_command_callback(void *userdata, struct aki_rpc_connection *conn, - struct aki_packet *packet, struct aki_packet *rpacket) -{ - struct tree_server *tree = (struct tree_server *)userdata; - (void)rpacket; - struct tree_user *user = get_user_by_connection(tree, conn); - if (!user) goto out; - - s32 n = aki_packet_read_s32(packet); - - struct tree_list *list = al_array_at(user->lists, 0); - tree_list_skip(list, n); - -out: - aki_packet_free(packet); - return false; -} - -static void connection_callback(void *userdata, struct aki_rpc_connection *conn) -{ - (void)userdata; - (void)conn; -} - -static void cleanup_node(struct tree_node *node) -{ - al_free(node); -} - -static void cleanup_client(struct tree_client *client) -{ - al_free(client); -} - -static void cleanup_sink(struct tree_sink *sink) -{ - al_free(sink); -} - -static void connection_closed_callback(void *userdata, struct aki_rpc_connection *conn) -{ - struct tree_server *tree = (struct tree_server *)userdata; - - struct tree_node *node; - al_array_foreach(tree->nodes, i, node) { - if (node->conn == conn) { - al_log_info("tree", "Node removed."); - cleanup_node(node); - al_array_remove_at_iter(tree->nodes, i); - break; - } - } - - struct tree_client *client; - al_array_foreach(tree->clients, i, client) { - if (client->conn == conn) { - al_log_info("tree", "User \"%.*s\" logged out.", AL_STR_PRINTF(&client->user->username)); - cleanup_client(client); - al_array_remove_at_iter(tree->clients, i); - break; - } - } - - struct tree_sink *sink; - al_array_foreach(tree->sinks, i, sink) { - if (sink->conn == conn) { - al_log_info("tree", "Sink removed."); - al_array_remove_at_iter(tree->sinks, i); - struct tree_user *user = al_array_at(tree->users, 0); - struct tree_list *list = al_array_at(user->lists, 0); - tree_list_remove_sink(list, sink); - cleanup_sink(sink); - break; - } - } -} - -static struct aki_rpc_command commands[] = { - { .op = TREE_CMD_IDENTIFY, .callback = identify_command_callback, .userdata = NULL }, - { .op = TREE_CMD_CREATE_SEARCH, .callback = create_search_command_callback, .userdata = NULL }, - { .op = TREE_CMD_GET_PAGE, .callback = get_page_command_callback, .userdata = NULL }, - { .op = TREE_CMD_ADD, .callback = add_command_callback, .userdata = NULL }, - { .op = TREE_CMD_SKIP, .callback = skip_command_callback, .userdata = NULL } -}; - -static bool open_user(struct tree_server *tree, struct aki_dir_entry *dir) -{ - struct aki_file file; - if (!aki_file_open(&file, &dir->path, 0)) { - return false; - } - str s; - aki_file_read_as_str(&file, &s); - json_error_t error; - json_t *root = json_loadb(s.data, s.len, 0, &error); - if (!root) { - al_log_error("tree", "Failed to parse json: %.*s:%d:%d (%s).", AL_STR_PRINTF(&dir->path), - error.line, error.column, error.text); - return false; - } - struct tree_user *user = al_alloc_object(struct tree_user); - al_str_from(&user->username, json_string_value(json_object_get(root, "username"))); - sho_client_init(&user->search, &tree->resources.cache); - al_array_init(user->lists); - struct tree_list *default_list = al_alloc_object(struct tree_list); - tree_list_init(default_list, tree, al_str_c("default")); - al_array_push(user->lists, default_list); - al_array_init(user->clients); - al_log_info("tree", "Loaded user \"%.*s\"", AL_STR_PRINTF(&user->username)); - al_array_push(tree->users, user); - return true; -} - -static bool open_db(struct tree_server *tree, str *path) -{ - struct aki_dir camu_db; - if (!aki_dir_open(&camu_db, path)) { - return false; - } - struct aki_dir_entry entry; - while (aki_dir_read(&camu_db, &entry)) { - if (al_str_eq(&entry.name, al_str_c("users"))) { - struct aki_dir users; - if (aki_dir_open(&users, &entry.path)) { - struct aki_dir_entry user; - while (aki_dir_read(&users, &user)) { - if (user.type == AKI_ENTRY_FILE) { - open_user(tree, &user); - } - } - aki_dir_close(&users); - } - } - aki_dir_entry_free(&entry); - } - aki_dir_close(&camu_db); - return true; -} - -#if TREE_USE_SOCKET -static u8 line_callback(void *userdata, str *line) -{ - struct tree_server *tree = (struct tree_server *)userdata; - struct tree_user *user = al_array_at(tree->users, 0); - struct tree_list *list = al_array_at(user->lists, 0); - if (al_str_eq(line, al_str_c(";NEXT"))) { - tree_list_skip(list, 1); - } else if (al_str_eq(line, al_str_c(";PREV"))) { - tree_list_skip(list, -1); - } else if (al_str_eq(line, al_str_c(";SHUFFLE"))) { - tree_list_shuffle(list); - } else if (al_str_eq(line, al_str_c(";SORT"))) { - tree_list_sort(list); - } else if (al_str_eq(line, al_str_c(";CLEAR"))) { - tree_list_clear(list); - } else { - tree_list_add_external(list, line); - } - return AKI_LINE_PROCESSOR_CONTINUE; -} -#endif - -static void sigint_handler(s32 signum) -{ - (void)signum; - // explode. - exit(EXIT_SUCCESS); -} - -static struct tree_server tree = { 0 }; - -s32 main(void) -{ - aki_common_init(); - - signal(SIGINT, sigint_handler); - - al_array_init(tree.nodes); - al_array_init(tree.clients); - al_array_init(tree.sinks); - al_array_init(tree.users); - - if (!open_db(&tree, CAMU_DB_PATH)) return EXIT_FAILURE; - - sho_post_cache_init(&tree.resources.cache); - - bool py_init = sho_python_init(); - - aki_event_loop_init(&tree.loop); - - aki_rpc_init(&tree.server, AKI_SOCKET_TCP, connection_callback, - connection_closed_callback, &tree); - for (u32 i = 0; i < AL_ARRAY_SIZE(commands); i++) { - commands[i].userdata = &tree; - aki_rpc_add_command(&tree.server, &commands[i]); - } - aki_rpc_listen(&tree.server, &tree.loop, al_str_c("0.0.0.0"), TREE_PORT); - - tree_resource_server_init(&tree.resources.server, request_resource, &tree); - tree_resource_server_listen(&tree.resources.server, &tree.loop, al_str_c("0.0.0.0"), TREE_RESOURCE_PORT); - - bmu_server_init(&tree.streams.server); - bmu_server_listen(&tree.streams.server, &tree.loop, al_str_c("0.0.0.0"), TREE_STREAM_PORT); - -#if TREE_USE_SOCKET - tree.socket.type = AKI_SOCKET_UNIX; - aki_socket_init(&tree.socket); - aki_socket_set_blocking(&tree.socket, false); - tree.cli.callback = line_callback; - tree.cli.userdata = &tree; - aki_line_processor_init(&tree.cli, al_str_c("\n")); - aki_line_processor_open_socket(&tree.cli, &tree.socket); - if (aki_socket_listen(&tree.socket, al_str_c("/tmp/tree_sock"), 0)) { - aki_line_processor_run(&tree.cli, &tree.loop); - } -#endif - - aki_event_loop_run(&tree.loop); - - if (py_init) sho_python_close(); - - aki_common_close(); - - return EXIT_SUCCESS; -} diff --git a/src/tree/tree.h b/src/tree/tree.h deleted file mode 100644 index b8edaad..0000000 --- a/src/tree/tree.h +++ /dev/null @@ -1,56 +0,0 @@ -#pragma once - -#define TREE_USE_SOCKET 1 - -#include <aki/rpc2.h> -#include <sho/post.h> -#include <sho/post_cache.h> -#include <sho/search.h> -#if TREE_USE_SOCKET -#include <aki/line_processor.h> -#endif - -#include "../bimu/server.h" - -#include "resource_manager.h" -#include "list.h" - -struct tree_node { - struct aki_rpc_connection *conn; -}; - -struct tree_client { - struct aki_rpc_connection *conn; - struct tree_user *user; -}; - -struct tree_sink { - struct aki_rpc_connection *conn; -}; - -struct tree_user { - str username; - struct sho_client search; - array(struct tree_list *) lists; - array(struct tree_client *) clients; -}; - -struct tree_server { - struct aki_event_loop loop; - struct aki_rpc server; - array(struct tree_node *) nodes; - array(struct tree_client *) clients; - array(struct tree_sink *) sinks; - array(struct tree_user *) users; - struct { - struct sho_post_cache cache; - struct tree_resource_server server; - } resources; - struct { - struct bmu_server server; - } streams; -#if TREE_USE_SOCKET - struct aki_socket socket; - struct aki_line_processor cli; -#endif -}; diff --git a/subprojects/ffmpeg.wrap b/subprojects/ffmpeg.wrap index c030027..8a97530 100644 --- a/subprojects/ffmpeg.wrap +++ b/subprojects/ffmpeg.wrap @@ -1,10 +1,14 @@ [wrap-file] -directory = ffmpeg-6.1.1 +directory = ffmpeg-7.0 +#directory = ffmpeg-6.1.1 #directory = ffmpeg-5.1.4 -source_url = https://www.ffmpeg.org/releases/ffmpeg-6.1.1.tar.gz +source_url = https://www.ffmpeg.org/releases/ffmpeg-7.0.tar.gz +#source_url = https://www.ffmpeg.org/releases/ffmpeg-6.1.1.tar.gz #source_url = https://www.ffmpeg.org/releases/ffmpeg-5.1.4.tar.gz -source_filename = ffmpeg-6.1.1.tar.gz +source_filename = ffmpeg-7.0.tar.gz +#source_filename = ffmpeg-6.1.1.tar.gz #source_filename = ffmpeg-5.1.4.tar.gz -source_hash = b0aee531e4a91fed6bc031445dfc3ad8eb36de1fcb0fa2acdb589ecc9f8dc9cb +source_hash = 943a2a28044947c17a905c39075494b0da46ec0795224c2c61eff986518321eb +#source_hash = b0aee531e4a91fed6bc031445dfc3ad8eb36de1fcb0fa2acdb589ecc9f8dc9cb #source_hash = ad1d83e0f022ba405928ea4985c9cab978c4698bb7937738a1085808c3eb78b4 patch_directory = ffmpeg diff --git a/subprojects/libass.wrap b/subprojects/libass.wrap index 37282a9..8bf38e4 100644 --- a/subprojects/libass.wrap +++ b/subprojects/libass.wrap @@ -1,6 +1,6 @@ [wrap-git] -url = https://github.com/libass/libass +url = https://github.com/libass/libass.git revision = 0.17.1 -depth = 1 clone-recursive = true +depth = 1 patch_directory = libass diff --git a/subprojects/libplacebo.wrap b/subprojects/libplacebo.wrap index e0f5e83..ca8053f 100644 --- a/subprojects/libplacebo.wrap +++ b/subprojects/libplacebo.wrap @@ -1,8 +1,10 @@ [wrap-git] url = https://code.videolan.org/videolan/libplacebo.git revision = 52314e0e435fbcb731e326815d4091ed0ba27475 -depth = 1 clone-recursive = true +depth = 1 + +diff_files = libplacebo/stretch.diff, libplacebo/no_dlltool.diff [provide] dependency_names = libplacebo diff --git a/subprojects/miniaudio.wrap b/subprojects/miniaudio.wrap index 3b4e3be..984b6a5 100644 --- a/subprojects/miniaudio.wrap +++ b/subprojects/miniaudio.wrap @@ -1,5 +1,5 @@ [wrap-git] -url = https://github.com/mackron/miniaudio +url = https://github.com/mackron/miniaudio.git revision = 0.11.21 depth = 1 patch_directory = miniaudio diff --git a/subprojects/packagefiles/ffmpeg/meson.build b/subprojects/packagefiles/ffmpeg/meson.build index 6700f5d..f47d35e 100644 --- a/subprojects/packagefiles/ffmpeg/meson.build +++ b/subprojects/packagefiles/ffmpeg/meson.build @@ -5,6 +5,7 @@ compiler = meson.get_compiler('c') cpu = target_machine.cpu_family() is_64bit = (cpu == 'x86_64' or cpu == 'aarch64') is_debug = get_option('buildtype').startswith('debug') +is_minsize = get_option('buildtype') == 'minsize' is_windows = host_machine.system() == 'windows' is_msvc = compiler.get_id() == 'msvc' or compiler.get_id() == 'clang-cl' is_mingw = is_windows and not is_msvc @@ -12,7 +13,7 @@ is_mingw = is_windows and not is_msvc extra_options = [] if not is_debug - extra_options += ['--disable-debug'] + extra_options += ['--disable-debug', '--enable-small'] endif extra_options += ['--cc=' + compiler.cmd_array()[0]] @@ -32,7 +33,8 @@ endif #'--malloc-prefix=al_', decoders = 'flac,mp3,mp3float,aac,opus,alac,mjpeg,jpeg2000,h264,hevc,vp9' -#decoders += ',pcm_f32be,pcm_f32le,pcm_s16be,pcm_s16be_planar,pcm_s16le,pcm_s16le_planar,pcm_s32be,pcm_s32le,pcm_s32le_planar,pcm_dvd,apng,mpegvideo,mpeg2video' +decoders += ',pcm_f32be,pcm_s32be,pcm_s32le,pcm_s32le_planar,pcm_f32le,pcm_s24be,pcm_s24le,pcm_s16be,pcm_s16be_planar,pcm_s16le,pcm_s16le_planar' +#decoders += ',pcm_dvd,mpegvideo,mpeg2video' #decoders += ',ass,srt' demuxers = 'flac,mp3,aac,wav,image2,mjpeg,image2pipe,image_jpeg_pipe,matroska,mov' @@ -44,7 +46,7 @@ parsers = 'mjpeg,jpeg2000' #extra_options += ['--enable-zlib', '--enable-libsoxr'] extra_options += ['--enable-zlib'] -decoders += ',png' +decoders += ',png,apng' demuxers += ',image_png_pipe' parsers += ',png' @@ -107,4 +109,4 @@ avformat = proj.dependency('avformat') avcodec = proj.dependency('avcodec') swresample = proj.dependency('swresample') swscale = proj.dependency('swscale') -av_inc_dep = declare_dependency(include_directories: include_directories('.', 'build/')) +ffmpeg_inc_dep = declare_dependency(include_directories: include_directories('.', 'build/')) diff --git a/subprojects/packagefiles/libplacebo/no_dlltool.diff b/subprojects/packagefiles/libplacebo/no_dlltool.diff new file mode 100644 index 0000000..e5c017c --- /dev/null +++ b/subprojects/packagefiles/libplacebo/no_dlltool.diff @@ -0,0 +1,47 @@ +diff --git a/src/meson.build b/src/meson.build +index 5958254..ee33055 100644 +--- a/src/meson.build ++++ b/src/meson.build +@@ -18,24 +18,24 @@ link_args = [] + link_depends = [] + + # Looks like meson in certain configuration returns ' ' instead of empty string +-mingw32 = cc.get_define('__MINGW32__').strip() +-if host_machine.system() == 'windows' and mingw32 != '' and host_machine.cpu() in ['aarch64', 'arm', 'x86_64'] +- # MinGW-w64 math functions are significantly slower than the UCRT ones. +- # In particular powf is over 7 times slower than UCRT counterpart. +- # MinGW-w64 explicitly excludes some math functions from their ucrtbase def +- # file and replaces with own versions. To workaround the issue, generate the +- # import library and link it with UCRT versions of math functions. +- dlltool = find_program('llvm-dlltool', 'dlltool') +- ucrt_math = custom_target('ucrt_math.lib', +- output : ['ucrt_math.lib'], +- input : 'ucrt_math.def', +- command : [dlltool, '-d', '@INPUT@', '-l', '@OUTPUT@']) +- link_args += ucrt_math.full_path() +- link_depends += ucrt_math +- # MinGW-w64 inlines functions like powf, rewriting them to pow. We want to use +- # the powf specialization from UCRT, so disable inlining. +- add_project_arguments(['-D__CRT__NO_INLINE'], language: ['c', 'cpp']) +-endif ++#mingw32 = cc.get_define('__MINGW32__').strip() ++#if host_machine.system() == 'windows' and mingw32 != '' and host_machine.cpu() in ['aarch64', 'arm', 'x86_64'] ++# # MinGW-w64 math functions are significantly slower than the UCRT ones. ++# # In particular powf is over 7 times slower than UCRT counterpart. ++# # MinGW-w64 explicitly excludes some math functions from their ucrtbase def ++# # file and replaces with own versions. To workaround the issue, generate the ++# # import library and link it with UCRT versions of math functions. ++# dlltool = find_program('llvm-dlltool', 'dlltool') ++# ucrt_math = custom_target('ucrt_math.lib', ++# output : ['ucrt_math.lib'], ++# input : 'ucrt_math.def', ++# command : [dlltool, '-d', '@INPUT@', '-l', '@OUTPUT@']) ++# link_args += ucrt_math.full_path() ++# link_depends += ucrt_math ++# # MinGW-w64 inlines functions like powf, rewriting them to pow. We want to use ++# # the powf specialization from UCRT, so disable inlining. ++# add_project_arguments(['-D__CRT__NO_INLINE'], language: ['c', 'cpp']) ++#endif + + # Work around missing atomics on some (obscure) platforms + atomic_test = ''' diff --git a/subprojects/packagefiles/libplacebo/stretch.diff b/subprojects/packagefiles/libplacebo/stretch.diff new file mode 100644 index 0000000..40544bc --- /dev/null +++ b/subprojects/packagefiles/libplacebo/stretch.diff @@ -0,0 +1,15 @@ +diff --git a/src/renderer.c b/src/renderer.c +index a052bc5..639fbfd 100644 +--- a/src/renderer.c ++++ b/src/renderer.c +@@ -1845,8 +1845,8 @@ static bool pass_read_image(struct pass_state *pass) + .rect = { + off_x, + off_y, +- off_x + pl_rect_w(ref->img.rect), +- off_y + pl_rect_h(ref->img.rect), ++ off_x + pl_rect_w(ref->img.rect) / stretch_x, ++ off_y + pl_rect_h(ref->img.rect) / stretch_y, + }, + }; + diff --git a/subprojects/shoki.wrap b/subprojects/shoki.wrap deleted file mode 100644 index 7d0f1a1..0000000 --- a/subprojects/shoki.wrap +++ /dev/null @@ -1,4 +0,0 @@ -[wrap-git] -url = https://git.akon.city/shoki -revision = master -depth = 1 diff --git a/subprojects/stb.wrap b/subprojects/stb.wrap index 24fc46e..f8193df 100644 --- a/subprojects/stb.wrap +++ b/subprojects/stb.wrap @@ -1,5 +1,5 @@ [wrap-git] url = https://github.com/nothings/stb.git -revision = f4a71b13373436a2866c5d68f8f80ac6f0bc1ffe +revision = ae721c50eaf761660b4f90cc590453cdb0c2acd0 depth = 1 patch_directory = stb |