From 20f9af6cbacad337c9a9d6a1f4d5d81ff93935d8 Mon Sep 17 00:00:00 2001 From: Froidefond Julien Date: Thu, 5 Mar 2026 15:26:47 +0100 Subject: [PATCH] add page streaming, admin ui flows, and runtime hardening --- .env.example | 1 + Cargo.lock | 128 +++++++++++++++ Cargo.toml | 4 +- PLAN.md | 35 +++-- apps/admin-ui/Cargo.toml | 4 + apps/admin-ui/src/main.rs | 317 +++++++++++++++++++++++++++++++++++++- apps/api/Cargo.toml | 4 + apps/api/Dockerfile | 2 +- apps/api/src/main.rs | 113 +++++++++++++- apps/api/src/pages.rs | 281 +++++++++++++++++++++++++++++++++ apps/indexer/src/main.rs | 16 +- crates/core/src/config.rs | 10 +- infra/bench.sh | 33 ++++ infra/smoke.sh | 42 +++++ 14 files changed, 957 insertions(+), 33 deletions(-) create mode 100644 apps/api/src/pages.rs create mode 100755 infra/bench.sh create mode 100755 infra/smoke.sh diff --git a/.env.example b/.env.example index 7b98547..8c6891a 100644 --- a/.env.example +++ b/.env.example @@ -1,5 +1,6 @@ API_LISTEN_ADDR=0.0.0.0:8080 ADMIN_UI_LISTEN_ADDR=0.0.0.0:8082 +API_BASE_URL=http://api:8080 INDEXER_LISTEN_ADDR=0.0.0.0:8081 INDEXER_SCAN_INTERVAL_SECONDS=5 DATABASE_URL=postgres://stripstream:stripstream@postgres:5432/stripstream diff --git a/Cargo.lock b/Cargo.lock index a9c436a..ceff366 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -14,10 +14,14 @@ version = "0.1.0" dependencies = [ "anyhow", "axum", + "reqwest", + "serde", + "serde_json", "stripstream-core", "tokio", "tracing", "tracing-subscriber", + "uuid", ] [[package]] @@ -70,10 +74,13 @@ dependencies = [ "axum", "base64", "chrono", + "image", + "lru", "rand 0.8.5", "reqwest", "serde", "serde_json", + "sha2", "sqlx", "stripstream-core", "tokio", @@ -81,6 +88,7 @@ dependencies = [ "tracing", "tracing-subscriber", "uuid", + "zip", ] [[package]] @@ -251,12 +259,24 @@ version = "0.6.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "175812e0be2bccb6abe50bb8d566126198344f707e304f45c648fd8f2cc0365e" +[[package]] +name = "bytemuck" +version = "1.25.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c8efb64bd706a16a1bdde310ae86b351e4d21550d98d056f22f8a7f7a2183fec" + [[package]] name = "byteorder" version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" +[[package]] +name = "byteorder-lite" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f1fe948ff07f4bd06c30984e69f5b4899c516a3ef74f34df92a2df2ab535495" + [[package]] name = "bytes" version = "1.11.1" @@ -532,6 +552,15 @@ dependencies = [ "pin-project-lite", ] +[[package]] +name = "fdeflate" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e6853b52649d4ac5c0bd02320cddc5ba956bdb407c4b75a2c6b75bf51500f8c" +dependencies = [ + "simd-adler32", +] + [[package]] name = "find-msvc-tools" version = "0.1.9" @@ -999,6 +1028,32 @@ dependencies = [ "icu_properties", ] +[[package]] +name = "image" +version = "0.25.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6506c6c10786659413faa717ceebcb8f70731c0a60cbae39795fdf114519c1a" +dependencies = [ + "bytemuck", + "byteorder-lite", + "image-webp", + "moxcms", + "num-traits", + "png", + "zune-core", + "zune-jpeg", +] + +[[package]] +name = "image-webp" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "525e9ff3e1a4be2fbea1fdf0e98686a6d98b4d8f937e1bf7402245af1909e8c3" +dependencies = [ + "byteorder-lite", + "quick-error", +] + [[package]] name = "indexer" version = "0.1.0" @@ -1168,6 +1223,15 @@ dependencies = [ "weezl", ] +[[package]] +name = "lru" +version = "0.12.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "234cf4f4a04dc1f57e24b96cc0cd600cf2af460d4161ac5ecdd0af8e1f3b2a38" +dependencies = [ + "hashbrown 0.15.5", +] + [[package]] name = "lru-slab" version = "0.1.2" @@ -1238,6 +1302,16 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "moxcms" +version = "0.7.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac9557c559cd6fc9867e122e20d2cbefc9ca29d80d027a8e39310920ed2f0a97" +dependencies = [ + "num-traits", + "pxfm", +] + [[package]] name = "nom" version = "7.1.3" @@ -1435,6 +1509,19 @@ version = "0.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b4596b6d070b27117e987119b4dac604f3c58cfb0b191112e24771b2faeac1a6" +[[package]] +name = "png" +version = "0.18.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "60769b8b31b2a9f263dae2776c37b1b28ae246943cf719eb6946a1db05128a61" +dependencies = [ + "bitflags", + "crc32fast", + "fdeflate", + "flate2", + "miniz_oxide", +] + [[package]] name = "potential_utf" version = "0.1.4" @@ -1478,6 +1565,18 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "pxfm" +version = "0.1.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b5a041e753da8b807c9255f28de81879c78c876392ff2469cde94799b2896b9d" + +[[package]] +name = "quick-error" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a993555f31e5a609f617c12db6250dedcac1b0a85076912c436e6fc9b2c8e6a3" + [[package]] name = "quinn" version = "0.11.9" @@ -2386,6 +2485,19 @@ dependencies = [ "tokio", ] +[[package]] +name = "tokio-util" +version = "0.7.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ae9cec805b01e8fc3fd2fe289f89149a9b66dd16786abd8b19cfa7b48cb0098" +dependencies = [ + "bytes", + "futures-core", + "futures-sink", + "pin-project-lite", + "tokio", +] + [[package]] name = "tower" version = "0.5.3" @@ -2397,6 +2509,7 @@ dependencies = [ "pin-project-lite", "sync_wrapper", "tokio", + "tokio-util", "tower-layer", "tower-service", "tracing", @@ -3310,3 +3423,18 @@ dependencies = [ "log", "simd-adler32", ] + +[[package]] +name = "zune-core" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb8a0807f7c01457d0379ba880ba6322660448ddebc890ce29bb64da71fb40f9" + +[[package]] +name = "zune-jpeg" +version = "0.5.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "410e9ecef634c709e3831c2cfdb8d9c32164fae1c67496d5b68fff728eec37fe" +dependencies = [ + "zune-core", +] diff --git a/Cargo.toml b/Cargo.toml index fa53f64..41308f0 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -19,6 +19,8 @@ argon2 = "0.5" axum = "0.7" base64 = "0.22" chrono = { version = "0.4", features = ["serde"] } +image = { version = "0.25", default-features = false, features = ["jpeg", "png", "webp"] } +lru = "0.12" reqwest = { version = "0.12", default-features = false, features = ["json", "rustls-tls"] } rand = "0.8" serde = { version = "1.0", features = ["derive"] } @@ -26,7 +28,7 @@ serde_json = "1.0" sha2 = "0.10" sqlx = { version = "0.8", features = ["runtime-tokio-rustls", "postgres", "uuid", "chrono"] } tokio = { version = "1.43", features = ["macros", "rt-multi-thread", "signal"] } -tower = "0.5" +tower = { version = "0.5", features = ["limit"] } tracing = "0.1" tracing-subscriber = { version = "0.3", features = ["env-filter", "fmt"] } uuid = { version = "1.12", features = ["serde", "v4"] } diff --git a/PLAN.md b/PLAN.md index dd891c5..655e7e3 100644 --- a/PLAN.md +++ b/PLAN.md @@ -120,39 +120,39 @@ Construire un serveur ultra performant pour indexer et servir des bibliotheques **DoD:** Recherche rapide et pertinente. ### T14 - Streaming pages multi-format -- [ ] `GET /books/:id/pages/:n` -- [ ] Query: `format=webp|jpeg|png`, `quality=1..100`, `width` -- [ ] Headers cache (`ETag`, `Cache-Control`) -- [ ] Validation stricte params +- [x] `GET /books/:id/pages/:n` +- [x] Query: `format=webp|jpeg|png`, `quality=1..100`, `width` +- [x] Headers cache (`ETag`, `Cache-Control`) +- [x] Validation stricte params **DoD:** Pages servies correctement dans les 3 formats. ### T15 - Perf guards -- [ ] Cache LRU en memoire (cle: `book:page:format:quality:width`) -- [ ] Limite concurrence rendu PDF -- [ ] Timeouts et bornes (`width` max) +- [x] Cache LRU en memoire (cle: `book:page:format:quality:width`) +- [x] Limite concurrence rendu PDF +- [x] Timeouts et bornes (`width` max) **DoD:** Service stable sous charge homelab. ### T16 - Admin UI Rust SSR -- [ ] Vue Libraries -- [ ] Vue Jobs -- [ ] Vue API Tokens (create/list/revoke) +- [x] Vue Libraries +- [x] Vue Jobs +- [x] Vue API Tokens (create/list/revoke) **DoD:** Admin complet utilisable sans SPA lourde. ### T17 - Observabilite et hardening -- [ ] Logs structures `tracing` -- [ ] Metriques Prometheus -- [ ] Health/readiness endpoints -- [ ] Rate limiting leger +- [x] Logs structures `tracing` +- [x] Metriques Prometheus +- [x] Health/readiness endpoints +- [x] Rate limiting leger **DoD:** Diagnostics et exploitation simples. ### T18 - Validation MVP - [ ] Tests d'integration API -- [ ] Smoke tests compose -- [ ] Bench p95/p99 basiques +- [x] Smoke tests compose +- [x] Bench p95/p99 basiques **DoD:** Checklist MVP validee de bout en bout. @@ -203,3 +203,6 @@ Construire un serveur ultra performant pour indexer et servir des bibliotheques - 2026-03-05: ajout d'un service `migrate` dans Compose pour executer automatiquement `infra/migrations/0001_init.sql` au demarrage. - 2026-03-05: Lot 2 termine (jobs, scan incremental, parsers `cbz/cbr/pdf`, API livres, sync + recherche Meilisearch). - 2026-03-05: verification de bout en bout OK sur une librairie de test (`/libraries/demo`) avec indexation, listing `/books` et recherche `/search` (1 CBZ detecte). +- 2026-03-05: Lot 3 avancee: endpoint pages (`/books/:id/pages/:n`) actif avec cache LRU, ETag/Cache-Control, limite concurrence rendu et timeouts. +- 2026-03-05: hardening API: readiness expose sans auth via `route_layer`, metriques simples `/metrics`, rate limiting lecture (120 req/s). +- 2026-03-05: smoke + bench scripts corriges et verifies (`infra/smoke.sh`, `infra/bench.sh`). diff --git a/apps/admin-ui/Cargo.toml b/apps/admin-ui/Cargo.toml index 8615545..05f7842 100644 --- a/apps/admin-ui/Cargo.toml +++ b/apps/admin-ui/Cargo.toml @@ -7,7 +7,11 @@ license.workspace = true [dependencies] anyhow.workspace = true axum.workspace = true +reqwest.workspace = true +serde.workspace = true +serde_json.workspace = true stripstream-core = { path = "../../crates/core" } tokio.workspace = true tracing.workspace = true tracing-subscriber.workspace = true +uuid.workspace = true diff --git a/apps/admin-ui/src/main.rs b/apps/admin-ui/src/main.rs index 2922601..7c71b5e 100644 --- a/apps/admin-ui/src/main.rs +++ b/apps/admin-ui/src/main.rs @@ -1,6 +1,79 @@ -use axum::{response::Html, routing::get, Router}; +use axum::{ + extract::{Form, State}, + response::{Html, Redirect}, + routing::{get, post}, + Router, +}; +use reqwest::Client; +use serde::Deserialize; use stripstream_core::config::AdminUiConfig; use tracing::info; +use uuid::Uuid; + +#[derive(Clone)] +struct AppState { + api_base_url: String, + api_token: String, + client: Client, +} + +#[derive(Deserialize)] +struct LibraryDto { + id: Uuid, + name: String, + root_path: String, + enabled: bool, +} + +#[derive(Deserialize)] +struct IndexJobDto { + id: Uuid, + #[serde(rename = "type")] + kind: String, + status: String, + created_at: String, +} + +#[derive(Deserialize)] +struct TokenDto { + id: Uuid, + name: String, + scope: String, + prefix: String, + revoked_at: Option, +} + +#[derive(Deserialize)] +struct CreatedToken { + token: String, +} + +#[derive(Deserialize)] +struct AddLibraryForm { + name: String, + root_path: String, +} + +#[derive(Deserialize)] +struct DeleteLibraryForm { + id: Uuid, +} + +#[derive(Deserialize)] +struct RebuildForm { + library_id: String, +} + +#[derive(Deserialize)] +struct CreateTokenForm { + name: String, + scope: String, +} + +#[derive(Deserialize)] +struct RevokeTokenForm { + id: Uuid, +} #[tokio::main] async fn main() -> anyhow::Result<()> { @@ -10,10 +83,25 @@ async fn main() -> anyhow::Result<()> { ) .init(); - let config = AdminUiConfig::from_env(); + let config = AdminUiConfig::from_env()?; + let state = AppState { + api_base_url: config.api_base_url, + api_token: config.api_token, + client: Client::new(), + }; + let app = Router::new() .route("/health", get(health)) - .route("/", get(index)); + .route("/", get(index)) + .route("/libraries", get(libraries_page)) + .route("/libraries/add", post(add_library)) + .route("/libraries/delete", post(delete_library)) + .route("/jobs", get(jobs_page)) + .route("/jobs/rebuild", post(rebuild_jobs)) + .route("/tokens", get(tokens_page)) + .route("/tokens/create", post(create_token)) + .route("/tokens/revoke", post(revoke_token)) + .with_state(state); let listener = tokio::net::TcpListener::bind(&config.listen_addr).await?; info!(addr = %config.listen_addr, "admin ui listening"); @@ -25,8 +113,225 @@ async fn health() -> &'static str { "ok" } -async fn index() -> Html<&'static str> { - Html( - "

Stripstream Admin

UI skeleton ready. Next: libraries, jobs, tokens screens.

", +async fn index() -> Html { + Html(layout( + "Dashboard", + "

Stripstream Admin

Gestion des librairies, jobs d'indexation et tokens API.

Libraries | Jobs | Tokens

", + )) +} + +async fn libraries_page(State(state): State) -> Html { + let libraries = fetch_libraries(&state).await.unwrap_or_default(); + let mut rows = String::new(); + for lib in libraries { + rows.push_str(&format!( + "{}{}{}
", + html_escape(&lib.name), + html_escape(&lib.root_path), + if lib.enabled { "yes" } else { "no" }, + lib.id + )); + } + + let content = format!( + "

Libraries

+
+ + + +
+ + + {} +
NameRoot PathEnabledActions
", + rows + ); + + Html(layout("Libraries", &content)) +} + +async fn add_library(State(state): State, Form(form): Form) -> Redirect { + let _ = state + .client + .post(format!("{}/libraries", state.api_base_url)) + .bearer_auth(&state.api_token) + .json(&serde_json::json!({"name": form.name, "root_path": form.root_path})) + .send() + .await; + Redirect::to("/libraries") +} + +async fn delete_library(State(state): State, Form(form): Form) -> Redirect { + let _ = state + .client + .delete(format!("{}/libraries/{}", state.api_base_url, form.id)) + .bearer_auth(&state.api_token) + .send() + .await; + Redirect::to("/libraries") +} + +async fn jobs_page(State(state): State) -> Html { + let jobs = fetch_jobs(&state).await.unwrap_or_default(); + let mut rows = String::new(); + for job in jobs { + rows.push_str(&format!( + "{}{}{}{}", + job.id, + html_escape(&job.kind), + html_escape(&job.status), + html_escape(&job.created_at) + )); + } + + let content = format!( + "

Index Jobs

+
+ + +
+ + + {} +
IDTypeStatusCreated
", + rows + ); + + Html(layout("Jobs", &content)) +} + +async fn rebuild_jobs(State(state): State, Form(form): Form) -> Redirect { + let body = if form.library_id.trim().is_empty() { + serde_json::json!({}) + } else { + serde_json::json!({"library_id": form.library_id.trim()}) + }; + let _ = state + .client + .post(format!("{}/index/rebuild", state.api_base_url)) + .bearer_auth(&state.api_token) + .json(&body) + .send() + .await; + Redirect::to("/jobs") +} + +async fn tokens_page(State(state): State) -> Html { + let tokens = fetch_tokens(&state).await.unwrap_or_default(); + let mut rows = String::new(); + for token in tokens { + rows.push_str(&format!( + "{}{}{}{}
", + html_escape(&token.name), + html_escape(&token.scope), + html_escape(&token.prefix), + if token.revoked_at.is_some() { "yes" } else { "no" }, + token.id + )); + } + + let content = format!( + "

API Tokens

+
+ + + +
+ + + {} +
NameScopePrefixRevokedActions
", + rows + ); + + Html(layout("Tokens", &content)) +} + +async fn create_token(State(state): State, Form(form): Form) -> Html { + let response = state + .client + .post(format!("{}/admin/tokens", state.api_base_url)) + .bearer_auth(&state.api_token) + .json(&serde_json::json!({"name": form.name, "scope": form.scope})) + .send() + .await; + + match response { + Ok(resp) if resp.status().is_success() => { + let created: Result = resp.json().await; + match created { + Ok(token) => Html(layout( + "Token Created", + &format!( + "

Token Created

Copie ce token maintenant (il ne sera plus affiche):

{}

Back to tokens

", + html_escape(&token.token) + ), + )), + Err(_) => Html(layout("Error", "

Token created but response parse failed.

Back

")), + } + } + _ => Html(layout("Error", "

Token creation failed.

Back

")), + } +} + +async fn revoke_token(State(state): State, Form(form): Form) -> Redirect { + let _ = state + .client + .delete(format!("{}/admin/tokens/{}", state.api_base_url, form.id)) + .bearer_auth(&state.api_token) + .send() + .await; + Redirect::to("/tokens") +} + +async fn fetch_libraries(state: &AppState) -> Result, reqwest::Error> { + state + .client + .get(format!("{}/libraries", state.api_base_url)) + .bearer_auth(&state.api_token) + .send() + .await? + .json::>() + .await +} + +async fn fetch_jobs(state: &AppState) -> Result, reqwest::Error> { + state + .client + .get(format!("{}/index/status", state.api_base_url)) + .bearer_auth(&state.api_token) + .send() + .await? + .json::>() + .await +} + +async fn fetch_tokens(state: &AppState) -> Result, reqwest::Error> { + state + .client + .get(format!("{}/admin/tokens", state.api_base_url)) + .bearer_auth(&state.api_token) + .send() + .await? + .json::>() + .await +} + +fn layout(title: &str, content: &str) -> String { + format!( + "{}
{}", + html_escape(title), + content ) } + +fn html_escape(value: &str) -> String { + value + .replace('&', "&") + .replace('<', "<") + .replace('>', ">") + .replace('"', """) +} diff --git a/apps/api/Cargo.toml b/apps/api/Cargo.toml index 09bde20..e56ced4 100644 --- a/apps/api/Cargo.toml +++ b/apps/api/Cargo.toml @@ -10,14 +10,18 @@ argon2.workspace = true axum.workspace = true base64.workspace = true chrono.workspace = true +image.workspace = true +lru.workspace = true stripstream-core = { path = "../../crates/core" } rand.workspace = true reqwest.workspace = true serde.workspace = true serde_json.workspace = true +sha2.workspace = true sqlx.workspace = true tokio.workspace = true tower.workspace = true tracing.workspace = true tracing-subscriber.workspace = true uuid.workspace = true +zip = { version = "2.2", default-features = false, features = ["deflate"] } diff --git a/apps/api/Dockerfile b/apps/api/Dockerfile index 588057f..99db826 100644 --- a/apps/api/Dockerfile +++ b/apps/api/Dockerfile @@ -16,7 +16,7 @@ COPY crates/parsers/src crates/parsers/src RUN cargo build --release -p api FROM debian:bookworm-slim -RUN apt-get update && apt-get install -y --no-install-recommends ca-certificates wget && rm -rf /var/lib/apt/lists/* +RUN apt-get update && apt-get install -y --no-install-recommends ca-certificates wget unrar-free poppler-utils && rm -rf /var/lib/apt/lists/* COPY --from=builder /app/target/release/api /usr/local/bin/api EXPOSE 8080 CMD ["/usr/local/bin/api"] diff --git a/apps/api/src/main.rs b/apps/api/src/main.rs index b86e0a6..95636a5 100644 --- a/apps/api/src/main.rs +++ b/apps/api/src/main.rs @@ -3,14 +3,29 @@ mod books; mod error; mod index_jobs; mod libraries; +mod pages; mod search; mod tokens; -use std::sync::Arc; +use std::{ + num::NonZeroUsize, + sync::{ + atomic::{AtomicU64, Ordering}, + Arc, + }, + time::{Duration, Instant}, +}; -use axum::{middleware, routing::{delete, get}, Router}; +use axum::{ + middleware, + response::IntoResponse, + routing::{delete, get}, + Json, Router, +}; +use lru::LruCache; use stripstream_core::config::ApiConfig; use sqlx::postgres::PgPoolOptions; +use tokio::sync::{Mutex, Semaphore}; use tracing::info; #[derive(Clone)] @@ -19,6 +34,31 @@ struct AppState { bootstrap_token: Arc, meili_url: Arc, meili_master_key: Arc, + page_cache: Arc>>>>, + page_render_limit: Arc, + metrics: Arc, + read_rate_limit: Arc>, +} + +struct Metrics { + requests_total: AtomicU64, + page_cache_hits: AtomicU64, + page_cache_misses: AtomicU64, +} + +struct ReadRateLimit { + window_started_at: Instant, + requests_in_window: u32, +} + +impl Metrics { + fn new() -> Self { + Self { + requests_total: AtomicU64::new(0), + page_cache_hits: AtomicU64::new(0), + page_cache_misses: AtomicU64::new(0), + } + } } #[tokio::main] @@ -40,6 +80,13 @@ async fn main() -> anyhow::Result<()> { bootstrap_token: Arc::from(config.api_bootstrap_token), meili_url: Arc::from(config.meili_url), meili_master_key: Arc::from(config.meili_master_key), + page_cache: Arc::new(Mutex::new(LruCache::new(NonZeroUsize::new(512).expect("non-zero")))), + page_render_limit: Arc::new(Semaphore::new(4)), + metrics: Arc::new(Metrics::new()), + read_rate_limit: Arc::new(Mutex::new(ReadRateLimit { + window_started_at: Instant::now(), + requests_in_window: 0, + })), }; let admin_routes = Router::new() @@ -49,18 +96,29 @@ async fn main() -> anyhow::Result<()> { .route("/index/status", get(index_jobs::list_index_jobs)) .route("/admin/tokens", get(tokens::list_tokens).post(tokens::create_token)) .route("/admin/tokens/:id", delete(tokens::revoke_token)) - .layer(middleware::from_fn_with_state(state.clone(), auth::require_admin)); + .route_layer(middleware::from_fn_with_state( + state.clone(), + auth::require_admin, + )); let read_routes = Router::new() .route("/books", get(books::list_books)) .route("/books/:id", get(books::get_book)) + .route("/books/:id/pages/:n", get(pages::get_page)) .route("/search", get(search::search_books)) - .layer(middleware::from_fn_with_state(state.clone(), auth::require_read)); + .route_layer(middleware::from_fn_with_state(state.clone(), read_rate_limit)) + .route_layer(middleware::from_fn_with_state( + state.clone(), + auth::require_read, + )); let app = Router::new() .route("/health", get(health)) + .route("/ready", get(ready)) + .route("/metrics", get(metrics)) .merge(admin_routes) .merge(read_routes) + .layer(middleware::from_fn_with_state(state.clone(), request_counter)) .with_state(state); let listener = tokio::net::TcpListener::bind(&config.listen_addr).await?; @@ -72,3 +130,50 @@ async fn main() -> anyhow::Result<()> { async fn health() -> &'static str { "ok" } + +async fn ready(axum::extract::State(state): axum::extract::State) -> Result, error::ApiError> { + sqlx::query("SELECT 1").execute(&state.pool).await?; + Ok(Json(serde_json::json!({"status": "ready"}))) +} + +async fn metrics(axum::extract::State(state): axum::extract::State) -> String { + format!( + "requests_total {}\npage_cache_hits {}\npage_cache_misses {}\n", + state.metrics.requests_total.load(Ordering::Relaxed), + state.metrics.page_cache_hits.load(Ordering::Relaxed), + state.metrics.page_cache_misses.load(Ordering::Relaxed), + ) +} + +async fn request_counter( + axum::extract::State(state): axum::extract::State, + req: axum::extract::Request, + next: axum::middleware::Next, +) -> axum::response::Response { + state.metrics.requests_total.fetch_add(1, Ordering::Relaxed); + next.run(req).await +} + +async fn read_rate_limit( + axum::extract::State(state): axum::extract::State, + req: axum::extract::Request, + next: axum::middleware::Next, +) -> axum::response::Response { + let mut limiter = state.read_rate_limit.lock().await; + if limiter.window_started_at.elapsed() >= Duration::from_secs(1) { + limiter.window_started_at = Instant::now(); + limiter.requests_in_window = 0; + } + + if limiter.requests_in_window >= 120 { + return ( + axum::http::StatusCode::TOO_MANY_REQUESTS, + "rate limit exceeded", + ) + .into_response(); + } + + limiter.requests_in_window += 1; + drop(limiter); + next.run(req).await +} diff --git a/apps/api/src/pages.rs b/apps/api/src/pages.rs new file mode 100644 index 0000000..6191af7 --- /dev/null +++ b/apps/api/src/pages.rs @@ -0,0 +1,281 @@ +use std::{ + io::Read, + path::Path, + sync::{atomic::Ordering, Arc}, + time::Duration, +}; + +use axum::{ + body::Body, + extract::{Path as AxumPath, Query, State}, + http::{header, HeaderMap, HeaderValue, StatusCode}, + response::{IntoResponse, Response}, +}; +use image::{codecs::jpeg::JpegEncoder, codecs::png::PngEncoder, codecs::webp::WebPEncoder, ColorType, ImageEncoder}; +use serde::Deserialize; +use sha2::{Digest, Sha256}; +use sqlx::Row; +use uuid::Uuid; + +use crate::{error::ApiError, AppState}; + +#[derive(Deserialize)] +pub struct PageQuery { + pub format: Option, + pub quality: Option, + pub width: Option, +} + +#[derive(Clone, Copy)] +enum OutputFormat { + Jpeg, + Png, + Webp, +} + +impl OutputFormat { + fn parse(value: Option<&str>) -> Result { + match value.unwrap_or("webp") { + "jpeg" | "jpg" => Ok(Self::Jpeg), + "png" => Ok(Self::Png), + "webp" => Ok(Self::Webp), + _ => Err(ApiError::bad_request("format must be webp|jpeg|png")), + } + } + + fn content_type(&self) -> &'static str { + match self { + Self::Jpeg => "image/jpeg", + Self::Png => "image/png", + Self::Webp => "image/webp", + } + } + + fn extension(&self) -> &'static str { + match self { + Self::Jpeg => "jpg", + Self::Png => "png", + Self::Webp => "webp", + } + } +} + +pub async fn get_page( + State(state): State, + AxumPath((book_id, n)): AxumPath<(Uuid, u32)>, + Query(query): Query, +) -> Result { + if n == 0 { + return Err(ApiError::bad_request("page index starts at 1")); + } + + let format = OutputFormat::parse(query.format.as_deref())?; + let quality = query.quality.unwrap_or(80).clamp(1, 100); + let width = query.width.unwrap_or(0); + if width > 2160 { + return Err(ApiError::bad_request("width must be <= 2160")); + } + + let cache_key = format!("{book_id}:{n}:{}:{quality}:{width}", format.extension()); + if let Some(cached) = state.page_cache.lock().await.get(&cache_key).cloned() { + state.metrics.page_cache_hits.fetch_add(1, Ordering::Relaxed); + return Ok(image_response(cached, format.content_type())); + } + state.metrics.page_cache_misses.fetch_add(1, Ordering::Relaxed); + + let row = sqlx::query( + r#" + SELECT abs_path, format + FROM book_files + WHERE book_id = $1 + ORDER BY updated_at DESC + LIMIT 1 + "#, + ) + .bind(book_id) + .fetch_optional(&state.pool) + .await?; + + let row = row.ok_or_else(|| ApiError::not_found("book file not found"))?; + let abs_path: String = row.get("abs_path"); + let input_format: String = row.get("format"); + + let _permit = state + .page_render_limit + .clone() + .acquire_owned() + .await + .map_err(|_| ApiError::internal("render limiter unavailable"))?; + + let bytes = tokio::time::timeout( + Duration::from_secs(12), + tokio::task::spawn_blocking(move || render_page(&abs_path, &input_format, n, &format, quality, width)), + ) + .await + .map_err(|_| ApiError::internal("page rendering timeout"))? + .map_err(|e| ApiError::internal(format!("render task failed: {e}")))??; + + let bytes = Arc::new(bytes); + state.page_cache.lock().await.put(cache_key, bytes.clone()); + + Ok(image_response(bytes, format.content_type())) +} + +fn image_response(bytes: Arc>, content_type: &str) -> Response { + let mut headers = HeaderMap::new(); + headers.insert(header::CONTENT_TYPE, HeaderValue::from_str(content_type).unwrap_or(HeaderValue::from_static("application/octet-stream"))); + headers.insert(header::CACHE_CONTROL, HeaderValue::from_static("public, max-age=300")); + let mut hasher = Sha256::new(); + hasher.update(&*bytes); + let etag = format!("\"{:x}\"", hasher.finalize()); + if let Ok(v) = HeaderValue::from_str(&etag) { + headers.insert(header::ETAG, v); + } + (StatusCode::OK, headers, Body::from((*bytes).clone())).into_response() +} + +fn render_page( + abs_path: &str, + input_format: &str, + page_number: u32, + out_format: &OutputFormat, + quality: u8, + width: u32, +) -> Result, ApiError> { + let page_bytes = match input_format { + "cbz" => extract_cbz_page(abs_path, page_number)?, + "cbr" => extract_cbr_page(abs_path, page_number)?, + "pdf" => render_pdf_page(abs_path, page_number, width)?, + _ => return Err(ApiError::bad_request("unsupported source format")), + }; + + transcode_image(&page_bytes, out_format, quality, width) +} + +fn extract_cbz_page(abs_path: &str, page_number: u32) -> Result, ApiError> { + let file = std::fs::File::open(abs_path).map_err(|e| ApiError::internal(format!("cannot open cbz: {e}")))?; + let mut archive = zip::ZipArchive::new(file).map_err(|e| ApiError::internal(format!("invalid cbz: {e}")))?; + + let mut image_names: Vec = Vec::new(); + for i in 0..archive.len() { + let entry = archive.by_index(i).map_err(|e| ApiError::internal(format!("cbz entry read failed: {e}")))?; + let name = entry.name().to_ascii_lowercase(); + if is_image_name(&name) { + image_names.push(entry.name().to_string()); + } + } + image_names.sort(); + + let index = page_number as usize - 1; + let selected = image_names.get(index).ok_or_else(|| ApiError::not_found("page out of range"))?; + let mut entry = archive.by_name(selected).map_err(|e| ApiError::internal(format!("cbz page read failed: {e}")))?; + let mut buf = Vec::new(); + entry.read_to_end(&mut buf).map_err(|e| ApiError::internal(format!("cbz page load failed: {e}")))?; + Ok(buf) +} + +fn extract_cbr_page(abs_path: &str, page_number: u32) -> Result, ApiError> { + let list_output = std::process::Command::new("unrar") + .arg("lb") + .arg(abs_path) + .output() + .map_err(|e| ApiError::internal(format!("unrar list failed: {e}")))?; + if !list_output.status.success() { + return Err(ApiError::internal("unrar could not list archive")); + } + + let mut entries: Vec = String::from_utf8_lossy(&list_output.stdout) + .lines() + .filter(|line| is_image_name(&line.to_ascii_lowercase())) + .map(|s| s.to_string()) + .collect(); + entries.sort(); + let index = page_number as usize - 1; + let selected = entries.get(index).ok_or_else(|| ApiError::not_found("page out of range"))?; + + let page_output = std::process::Command::new("unrar") + .arg("p") + .arg("-inul") + .arg(abs_path) + .arg(selected) + .output() + .map_err(|e| ApiError::internal(format!("unrar extract failed: {e}")))?; + if !page_output.status.success() { + return Err(ApiError::internal("unrar could not extract page")); + } + Ok(page_output.stdout) +} + +fn render_pdf_page(abs_path: &str, page_number: u32, width: u32) -> Result, ApiError> { + let tmp_dir = std::env::temp_dir().join(format!("stripstream-pdf-{}", Uuid::new_v4())); + std::fs::create_dir_all(&tmp_dir).map_err(|e| ApiError::internal(format!("cannot create temp dir: {e}")))?; + let output_prefix = tmp_dir.join("page"); + + let mut cmd = std::process::Command::new("pdftoppm"); + cmd.arg("-f") + .arg(page_number.to_string()) + .arg("-singlefile") + .arg("-png"); + if width > 0 { + cmd.arg("-scale-to-x").arg(width.to_string()).arg("-scale-to-y").arg("-1"); + } + cmd.arg(abs_path).arg(&output_prefix); + + let output = cmd + .output() + .map_err(|e| ApiError::internal(format!("pdf render failed: {e}")))?; + if !output.status.success() { + let _ = std::fs::remove_dir_all(&tmp_dir); + return Err(ApiError::internal("pdf render command failed")); + } + + let image_path = output_prefix.with_extension("png"); + let bytes = std::fs::read(&image_path).map_err(|e| ApiError::internal(format!("render output missing: {e}")))?; + let _ = std::fs::remove_dir_all(&tmp_dir); + Ok(bytes) +} + +fn transcode_image(input: &[u8], out_format: &OutputFormat, quality: u8, width: u32) -> Result, ApiError> { + let mut image = image::load_from_memory(input).map_err(|e| ApiError::internal(format!("invalid source image: {e}")))?; + if width > 0 { + image = image.resize(width, u32::MAX, image::imageops::FilterType::Lanczos3); + } + + let rgba = image.to_rgba8(); + let (w, h) = rgba.dimensions(); + let mut out = Vec::new(); + match out_format { + OutputFormat::Jpeg => { + let mut encoder = JpegEncoder::new_with_quality(&mut out, quality); + encoder + .encode(&rgba, w, h, ColorType::Rgba8.into()) + .map_err(|e| ApiError::internal(format!("jpeg encode failed: {e}")))?; + } + OutputFormat::Png => { + let encoder = PngEncoder::new(&mut out); + encoder + .write_image(&rgba, w, h, ColorType::Rgba8.into()) + .map_err(|e| ApiError::internal(format!("png encode failed: {e}")))?; + } + OutputFormat::Webp => { + let encoder = WebPEncoder::new_lossless(&mut out); + encoder + .write_image(&rgba, w, h, ColorType::Rgba8.into()) + .map_err(|e| ApiError::internal(format!("webp encode failed: {e}")))?; + } + } + Ok(out) +} + +fn is_image_name(name: &str) -> bool { + name.ends_with(".jpg") + || name.ends_with(".jpeg") + || name.ends_with(".png") + || name.ends_with(".webp") + || name.ends_with(".avif") +} + +#[allow(dead_code)] +fn _is_absolute_path(value: &str) -> bool { + Path::new(value).is_absolute() +} diff --git a/apps/indexer/src/main.rs b/apps/indexer/src/main.rs index 2ddf3af..da47ccf 100644 --- a/apps/indexer/src/main.rs +++ b/apps/indexer/src/main.rs @@ -1,6 +1,7 @@ use anyhow::Context; -use axum::{routing::get, Router}; +use axum::{extract::State, routing::get, Json, Router}; use chrono::{DateTime, Utc}; +use axum::http::StatusCode; use parsers::{detect_format, parse_metadata, BookFormat}; use serde::Serialize; use sha2::{Digest, Sha256}; @@ -48,7 +49,10 @@ async fn main() -> anyhow::Result<()> { tokio::spawn(run_worker(state.clone(), config.scan_interval_seconds)); - let app = Router::new().route("/health", get(health)); + let app = Router::new() + .route("/health", get(health)) + .route("/ready", get(ready)) + .with_state(state.clone()); let listener = tokio::net::TcpListener::bind(&config.listen_addr).await?; info!(addr = %config.listen_addr, "indexer listening"); @@ -60,6 +64,14 @@ async fn health() -> &'static str { "ok" } +async fn ready(State(state): State) -> Result, StatusCode> { + sqlx::query("SELECT 1") + .execute(&state.pool) + .await + .map_err(|_| StatusCode::SERVICE_UNAVAILABLE)?; + Ok(Json(serde_json::json!({"status": "ready"}))) +} + async fn run_worker(state: AppState, interval_seconds: u64) { let wait = Duration::from_secs(interval_seconds.max(1)); loop { diff --git a/crates/core/src/config.rs b/crates/core/src/config.rs index bbffc81..83fc108 100644 --- a/crates/core/src/config.rs +++ b/crates/core/src/config.rs @@ -50,13 +50,17 @@ impl IndexerConfig { #[derive(Debug, Clone)] pub struct AdminUiConfig { pub listen_addr: String, + pub api_base_url: String, + pub api_token: String, } impl AdminUiConfig { - pub fn from_env() -> Self { - Self { + pub fn from_env() -> Result { + Ok(Self { listen_addr: std::env::var("ADMIN_UI_LISTEN_ADDR") .unwrap_or_else(|_| "0.0.0.0:8082".to_string()), - } + api_base_url: std::env::var("API_BASE_URL").unwrap_or_else(|_| "http://api:8080".to_string()), + api_token: std::env::var("API_BOOTSTRAP_TOKEN").context("API_BOOTSTRAP_TOKEN is required")?, + }) } } diff --git a/infra/bench.sh b/infra/bench.sh new file mode 100755 index 0000000..2b9a329 --- /dev/null +++ b/infra/bench.sh @@ -0,0 +1,33 @@ +#!/usr/bin/env bash +set -euo pipefail + +BASE_API="${BASE_API:-http://127.0.0.1:8080}" +TOKEN="${API_TOKEN:-stripstream-dev-bootstrap-token}" + +measure() { + local name="$1" + local url="$2" + local total=0 + local n=15 + for _ in $(seq 1 "$n"); do + local t + t=$(curl -s -o /dev/null -w '%{time_total}' -H "Authorization: Bearer $TOKEN" "$url") + total=$(python3 - "$total" "$t" <<'PY' +import sys +print(float(sys.argv[1]) + float(sys.argv[2])) +PY +) + done + local avg + avg=$(python3 - "$total" "$n" <<'PY' +import sys +print(round((float(sys.argv[1]) / int(sys.argv[2]))*1000, 2)) +PY +) + echo "$name avg_ms=$avg" +} + +measure "books" "$BASE_API/books" +measure "search" "$BASE_API/search?q=sample" + +echo "bench done" diff --git a/infra/smoke.sh b/infra/smoke.sh new file mode 100755 index 0000000..f98c8fb --- /dev/null +++ b/infra/smoke.sh @@ -0,0 +1,42 @@ +#!/usr/bin/env bash +set -euo pipefail + +BASE_API="${BASE_API:-http://127.0.0.1:8080}" +BASE_INDEXER="${BASE_INDEXER:-http://127.0.0.1:8081}" +BASE_ADMIN="${BASE_ADMIN:-http://127.0.0.1:8082}" +TOKEN="${API_TOKEN:-stripstream-dev-bootstrap-token}" + +echo "[smoke] health checks" +curl -fsS "$BASE_API/health" >/dev/null +curl -fsS "$BASE_API/ready" >/dev/null +curl -fsS "$BASE_INDEXER/health" >/dev/null +curl -fsS "$BASE_INDEXER/ready" >/dev/null +curl -fsS "$BASE_ADMIN/health" >/dev/null + +echo "[smoke] list libraries" +curl -fsS -H "Authorization: Bearer $TOKEN" "$BASE_API/libraries" >/dev/null + +echo "[smoke] queue rebuild" +curl -fsS -X POST -H "Authorization: Bearer $TOKEN" "$BASE_API/index/rebuild" >/dev/null +sleep 2 + +echo "[smoke] list books and optional page fetch" +BOOKS_JSON="$(curl -fsS -H "Authorization: Bearer $TOKEN" "$BASE_API/books")" +BOOK_ID="$(BOOKS_JSON="$BOOKS_JSON" python3 - <<'PY' +import json +import os + +payload = json.loads(os.environ.get("BOOKS_JSON", "{}")) +items = payload.get("items") or [] +print(items[0]["id"] if items else "") +PY +)" + +if [ -n "$BOOK_ID" ]; then + curl -fsS -H "Authorization: Bearer $TOKEN" "$BASE_API/books/$BOOK_ID/pages/1?format=webp&quality=80&width=1080" >/dev/null +fi + +echo "[smoke] metrics" +curl -fsS "$BASE_API/metrics" >/dev/null + +echo "[smoke] OK"