From e64848a2160db090f51f9175cdd3d9d20bad797b Mon Sep 17 00:00:00 2001 From: Froidefond Julien Date: Sun, 8 Mar 2026 20:55:12 +0100 Subject: [PATCH] feat: implement thumbnail generation and management - Remove unused image dependencies from Cargo.lock. - Update API to handle thumbnail generation and checkup processes. - Introduce new routes for rebuilding and regenerating thumbnails. - Enhance job tracking with progress indicators for thumbnail jobs. - Update front-end components to display thumbnail job status and progress. - Add backend logic for managing thumbnail jobs and integrating with the API. - Refactor existing code to accommodate new thumbnail functionalities. --- Cargo.lock | 2 - PLAN_THUMBNAILS.md | 6 + apps/api/src/books.rs | 28 +- apps/api/src/index_jobs.rs | 16 +- apps/api/src/main.rs | 4 + apps/api/src/openapi.rs | 3 + apps/api/src/pages.rs | 48 +++ apps/api/src/thumbnails.rs | 284 ++++++++++++++++++ .../backoffice/app/components/JobProgress.tsx | 6 +- apps/backoffice/app/components/JobRow.tsx | 68 +++-- .../app/components/JobsIndicator.tsx | 10 +- apps/backoffice/app/components/JobsList.tsx | 1 + apps/backoffice/app/components/ui/Badge.tsx | 24 +- apps/backoffice/app/jobs/[id]/page.tsx | 12 +- apps/backoffice/app/jobs/page.tsx | 62 +++- apps/backoffice/lib/api.ts | 18 ++ apps/indexer/Cargo.toml | 2 - apps/indexer/src/main.rs | 265 +++++----------- crates/core/src/config.rs | 8 + .../0010_index_job_thumbnails_phase.sql | 6 + .../0011_thumbnail_rebuild_type.sql | 6 + 21 files changed, 625 insertions(+), 254 deletions(-) create mode 100644 apps/api/src/thumbnails.rs create mode 100644 infra/migrations/0010_index_job_thumbnails_phase.sql create mode 100644 infra/migrations/0011_thumbnail_rebuild_type.sql diff --git a/Cargo.lock b/Cargo.lock index 823c35e..07a6b3c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1146,7 +1146,6 @@ dependencies = [ "anyhow", "axum", "chrono", - "image", "notify", "parsers", "rand 0.8.5", @@ -1162,7 +1161,6 @@ dependencies = [ "tracing-subscriber", "uuid", "walkdir", - "webp", ] [[package]] diff --git a/PLAN_THUMBNAILS.md b/PLAN_THUMBNAILS.md index 2553c60..56359df 100644 --- a/PLAN_THUMBNAILS.md +++ b/PLAN_THUMBNAILS.md @@ -95,6 +95,12 @@ La recupération des thumbnail est fait par une route page/1. En fait l'indexer pourrait appeler l'api pour qu'il fasse les vignettes et c'est l'api qui est responsable des images et des lectures ebooks. Je préfère que chaque domaine soit bien respecté. A la fin d'une build, on appelle l'api pour faire le checkup des thumbnails. Il faudra que coté backoffice on voit partout ou on peut voir le traitement live des jobs, une phase ou on voit en sse le traitement des thumbnails. Coté api, si on a pas de thumbnail on passe par le code actuel de pages. +- [x] Migration `0010_index_job_thumbnails_phase.sql`: status `generating_thumbnails` dans index_jobs +- [x] API: `get_thumbnail` fallback sur page 1 si pas de thumbnail_path (via `pages::render_book_page_1`) +- [x] API: module `thumbnails.rs`, POST `/index/jobs/:id/thumbnails/checkup` (admin), lance la génération en tâche de fond et met à jour la job +- [x] Indexer: plus de génération de thumbnails; en fin de build: status = `generating_thumbnails`, puis appel API checkup; config `api_base_url` + `api_bootstrap_token` (core) +- [x] Backoffice: StatusBadge "Thumbnails" pour `generating_thumbnails`; JobProgress/JobRow/JobsIndicator/page job détail: phase thumbnails visible en SSE (X/Y thumbnails, barre de progression) + --- ## 6. Settings API diff --git a/apps/api/src/books.rs b/apps/api/src/books.rs index 863e332..5f0d1c0 100644 --- a/apps/api/src/books.rs +++ b/apps/api/src/books.rs @@ -351,25 +351,29 @@ pub async fn get_thumbnail( State(state): State, Path(book_id): Path, ) -> Result { - let row = sqlx::query( - "SELECT thumbnail_path FROM books WHERE id = $1" - ) - .bind(book_id) - .fetch_optional(&state.pool) - .await - .map_err(|e| ApiError::internal(e.to_string()))?; + let row = sqlx::query("SELECT thumbnail_path FROM books WHERE id = $1") + .bind(book_id) + .fetch_optional(&state.pool) + .await + .map_err(|e| ApiError::internal(e.to_string()))?; let row = row.ok_or_else(|| ApiError::not_found("book not found"))?; let thumbnail_path: Option = row.get("thumbnail_path"); - let path = thumbnail_path.ok_or_else(|| ApiError::not_found("thumbnail not found"))?; - - let data = std::fs::read(&path) - .map_err(|e| ApiError::internal(format!("cannot read thumbnail: {}", e)))?; + let data = if let Some(ref path) = thumbnail_path { + std::fs::read(path) + .map_err(|e| ApiError::internal(format!("cannot read thumbnail: {}", e)))? + } else { + // Fallback: render page 1 on the fly (same as pages logic) + crate::pages::render_book_page_1(&state, book_id, 300, 80).await? + }; let mut headers = HeaderMap::new(); headers.insert(header::CONTENT_TYPE, HeaderValue::from_static("image/webp")); - headers.insert(header::CACHE_CONTROL, HeaderValue::from_static("public, max-age=31536000, immutable")); + headers.insert( + header::CACHE_CONTROL, + HeaderValue::from_static("public, max-age=31536000, immutable"), + ); Ok((StatusCode::OK, headers, Body::from(data))) } diff --git a/apps/api/src/index_jobs.rs b/apps/api/src/index_jobs.rs index 9b2fe53..7439cd5 100644 --- a/apps/api/src/index_jobs.rs +++ b/apps/api/src/index_jobs.rs @@ -34,6 +34,9 @@ pub struct IndexJobResponse { pub error_opt: Option, #[schema(value_type = String)] pub created_at: DateTime, + pub progress_percent: Option, + pub processed_files: Option, + pub total_files: Option, } #[derive(Serialize, ToSchema)] @@ -142,7 +145,7 @@ pub async fn enqueue_rebuild( )] pub async fn list_index_jobs(State(state): State) -> Result>, ApiError> { let rows = sqlx::query( - "SELECT id, library_id, type, status, started_at, finished_at, stats_json, error_opt, created_at FROM index_jobs ORDER BY created_at DESC LIMIT 100", + "SELECT id, library_id, type, status, started_at, finished_at, stats_json, error_opt, created_at, progress_percent, processed_files, total_files FROM index_jobs ORDER BY created_at DESC LIMIT 100", ) .fetch_all(&state.pool) .await?; @@ -171,7 +174,7 @@ pub async fn cancel_job( id: axum::extract::Path, ) -> Result, ApiError> { let rows_affected = sqlx::query( - "UPDATE index_jobs SET status = 'cancelled' WHERE id = $1 AND status IN ('pending', 'running')", + "UPDATE index_jobs SET status = 'cancelled' WHERE id = $1 AND status IN ('pending', 'running', 'generating_thumbnails')", ) .bind(id.0) .execute(&state.pool) @@ -182,7 +185,7 @@ pub async fn cancel_job( } let row = sqlx::query( - "SELECT id, library_id, type, status, started_at, finished_at, stats_json, error_opt, created_at FROM index_jobs WHERE id = $1", + "SELECT id, library_id, type, status, started_at, finished_at, stats_json, error_opt, created_at, progress_percent, processed_files, total_files FROM index_jobs WHERE id = $1", ) .bind(id.0) .fetch_one(&state.pool) @@ -298,6 +301,9 @@ pub fn map_row(row: sqlx::postgres::PgRow) -> IndexJobResponse { stats_json: row.get("stats_json"), error_opt: row.get("error_opt"), created_at: row.get("created_at"), + progress_percent: row.try_get("progress_percent").ok(), + processed_files: row.try_get("processed_files").ok(), + total_files: row.try_get("total_files").ok(), } } @@ -333,9 +339,9 @@ fn map_row_detail(row: sqlx::postgres::PgRow) -> IndexJobDetailResponse { )] pub async fn get_active_jobs(State(state): State) -> Result>, ApiError> { let rows = sqlx::query( - "SELECT id, library_id, type, status, started_at, finished_at, stats_json, error_opt, created_at + "SELECT id, library_id, type, status, started_at, finished_at, stats_json, error_opt, created_at, progress_percent, processed_files, total_files FROM index_jobs - WHERE status IN ('pending', 'running') + WHERE status IN ('pending', 'running', 'generating_thumbnails') ORDER BY created_at ASC" ) .fetch_all(&state.pool) diff --git a/apps/api/src/main.rs b/apps/api/src/main.rs index c758de4..05f2ae3 100644 --- a/apps/api/src/main.rs +++ b/apps/api/src/main.rs @@ -7,6 +7,7 @@ mod openapi; mod pages; mod search; mod settings; +mod thumbnails; mod tokens; use std::{ @@ -99,10 +100,13 @@ async fn main() -> anyhow::Result<()> { .route("/libraries/:id/scan", axum::routing::post(libraries::scan_library)) .route("/libraries/:id/monitoring", axum::routing::patch(libraries::update_monitoring)) .route("/index/rebuild", axum::routing::post(index_jobs::enqueue_rebuild)) + .route("/index/thumbnails/rebuild", axum::routing::post(thumbnails::start_thumbnails_rebuild)) + .route("/index/thumbnails/regenerate", axum::routing::post(thumbnails::start_thumbnails_regenerate)) .route("/index/status", get(index_jobs::list_index_jobs)) .route("/index/jobs/active", get(index_jobs::get_active_jobs)) .route("/index/jobs/:id", get(index_jobs::get_job_details)) .route("/index/jobs/:id/stream", get(index_jobs::stream_job_progress)) + .route("/index/jobs/:id/thumbnails/checkup", axum::routing::post(thumbnails::start_checkup)) .route("/index/jobs/:id/errors", get(index_jobs::get_job_errors)) .route("/index/cancel/:id", axum::routing::post(index_jobs::cancel_job)) .route("/folders", get(index_jobs::list_folders)) diff --git a/apps/api/src/openapi.rs b/apps/api/src/openapi.rs index b55c5b0..6d1c6fc 100644 --- a/apps/api/src/openapi.rs +++ b/apps/api/src/openapi.rs @@ -10,6 +10,8 @@ use utoipa::OpenApi; crate::pages::get_page, crate::search::search_books, crate::index_jobs::enqueue_rebuild, + crate::thumbnails::start_thumbnails_rebuild, + crate::thumbnails::start_thumbnails_regenerate, crate::index_jobs::list_index_jobs, crate::index_jobs::get_active_jobs, crate::index_jobs::get_job_details, @@ -37,6 +39,7 @@ use utoipa::OpenApi; crate::search::SearchQuery, crate::search::SearchResponse, crate::index_jobs::RebuildRequest, + crate::thumbnails::ThumbnailsRebuildRequest, crate::index_jobs::IndexJobResponse, crate::index_jobs::IndexJobDetailResponse, crate::index_jobs::JobErrorResponse, diff --git a/apps/api/src/pages.rs b/apps/api/src/pages.rs index f50a935..7c1730a 100644 --- a/apps/api/src/pages.rs +++ b/apps/api/src/pages.rs @@ -279,6 +279,54 @@ fn image_response(bytes: Arc>, content_type: &str, etag_suffix: Option<& (StatusCode::OK, headers, Body::from((*bytes).clone())).into_response() } +/// Render page 1 of a book (for thumbnail fallback or thumbnail checkup). Uses thumbnail dimensions by default. +pub async fn render_book_page_1( + state: &AppState, + book_id: Uuid, + width: u32, + quality: u8, +) -> Result, ApiError> { + let row = sqlx::query( + r#"SELECT abs_path, format FROM book_files WHERE book_id = $1 ORDER BY updated_at DESC LIMIT 1"#, + ) + .bind(book_id) + .fetch_optional(&state.pool) + .await + .map_err(|e| ApiError::internal(e.to_string()))?; + + let row = row.ok_or_else(|| ApiError::not_found("book file not found"))?; + let abs_path: String = row.get("abs_path"); + let abs_path = remap_libraries_path(&abs_path); + let input_format: String = row.get("format"); + + let _permit = state + .page_render_limit + .clone() + .acquire_owned() + .await + .map_err(|_| ApiError::internal("render limiter unavailable"))?; + + let abs_path_clone = abs_path.clone(); + let bytes = tokio::time::timeout( + Duration::from_secs(60), + tokio::task::spawn_blocking(move || { + render_page( + &abs_path_clone, + &input_format, + 1, + &OutputFormat::Webp, + quality, + width, + ) + }), + ) + .await + .map_err(|_| ApiError::internal("page rendering timeout"))? + .map_err(|e| ApiError::internal(format!("render task failed: {e}")))?; + + bytes +} + fn render_page( abs_path: &str, input_format: &str, diff --git a/apps/api/src/thumbnails.rs b/apps/api/src/thumbnails.rs new file mode 100644 index 0000000..8a8380e --- /dev/null +++ b/apps/api/src/thumbnails.rs @@ -0,0 +1,284 @@ +use std::path::Path; + +use anyhow::Context; +use axum::{ + extract::{Path as AxumPath, State}, + http::StatusCode, + Json, +}; +use image::GenericImageView; +use serde::Deserialize; +use sqlx::Row; +use tracing::{info, warn}; +use uuid::Uuid; +use utoipa::ToSchema; + +use crate::{error::ApiError, index_jobs, pages, AppState}; + +#[derive(Clone)] +struct ThumbnailConfig { + enabled: bool, + width: u32, + height: u32, + quality: u8, + directory: String, +} + +async fn load_thumbnail_config(pool: &sqlx::PgPool) -> ThumbnailConfig { + let fallback = ThumbnailConfig { + enabled: true, + width: 300, + height: 400, + quality: 80, + directory: "/data/thumbnails".to_string(), + }; + let row = sqlx::query(r#"SELECT value FROM app_settings WHERE key = 'thumbnail'"#) + .fetch_optional(pool) + .await; + + match row { + Ok(Some(row)) => { + let value: serde_json::Value = row.get("value"); + ThumbnailConfig { + enabled: value + .get("enabled") + .and_then(|v| v.as_bool()) + .unwrap_or(fallback.enabled), + width: value + .get("width") + .and_then(|v| v.as_u64()) + .map(|v| v as u32) + .unwrap_or(fallback.width), + height: value + .get("height") + .and_then(|v| v.as_u64()) + .map(|v| v as u32) + .unwrap_or(fallback.height), + quality: value + .get("quality") + .and_then(|v| v.as_u64()) + .map(|v| v as u8) + .unwrap_or(fallback.quality), + directory: value + .get("directory") + .and_then(|v| v.as_str()) + .map(|s| s.to_string()) + .unwrap_or_else(|| fallback.directory.clone()), + } + } + _ => fallback, + } +} + +fn generate_thumbnail(image_bytes: &[u8], config: &ThumbnailConfig) -> anyhow::Result> { + let img = image::load_from_memory(image_bytes).context("failed to load image")?; + let (orig_w, orig_h) = img.dimensions(); + let ratio_w = config.width as f32 / orig_w as f32; + let ratio_h = config.height as f32 / orig_h as f32; + let ratio = ratio_w.min(ratio_h); + let new_w = (orig_w as f32 * ratio) as u32; + let new_h = (orig_h as f32 * ratio) as u32; + let resized = img.resize(new_w, new_h, image::imageops::FilterType::Lanczos3); + let rgba = resized.to_rgba8(); + let (w, h) = rgba.dimensions(); + let rgb_data: Vec = rgba.pixels().flat_map(|p| [p[0], p[1], p[2]]).collect(); + let quality = f32::max(config.quality as f32, 85.0); + let webp_data = + webp::Encoder::new(&rgb_data, webp::PixelLayout::Rgb, w, h).encode(quality); + Ok(webp_data.to_vec()) +} + +fn save_thumbnail(book_id: Uuid, thumbnail_bytes: &[u8], config: &ThumbnailConfig) -> anyhow::Result { + let dir = Path::new(&config.directory); + std::fs::create_dir_all(dir)?; + let filename = format!("{}.webp", book_id); + let path = dir.join(&filename); + std::fs::write(&path, thumbnail_bytes)?; + Ok(path.to_string_lossy().to_string()) +} + +async fn run_checkup(state: AppState, job_id: Uuid) { + let pool = &state.pool; + let row = sqlx::query("SELECT library_id, type FROM index_jobs WHERE id = $1") + .bind(job_id) + .fetch_optional(pool) + .await; + + let (library_id, job_type) = match row { + Ok(Some(r)) => ( + r.get::, _>("library_id"), + r.get::("type"), + ), + _ => { + warn!("thumbnails checkup: job {} not found", job_id); + return; + } + }; + + // Regenerate: clear existing thumbnails in scope so they get regenerated + if job_type == "thumbnail_regenerate" { + let cleared = sqlx::query( + r#"UPDATE books SET thumbnail_path = NULL WHERE (library_id = $1 OR $1 IS NULL)"#, + ) + .bind(library_id) + .execute(pool) + .await; + if let Ok(res) = cleared { + info!("thumbnails regenerate: cleared {} books", res.rows_affected()); + } + } + + let book_ids: Vec = sqlx::query_scalar( + r#"SELECT id FROM books WHERE (library_id = $1 OR $1 IS NULL) AND thumbnail_path IS NULL"#, + ) + .bind(library_id) + .fetch_all(pool) + .await + .unwrap_or_default(); + + let config = load_thumbnail_config(pool).await; + if !config.enabled || book_ids.is_empty() { + let _ = sqlx::query( + "UPDATE index_jobs SET status = 'success', finished_at = NOW(), progress_percent = 100, current_file = NULL WHERE id = $1", + ) + .bind(job_id) + .execute(pool) + .await; + return; + } + + let total = book_ids.len() as i32; + let _ = sqlx::query( + "UPDATE index_jobs SET status = 'generating_thumbnails', total_files = $2, processed_files = 0, current_file = NULL WHERE id = $1", + ) + .bind(job_id) + .bind(total) + .execute(pool) + .await; + + for (i, &book_id) in book_ids.iter().enumerate() { + match pages::render_book_page_1(&state, book_id, config.width, config.quality).await { + Ok(page_bytes) => { + match generate_thumbnail(&page_bytes, &config) { + Ok(thumb_bytes) => { + if let Ok(path) = save_thumbnail(book_id, &thumb_bytes, &config) { + if sqlx::query("UPDATE books SET thumbnail_path = $1 WHERE id = $2") + .bind(&path) + .bind(book_id) + .execute(pool) + .await + .is_ok() + { + let processed = (i + 1) as i32; + let percent = ((i + 1) as f64 / total as f64 * 100.0) as i32; + let _ = sqlx::query( + "UPDATE index_jobs SET processed_files = $2, progress_percent = $3 WHERE id = $1", + ) + .bind(job_id) + .bind(processed) + .bind(percent) + .execute(pool) + .await; + } + } + } + Err(e) => warn!("thumbnail generate failed for book {}: {:?}", book_id, e), + } + } + Err(e) => warn!("render page 1 failed for book {}: {:?}", book_id, e), + } + } + + let _ = sqlx::query( + "UPDATE index_jobs SET status = 'success', finished_at = NOW(), progress_percent = 100, current_file = NULL WHERE id = $1", + ) + .bind(job_id) + .execute(pool) + .await; + + info!("thumbnails checkup finished for job {} ({} books)", job_id, total); +} + +#[derive(Deserialize, ToSchema)] +pub struct ThumbnailsRebuildRequest { + #[schema(value_type = Option)] + pub library_id: Option, +} + +/// POST /index/thumbnails/rebuild — create a job and generate thumbnails for books that don't have one (optional library scope). +#[utoipa::path( + post, + path = "/index/thumbnails/rebuild", + tag = "indexing", + request_body = Option, + responses( + (status = 200, body = index_jobs::IndexJobResponse), + (status = 401, description = "Unauthorized"), + (status = 403, description = "Forbidden - Admin scope required"), + ), + security(("Bearer" = [])) +)] +pub async fn start_thumbnails_rebuild( + State(state): State, + payload: Option>, +) -> Result, ApiError> { + let library_id = payload.as_ref().and_then(|p| p.0.library_id); + let job_id = Uuid::new_v4(); + + let row = sqlx::query( + r#"INSERT INTO index_jobs (id, library_id, type, status) + VALUES ($1, $2, 'thumbnail_rebuild', 'pending') + RETURNING id, library_id, type, status, started_at, finished_at, stats_json, error_opt, created_at"#, + ) + .bind(job_id) + .bind(library_id) + .fetch_one(&state.pool) + .await + .map_err(|e| ApiError::internal(e.to_string()))?; + + Ok(Json(index_jobs::map_row(row))) +} + +/// POST /index/thumbnails/regenerate — create a job and regenerate all thumbnails in scope (clears then regenerates). +#[utoipa::path( + post, + path = "/index/thumbnails/regenerate", + tag = "indexing", + request_body = Option, + responses( + (status = 200, body = index_jobs::IndexJobResponse), + (status = 401, description = "Unauthorized"), + (status = 403, description = "Forbidden - Admin scope required"), + ), + security(("Bearer" = [])) +)] +pub async fn start_thumbnails_regenerate( + State(state): State, + payload: Option>, +) -> Result, ApiError> { + let library_id = payload.as_ref().and_then(|p| p.0.library_id); + let job_id = Uuid::new_v4(); + + let row = sqlx::query( + r#"INSERT INTO index_jobs (id, library_id, type, status) + VALUES ($1, $2, 'thumbnail_regenerate', 'pending') + RETURNING id, library_id, type, status, started_at, finished_at, stats_json, error_opt, created_at"#, + ) + .bind(job_id) + .bind(library_id) + .fetch_one(&state.pool) + .await + .map_err(|e| ApiError::internal(e.to_string()))?; + + Ok(Json(index_jobs::map_row(row))) +} + +/// POST /index/jobs/:id/thumbnails/checkup — start thumbnail generation for books missing thumbnails (called by indexer at end of build). +pub async fn start_checkup( + State(state): State, + AxumPath(job_id): AxumPath, +) -> Result { + let state = state.clone(); + tokio::spawn(async move { run_checkup(state, job_id).await }); + Ok(StatusCode::ACCEPTED) +} diff --git a/apps/backoffice/app/components/JobProgress.tsx b/apps/backoffice/app/components/JobProgress.tsx index e12f5f3..c74fc97 100644 --- a/apps/backoffice/app/components/JobProgress.tsx +++ b/apps/backoffice/app/components/JobProgress.tsx @@ -87,6 +87,8 @@ export function JobProgress({ jobId, onComplete }: JobProgressProps) { const percent = progress.progress_percent ?? 0; const processed = progress.processed_files ?? 0; const total = progress.total_files ?? 0; + const isThumbnailsPhase = progress.status === "generating_thumbnails"; + const unitLabel = isThumbnailsPhase ? "thumbnails" : "files"; return (
@@ -100,7 +102,7 @@ export function JobProgress({ jobId, onComplete }: JobProgressProps) {
- {processed} / {total} files + {processed} / {total} {unitLabel} {progress.current_file && ( Current: {progress.current_file.length > 40 @@ -110,7 +112,7 @@ export function JobProgress({ jobId, onComplete }: JobProgressProps) { )}
- {progress.stats_json && ( + {progress.stats_json && !isThumbnailsPhase && (
Scanned: {progress.stats_json.scanned_files} Indexed: {progress.stats_json.indexed_files} diff --git a/apps/backoffice/app/components/JobRow.tsx b/apps/backoffice/app/components/JobRow.tsx index 55b46df..527d47c 100644 --- a/apps/backoffice/app/components/JobRow.tsx +++ b/apps/backoffice/app/components/JobRow.tsx @@ -33,9 +33,8 @@ interface JobRowProps { } export function JobRow({ job, libraryName, highlighted, onCancel, formatDate, formatDuration }: JobRowProps) { - const [showProgress, setShowProgress] = useState( - highlighted || job.status === "running" || job.status === "pending" - ); + const isActive = job.status === "running" || job.status === "pending" || job.status === "generating_thumbnails"; + const [showProgress, setShowProgress] = useState(highlighted || isActive); const handleComplete = () => { setShowProgress(false); @@ -53,12 +52,32 @@ export function JobRow({ job, libraryName, highlighted, onCancel, formatDate, fo const removed = job.stats_json?.removed_files ?? 0; const errors = job.stats_json?.errors ?? 0; - // Format files display - const filesDisplay = job.status === "running" && job.total_files - ? `${job.processed_files || 0}/${job.total_files}` - : scanned > 0 - ? `${scanned} scanned` - : "-"; + const isThumbnailPhase = job.status === "generating_thumbnails"; + const isThumbnailJob = job.type === "thumbnail_rebuild" || job.type === "thumbnail_regenerate"; + const hasThumbnailPhase = isThumbnailPhase || isThumbnailJob; + + // Files column: index-phase stats only + const filesDisplay = + job.status === "running" && !isThumbnailPhase + ? job.total_files != null + ? `${job.processed_files ?? 0}/${job.total_files}` + : scanned > 0 + ? `${scanned} scanned` + : "-" + : job.status === "success" && (indexed > 0 || removed > 0 || errors > 0) + ? null // rendered below as ✓ / − / ⚠ + : scanned > 0 + ? `${scanned} scanned` + : "—"; + + // Thumbnails column + const thumbInProgress = hasThumbnailPhase && (job.status === "running" || isThumbnailPhase); + const thumbDisplay = + thumbInProgress && job.total_files != null + ? `${job.processed_files ?? 0}/${job.total_files}` + : job.status === "success" && job.total_files != null && hasThumbnailPhase + ? `✓ ${job.total_files}` + : "—"; return ( <> @@ -86,7 +105,7 @@ export function JobRow({ job, libraryName, highlighted, onCancel, formatDate, fo ! )} - {(job.status === "running" || job.status === "pending") && ( + {isActive && ( + +
+ + + + + {libraries.map((lib) => ( + + ))} + + + + +
+ +
+ + + + + {libraries.map((lib) => ( + + ))} + + + + +
diff --git a/apps/backoffice/lib/api.ts b/apps/backoffice/lib/api.ts index 78ee8b0..6fdb801 100644 --- a/apps/backoffice/lib/api.ts +++ b/apps/backoffice/lib/api.ts @@ -187,6 +187,24 @@ export async function rebuildIndex(libraryId?: string, full?: boolean) { }); } +export async function rebuildThumbnails(libraryId?: string) { + const body: { library_id?: string } = {}; + if (libraryId) body.library_id = libraryId; + return apiFetch("/index/thumbnails/rebuild", { + method: "POST", + body: JSON.stringify(body), + }); +} + +export async function regenerateThumbnails(libraryId?: string) { + const body: { library_id?: string } = {}; + if (libraryId) body.library_id = libraryId; + return apiFetch("/index/thumbnails/regenerate", { + method: "POST", + body: JSON.stringify(body), + }); +} + export async function cancelJob(id: string) { return apiFetch(`/index/cancel/${id}`, { method: "POST" }); } diff --git a/apps/indexer/Cargo.toml b/apps/indexer/Cargo.toml index bce6494..e6da562 100644 --- a/apps/indexer/Cargo.toml +++ b/apps/indexer/Cargo.toml @@ -8,7 +8,6 @@ license.workspace = true anyhow.workspace = true axum.workspace = true chrono.workspace = true -image.workspace = true notify = "6.1" parsers = { path = "../../crates/parsers" } rand.workspace = true @@ -24,4 +23,3 @@ tracing.workspace = true tracing-subscriber.workspace = true uuid.workspace = true walkdir.workspace = true -webp = "0.3" diff --git a/apps/indexer/src/main.rs b/apps/indexer/src/main.rs index 09bc849..c99edaa 100644 --- a/apps/indexer/src/main.rs +++ b/apps/indexer/src/main.rs @@ -2,9 +2,8 @@ use anyhow::Context; use axum::{extract::State, routing::get, Json, Router}; use chrono::{DateTime, Utc}; use axum::http::StatusCode; -use image::GenericImageView; use notify::{Event, RecommendedWatcher, RecursiveMode, Watcher}; -use parsers::{detect_format, parse_metadata, BookFormat, extract_first_page}; +use parsers::{detect_format, parse_metadata, BookFormat}; use rayon::prelude::*; use serde::Serialize; use sha2::{Digest, Sha256}; @@ -40,6 +39,8 @@ struct AppState { meili_url: String, meili_master_key: String, thumbnail_config: ThumbnailConfig, + api_base_url: String, + api_bootstrap_token: String, } #[derive(Serialize)] @@ -69,6 +70,8 @@ async fn main() -> anyhow::Result<()> { meili_url: config.meili_url.clone(), meili_master_key: config.meili_master_key.clone(), thumbnail_config: config.thumbnail_config.clone(), + api_base_url: config.api_base_url.clone(), + api_bootstrap_token: config.api_bootstrap_token.clone(), }; tokio::spawn(run_worker(state.clone(), config.scan_interval_seconds)); @@ -416,50 +419,54 @@ async fn claim_next_job(pool: &sqlx::PgPool) -> anyhow::Result) -> anyhow::Result<()> { info!("[JOB] Processing {} library={:?}", job_id, target_library_id); - // Load thumbnail config from database (fallback to env/default) - let thumbnail_config = load_thumbnail_config(&state.pool, &state.thumbnail_config).await; - info!("[THUMB] Config: enabled={}, dir={}", thumbnail_config.enabled, thumbnail_config.directory); - - // Get job type to check if it's a full rebuild let job_type: String = sqlx::query_scalar("SELECT type FROM index_jobs WHERE id = $1") .bind(job_id) .fetch_one(&state.pool) .await?; + + // Thumbnail jobs: hand off to API and wait for completion (same queue as rebuilds) + if job_type == "thumbnail_rebuild" || job_type == "thumbnail_regenerate" { + sqlx::query( + "UPDATE index_jobs SET status = 'generating_thumbnails', started_at = NOW() WHERE id = $1", + ) + .bind(job_id) + .execute(&state.pool) + .await?; + + let api_base = state.api_base_url.trim_end_matches('/'); + let url = format!("{}/index/jobs/{}/thumbnails/checkup", api_base, job_id); + let client = reqwest::Client::new(); + let res = client + .post(&url) + .header("Authorization", format!("Bearer {}", state.api_bootstrap_token)) + .send() + .await?; + if !res.status().is_success() { + anyhow::bail!("thumbnail checkup API returned {}", res.status()); + } + + // Poll until job is finished (API updates the same row) + let poll_interval = Duration::from_secs(1); + loop { + tokio::time::sleep(poll_interval).await; + let status: String = sqlx::query_scalar("SELECT status FROM index_jobs WHERE id = $1") + .bind(job_id) + .fetch_one(&state.pool) + .await?; + if status == "success" || status == "failed" { + info!("[JOB] Thumbnail job {} finished with status {}", job_id, status); + return Ok(()); + } + } + } + let is_full_rebuild = job_type == "full_rebuild"; info!("[JOB] {} type={} full_rebuild={}", job_id, job_type, is_full_rebuild); // For full rebuilds, delete existing data first if is_full_rebuild { info!("[JOB] Full rebuild: deleting existing data"); - - // Clean thumbnail directory - only for affected books - let thumb_dir = Path::new(&thumbnail_config.directory); - if thumb_dir.exists() { - if let Some(library_id) = target_library_id { - // Get book IDs for this library to delete their thumbnails - let book_ids: Vec = sqlx::query_scalar( - "SELECT id FROM books WHERE library_id = $1" - ) - .bind(target_library_id) - .fetch_all(&state.pool) - .await?; - - for book_id in &book_ids { - let thumb_path = thumb_dir.join(format!("{}.webp", book_id)); - let _ = std::fs::remove_file(thumb_path); - } - info!("[JOB] Cleaned {} thumbnails for library {}", book_ids.len(), library_id); - } else { - // Delete all thumbnails - if let Ok(entries) = std::fs::read_dir(thumb_dir) { - for entry in entries.flatten() { - let _ = std::fs::remove_file(entry.path()); - } - } - info!("[JOB] Cleaned all thumbnails"); - } - } - + if let Some(library_id) = target_library_id { // Delete books and files for specific library sqlx::query("DELETE FROM book_files WHERE book_id IN (SELECT id FROM books WHERE library_id = $1)") @@ -528,7 +535,7 @@ async fn process_job(state: &AppState, job_id: Uuid, target_library_id: Option {} Err(err) => { stats.errors += 1; @@ -539,12 +546,33 @@ async fn process_job(state: &AppState, job_id: Uuid, target_library_id: Option anyhow::Result<()> { info!("[SCAN] Starting scan of library {} at path: {} (full_rebuild={})", library_id, root.display(), is_full_rebuild); @@ -928,36 +955,6 @@ async fn scan_library( info!("[PROCESS] Updating existing file: {} (full_rebuild={}, fingerprint_match={})", file_name, is_full_rebuild, old_fingerprint == fingerprint); - // Generate thumbnail for existing files if enabled and fingerprint changed - let thumbnail_path = if thumbnail_config.enabled && fingerprint != old_fingerprint { - info!("[THUMB] Generating thumbnail for updated file: {}", file_name); - match extract_first_page(path, format) { - Ok(page_bytes) => { - match generate_thumbnail(&page_bytes, &thumbnail_config) { - Ok(thumb_bytes) => { - match save_thumbnail(book_id, &thumb_bytes, &thumbnail_config) { - Ok(path) => Some(path), - Err(e) => { - warn!("[THUMB] Failed to save thumbnail for {}: {}", file_name, e); - None - } - } - } - Err(e) => { - warn!("[THUMB] Failed to generate thumbnail for {}: {}", file_name, e); - None - } - } - } - Err(e) => { - warn!("[THUMB] Failed to extract first page for {}: {}", file_name, e); - None - } - } - } else { - None - }; - match parse_metadata(path, format, root) { Ok(parsed) => { books_to_update.push(BookUpdate { @@ -977,17 +974,6 @@ async fn scan_library( fingerprint, }); - // Update thumbnail_path if we generated one - if let Some(thumb_path) = thumbnail_path { - let book_id_for_update = book_id; - let thumb_path_clone = thumb_path.clone(); - sqlx::query("UPDATE books SET thumbnail_path = $1 WHERE id = $2") - .bind(thumb_path_clone) - .bind(book_id_for_update) - .execute(&state.pool) - .await?; - } - stats.indexed_files += 1; } Err(err) => { @@ -1027,46 +1013,9 @@ async fn scan_library( continue; } - // New file + // New file (thumbnails generated by API after job handoff) info!("[PROCESS] Inserting new file: {}", file_name); - - // Generate book_id early for thumbnail naming let book_id = Uuid::new_v4(); - - let thumbnail_path = if thumbnail_config.enabled { - info!("[THUMB] Generating thumbnail for {} (enabled={}, dir={})", file_name, thumbnail_config.enabled, thumbnail_config.directory); - match extract_first_page(path, format) { - Ok(page_bytes) => { - info!("[THUMB] Extracted first page: {} bytes", page_bytes.len()); - match generate_thumbnail(&page_bytes, &thumbnail_config) { - Ok(thumb_bytes) => { - info!("[THUMB] Generated thumbnail: {} bytes", thumb_bytes.len()); - match save_thumbnail(book_id, &thumb_bytes, &thumbnail_config) { - Ok(path) => { - info!("[THUMB] Saved thumbnail to {}", path); - Some(path) - }, - Err(e) => { - warn!("[THUMB] Failed to save thumbnail for {}: {}", file_name, e); - None - } - } - } - Err(e) => { - warn!("[THUMB] Failed to generate thumbnail for {}: {}", file_name, e); - None - } - } - } - Err(e) => { - warn!("[THUMB] Failed to extract first page for {}: {}", file_name, e); - None - } - } - } else { - info!("[THUMB] Skipping thumbnail (disabled)"); - None - }; match parse_metadata(path, format, root) { Ok(parsed) => { @@ -1080,7 +1029,7 @@ async fn scan_library( series: parsed.series, volume: parsed.volume, page_count: parsed.page_count, - thumbnail_path, + thumbnail_path: None, }); files_to_insert.push(FileInsert { @@ -1188,30 +1137,6 @@ fn compute_fingerprint(path: &Path, size: u64, mtime: &DateTime) -> anyhow: Ok(format!("{:x}", hasher.finalize())) } -async fn load_thumbnail_config(pool: &sqlx::PgPool, fallback: &ThumbnailConfig) -> ThumbnailConfig { - let row = sqlx::query(r#"SELECT value FROM app_settings WHERE key = 'thumbnail'"#) - .fetch_optional(pool) - .await; - - match row { - Ok(Some(row)) => { - let value: serde_json::Value = row.get("value"); - ThumbnailConfig { - enabled: value.get("enabled").and_then(|v| v.as_bool()).unwrap_or(fallback.enabled), - width: value.get("width").and_then(|v| v.as_u64()).map(|v| v as u32).unwrap_or(fallback.width), - height: value.get("height").and_then(|v| v.as_u64()).map(|v| v as u32).unwrap_or(fallback.height), - quality: value.get("quality").and_then(|v| v.as_u64()).map(|v| v as u8).unwrap_or(fallback.quality), - format: value.get("format").and_then(|v| v.as_str()).map(|s| s.to_string()).unwrap_or_else(|| fallback.format.clone()), - directory: value.get("directory").and_then(|v| v.as_str()).map(|s| s.to_string()).unwrap_or_else(|| fallback.directory.clone()), - } - } - _ => { - warn!("[THUMB] Could not load thumbnail config from DB, using fallback"); - fallback.clone() - } - } -} - fn kind_from_format(format: BookFormat) -> &'static str { match format { BookFormat::Pdf => "ebook", @@ -1225,50 +1150,6 @@ fn file_display_name(path: &Path) -> String { .unwrap_or_else(|| "Untitled".to_string()) } -fn generate_thumbnail(image_bytes: &[u8], config: &ThumbnailConfig) -> anyhow::Result> { - let img = image::load_from_memory(image_bytes) - .context("failed to load image")?; - - let (orig_w, orig_h) = img.dimensions(); - let target_w = config.width; - let target_h = config.height; - - let ratio_w = target_w as f32 / orig_w as f32; - let ratio_h = target_h as f32 / orig_h as f32; - let ratio = ratio_w.min(ratio_h); - - let new_w = (orig_w as f32 * ratio) as u32; - let new_h = (orig_h as f32 * ratio) as u32; - - let resized = img.resize(new_w, new_h, image::imageops::FilterType::Lanczos3); - - let rgba = resized.to_rgba8(); - let (w, h) = rgba.dimensions(); - - let rgb_data: Vec = rgba - .pixels() - .flat_map(|p| [p[0], p[1], p[2]]) - .collect(); - - let quality = f32::max(config.quality as f32, 85.0); - let webp_data = webp::Encoder::new(&rgb_data, webp::PixelLayout::Rgb, w, h) - .encode(quality); - - Ok(webp_data.to_vec()) -} - -fn save_thumbnail(book_id: Uuid, thumbnail_bytes: &[u8], config: &ThumbnailConfig) -> anyhow::Result { - let dir = Path::new(&config.directory); - std::fs::create_dir_all(dir)?; - - let filename = format!("{}.webp", book_id); - let path = dir.join(&filename); - - std::fs::write(&path, thumbnail_bytes)?; - - Ok(path.to_string_lossy().to_string()) -} - #[derive(Serialize)] struct SearchDoc { id: String, diff --git a/crates/core/src/config.rs b/crates/core/src/config.rs index e5a0df6..20271d3 100644 --- a/crates/core/src/config.rs +++ b/crates/core/src/config.rs @@ -32,6 +32,10 @@ pub struct IndexerConfig { pub meili_master_key: String, pub scan_interval_seconds: u64, pub thumbnail_config: ThumbnailConfig, + /// API base URL for thumbnail checkup at end of build (e.g. http://api:8080) + pub api_base_url: String, + /// Token to call API (e.g. API_BOOTSTRAP_TOKEN) + pub api_bootstrap_token: String, } #[derive(Debug, Clone)] @@ -93,6 +97,10 @@ impl IndexerConfig { .and_then(|v| v.parse::().ok()) .unwrap_or(5), thumbnail_config, + api_base_url: std::env::var("API_BASE_URL") + .unwrap_or_else(|_| "http://api:8080".to_string()), + api_bootstrap_token: std::env::var("API_BOOTSTRAP_TOKEN") + .context("API_BOOTSTRAP_TOKEN is required for thumbnail checkup")?, }) } } diff --git a/infra/migrations/0010_index_job_thumbnails_phase.sql b/infra/migrations/0010_index_job_thumbnails_phase.sql new file mode 100644 index 0000000..c6eebed --- /dev/null +++ b/infra/migrations/0010_index_job_thumbnails_phase.sql @@ -0,0 +1,6 @@ +-- Migration: Add status 'generating_thumbnails' for thumbnail phase after indexing + +ALTER TABLE index_jobs + DROP CONSTRAINT IF EXISTS index_jobs_status_check, + ADD CONSTRAINT index_jobs_status_check + CHECK (status IN ('pending', 'running', 'generating_thumbnails', 'success', 'failed')); diff --git a/infra/migrations/0011_thumbnail_rebuild_type.sql b/infra/migrations/0011_thumbnail_rebuild_type.sql new file mode 100644 index 0000000..cf01508 --- /dev/null +++ b/infra/migrations/0011_thumbnail_rebuild_type.sql @@ -0,0 +1,6 @@ +-- Migration: Add job type 'thumbnail_rebuild' for manual thumbnail generation + +ALTER TABLE index_jobs + DROP CONSTRAINT IF EXISTS index_jobs_type_check, + ADD CONSTRAINT index_jobs_type_check + CHECK (type IN ('scan', 'rebuild', 'full_rebuild', 'thumbnail_rebuild', 'thumbnail_regenerate'));