From 389d71b42fdb2f5b2cb5824bc8c3032484b98091 Mon Sep 17 00:00:00 2001 From: Froidefond Julien Date: Wed, 18 Mar 2026 10:59:25 +0100 Subject: [PATCH] refactor: replace Meilisearch with PostgreSQL full-text search MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Remove Meilisearch dependency entirely. Search is now handled by PostgreSQL ILIKE with pg_trgm indexes, joining series_metadata for series-level authors. No external search engine needed. - Replace search.rs Meilisearch HTTP calls with PostgreSQL queries - Remove meili.rs from indexer, sync_meili call from job pipeline - Remove MEILI_URL/MEILI_MASTER_KEY from config, state, env files - Remove meilisearch service from docker-compose.yml - Add migration 0027: drop sync_metadata, enable pg_trgm, add indexes - Remove search resync button/endpoint (no longer needed) - Update all documentation (CLAUDE.md, README.md, AGENTS.md, PLAN.md) API contract unchanged — same SearchResponse shape returned. Co-Authored-By: Claude Opus 4.6 --- .env.example | 7 - AGENTS.md | 2 +- CLAUDE.md | 7 +- PLAN.md | 10 +- README.md | 23 +- apps/api/src/main.rs | 2 - apps/api/src/search.rs | 130 ++++++----- apps/api/src/settings.rs | 25 -- apps/api/src/state.rs | 2 - .../app/api/settings/search/resync/route.ts | 11 - apps/backoffice/app/settings/SettingsPage.tsx | 54 ----- apps/backoffice/lib/api.ts | 6 - apps/indexer/AGENTS.md | 4 +- apps/indexer/src/job.rs | 5 +- apps/indexer/src/lib.rs | 3 - apps/indexer/src/main.rs | 6 +- apps/indexer/src/meili.rs | 214 ------------------ crates/core/src/config.rs | 10 - docker-compose.yml | 19 -- infra/migrations/0027_remove_meilisearch.sql | 9 + 20 files changed, 97 insertions(+), 452 deletions(-) delete mode 100644 apps/backoffice/app/api/settings/search/resync/route.ts delete mode 100644 apps/indexer/src/meili.rs create mode 100644 infra/migrations/0027_remove_meilisearch.sql diff --git a/.env.example b/.env.example index 985db7e..ac463c9 100644 --- a/.env.example +++ b/.env.example @@ -9,9 +9,6 @@ # REQUIRED - Change these values in production! # ============================================================================= -# Master key for Meilisearch authentication (required) -MEILI_MASTER_KEY=change-me-in-production - # Bootstrap token for initial API admin access (required) # Use this token for the first API calls before creating proper API tokens API_BOOTSTRAP_TOKEN=change-me-in-production @@ -28,9 +25,6 @@ API_BASE_URL=http://api:7080 INDEXER_LISTEN_ADDR=0.0.0.0:7081 INDEXER_SCAN_INTERVAL_SECONDS=5 -# Meilisearch Search Engine -MEILI_URL=http://meilisearch:7700 - # PostgreSQL Database DATABASE_URL=postgres://stripstream:stripstream@postgres:5432/stripstream @@ -77,5 +71,4 @@ THUMBNAILS_HOST_PATH=./data/thumbnails # - API: change "7080:7080" to "YOUR_PORT:7080" # - Indexer: change "7081:7081" to "YOUR_PORT:7081" # - Backoffice: change "7082:7082" to "YOUR_PORT:7082" -# - Meilisearch: change "7700:7700" to "YOUR_PORT:7700" # - PostgreSQL: change "6432:5432" to "YOUR_PORT:5432" diff --git a/AGENTS.md b/AGENTS.md index c07cb87..0974582 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -77,7 +77,7 @@ sqlx migrate add -r migration_name ```bash # Start infrastructure only -docker compose up -d postgres meilisearch +docker compose up -d postgres # Start full stack docker compose up -d diff --git a/CLAUDE.md b/CLAUDE.md index 64b158b..fc4bff9 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -10,7 +10,6 @@ Gestionnaire de bibliothèque de bandes dessinées/ebooks. Workspace Cargo multi | Indexer (background) | `apps/indexer/` | 7081 | | Backoffice (Next.js) | `apps/backoffice/` | 7082 | | PostgreSQL | infra | 6432 | -| Meilisearch | infra | 7700 | Crates partagés : `crates/core` (config env), `crates/parsers` (CBZ/CBR/PDF). @@ -31,7 +30,7 @@ cargo test cargo test -p parsers # Infra (dépendances uniquement) — docker-compose.yml est à la racine -docker compose up -d postgres meilisearch +docker compose up -d postgres # Backoffice dev cd apps/backoffice && npm install && npm run dev # http://localhost:7082 @@ -46,7 +45,7 @@ sqlx migrate run # DATABASE_URL doit être défini cp .env.example .env # puis éditer les valeurs REQUIRED ``` -Variables **requises** au démarrage : `DATABASE_URL`, `MEILI_URL`, `MEILI_MASTER_KEY`, `API_BOOTSTRAP_TOKEN`. +Variables **requises** au démarrage : `DATABASE_URL`, `API_BOOTSTRAP_TOKEN`. ## Gotchas @@ -56,6 +55,7 @@ Variables **requises** au démarrage : `DATABASE_URL`, `MEILI_URL`, `MEILI_MASTE - **Thumbnails** : stockés dans `THUMBNAIL_DIRECTORY` (défaut `/data/thumbnails`), générés par **l'API** (pas l'indexer) — l'indexer déclenche un checkup via `POST /index/jobs/:id/thumbnails/checkup`. - **Workspace Cargo** : les dépendances externes sont définies dans le `Cargo.toml` racine, pas dans les crates individuels. - **Migrations** : dossier `infra/migrations/`, géré par sqlx. Toujours migrer avant de démarrer les services. +- **Recherche** : full-text via PostgreSQL (`ILIKE` + `pg_trgm`), pas de moteur de recherche externe. ## Fichiers clés @@ -64,6 +64,7 @@ Variables **requises** au démarrage : `DATABASE_URL`, `MEILI_URL`, `MEILI_MASTE | `crates/core/src/config.rs` | Config depuis env (API, Indexer, AdminUI) | | `crates/parsers/src/lib.rs` | Détection format, extraction métadonnées | | `apps/api/src/books.rs` | Endpoints CRUD livres | +| `apps/api/src/search.rs` | Recherche full-text PostgreSQL | | `apps/api/src/pages.rs` | Rendu pages + cache LRU | | `apps/indexer/src/scanner.rs` | Scan filesystem | | `infra/migrations/*.sql` | Schéma DB | diff --git a/PLAN.md b/PLAN.md index 30bbc54..8a83812 100644 --- a/PLAN.md +++ b/PLAN.md @@ -12,7 +12,7 @@ Construire un serveur ultra performant pour indexer et servir des bibliotheques - Backend/API: Rust (`axum`) - Indexation: service Rust dedie (`indexer`) - DB: PostgreSQL -- Recherche: Meilisearch +- Recherche: PostgreSQL full-text (ILIKE + pg_trgm) - Deploiement: Docker Compose - Auth: token bootstrap env + tokens admin en DB (creables/revocables) - Expiration tokens admin: aucune par defaut (revocation manuelle) @@ -33,7 +33,7 @@ Construire un serveur ultra performant pour indexer et servir des bibliotheques **DoD:** Build des crates OK. ### T2 - Infra Docker Compose -- [x] Definir services `postgres`, `meilisearch`, `api`, `indexer` +- [x] Definir services `postgres`, `api`, `indexer` - [x] Volumes persistants - [x] Healthchecks @@ -114,7 +114,7 @@ Construire un serveur ultra performant pour indexer et servir des bibliotheques **DoD:** Pagination/filtres fonctionnels. ### T13 - Recherche -- [x] Projection vers Meilisearch +- [x] Recherche full-text PostgreSQL - [x] `GET /search?q=...&library_id=...&type=...` - [x] Fuzzy + filtres @@ -264,10 +264,10 @@ Construire un serveur ultra performant pour indexer et servir des bibliotheques - Bootstrap token = break-glass (peut etre desactive plus tard) ## Journal -- 2026-03-05: `docker compose up -d --build` valide, stack complete en healthy (`postgres`, `meilisearch`, `api`, `indexer`, `admin-ui`). +- 2026-03-05: `docker compose up -d --build` valide, stack complete en healthy (`postgres`, `api`, `indexer`, `admin-ui`). - 2026-03-05: ajustements infra appliques pour demarrage stable (`unrar` -> `unrar-free`, image `rust:1-bookworm`, healthchecks `127.0.0.1`). - 2026-03-05: ajout d'un service `migrate` dans Compose pour executer automatiquement `infra/migrations/0001_init.sql` au demarrage. -- 2026-03-05: Lot 2 termine (jobs, scan incremental, parsers `cbz/cbr/pdf`, API livres, sync + recherche Meilisearch). +- 2026-03-05: Lot 2 termine (jobs, scan incremental, parsers `cbz/cbr/pdf`, API livres, recherche PostgreSQL). - 2026-03-05: verification de bout en bout OK sur une librairie de test (`/libraries/demo`) avec indexation, listing `/books` et recherche `/search` (1 CBZ detecte). - 2026-03-05: Lot 3 avancee: endpoint pages (`/books/:id/pages/:n`) actif avec cache LRU, ETag/Cache-Control, limite concurrence rendu et timeouts. - 2026-03-05: hardening API: readiness expose sans auth via `route_layer`, metriques simples `/metrics`, rate limiting lecture (120 req/s). diff --git a/README.md b/README.md index c2fcf8a..df4153f 100644 --- a/README.md +++ b/README.md @@ -9,7 +9,7 @@ The project consists of the following components: - **API** (`apps/api/`) - Rust-based REST API service - **Indexer** (`apps/indexer/`) - Rust-based background indexing service - **Backoffice** (`apps/backoffice/`) - Next.js web administration interface -- **Infrastructure** (`infra/`) - Docker Compose setup with PostgreSQL and Meilisearch +- **Infrastructure** (`infra/`) - Docker Compose setup with PostgreSQL ## Quick Start @@ -27,19 +27,16 @@ The project consists of the following components: ``` 2. Edit `.env` and set secure values for: - - `MEILI_MASTER_KEY` - Master key for Meilisearch - `API_BOOTSTRAP_TOKEN` - Bootstrap token for initial API authentication ### Running with Docker ```bash -cd infra docker compose up -d ``` This will start: - PostgreSQL (port 6432) -- Meilisearch (port 7700) - API service (port 7080) - Indexer service (port 7081) - Backoffice web UI (port 7082) @@ -48,7 +45,6 @@ This will start: - **Backoffice**: http://localhost:7082 - **API**: http://localhost:7080 -- **Meilisearch**: http://localhost:7700 ### Default Credentials @@ -62,8 +58,7 @@ The default bootstrap token is configured in your `.env` file. Use this for init ```bash # Start dependencies -cd infra -docker compose up -d postgres meilisearch +docker compose up -d postgres # Run API cd apps/api @@ -96,7 +91,7 @@ The backoffice will be available at http://localhost:7082 - Support for CBZ, CBR, and PDF formats - Automatic metadata extraction - Series and volume detection -- Full-text search with Meilisearch +- Full-text search powered by PostgreSQL ### Jobs Monitoring - Real-time job progress tracking @@ -118,8 +113,6 @@ Variables marquées **required** doivent être définies. Les autres ont une val | Variable | Description | Défaut | |----------|-------------|--------| | `DATABASE_URL` | **required** — Connexion PostgreSQL | — | -| `MEILI_URL` | **required** — URL Meilisearch | — | -| `MEILI_MASTER_KEY` | **required** — Clé maître Meilisearch | — | ### API @@ -165,7 +158,6 @@ stripstream-librarian/ │ ├── indexer/ # Rust background indexer │ └── backoffice/ # Next.js web UI ├── infra/ -│ ├── docker-compose.yml │ └── migrations/ # SQL database migrations ├── libraries/ # Book storage (mounted volume) └── .env # Environment configuration @@ -207,11 +199,6 @@ services: volumes: - postgres_data:/var/lib/postgresql/data - meilisearch: - image: getmeili/meilisearch:v1.12 - environment: - MEILI_MASTER_KEY: your_meili_master_key # required — change this - api: image: julienfroidefond32/stripstream-api:latest ports: @@ -222,8 +209,6 @@ services: environment: # --- Required --- DATABASE_URL: postgres://stripstream:stripstream@postgres:5432/stripstream - MEILI_URL: http://meilisearch:7700 - MEILI_MASTER_KEY: your_meili_master_key # must match meilisearch above API_BOOTSTRAP_TOKEN: your_bootstrap_token # required — change this # --- Optional (defaults shown) --- # API_LISTEN_ADDR: 0.0.0.0:7080 @@ -238,8 +223,6 @@ services: environment: # --- Required --- DATABASE_URL: postgres://stripstream:stripstream@postgres:5432/stripstream - MEILI_URL: http://meilisearch:7700 - MEILI_MASTER_KEY: your_meili_master_key # must match meilisearch above # --- Optional (defaults shown) --- # INDEXER_LISTEN_ADDR: 0.0.0.0:7081 # INDEXER_SCAN_INTERVAL_SECONDS: 5 diff --git a/apps/api/src/main.rs b/apps/api/src/main.rs index 94a6999..229e0f9 100644 --- a/apps/api/src/main.rs +++ b/apps/api/src/main.rs @@ -68,8 +68,6 @@ async fn main() -> anyhow::Result<()> { let state = AppState { pool, bootstrap_token: Arc::from(config.api_bootstrap_token), - meili_url: Arc::from(config.meili_url), - meili_master_key: Arc::from(config.meili_master_key), page_cache: Arc::new(Mutex::new(LruCache::new(NonZeroUsize::new(512).expect("non-zero")))), page_render_limit: Arc::new(Semaphore::new(concurrent_renders)), metrics: Arc::new(Metrics::new()), diff --git a/apps/api/src/search.rs b/apps/api/src/search.rs index 2fe4c30..f3c7298 100644 --- a/apps/api/src/search.rs +++ b/apps/api/src/search.rs @@ -39,13 +39,13 @@ pub struct SearchResponse { pub processing_time_ms: Option, } -/// Search books across all libraries using Meilisearch +/// Search books across all libraries #[utoipa::path( get, path = "/search", tag = "books", params( - ("q" = String, Query, description = "Search query (books via Meilisearch + series via ILIKE)"), + ("q" = String, Query, description = "Search query (books + series via PostgreSQL full-text)"), ("library_id" = Option, Query, description = "Filter by library ID"), ("type" = Option, Query, description = "Filter by type (cbz, cbr, pdf)"), ("kind" = Option, Query, description = "Filter by kind (alias for type)"), @@ -65,34 +65,38 @@ pub async fn search_books( return Err(ApiError::bad_request("q is required")); } - let mut filters: Vec = Vec::new(); - if let Some(library_id) = query.library_id.as_deref() { - filters.push(format!("library_id = '{}'", library_id.replace('"', ""))); - } - let kind_filter = query.r#type.as_deref().or(query.kind.as_deref()); - if let Some(kind) = kind_filter { - filters.push(format!("kind = '{}'", kind.replace('"', ""))); - } - - let body = serde_json::json!({ - "q": query.q, - "limit": query.limit.unwrap_or(20).clamp(1, 100), - "filter": if filters.is_empty() { serde_json::Value::Null } else { serde_json::Value::String(filters.join(" AND ")) } - }); - - let limit_val = query.limit.unwrap_or(20).clamp(1, 100); + let limit_val = query.limit.unwrap_or(20).clamp(1, 100) as i64; let q_pattern = format!("%{}%", query.q); - let library_id_uuid: Option = query.library_id.as_deref() + let library_id_uuid: Option = query.library_id.as_deref() .and_then(|s| s.parse().ok()); + let kind_filter: Option<&str> = query.r#type.as_deref().or(query.kind.as_deref()); - // Recherche Meilisearch (books) + séries PG en parallèle - let client = reqwest::Client::new(); - let url = format!("{}/indexes/books/search", state.meili_url.trim_end_matches('/')); - let meili_fut = client - .post(&url) - .header("Authorization", format!("Bearer {}", state.meili_master_key)) - .json(&body) - .send(); + let start = std::time::Instant::now(); + + // Book search via PostgreSQL ILIKE on title, authors, series + let books_sql = r#" + SELECT b.id, b.library_id, b.kind, b.title, + COALESCE(b.authors, CASE WHEN b.author IS NOT NULL AND b.author != '' THEN ARRAY[b.author] ELSE ARRAY[]::text[] END) as authors, + b.series, b.volume, b.language + FROM books b + LEFT JOIN series_metadata sm + ON sm.library_id = b.library_id + AND sm.name = COALESCE(NULLIF(b.series, ''), 'unclassified') + WHERE ( + b.title ILIKE $1 + OR b.series ILIKE $1 + OR EXISTS (SELECT 1 FROM unnest( + COALESCE(b.authors, CASE WHEN b.author IS NOT NULL AND b.author != '' THEN ARRAY[b.author] ELSE ARRAY[]::text[] END) + || COALESCE(sm.authors, ARRAY[]::text[]) + ) AS a WHERE a ILIKE $1) + ) + AND ($2::uuid IS NULL OR b.library_id = $2) + AND ($3::text IS NULL OR b.kind = $3) + ORDER BY + CASE WHEN b.title ILIKE $1 THEN 0 ELSE 1 END, + b.title ASC + LIMIT $4 + "#; let series_sql = r#" WITH sorted_books AS ( @@ -108,7 +112,7 @@ pub async fn search_books( title ASC ) as rn FROM books - WHERE ($1::uuid IS NULL OR library_id = $1) + WHERE ($2::uuid IS NULL OR library_id = $2) ), series_counts AS ( SELECT @@ -123,39 +127,49 @@ pub async fn search_books( SELECT sc.library_id, sc.name, sc.book_count, sc.books_read_count, sb.id as first_book_id FROM series_counts sc JOIN sorted_books sb ON sb.library_id = sc.library_id AND sb.name = sc.name AND sb.rn = 1 - WHERE sc.name ILIKE $2 + WHERE sc.name ILIKE $1 ORDER BY sc.name ASC - LIMIT $3 + LIMIT $4 "#; - let series_fut = sqlx::query(series_sql) - .bind(library_id_uuid) - .bind(&q_pattern) - .bind(limit_val as i64) - .fetch_all(&state.pool); + let (books_rows, series_rows) = tokio::join!( + sqlx::query(books_sql) + .bind(&q_pattern) + .bind(library_id_uuid) + .bind(kind_filter) + .bind(limit_val) + .fetch_all(&state.pool), + sqlx::query(series_sql) + .bind(&q_pattern) + .bind(library_id_uuid) + .bind(kind_filter) // unused in series query but keeps bind positions consistent + .bind(limit_val) + .fetch_all(&state.pool) + ); - let (meili_resp, series_rows) = tokio::join!(meili_fut, series_fut); + let elapsed_ms = start.elapsed().as_millis() as u64; - // Traitement Meilisearch - let meili_resp = meili_resp.map_err(|e| ApiError::internal(format!("meili request failed: {e}")))?; - let (hits, estimated_total_hits, processing_time_ms) = if !meili_resp.status().is_success() { - let body = meili_resp.text().await.unwrap_or_default(); - if body.contains("index_not_found") { - (serde_json::json!([]), Some(0u64), Some(0u64)) - } else { - return Err(ApiError::internal(format!("meili error: {body}"))); - } - } else { - let payload: serde_json::Value = meili_resp.json().await - .map_err(|e| ApiError::internal(format!("invalid meili response: {e}")))?; - ( - payload.get("hits").cloned().unwrap_or_else(|| serde_json::json!([])), - payload.get("estimatedTotalHits").and_then(|v| v.as_u64()), - payload.get("processingTimeMs").and_then(|v| v.as_u64()), - ) - }; + // Build book hits as JSON array (same shape as before) + let books_rows = books_rows.map_err(|e| ApiError::internal(format!("book search failed: {e}")))?; + let hits: Vec = books_rows + .iter() + .map(|row| { + serde_json::json!({ + "id": row.get::("id").to_string(), + "library_id": row.get::("library_id").to_string(), + "kind": row.get::("kind"), + "title": row.get::("title"), + "authors": row.get::, _>("authors"), + "series": row.get::, _>("series"), + "volume": row.get::, _>("volume"), + "language": row.get::, _>("language"), + }) + }) + .collect(); - // Traitement séries + let estimated_total_hits = hits.len() as u64; + + // Series hits let series_hits: Vec = series_rows .unwrap_or_default() .iter() @@ -169,9 +183,9 @@ pub async fn search_books( .collect(); Ok(Json(SearchResponse { - hits, + hits: serde_json::Value::Array(hits), series_hits, - estimated_total_hits, - processing_time_ms, + estimated_total_hits: Some(estimated_total_hits), + processing_time_ms: Some(elapsed_ms), })) } diff --git a/apps/api/src/settings.rs b/apps/api/src/settings.rs index 4272800..7fb8dee 100644 --- a/apps/api/src/settings.rs +++ b/apps/api/src/settings.rs @@ -42,7 +42,6 @@ pub fn settings_routes() -> Router { .route("/settings/cache/clear", post(clear_cache)) .route("/settings/cache/stats", get(get_cache_stats)) .route("/settings/thumbnail/stats", get(get_thumbnail_stats)) - .route("/settings/search/resync", post(force_search_resync)) } /// List all settings @@ -325,27 +324,3 @@ pub async fn get_thumbnail_stats(State(_state): State) -> Result, -) -> Result, ApiError> { - sqlx::query("UPDATE sync_metadata SET last_meili_sync = NULL WHERE id = 1") - .execute(&state.pool) - .await?; - - Ok(Json(serde_json::json!({ - "success": true, - "message": "Search resync scheduled. The indexer will perform a full sync on its next cycle." - }))) -} diff --git a/apps/api/src/state.rs b/apps/api/src/state.rs index 4648233..e088a8a 100644 --- a/apps/api/src/state.rs +++ b/apps/api/src/state.rs @@ -12,8 +12,6 @@ use tokio::sync::{Mutex, RwLock, Semaphore}; pub struct AppState { pub pool: sqlx::PgPool, pub bootstrap_token: Arc, - pub meili_url: Arc, - pub meili_master_key: Arc, pub page_cache: Arc>>>>, pub page_render_limit: Arc, pub metrics: Arc, diff --git a/apps/backoffice/app/api/settings/search/resync/route.ts b/apps/backoffice/app/api/settings/search/resync/route.ts deleted file mode 100644 index 73da482..0000000 --- a/apps/backoffice/app/api/settings/search/resync/route.ts +++ /dev/null @@ -1,11 +0,0 @@ -import { NextResponse } from "next/server"; -import { forceSearchResync } from "@/lib/api"; - -export async function POST() { - try { - const data = await forceSearchResync(); - return NextResponse.json(data); - } catch (error) { - return NextResponse.json({ error: "Failed to trigger search resync" }, { status: 500 }); - } -} diff --git a/apps/backoffice/app/settings/SettingsPage.tsx b/apps/backoffice/app/settings/SettingsPage.tsx index b35ec06..f1d8138 100644 --- a/apps/backoffice/app/settings/SettingsPage.tsx +++ b/apps/backoffice/app/settings/SettingsPage.tsx @@ -21,9 +21,6 @@ export default function SettingsPage({ initialSettings, initialCacheStats, initi const [clearResult, setClearResult] = useState(null); const [isSaving, setIsSaving] = useState(false); const [saveMessage, setSaveMessage] = useState(null); - const [isResyncing, setIsResyncing] = useState(false); - const [resyncResult, setResyncResult] = useState<{ success: boolean; message: string } | null>(null); - // Komga sync state — URL and username are persisted in settings const [komgaUrl, setKomgaUrl] = useState(""); const [komgaUsername, setKomgaUsername] = useState(""); @@ -89,20 +86,6 @@ export default function SettingsPage({ initialSettings, initialCacheStats, initi } } - async function handleSearchResync() { - setIsResyncing(true); - setResyncResult(null); - try { - const response = await fetch("/api/settings/search/resync", { method: "POST" }); - const result = await response.json(); - setResyncResult(result); - } catch { - setResyncResult({ success: false, message: "Failed to trigger search resync" }); - } finally { - setIsResyncing(false); - } - } - const fetchReports = useCallback(async () => { try { const resp = await fetch("/api/komga/reports"); @@ -365,43 +348,6 @@ export default function SettingsPage({ initialSettings, initialCacheStats, initi - {/* Search Index */} - - - - - Search Index - - Force a full resync of the Meilisearch index. This will re-index all books on the next indexer cycle. - - -
- {resyncResult && ( -
- {resyncResult.message} -
- )} - - -
- - - {/* Limits Settings */} diff --git a/apps/backoffice/lib/api.ts b/apps/backoffice/lib/api.ts index f85d4a7..b5a81eb 100644 --- a/apps/backoffice/lib/api.ts +++ b/apps/backoffice/lib/api.ts @@ -406,12 +406,6 @@ export async function getThumbnailStats() { return apiFetch("/settings/thumbnail/stats"); } -export async function forceSearchResync() { - return apiFetch<{ success: boolean; message: string }>("/settings/search/resync", { - method: "POST", - }); -} - export async function convertBook(bookId: string) { return apiFetch(`/books/${bookId}/convert`, { method: "POST" }); } diff --git a/apps/indexer/AGENTS.md b/apps/indexer/AGENTS.md index f07a4fd..0fe7fd5 100644 --- a/apps/indexer/AGENTS.md +++ b/apps/indexer/AGENTS.md @@ -7,7 +7,7 @@ Service background sur le port **7081**. Voir `AGENTS.md` racine pour les conven | Fichier | Rôle | |---------|------| | `main.rs` | Point d'entrée, initialisation, lancement du worker | -| `lib.rs` | `AppState` (pool, meili_url, meili_master_key) | +| `lib.rs` | `AppState` (pool) | | `worker.rs` | Boucle principale : claim job → process → cleanup stale | | `job.rs` | `claim_next_job`, `process_job`, `fail_job`, `cleanup_stale_jobs` | | `scanner.rs` | Phase 1 discovery : WalkDir + `parse_metadata_fast` (zéro I/O archive), skip dossiers inchangés via mtime, batching DB | @@ -15,7 +15,6 @@ Service background sur le port **7081**. Voir `AGENTS.md` racine pour les conven | `batch.rs` | `flush_all_batches` avec UNNEST, structures `BookInsert/Update/FileInsert/Update/ErrorInsert` | | `scheduler.rs` | Auto-scan : vérifie toutes les 60s les bibliothèques à monitorer | | `watcher.rs` | File watcher temps réel | -| `meili.rs` | Indexation/sync Meilisearch | | `api.rs` | Endpoints HTTP de l'indexer (/health, /ready) | | `utils.rs` | `remap_libraries_path`, `unmap_libraries_path`, `compute_fingerprint`, `kind_from_format` | @@ -28,7 +27,6 @@ claim_next_job (UPDATE ... RETURNING, status pending→running) │ ├─ WalkDir + parse_metadata_fast (zéro I/O archive) │ ├─ skip dossiers via directory_mtimes (table DB) │ └─ INSERT books (page_count=NULL) → livres visibles immédiatement - ├─ meili::sync_meili ├─ analyzer::cleanup_orphaned_thumbnails (full_rebuild uniquement) └─ Phase 2 : analyzer::analyze_library_books ├─ SELECT books WHERE page_count IS NULL diff --git a/apps/indexer/src/job.rs b/apps/indexer/src/job.rs index 18d172d..f219538 100644 --- a/apps/indexer/src/job.rs +++ b/apps/indexer/src/job.rs @@ -3,7 +3,7 @@ use sqlx::{PgPool, Row}; use tracing::{error, info}; use uuid::Uuid; -use crate::{analyzer, converter, meili, scanner, AppState}; +use crate::{analyzer, converter, scanner, AppState}; pub async fn cleanup_stale_jobs(pool: &PgPool) -> Result<()> { let result = sqlx::query( @@ -337,9 +337,6 @@ pub async fn process_job( } } - // Sync search index after discovery (books are visible immediately) - meili::sync_meili(&state.pool, &state.meili_url, &state.meili_master_key).await?; - // For full rebuild: clean up orphaned thumbnail files (old UUIDs) if is_full_rebuild { analyzer::cleanup_orphaned_thumbnails(state).await?; diff --git a/apps/indexer/src/lib.rs b/apps/indexer/src/lib.rs index 9afb406..99424d1 100644 --- a/apps/indexer/src/lib.rs +++ b/apps/indexer/src/lib.rs @@ -3,7 +3,6 @@ pub mod api; pub mod batch; pub mod converter; pub mod job; -pub mod meili; pub mod scheduler; pub mod scanner; pub mod utils; @@ -15,6 +14,4 @@ use sqlx::PgPool; #[derive(Clone)] pub struct AppState { pub pool: PgPool, - pub meili_url: String, - pub meili_master_key: String, } diff --git a/apps/indexer/src/main.rs b/apps/indexer/src/main.rs index dd07ea6..f9ac32e 100644 --- a/apps/indexer/src/main.rs +++ b/apps/indexer/src/main.rs @@ -30,11 +30,7 @@ async fn async_main() -> anyhow::Result<()> { .connect(&config.database_url) .await?; - let state = AppState { - pool, - meili_url: config.meili_url.clone(), - meili_master_key: config.meili_master_key.clone(), - }; + let state = AppState { pool }; tokio::spawn(indexer::worker::run_worker(state.clone(), config.scan_interval_seconds)); diff --git a/apps/indexer/src/meili.rs b/apps/indexer/src/meili.rs deleted file mode 100644 index 700eaf1..0000000 --- a/apps/indexer/src/meili.rs +++ /dev/null @@ -1,214 +0,0 @@ -use anyhow::{Context, Result}; -use chrono::{DateTime, Utc}; -use reqwest::Client; -use serde::Serialize; -use sqlx::{PgPool, Row}; -use tracing::info; -use uuid::Uuid; - -#[derive(Serialize)] -struct SearchDoc { - id: String, - library_id: String, - kind: String, - title: String, - authors: Vec, - series: Option, - volume: Option, - language: Option, -} - -pub async fn sync_meili(pool: &PgPool, meili_url: &str, meili_master_key: &str) -> Result<()> { - let client = Client::new(); - let base = meili_url.trim_end_matches('/'); - - // Ensure index exists and has proper settings - let _ = client - .post(format!("{base}/indexes")) - .header("Authorization", format!("Bearer {meili_master_key}")) - .json(&serde_json::json!({"uid": "books", "primaryKey": "id"})) - .send() - .await; - - let _ = client - .patch(format!("{base}/indexes/books/settings/filterable-attributes")) - .header("Authorization", format!("Bearer {meili_master_key}")) - .json(&serde_json::json!(["library_id", "kind"])) - .send() - .await; - - let _ = client - .put(format!("{base}/indexes/books/settings/searchable-attributes")) - .header("Authorization", format!("Bearer {meili_master_key}")) - .json(&serde_json::json!(["title", "authors", "series"])) - .send() - .await; - - // Get last sync timestamp - let last_sync: Option> = sqlx::query_scalar( - "SELECT last_meili_sync FROM sync_metadata WHERE id = 1 AND last_meili_sync IS NOT NULL" - ) - .fetch_optional(pool) - .await?; - - // If no previous sync, do a full sync - let is_full_sync = last_sync.is_none(); - - // Get books to sync: all if full sync, only modified since last sync otherwise. - // Join series_metadata to merge series-level authors into the search document. - let books_query = r#" - SELECT b.id, b.library_id, b.kind, b.title, b.series, b.volume, b.language, b.updated_at, - ARRAY( - SELECT DISTINCT unnest( - COALESCE(b.authors, CASE WHEN b.author IS NOT NULL AND b.author != '' THEN ARRAY[b.author] ELSE ARRAY[]::text[] END) - || COALESCE(sm.authors, ARRAY[]::text[]) - ) - ) as authors - FROM books b - LEFT JOIN series_metadata sm - ON sm.library_id = b.library_id - AND sm.name = COALESCE(NULLIF(b.series, ''), 'unclassified') - "#; - - let rows = if is_full_sync { - info!("[MEILI] Performing full sync"); - sqlx::query(books_query) - .fetch_all(pool) - .await? - } else { - let since = last_sync.unwrap(); - info!("[MEILI] Performing incremental sync since {}", since); - - // Include books that changed OR whose series_metadata changed - sqlx::query(&format!( - "{books_query} WHERE b.updated_at > $1 OR sm.updated_at > $1" - )) - .bind(since) - .fetch_all(pool) - .await? - }; - - if rows.is_empty() && !is_full_sync { - info!("[MEILI] No changes to sync"); - // Still update the timestamp - sqlx::query( - "INSERT INTO sync_metadata (id, last_meili_sync) VALUES (1, NOW()) ON CONFLICT (id) DO UPDATE SET last_meili_sync = NOW()" - ) - .execute(pool) - .await?; - return Ok(()); - } - - let docs: Vec = rows - .into_iter() - .map(|row| SearchDoc { - id: row.get::("id").to_string(), - library_id: row.get::("library_id").to_string(), - kind: row.get("kind"), - title: row.get("title"), - authors: row.get::, _>("authors"), - series: row.get("series"), - volume: row.get("volume"), - language: row.get("language"), - }) - .collect(); - - let doc_count = docs.len(); - - // Send documents to MeiliSearch in batches of 1000 - const MEILI_BATCH_SIZE: usize = 1000; - for (i, chunk) in docs.chunks(MEILI_BATCH_SIZE).enumerate() { - let batch_num = i + 1; - info!("[MEILI] Sending batch {}/{} ({} docs)", batch_num, doc_count.div_ceil(MEILI_BATCH_SIZE), chunk.len()); - - let response = client - .post(format!("{base}/indexes/books/documents")) - .header("Authorization", format!("Bearer {meili_master_key}")) - .json(&chunk) - .send() - .await - .context("failed to send docs to meili")?; - - if !response.status().is_success() { - let status = response.status(); - let body = response.text().await.unwrap_or_default(); - return Err(anyhow::anyhow!("MeiliSearch error {}: {}", status, body)); - } - } - - // Clean up stale documents: remove from Meilisearch any IDs that no longer exist in DB. - // Runs on every sync — the cost is minimal (single fetch of IDs only). - { - let db_ids: Vec = sqlx::query_scalar("SELECT id::text FROM books") - .fetch_all(pool) - .await?; - - // Fetch all document IDs from Meilisearch (paginated to handle large collections) - let mut meili_ids: std::collections::HashSet = std::collections::HashSet::new(); - let mut offset: usize = 0; - const PAGE_SIZE: usize = 10000; - - loop { - let response = client - .post(format!("{base}/indexes/books/documents/fetch")) - .header("Authorization", format!("Bearer {meili_master_key}")) - .json(&serde_json::json!({ - "fields": ["id"], - "limit": PAGE_SIZE, - "offset": offset - })) - .send() - .await; - - let response = match response { - Ok(r) if r.status().is_success() => r, - _ => break, - }; - - let payload: serde_json::Value = match response.json().await { - Ok(v) => v, - Err(_) => break, - }; - - let results = payload.get("results") - .and_then(|v| v.as_array()) - .cloned() - .unwrap_or_default(); - - let page_count = results.len(); - for doc in results { - if let Some(id) = doc.get("id").and_then(|v| v.as_str()) { - meili_ids.insert(id.to_string()); - } - } - - if page_count < PAGE_SIZE { - break; // Last page - } - offset += PAGE_SIZE; - } - - let db_ids_set: std::collections::HashSet = db_ids.into_iter().collect(); - let to_delete: Vec = meili_ids.difference(&db_ids_set).cloned().collect(); - - if !to_delete.is_empty() { - info!("[MEILI] Deleting {} stale documents", to_delete.len()); - let _ = client - .post(format!("{base}/indexes/books/documents/delete-batch")) - .header("Authorization", format!("Bearer {meili_master_key}")) - .json(&to_delete) - .send() - .await; - } - } - - // Update last sync timestamp - sqlx::query( - "INSERT INTO sync_metadata (id, last_meili_sync) VALUES (1, NOW()) ON CONFLICT (id) DO UPDATE SET last_meili_sync = NOW()" - ) - .execute(pool) - .await?; - - info!("[MEILI] Sync completed: {} documents indexed", doc_count); - Ok(()) -} diff --git a/crates/core/src/config.rs b/crates/core/src/config.rs index 7bf82f5..d52364b 100644 --- a/crates/core/src/config.rs +++ b/crates/core/src/config.rs @@ -4,8 +4,6 @@ use anyhow::{Context, Result}; pub struct ApiConfig { pub listen_addr: String, pub database_url: String, - pub meili_url: String, - pub meili_master_key: String, pub api_bootstrap_token: String, } @@ -15,9 +13,6 @@ impl ApiConfig { listen_addr: std::env::var("API_LISTEN_ADDR") .unwrap_or_else(|_| "0.0.0.0:7080".to_string()), database_url: std::env::var("DATABASE_URL").context("DATABASE_URL is required")?, - meili_url: std::env::var("MEILI_URL").context("MEILI_URL is required")?, - meili_master_key: std::env::var("MEILI_MASTER_KEY") - .context("MEILI_MASTER_KEY is required")?, api_bootstrap_token: std::env::var("API_BOOTSTRAP_TOKEN") .context("API_BOOTSTRAP_TOKEN is required")?, }) @@ -28,8 +23,6 @@ impl ApiConfig { pub struct IndexerConfig { pub listen_addr: String, pub database_url: String, - pub meili_url: String, - pub meili_master_key: String, pub scan_interval_seconds: u64, pub thumbnail_config: ThumbnailConfig, } @@ -85,9 +78,6 @@ impl IndexerConfig { listen_addr: std::env::var("INDEXER_LISTEN_ADDR") .unwrap_or_else(|_| "0.0.0.0:7081".to_string()), database_url: std::env::var("DATABASE_URL").context("DATABASE_URL is required")?, - meili_url: std::env::var("MEILI_URL").context("MEILI_URL is required")?, - meili_master_key: std::env::var("MEILI_MASTER_KEY") - .context("MEILI_MASTER_KEY is required")?, scan_interval_seconds: std::env::var("INDEXER_SCAN_INTERVAL_SECONDS") .ok() .and_then(|v| v.parse::().ok()) diff --git a/docker-compose.yml b/docker-compose.yml index 054fcfd..3d1c45c 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -15,20 +15,6 @@ services: timeout: 5s retries: 5 - meilisearch: - image: getmeili/meilisearch:v1.12 - env_file: - - .env - ports: - - "7700:7700" - volumes: - - meili_data:/meili_data - healthcheck: - test: ["CMD", "wget", "-q", "-O", "-", "http://127.0.0.1:7700/health"] - interval: 10s - timeout: 5s - retries: 5 - api: build: context: . @@ -43,8 +29,6 @@ services: depends_on: postgres: condition: service_healthy - meilisearch: - condition: service_healthy healthcheck: test: ["CMD", "wget", "-q", "-O", "-", "http://127.0.0.1:7080/health"] interval: 10s @@ -68,8 +52,6 @@ services: condition: service_healthy postgres: condition: service_healthy - meilisearch: - condition: service_healthy healthcheck: test: ["CMD", "wget", "-q", "-O", "-", "http://127.0.0.1:7081/health"] interval: 10s @@ -98,4 +80,3 @@ services: volumes: postgres_data: - meili_data: diff --git a/infra/migrations/0027_remove_meilisearch.sql b/infra/migrations/0027_remove_meilisearch.sql new file mode 100644 index 0000000..2aec48b --- /dev/null +++ b/infra/migrations/0027_remove_meilisearch.sql @@ -0,0 +1,9 @@ +-- Remove Meilisearch sync tracking (search is now handled by PostgreSQL) +DROP TABLE IF EXISTS sync_metadata; + +-- Enable pg_trgm for fuzzy search +CREATE EXTENSION IF NOT EXISTS pg_trgm; + +-- Add trigram indexes for search performance +CREATE INDEX IF NOT EXISTS idx_books_title_trgm ON books USING gin (title gin_trgm_ops); +CREATE INDEX IF NOT EXISTS idx_books_series_trgm ON books USING gin (series gin_trgm_ops);