From 0f5094575a9d37e923f2490d566e69e44fb4c298 Mon Sep 17 00:00:00 2001 From: Froidefond Julien Date: Mon, 9 Mar 2026 13:57:39 +0100 Subject: [PATCH] docs: add AGENTS.md per module and unify ports to 70XX - Add CLAUDE.md at root and AGENTS.md in apps/api, apps/indexer, apps/backoffice, crates/parsers with module-specific guidelines - Unify all service ports to 70XX (no more internal/external split): API 7080, Indexer 7081, Backoffice 7082 - Update docker-compose.yml, Dockerfiles, config.rs defaults, .env.example, backoffice routes, bench.sh, smoke.sh Co-Authored-By: Claude Sonnet 4.6 --- .env.example | 12 +-- AGENTS.md | 45 ++++++----- CLAUDE.md | 72 ++++++++++++++++++ README.md | 22 +++--- apps/api/AGENTS.md | 73 ++++++++++++++++++ apps/api/Dockerfile | 2 +- apps/backoffice/.env.local | 4 +- apps/backoffice/AGENTS.md | 66 ++++++++++++++++ apps/backoffice/Dockerfile | 4 +- .../books/[bookId]/pages/[pageNum]/route.ts | 2 +- .../app/api/books/[bookId]/thumbnail/route.ts | 2 +- apps/backoffice/app/api/folders/route.ts | 2 +- .../app/api/jobs/[id]/cancel/route.ts | 2 +- apps/backoffice/app/api/jobs/[id]/route.ts | 2 +- .../app/api/jobs/[id]/stream/route.ts | 2 +- apps/backoffice/app/api/jobs/route.ts | 2 +- apps/backoffice/app/api/jobs/stream/route.ts | 2 +- .../app/api/settings/[key]/route.ts | 4 +- .../app/api/settings/cache/clear/route.ts | 2 +- .../app/api/settings/cache/stats/route.ts | 2 +- apps/backoffice/app/api/settings/route.ts | 2 +- apps/backoffice/lib/api.ts | 2 +- apps/indexer/AGENTS.md | 75 ++++++++++++++++++ apps/indexer/Dockerfile | 2 +- crates/core/src/config.rs | 12 +-- crates/parsers/AGENTS.md | 76 +++++++++++++++++++ docker-compose.yml | 14 ++-- infra/bench.sh | 2 +- infra/smoke.sh | 6 +- 29 files changed, 441 insertions(+), 74 deletions(-) create mode 100644 CLAUDE.md create mode 100644 apps/api/AGENTS.md create mode 100644 apps/backoffice/AGENTS.md create mode 100644 apps/indexer/AGENTS.md create mode 100644 crates/parsers/AGENTS.md diff --git a/.env.example b/.env.example index afb4cef..bb5730e 100644 --- a/.env.example +++ b/.env.example @@ -21,11 +21,11 @@ API_BOOTSTRAP_TOKEN=change-me-in-production # ============================================================================= # API Service -API_LISTEN_ADDR=0.0.0.0:8080 -API_BASE_URL=http://api:8080 +API_LISTEN_ADDR=0.0.0.0:7080 +API_BASE_URL=http://api:7080 # Indexer Service -INDEXER_LISTEN_ADDR=0.0.0.0:8081 +INDEXER_LISTEN_ADDR=0.0.0.0:7081 INDEXER_SCAN_INTERVAL_SECONDS=5 # Meilisearch Search Engine @@ -56,8 +56,8 @@ THUMBNAILS_HOST_PATH=../data/thumbnails # Port Configuration # ============================================================================= # To change ports, edit docker-compose.yml directly: -# - API: change "7080:8080" to "YOUR_PORT:8080" -# - Indexer: change "7081:8081" to "YOUR_PORT:8081" -# - Backoffice: change "7082:8082" to "YOUR_PORT:8082" +# - API: change "7080:7080" to "YOUR_PORT:7080" +# - Indexer: change "7081:7081" to "YOUR_PORT:7081" +# - Backoffice: change "7082:7082" to "YOUR_PORT:7082" # - Meilisearch: change "7700:7700" to "YOUR_PORT:7700" # - PostgreSQL: change "6432:5432" to "YOUR_PORT:5432" diff --git a/AGENTS.md b/AGENTS.md index 82293d4..41e5260 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -73,12 +73,14 @@ sqlx migrate add -r migration_name ### Docker Development +`docker-compose.yml` est à la **racine** du projet (pas dans `infra/`). + ```bash # Start infrastructure only -cd infra && docker compose up -d postgres meilisearch +docker compose up -d postgres meilisearch # Start full stack -cd infra && docker compose up -d +docker compose up -d # View logs docker compose logs -f api @@ -226,24 +228,21 @@ pub struct BookItem { ``` stripstream-librarian/ ├── apps/ -│ ├── api/ # REST API (axum) -│ │ └── src/ -│ │ ├── main.rs -│ │ ├── books.rs -│ │ ├── pages.rs -│ │ └── ... -│ ├── indexer/ # Background indexing service -│ │ └── src/ -│ │ └── main.rs -│ └── backoffice/ # Next.js admin UI +│ ├── api/ # REST API (axum) — port 7080 +│ │ └── src/ # books.rs, pages.rs, thumbnails.rs, state.rs, auth.rs... +│ ├── indexer/ # Background indexing service — port 7081 +│ │ └── src/ # worker.rs, scanner.rs, batch.rs, scheduler.rs, watcher.rs... +│ └── backoffice/ # Next.js admin UI — port 7082 ├── crates/ -│ ├── core/ # Shared config +│ ├── core/ # Shared config (env vars) │ │ └── src/config.rs │ └── parsers/ # Book parsing (CBZ, CBR, PDF) ├── infra/ -│ ├── migrations/ # SQL migrations -│ └── docker-compose.yml -└── libraries/ # Book storage (mounted volume) +│ └── migrations/ # SQL migrations (sqlx) +├── data/ +│ └── thumbnails/ # Thumbnails générés par l'API +├── libraries/ # Book storage (mounted volume) +└── docker-compose.yml # À la racine (pas dans infra/) ``` ### Key Files @@ -251,8 +250,12 @@ stripstream-librarian/ | File | Purpose | |------|---------| | `apps/api/src/books.rs` | Book CRUD endpoints | -| `apps/api/src/pages.rs` | Page rendering & caching | -| `apps/indexer/src/main.rs` | Indexing logic, batch processing | +| `apps/api/src/pages.rs` | Page rendering & caching (LRU + disk) | +| `apps/api/src/thumbnails.rs` | Thumbnail generation (triggered by indexer) | +| `apps/api/src/state.rs` | AppState, Semaphore concurrent_renders | +| `apps/indexer/src/scanner.rs` | Filesystem scan, rayon parallel parsing | +| `apps/indexer/src/batch.rs` | Bulk DB ops via UNNEST | +| `apps/indexer/src/worker.rs` | Job loop, watcher, scheduler orchestration | | `crates/parsers/src/lib.rs` | Format detection, metadata parsing | | `crates/core/src/config.rs` | Configuration from environment | | `infra/migrations/*.sql` | Database schema | @@ -269,7 +272,7 @@ impl IndexerConfig { pub fn from_env() -> Result { Ok(Self { listen_addr: std::env::var("INDEXER_LISTEN_ADDR") - .unwrap_or_else(|_| "0.0.0.0:8081".to_string()), + .unwrap_or_else(|_| "0.0.0.0:7081".to_string()), database_url: std::env::var("DATABASE_URL") .context("DATABASE_URL is required")?, // ... @@ -298,4 +301,6 @@ fn remap_libraries_path(path: &str) -> String { - **Workspace**: This is a Cargo workspace. Always specify the package when building specific apps. - **Dependencies**: External crates are defined in workspace `Cargo.toml`, not individual `Cargo.toml`. - **Database**: PostgreSQL is required. Run migrations before starting services. -- **External Tools**: The indexer relies on `unar` (for CBR) and `pdftoppm` (for PDF) being installed on the system. +- **External Tools**: 4 system tools required — `unrar` (CBR page count), `unar` (CBR extraction), `pdfinfo` (PDF page count), `pdftoppm` (PDF page render). Note: `unrar` and `unar` are distinct tools. +- **Thumbnails**: generated by the **API** service (not the indexer). The indexer triggers a checkup via `POST /index/jobs/:id/thumbnails/checkup` after indexing. +- **Sub-AGENTS.md**: module-specific guidelines in `apps/api/`, `apps/indexer/`, `apps/backoffice/`, `crates/parsers/`. diff --git a/CLAUDE.md b/CLAUDE.md new file mode 100644 index 0000000..64b158b --- /dev/null +++ b/CLAUDE.md @@ -0,0 +1,72 @@ +# Stripstream Librarian + +Gestionnaire de bibliothèque de bandes dessinées/ebooks. Workspace Cargo multi-crates avec backoffice Next.js. + +## Architecture + +| Service | Dossier | Port local | +|---------|---------|------------| +| API REST (axum) | `apps/api/` | 7080 | +| Indexer (background) | `apps/indexer/` | 7081 | +| Backoffice (Next.js) | `apps/backoffice/` | 7082 | +| PostgreSQL | infra | 6432 | +| Meilisearch | infra | 7700 | + +Crates partagés : `crates/core` (config env), `crates/parsers` (CBZ/CBR/PDF). + +## Commandes + +```bash +# Build +cargo build # workspace entier +cargo build -p api # crate spécifique +cargo build --release # version optimisée + +# Linting / format +cargo clippy +cargo fmt + +# Tests +cargo test +cargo test -p parsers + +# Infra (dépendances uniquement) — docker-compose.yml est à la racine +docker compose up -d postgres meilisearch + +# Backoffice dev +cd apps/backoffice && npm install && npm run dev # http://localhost:7082 + +# Migrations +sqlx migrate run # DATABASE_URL doit être défini +``` + +## Environnement + +```bash +cp .env.example .env # puis éditer les valeurs REQUIRED +``` + +Variables **requises** au démarrage : `DATABASE_URL`, `MEILI_URL`, `MEILI_MASTER_KEY`, `API_BOOTSTRAP_TOKEN`. + +## Gotchas + +- **Dépendances système** : 4 outils requis — `unrar` (CBR listing), `unar` (CBR extraction), `pdfinfo` (PDF page count), `pdftoppm` (PDF rendu). `unrar` ≠ `unar`. +- **Port backoffice** : `npm run dev` écoute sur **7082**, pas 3000. +- **LIBRARIES_ROOT_PATH** : les chemins en DB commencent par `/libraries/` ; en dev local, définir cette variable pour remapper vers le dossier réel. +- **Thumbnails** : stockés dans `THUMBNAIL_DIRECTORY` (défaut `/data/thumbnails`), générés par **l'API** (pas l'indexer) — l'indexer déclenche un checkup via `POST /index/jobs/:id/thumbnails/checkup`. +- **Workspace Cargo** : les dépendances externes sont définies dans le `Cargo.toml` racine, pas dans les crates individuels. +- **Migrations** : dossier `infra/migrations/`, géré par sqlx. Toujours migrer avant de démarrer les services. + +## Fichiers clés + +| Fichier | Rôle | +|---------|------| +| `crates/core/src/config.rs` | Config depuis env (API, Indexer, AdminUI) | +| `crates/parsers/src/lib.rs` | Détection format, extraction métadonnées | +| `apps/api/src/books.rs` | Endpoints CRUD livres | +| `apps/api/src/pages.rs` | Rendu pages + cache LRU | +| `apps/indexer/src/scanner.rs` | Scan filesystem | +| `infra/migrations/*.sql` | Schéma DB | + +> Voir `AGENTS.md` pour les conventions de code détaillées (error handling, patterns sqlx, async/tokio). +> Des `AGENTS.md` spécifiques existent dans `apps/api/`, `apps/indexer/`, `apps/backoffice/`, `crates/parsers/`. diff --git a/README.md b/README.md index e2d58c5..5f43d7a 100644 --- a/README.md +++ b/README.md @@ -38,16 +38,16 @@ docker compose up -d ``` This will start: -- PostgreSQL (port 5432) -- Meilisearch (port 7700) -- API service (port 8080) -- Indexer service (port 8081) -- Backoffice web UI (port 8082) +- PostgreSQL (port 6432) +- Meilisearch (port 7700) +- API service (port 7080) +- Indexer service (port 7081) +- Backoffice web UI (port 7082) ### Accessing the Application -- **Backoffice**: http://localhost:8082 -- **API**: http://localhost:8080 +- **Backoffice**: http://localhost:7082 +- **API**: http://localhost:7080 - **Meilisearch**: http://localhost:7700 ### Default Credentials @@ -113,9 +113,9 @@ The backoffice will be available at http://localhost:3000 | Variable | Description | Default | |----------|-------------|---------| -| `API_LISTEN_ADDR` | API service bind address | `0.0.0.0:8080` | -| `INDEXER_LISTEN_ADDR` | Indexer service bind address | `0.0.0.0:8081` | -| `BACKOFFICE_PORT` | Backoffice web UI port | `8082` | +| `API_LISTEN_ADDR` | API service bind address | `0.0.0.0:7080` | +| `INDEXER_LISTEN_ADDR` | Indexer service bind address | `0.0.0.0:7081` | +| `BACKOFFICE_PORT` | Backoffice web UI port | `7082` | | `DATABASE_URL` | PostgreSQL connection string | `postgres://stripstream:stripstream@postgres:5432/stripstream` | | `MEILI_URL` | Meilisearch connection URL | `http://meilisearch:7700` | | `MEILI_MASTER_KEY` | Meilisearch master key (required) | - | @@ -128,7 +128,7 @@ The backoffice will be available at http://localhost:3000 The API is documented with OpenAPI/Swagger. When running locally, access the docs at: ``` -http://localhost:8080/api-docs +http://localhost:7080/swagger-ui ``` ## Project Structure diff --git a/apps/api/AGENTS.md b/apps/api/AGENTS.md new file mode 100644 index 0000000..cccf3d7 --- /dev/null +++ b/apps/api/AGENTS.md @@ -0,0 +1,73 @@ +# apps/api — REST API (axum) + +Service HTTP sur le port **7080**. Voir `AGENTS.md` racine pour les conventions globales. + +## Structure des fichiers + +| Fichier | Rôle | +|---------|------| +| `main.rs` | Routes, initialisation AppState, Semaphore concurrent_renders | +| `state.rs` | `AppState` (pool, caches, métriques), `load_concurrent_renders` | +| `auth.rs` | Middlewares `require_admin` / `require_read`, authentification tokens | +| `error.rs` | `ApiError` avec constructeurs `bad_request`, `not_found`, `internal`, etc. | +| `books.rs` | CRUD livres, thumbnails | +| `pages.rs` | Rendu page + double cache (mémoire LRU + disque) | +| `libraries.rs` | CRUD bibliothèques, déclenchement scans | +| `index_jobs.rs` | Suivi jobs, SSE streaming progression | +| `thumbnails.rs` | Rebuild/regénération thumbnails | +| `tokens.rs` | Gestion tokens API (create/revoke) | +| `settings.rs` | Paramètres applicatifs (stockés en DB, clé `limits`) | +| `openapi.rs` | Doc OpenAPI via utoipa, accessible sur `/swagger-ui` | + +## Patterns clés + +### Handler type +```rust +async fn my_handler( + State(state): State, + Path(id): Path, +) -> Result, ApiError> { + // ... +} +``` + +### Erreurs API +```rust +// Constructeurs disponibles dans error.rs +ApiError::bad_request("message") +ApiError::not_found("resource not found") +ApiError::internal("unexpected error") +ApiError::unauthorized("missing token") +ApiError::forbidden("admin required") + +// Conversion auto depuis sqlx::Error et std::io::Error +``` + +### Authentification +- **Bootstrap token** : comparaison directe (`API_BOOTSTRAP_TOKEN`), scope Admin +- **Tokens DB** : format `stl__`, hash argon2 en DB, scope `admin` ou `read` +- Middleware `require_admin` → routes admin ; `require_read` → routes lecture + +### OpenAPI (utoipa) +```rust +#[utoipa::path(get, path = "/books/{id}", ...)] +async fn get_book(...) { } +// Ajouter le handler dans openapi.rs (ApiDoc) +``` + +### Cache pages (`pages.rs`) +- **Cache mémoire** : LRU 512 entrées (`AppState.page_cache`) +- **Cache disque** : `IMAGE_CACHE_DIR` (défaut `/tmp/stripstream-image-cache`), clé SHA256 +- Concurrence limitée par `AppState.page_render_limit` (Semaphore, configurable en DB) +- `spawn_blocking` pour le rendu image (CPU-bound) + +### Paramètre concurrent_renders +Stocké en DB : `SELECT value FROM app_settings WHERE key = 'limits'` → JSON `{"concurrent_renders": N}`. +Chargé au démarrage dans `load_concurrent_renders`. + +## Gotchas + +- **LIBRARIES_ROOT_PATH** : les `abs_path` en DB commencent par `/libraries/`. Appeler `remap_libraries_path()` avant tout accès fichier. +- **Rate limit lecture** : middleware `read_rate_limit` sur les routes read (100 req/5s par défaut). +- **Métriques** : `/metrics` expose `requests_total`, `page_cache_hits`, `page_cache_misses` (atomics dans `AppState.metrics`). +- **Swagger** : accessible sur `/swagger-ui`, spec JSON sur `/openapi.json`. diff --git a/apps/api/Dockerfile b/apps/api/Dockerfile index 7ff27c6..868d5aa 100644 --- a/apps/api/Dockerfile +++ b/apps/api/Dockerfile @@ -26,5 +26,5 @@ RUN sed -i '/en_US.UTF-8/s/^# //g' /etc/locale.gen && locale-gen ENV LANG=en_US.UTF-8 ENV LC_ALL=en_US.UTF-8 COPY --from=builder /app/target/release/api /usr/local/bin/api -EXPOSE 8080 +EXPOSE 7080 CMD ["/usr/local/bin/api"] diff --git a/apps/backoffice/.env.local b/apps/backoffice/.env.local index aa836e3..dfbe796 100644 --- a/apps/backoffice/.env.local +++ b/apps/backoffice/.env.local @@ -1,4 +1,4 @@ -API_BASE_URL=http://localhost:8080 +API_BASE_URL=http://localhost:7080 API_BOOTSTRAP_TOKEN=stripstream-dev-bootstrap-token -NEXT_PUBLIC_API_BASE_URL=http://localhost:8080 +NEXT_PUBLIC_API_BASE_URL=http://localhost:7080 NEXT_PUBLIC_API_BOOTSTRAP_TOKEN=stripstream-dev-bootstrap-token diff --git a/apps/backoffice/AGENTS.md b/apps/backoffice/AGENTS.md new file mode 100644 index 0000000..1267105 --- /dev/null +++ b/apps/backoffice/AGENTS.md @@ -0,0 +1,66 @@ +# apps/backoffice — Interface d'administration (Next.js) + +App Next.js 16 avec React 19, Tailwind CSS v4, TypeScript. Port de dev : **7082** (`npm run dev`). + +## Structure + +``` +app/ +├── layout.tsx # Layout global (nav sticky glassmorphism, ThemeProvider) +├── page.tsx # Dashboard +├── books/ # Liste et détail des livres +├── libraries/ # Gestion bibliothèques +├── jobs/ # Monitoring jobs +├── tokens/ # Tokens API +├── settings/ # Paramètres +├── components/ # Composants métier +│ ├── ui/ # Composants génériques (Button, Card, Badge, Icon, Input, ProgressBar, StatBox...) +│ ├── BookCard.tsx +│ ├── JobProgress.tsx +│ ├── JobsList.tsx +│ ├── LibraryForm.tsx +│ ├── FolderBrowser.tsx / FolderPicker.tsx +│ └── ... +└── globals.css # Variables CSS, Tailwind base +lib/ +└── api.ts # Client API : types DTO + fonctions fetch vers l'API Rust +``` + +## Client API (lib/api.ts) + +Tous les appels vers l'API Rust passent par `lib/api.ts`. Les types DTO sont définis là : +- `LibraryDto`, `IndexJobDto`, `BookDto`, `TokenDto`, `FolderItem` + +Ajouter les nouveaux endpoints et types dans ce fichier. + +## Composants UI + +Les composants génériques sont dans `app/components/ui/`. Utiliser ces composants plutôt que des éléments HTML bruts : + +```tsx +import { Button, Card, Badge, Icon, Input, ProgressBar, StatBox } from "@/app/components/ui"; +``` + +## Conventions + +- **App Router** : toutes les pages sont des Server Components par défaut. Utiliser `"use client"` seulement pour l'interactivité. +- **Tailwind v4** : config dans `postcss.config.js` + `tailwind.config.js`. Variables CSS dans `globals.css`. +- **Thème** : `ThemeProvider` + `ThemeToggle` pour dark/light mode via `next-themes`. +- **Icônes** : composant `` dans `ui/Icon.tsx` — pas de librairie externe. +- **Navigation** : routes typées dans `layout.tsx` (`"/" | "/books" | "/libraries" | "/jobs" | "/tokens" | "/settings"`). + +## Commandes + +```bash +npm install +npm run dev # http://localhost:7082 +npm run build +npm run start # Production sur http://localhost:7082 +``` + +## Gotchas + +- **Port 7082** : pas le port Next.js par défaut (3000). Défini dans `package.json` scripts (`-p 7082`). +- **API_BASE_URL** : en prod, configuré via env. En dev local, l'API doit tourner sur `http://localhost:7080`. +- **React 19 + Next.js 16** : utiliser les nouvelles APIs (actions serveur, `use()` hook) si disponibles. +- **Pas de gestion d'état global** : fetch direct depuis les Server Components ou `useState`/`useEffect` dans les Client Components. diff --git a/apps/backoffice/Dockerfile b/apps/backoffice/Dockerfile index 48c9c29..4ffdd5f 100644 --- a/apps/backoffice/Dockerfile +++ b/apps/backoffice/Dockerfile @@ -12,11 +12,11 @@ RUN npm run build FROM node:22-alpine AS runner WORKDIR /app ENV NODE_ENV=production -ENV PORT=8082 +ENV PORT=7082 ENV HOST=0.0.0.0 RUN apk add --no-cache wget COPY --from=builder /app/.next/standalone ./ COPY --from=builder /app/.next/static ./.next/static COPY --from=builder /app/public ./public -EXPOSE 8082 +EXPOSE 7082 CMD ["node", "server.js"] diff --git a/apps/backoffice/app/api/books/[bookId]/pages/[pageNum]/route.ts b/apps/backoffice/app/api/books/[bookId]/pages/[pageNum]/route.ts index 5197c79..efc4702 100644 --- a/apps/backoffice/app/api/books/[bookId]/pages/[pageNum]/route.ts +++ b/apps/backoffice/app/api/books/[bookId]/pages/[pageNum]/route.ts @@ -13,7 +13,7 @@ export async function GET( const quality = searchParams.get("quality") || ""; // Construire l'URL vers l'API backend - const apiBaseUrl = process.env.API_BASE_URL || "http://api:8080"; + const apiBaseUrl = process.env.API_BASE_URL || "http://api:7080"; const apiUrl = new URL(`${apiBaseUrl}/books/${bookId}/pages/${pageNum}`); apiUrl.searchParams.set("format", format); if (width) apiUrl.searchParams.set("width", width); diff --git a/apps/backoffice/app/api/books/[bookId]/thumbnail/route.ts b/apps/backoffice/app/api/books/[bookId]/thumbnail/route.ts index ead50d1..22c2a96 100644 --- a/apps/backoffice/app/api/books/[bookId]/thumbnail/route.ts +++ b/apps/backoffice/app/api/books/[bookId]/thumbnail/route.ts @@ -6,7 +6,7 @@ export async function GET( ) { const { bookId } = await params; - const apiBaseUrl = process.env.API_BASE_URL || "http://api:8080"; + const apiBaseUrl = process.env.API_BASE_URL || "http://api:7080"; const apiUrl = `${apiBaseUrl}/books/${bookId}/thumbnail`; const token = process.env.API_BOOTSTRAP_TOKEN; diff --git a/apps/backoffice/app/api/folders/route.ts b/apps/backoffice/app/api/folders/route.ts index 87eaf6f..37fb8cb 100644 --- a/apps/backoffice/app/api/folders/route.ts +++ b/apps/backoffice/app/api/folders/route.ts @@ -1,7 +1,7 @@ import { NextRequest, NextResponse } from "next/server"; export async function GET(request: NextRequest) { - const apiBaseUrl = process.env.API_BASE_URL || "http://api:8080"; + const apiBaseUrl = process.env.API_BASE_URL || "http://api:7080"; const apiToken = process.env.API_BOOTSTRAP_TOKEN; if (!apiToken) { diff --git a/apps/backoffice/app/api/jobs/[id]/cancel/route.ts b/apps/backoffice/app/api/jobs/[id]/cancel/route.ts index fe2ef5d..7d47f12 100644 --- a/apps/backoffice/app/api/jobs/[id]/cancel/route.ts +++ b/apps/backoffice/app/api/jobs/[id]/cancel/route.ts @@ -5,7 +5,7 @@ export async function POST( { params }: { params: Promise<{ id: string }> } ) { const { id } = await params; - const apiBaseUrl = process.env.API_BASE_URL || "http://api:8080"; + const apiBaseUrl = process.env.API_BASE_URL || "http://api:7080"; const apiToken = process.env.API_BOOTSTRAP_TOKEN; if (!apiToken) { diff --git a/apps/backoffice/app/api/jobs/[id]/route.ts b/apps/backoffice/app/api/jobs/[id]/route.ts index 35eb915..eee6fc0 100644 --- a/apps/backoffice/app/api/jobs/[id]/route.ts +++ b/apps/backoffice/app/api/jobs/[id]/route.ts @@ -5,7 +5,7 @@ export async function GET( { params }: { params: Promise<{ id: string }> } ) { const { id } = await params; - const apiBaseUrl = process.env.API_BASE_URL || "http://api:8080"; + const apiBaseUrl = process.env.API_BASE_URL || "http://api:7080"; const apiToken = process.env.API_BOOTSTRAP_TOKEN; if (!apiToken) { diff --git a/apps/backoffice/app/api/jobs/[id]/stream/route.ts b/apps/backoffice/app/api/jobs/[id]/stream/route.ts index 45cd77a..33617e6 100644 --- a/apps/backoffice/app/api/jobs/[id]/stream/route.ts +++ b/apps/backoffice/app/api/jobs/[id]/stream/route.ts @@ -5,7 +5,7 @@ export async function GET( { params }: { params: Promise<{ id: string }> } ) { const { id } = await params; - const apiBaseUrl = process.env.API_BASE_URL || "http://api:8080"; + const apiBaseUrl = process.env.API_BASE_URL || "http://api:7080"; const apiToken = process.env.API_BOOTSTRAP_TOKEN; if (!apiToken) { diff --git a/apps/backoffice/app/api/jobs/route.ts b/apps/backoffice/app/api/jobs/route.ts index b3b2e79..15c2cb1 100644 --- a/apps/backoffice/app/api/jobs/route.ts +++ b/apps/backoffice/app/api/jobs/route.ts @@ -1,7 +1,7 @@ import { NextRequest, NextResponse } from "next/server"; export async function GET(request: NextRequest) { - const apiBaseUrl = process.env.API_BASE_URL || "http://api:8080"; + const apiBaseUrl = process.env.API_BASE_URL || "http://api:7080"; const apiToken = process.env.API_BOOTSTRAP_TOKEN; if (!apiToken) { diff --git a/apps/backoffice/app/api/jobs/stream/route.ts b/apps/backoffice/app/api/jobs/stream/route.ts index 857f10b..4e4e282 100644 --- a/apps/backoffice/app/api/jobs/stream/route.ts +++ b/apps/backoffice/app/api/jobs/stream/route.ts @@ -1,7 +1,7 @@ import { NextRequest } from "next/server"; export async function GET(request: NextRequest) { - const apiBaseUrl = process.env.API_BASE_URL || "http://api:8080"; + const apiBaseUrl = process.env.API_BASE_URL || "http://api:7080"; const apiToken = process.env.API_BOOTSTRAP_TOKEN; if (!apiToken) { diff --git a/apps/backoffice/app/api/settings/[key]/route.ts b/apps/backoffice/app/api/settings/[key]/route.ts index d98cd8f..a3891b1 100644 --- a/apps/backoffice/app/api/settings/[key]/route.ts +++ b/apps/backoffice/app/api/settings/[key]/route.ts @@ -6,7 +6,7 @@ export async function GET( ) { try { const { key } = await params; - const baseUrl = process.env.API_BASE_URL || "http://api:8080"; + const baseUrl = process.env.API_BASE_URL || "http://api:7080"; const token = process.env.API_BOOTSTRAP_TOKEN; const response = await fetch(`${baseUrl}/settings/${key}`, { @@ -33,7 +33,7 @@ export async function POST( ) { try { const { key } = await params; - const baseUrl = process.env.API_BASE_URL || "http://api:8080"; + const baseUrl = process.env.API_BASE_URL || "http://api:7080"; const token = process.env.API_BOOTSTRAP_TOKEN; const body = await request.json(); diff --git a/apps/backoffice/app/api/settings/cache/clear/route.ts b/apps/backoffice/app/api/settings/cache/clear/route.ts index 8a554c4..a462dfd 100644 --- a/apps/backoffice/app/api/settings/cache/clear/route.ts +++ b/apps/backoffice/app/api/settings/cache/clear/route.ts @@ -2,7 +2,7 @@ import { NextRequest, NextResponse } from "next/server"; export async function POST(request: NextRequest) { try { - const baseUrl = process.env.API_BASE_URL || "http://api:8080"; + const baseUrl = process.env.API_BASE_URL || "http://api:7080"; const token = process.env.API_BOOTSTRAP_TOKEN; const response = await fetch(`${baseUrl}/settings/cache/clear`, { diff --git a/apps/backoffice/app/api/settings/cache/stats/route.ts b/apps/backoffice/app/api/settings/cache/stats/route.ts index b73c7b7..6db496a 100644 --- a/apps/backoffice/app/api/settings/cache/stats/route.ts +++ b/apps/backoffice/app/api/settings/cache/stats/route.ts @@ -2,7 +2,7 @@ import { NextRequest, NextResponse } from "next/server"; export async function GET(request: NextRequest) { try { - const baseUrl = process.env.API_BASE_URL || "http://api:8080"; + const baseUrl = process.env.API_BASE_URL || "http://api:7080"; const token = process.env.API_BOOTSTRAP_TOKEN; const response = await fetch(`${baseUrl}/settings/cache/stats`, { diff --git a/apps/backoffice/app/api/settings/route.ts b/apps/backoffice/app/api/settings/route.ts index 1e48dae..41b838a 100644 --- a/apps/backoffice/app/api/settings/route.ts +++ b/apps/backoffice/app/api/settings/route.ts @@ -2,7 +2,7 @@ import { NextRequest, NextResponse } from "next/server"; export async function GET(request: NextRequest) { try { - const baseUrl = process.env.API_BASE_URL || "http://api:8080"; + const baseUrl = process.env.API_BASE_URL || "http://api:7080"; const token = process.env.API_BOOTSTRAP_TOKEN; const response = await fetch(`${baseUrl}/settings`, { diff --git a/apps/backoffice/lib/api.ts b/apps/backoffice/lib/api.ts index 6fdb801..e75cf75 100644 --- a/apps/backoffice/lib/api.ts +++ b/apps/backoffice/lib/api.ts @@ -90,7 +90,7 @@ export type SeriesDto = { }; function config() { - const baseUrl = process.env.API_BASE_URL || "http://api:8080"; + const baseUrl = process.env.API_BASE_URL || "http://api:7080"; const token = process.env.API_BOOTSTRAP_TOKEN; if (!token) { throw new Error("API_BOOTSTRAP_TOKEN is required for backoffice"); diff --git a/apps/indexer/AGENTS.md b/apps/indexer/AGENTS.md new file mode 100644 index 0000000..6dd0b3c --- /dev/null +++ b/apps/indexer/AGENTS.md @@ -0,0 +1,75 @@ +# apps/indexer — Service d'indexation + +Service background sur le port **7081**. Voir `AGENTS.md` racine pour les conventions globales. + +## Structure des fichiers + +| Fichier | Rôle | +|---------|------| +| `main.rs` | Point d'entrée, initialisation, lancement du worker | +| `lib.rs` | `AppState` (pool, meili, api_base_url) | +| `worker.rs` | Boucle principale : claim job → process → cleanup stale | +| `job.rs` | `claim_next_job`, `process_job`, `fail_job`, `cleanup_stale_jobs` | +| `scanner.rs` | Scan filesystem, parsing parallèle (rayon), batching DB | +| `batch.rs` | `flush_all_batches` avec UNNEST, structures `BookInsert/Update/FileInsert/Update/ErrorInsert` | +| `scheduler.rs` | Auto-scan : vérifie toutes les 60s les bibliothèques à monitorer | +| `watcher.rs` | File watcher temps réel | +| `meili.rs` | Indexation/sync Meilisearch | +| `api.rs` | Appels HTTP vers l'API (pour checkup thumbnails) | +| `utils.rs` | `remap_libraries_path`, `unmap_libraries_path`, `compute_fingerprint`, `kind_from_format` | + +## Cycle de vie d'un job + +``` +claim_next_job (UPDATE ... RETURNING, status pending→running) + └─ process_job + ├─ scanner::scan_library (rayon par_iter pour le parsing) + │ └─ flush_all_batches toutes les BATCH_SIZE=100 itérations + └─ meili sync + └─ api checkup thumbnails (POST /index/jobs/:id/thumbnails/checkup) +``` + +- Annulation : `is_job_cancelled` vérifié toutes les 10 fichiers ou 1s — retourne `Err("Job cancelled")` +- Jobs stale (running au redémarrage) → nettoyés par `cleanup_stale_jobs` au boot + +## Pattern batch (batch.rs) + +Toutes les opérations DB massives passent par `flush_all_batches` avec UNNEST : + +```rust +// Accumuler dans des Vec, Vec, etc. +books_to_insert.push(BookInsert { ... }); + +// Flush quand plein ou en fin de scan +if books_to_insert.len() >= BATCH_SIZE { + flush_all_batches(&pool, &mut books_update, &mut files_update, + &mut books_insert, &mut files_insert, &mut errors_insert).await?; +} +``` + +Toutes les opérations du flush sont dans une seule transaction. + +## Scan filesystem (scanner.rs) + +Pipeline en 3 étapes : +1. **Collect** : WalkDir → filtrer par format (CBZ/CBR/PDF) +2. **Parse** : `file_infos.into_par_iter().map(parse_metadata)` (rayon) +3. **Process** : séquentiel pour les inserts/updates DB + +Fingerprint = SHA256(taille + mtime) pour détecter les changements sans relire le fichier. + +## Path remapping + +```rust +// abs_path en DB = chemin conteneur (/libraries/...) +// Sur l'hôte : LIBRARIES_ROOT_PATH remplace /libraries +utils::remap_libraries_path(&abs_path) // DB → filesystem local +utils::unmap_libraries_path(&local_path) // filesystem local → DB +``` + +## Gotchas + +- **Thumbnails** : générés par l'API après handoff, pas par l'indexer directement. L'indexer appelle `/index/jobs/:id/thumbnails/checkup` via `api.rs`. +- **full_rebuild** : si `true`, ignore les fingerprints → tous les fichiers sont retraités. +- **Annulation** : vérifier `is_job_cancelled` régulièrement pour respecter les annulations utilisateur. +- **Watcher + scheduler** : tournent en tâches tokio séparées dans `worker.rs`, en parallèle de la boucle principale. diff --git a/apps/indexer/Dockerfile b/apps/indexer/Dockerfile index 4be7f35..3e20c52 100644 --- a/apps/indexer/Dockerfile +++ b/apps/indexer/Dockerfile @@ -23,5 +23,5 @@ RUN --mount=type=cache,target=/sccache \ FROM debian:bookworm-slim RUN apt-get update && apt-get install -y --no-install-recommends ca-certificates wget unrar-free && rm -rf /var/lib/apt/lists/* COPY --from=builder /app/target/release/indexer /usr/local/bin/indexer -EXPOSE 8081 +EXPOSE 7081 CMD ["/usr/local/bin/indexer"] diff --git a/crates/core/src/config.rs b/crates/core/src/config.rs index 20271d3..365d549 100644 --- a/crates/core/src/config.rs +++ b/crates/core/src/config.rs @@ -13,7 +13,7 @@ impl ApiConfig { pub fn from_env() -> Result { Ok(Self { listen_addr: std::env::var("API_LISTEN_ADDR") - .unwrap_or_else(|_| "0.0.0.0:8080".to_string()), + .unwrap_or_else(|_| "0.0.0.0:7080".to_string()), database_url: std::env::var("DATABASE_URL").context("DATABASE_URL is required")?, meili_url: std::env::var("MEILI_URL").context("MEILI_URL is required")?, meili_master_key: std::env::var("MEILI_MASTER_KEY") @@ -32,7 +32,7 @@ pub struct IndexerConfig { pub meili_master_key: String, pub scan_interval_seconds: u64, pub thumbnail_config: ThumbnailConfig, - /// API base URL for thumbnail checkup at end of build (e.g. http://api:8080) + /// API base URL for thumbnail checkup at end of build (e.g. http://api:7080) pub api_base_url: String, /// Token to call API (e.g. API_BOOTSTRAP_TOKEN) pub api_bootstrap_token: String, @@ -87,7 +87,7 @@ impl IndexerConfig { Ok(Self { listen_addr: std::env::var("INDEXER_LISTEN_ADDR") - .unwrap_or_else(|_| "0.0.0.0:8081".to_string()), + .unwrap_or_else(|_| "0.0.0.0:7081".to_string()), database_url: std::env::var("DATABASE_URL").context("DATABASE_URL is required")?, meili_url: std::env::var("MEILI_URL").context("MEILI_URL is required")?, meili_master_key: std::env::var("MEILI_MASTER_KEY") @@ -98,7 +98,7 @@ impl IndexerConfig { .unwrap_or(5), thumbnail_config, api_base_url: std::env::var("API_BASE_URL") - .unwrap_or_else(|_| "http://api:8080".to_string()), + .unwrap_or_else(|_| "http://api:7080".to_string()), api_bootstrap_token: std::env::var("API_BOOTSTRAP_TOKEN") .context("API_BOOTSTRAP_TOKEN is required for thumbnail checkup")?, }) @@ -116,9 +116,9 @@ impl AdminUiConfig { pub fn from_env() -> Result { Ok(Self { listen_addr: std::env::var("ADMIN_UI_LISTEN_ADDR") - .unwrap_or_else(|_| "0.0.0.0:8082".to_string()), + .unwrap_or_else(|_| "0.0.0.0:7082".to_string()), api_base_url: std::env::var("API_BASE_URL") - .unwrap_or_else(|_| "http://api:8080".to_string()), + .unwrap_or_else(|_| "http://api:7080".to_string()), api_token: std::env::var("API_BOOTSTRAP_TOKEN") .context("API_BOOTSTRAP_TOKEN is required")?, }) diff --git a/crates/parsers/AGENTS.md b/crates/parsers/AGENTS.md new file mode 100644 index 0000000..47a89a0 --- /dev/null +++ b/crates/parsers/AGENTS.md @@ -0,0 +1,76 @@ +# crates/parsers — Parsing de livres (CBZ, CBR, PDF) + +Crate utilitaire sans état, utilisée par `apps/api` et `apps/indexer`. + +## API publique (lib.rs) + +```rust +// Détection du format par extension +pub fn detect_format(path: &Path) -> Option // .cbz | .cbr | .pdf + +// Extraction des métadonnées +pub fn parse_metadata(path: &Path, format: BookFormat, library_root: &Path) -> Result + +// Extraction de la première page (pour thumbnails) +pub fn extract_first_page(path: &Path, format: BookFormat) -> Result> + +pub enum BookFormat { Cbz, Cbr, Pdf } + +pub struct ParsedMetadata { + pub title: String, // = nom de fichier (sans extension) + pub series: Option, // = premier dossier relatif à library_root + pub volume: Option, // extrait du nom de fichier + pub page_count: Option, +} +``` + +## Logique de parsing + +### Titre +Nom de fichier sans extension, conservé tel quel (pas de nettoyage). + +### Série +Premier composant du chemin relatif entre `library_root` et le fichier : +- `/libraries/One Piece/T01.cbz` → série = `"One Piece"` +- `/libraries/one-shot.cbz` → série = `None` + +### Volume (`extract_volume`) +Patterns reconnus dans le nom de fichier (dans l'ordre de priorité) : +- `T01`, `T1` (manga/comics français) +- `Vol. 1`, `Vol 1`, `Volume 1` +- `#1`, `#01` +- `- 1`, `- 01` (en fin de nom) + +### Nombre de pages +| Format | Outil | +|--------|-------| +| CBZ | `zip::ZipArchive` — compte les entrées image (jpg/jpeg/png/webp/avif) | +| CBR | `unrar lb ` — liste les fichiers, filtre les images | +| PDF | `pdfinfo ` — lit la ligne `Pages:` | + +## Dépendances système requises + +| Outil | Utilisé pour | Installation | +|-------|-------------|-------------| +| `unrar` | CBR page count | `brew install rar` / `apt install unrar` | +| `unar` | CBR first page extraction | `brew install unar` / `apt install unar` | +| `pdfinfo` | PDF page count | inclus dans `poppler-utils` | +| `pdftoppm` | PDF first page render | inclus dans `poppler-utils` | + +**Important** : `unrar` (pour le listing) et `unar` (pour l'extraction) sont deux outils différents. + +## Extraction première page + +- **CBZ** : `zip::ZipArchive`, trie les noms d'images, lit la première +- **CBR** : `unar -o `, `WalkDir` récursif, trie, lit la première — nettoie `tmp_dir` ensuite +- **PDF** : `pdftoppm -f 1 -singlefile -png -scale-to 800` → fichier PNG temporaire — nettoie `tmp_dir` ensuite + +Répertoire temp : `std::env::temp_dir()/stripstream-{cbr|pdf}-thumb-`. + +## Gotchas + +- `clean_title()` existe mais est marqué `#[allow(dead_code)]` — le titre n'est **pas** nettoyé (décision volontaire). +- Les CBR peuvent avoir des sous-dossiers internes → WalkDir nécessaire (pas de listing plat). +- La détection du format est **uniquement par extension** (pas de magic bytes). +- `pdfinfo` et `pdftoppm` doivent être du paquet `poppler-utils` (pas `poppler` seul). +- En cas d'échec de parsing, l'appelant (indexer/api) stocke `parse_status = 'error'` en DB mais continue. diff --git a/docker-compose.yml b/docker-compose.yml index 98add3c..e7071a0 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -54,7 +54,7 @@ services: env_file: - .env ports: - - "7080:8080" + - "7080:7080" volumes: - ${LIBRARIES_HOST_PATH:-./libraries}:/libraries - ${THUMBNAILS_HOST_PATH:-./data/thumbnails}:/data/thumbnails @@ -66,7 +66,7 @@ services: meilisearch: condition: service_healthy healthcheck: - test: ["CMD", "wget", "-q", "-O", "-", "http://127.0.0.1:8080/health"] + test: ["CMD", "wget", "-q", "-O", "-", "http://127.0.0.1:7080/health"] interval: 10s timeout: 5s retries: 5 @@ -78,7 +78,7 @@ services: env_file: - .env ports: - - "7081:8081" + - "7081:7081" volumes: - ${LIBRARIES_HOST_PATH:-./libraries}:/libraries - ${THUMBNAILS_HOST_PATH:-./data/thumbnails}:/data/thumbnails @@ -90,7 +90,7 @@ services: meilisearch: condition: service_healthy healthcheck: - test: ["CMD", "wget", "-q", "-O", "-", "http://127.0.0.1:8081/health"] + test: ["CMD", "wget", "-q", "-O", "-", "http://127.0.0.1:7081/health"] interval: 10s timeout: 5s retries: 5 @@ -102,15 +102,15 @@ services: env_file: - .env environment: - - PORT=8082 + - PORT=7082 - HOST=0.0.0.0 ports: - - "7082:8082" + - "7082:7082" depends_on: api: condition: service_healthy healthcheck: - test: ["CMD", "wget", "-q", "-O", "-", "http://127.0.0.1:8082/health"] + test: ["CMD", "wget", "-q", "-O", "-", "http://127.0.0.1:7082/health"] interval: 10s timeout: 5s retries: 5 diff --git a/infra/bench.sh b/infra/bench.sh index 2b9a329..65d57ce 100755 --- a/infra/bench.sh +++ b/infra/bench.sh @@ -1,7 +1,7 @@ #!/usr/bin/env bash set -euo pipefail -BASE_API="${BASE_API:-http://127.0.0.1:8080}" +BASE_API="${BASE_API:-http://127.0.0.1:7080}" TOKEN="${API_TOKEN:-stripstream-dev-bootstrap-token}" measure() { diff --git a/infra/smoke.sh b/infra/smoke.sh index 70f51a1..9b57053 100755 --- a/infra/smoke.sh +++ b/infra/smoke.sh @@ -1,9 +1,9 @@ #!/usr/bin/env bash set -euo pipefail -BASE_API="${BASE_API:-http://127.0.0.1:8080}" -BASE_INDEXER="${BASE_INDEXER:-http://127.0.0.1:8081}" -BASE_BACKOFFICE="${BASE_BACKOFFICE:-${BASE_ADMIN:-http://127.0.0.1:8082}}" +BASE_API="${BASE_API:-http://127.0.0.1:7080}" +BASE_INDEXER="${BASE_INDEXER:-http://127.0.0.1:7081}" +BASE_BACKOFFICE="${BASE_BACKOFFICE:-${BASE_ADMIN:-http://127.0.0.1:7082}}" TOKEN="${API_TOKEN:-stripstream-dev-bootstrap-token}" echo "[smoke] health checks"