docs: add AGENTS.md per module and unify ports to 70XX
- Add CLAUDE.md at root and AGENTS.md in apps/api, apps/indexer, apps/backoffice, crates/parsers with module-specific guidelines - Unify all service ports to 70XX (no more internal/external split): API 7080, Indexer 7081, Backoffice 7082 - Update docker-compose.yml, Dockerfiles, config.rs defaults, .env.example, backoffice routes, bench.sh, smoke.sh Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
12
.env.example
12
.env.example
@@ -21,11 +21,11 @@ API_BOOTSTRAP_TOKEN=change-me-in-production
|
||||
# =============================================================================
|
||||
|
||||
# API Service
|
||||
API_LISTEN_ADDR=0.0.0.0:8080
|
||||
API_BASE_URL=http://api:8080
|
||||
API_LISTEN_ADDR=0.0.0.0:7080
|
||||
API_BASE_URL=http://api:7080
|
||||
|
||||
# Indexer Service
|
||||
INDEXER_LISTEN_ADDR=0.0.0.0:8081
|
||||
INDEXER_LISTEN_ADDR=0.0.0.0:7081
|
||||
INDEXER_SCAN_INTERVAL_SECONDS=5
|
||||
|
||||
# Meilisearch Search Engine
|
||||
@@ -56,8 +56,8 @@ THUMBNAILS_HOST_PATH=../data/thumbnails
|
||||
# Port Configuration
|
||||
# =============================================================================
|
||||
# To change ports, edit docker-compose.yml directly:
|
||||
# - API: change "7080:8080" to "YOUR_PORT:8080"
|
||||
# - Indexer: change "7081:8081" to "YOUR_PORT:8081"
|
||||
# - Backoffice: change "7082:8082" to "YOUR_PORT:8082"
|
||||
# - API: change "7080:7080" to "YOUR_PORT:7080"
|
||||
# - Indexer: change "7081:7081" to "YOUR_PORT:7081"
|
||||
# - Backoffice: change "7082:7082" to "YOUR_PORT:7082"
|
||||
# - Meilisearch: change "7700:7700" to "YOUR_PORT:7700"
|
||||
# - PostgreSQL: change "6432:5432" to "YOUR_PORT:5432"
|
||||
|
||||
45
AGENTS.md
45
AGENTS.md
@@ -73,12 +73,14 @@ sqlx migrate add -r migration_name
|
||||
|
||||
### Docker Development
|
||||
|
||||
`docker-compose.yml` est à la **racine** du projet (pas dans `infra/`).
|
||||
|
||||
```bash
|
||||
# Start infrastructure only
|
||||
cd infra && docker compose up -d postgres meilisearch
|
||||
docker compose up -d postgres meilisearch
|
||||
|
||||
# Start full stack
|
||||
cd infra && docker compose up -d
|
||||
docker compose up -d
|
||||
|
||||
# View logs
|
||||
docker compose logs -f api
|
||||
@@ -226,24 +228,21 @@ pub struct BookItem {
|
||||
```
|
||||
stripstream-librarian/
|
||||
├── apps/
|
||||
│ ├── api/ # REST API (axum)
|
||||
│ │ └── src/
|
||||
│ │ ├── main.rs
|
||||
│ │ ├── books.rs
|
||||
│ │ ├── pages.rs
|
||||
│ │ └── ...
|
||||
│ ├── indexer/ # Background indexing service
|
||||
│ │ └── src/
|
||||
│ │ └── main.rs
|
||||
│ └── backoffice/ # Next.js admin UI
|
||||
│ ├── api/ # REST API (axum) — port 7080
|
||||
│ │ └── src/ # books.rs, pages.rs, thumbnails.rs, state.rs, auth.rs...
|
||||
│ ├── indexer/ # Background indexing service — port 7081
|
||||
│ │ └── src/ # worker.rs, scanner.rs, batch.rs, scheduler.rs, watcher.rs...
|
||||
│ └── backoffice/ # Next.js admin UI — port 7082
|
||||
├── crates/
|
||||
│ ├── core/ # Shared config
|
||||
│ ├── core/ # Shared config (env vars)
|
||||
│ │ └── src/config.rs
|
||||
│ └── parsers/ # Book parsing (CBZ, CBR, PDF)
|
||||
├── infra/
|
||||
│ ├── migrations/ # SQL migrations
|
||||
│ └── docker-compose.yml
|
||||
└── libraries/ # Book storage (mounted volume)
|
||||
│ └── migrations/ # SQL migrations (sqlx)
|
||||
├── data/
|
||||
│ └── thumbnails/ # Thumbnails générés par l'API
|
||||
├── libraries/ # Book storage (mounted volume)
|
||||
└── docker-compose.yml # À la racine (pas dans infra/)
|
||||
```
|
||||
|
||||
### Key Files
|
||||
@@ -251,8 +250,12 @@ stripstream-librarian/
|
||||
| File | Purpose |
|
||||
|------|---------|
|
||||
| `apps/api/src/books.rs` | Book CRUD endpoints |
|
||||
| `apps/api/src/pages.rs` | Page rendering & caching |
|
||||
| `apps/indexer/src/main.rs` | Indexing logic, batch processing |
|
||||
| `apps/api/src/pages.rs` | Page rendering & caching (LRU + disk) |
|
||||
| `apps/api/src/thumbnails.rs` | Thumbnail generation (triggered by indexer) |
|
||||
| `apps/api/src/state.rs` | AppState, Semaphore concurrent_renders |
|
||||
| `apps/indexer/src/scanner.rs` | Filesystem scan, rayon parallel parsing |
|
||||
| `apps/indexer/src/batch.rs` | Bulk DB ops via UNNEST |
|
||||
| `apps/indexer/src/worker.rs` | Job loop, watcher, scheduler orchestration |
|
||||
| `crates/parsers/src/lib.rs` | Format detection, metadata parsing |
|
||||
| `crates/core/src/config.rs` | Configuration from environment |
|
||||
| `infra/migrations/*.sql` | Database schema |
|
||||
@@ -269,7 +272,7 @@ impl IndexerConfig {
|
||||
pub fn from_env() -> Result<Self> {
|
||||
Ok(Self {
|
||||
listen_addr: std::env::var("INDEXER_LISTEN_ADDR")
|
||||
.unwrap_or_else(|_| "0.0.0.0:8081".to_string()),
|
||||
.unwrap_or_else(|_| "0.0.0.0:7081".to_string()),
|
||||
database_url: std::env::var("DATABASE_URL")
|
||||
.context("DATABASE_URL is required")?,
|
||||
// ...
|
||||
@@ -298,4 +301,6 @@ fn remap_libraries_path(path: &str) -> String {
|
||||
- **Workspace**: This is a Cargo workspace. Always specify the package when building specific apps.
|
||||
- **Dependencies**: External crates are defined in workspace `Cargo.toml`, not individual `Cargo.toml`.
|
||||
- **Database**: PostgreSQL is required. Run migrations before starting services.
|
||||
- **External Tools**: The indexer relies on `unar` (for CBR) and `pdftoppm` (for PDF) being installed on the system.
|
||||
- **External Tools**: 4 system tools required — `unrar` (CBR page count), `unar` (CBR extraction), `pdfinfo` (PDF page count), `pdftoppm` (PDF page render). Note: `unrar` and `unar` are distinct tools.
|
||||
- **Thumbnails**: generated by the **API** service (not the indexer). The indexer triggers a checkup via `POST /index/jobs/:id/thumbnails/checkup` after indexing.
|
||||
- **Sub-AGENTS.md**: module-specific guidelines in `apps/api/`, `apps/indexer/`, `apps/backoffice/`, `crates/parsers/`.
|
||||
|
||||
72
CLAUDE.md
Normal file
72
CLAUDE.md
Normal file
@@ -0,0 +1,72 @@
|
||||
# Stripstream Librarian
|
||||
|
||||
Gestionnaire de bibliothèque de bandes dessinées/ebooks. Workspace Cargo multi-crates avec backoffice Next.js.
|
||||
|
||||
## Architecture
|
||||
|
||||
| Service | Dossier | Port local |
|
||||
|---------|---------|------------|
|
||||
| API REST (axum) | `apps/api/` | 7080 |
|
||||
| Indexer (background) | `apps/indexer/` | 7081 |
|
||||
| Backoffice (Next.js) | `apps/backoffice/` | 7082 |
|
||||
| PostgreSQL | infra | 6432 |
|
||||
| Meilisearch | infra | 7700 |
|
||||
|
||||
Crates partagés : `crates/core` (config env), `crates/parsers` (CBZ/CBR/PDF).
|
||||
|
||||
## Commandes
|
||||
|
||||
```bash
|
||||
# Build
|
||||
cargo build # workspace entier
|
||||
cargo build -p api # crate spécifique
|
||||
cargo build --release # version optimisée
|
||||
|
||||
# Linting / format
|
||||
cargo clippy
|
||||
cargo fmt
|
||||
|
||||
# Tests
|
||||
cargo test
|
||||
cargo test -p parsers
|
||||
|
||||
# Infra (dépendances uniquement) — docker-compose.yml est à la racine
|
||||
docker compose up -d postgres meilisearch
|
||||
|
||||
# Backoffice dev
|
||||
cd apps/backoffice && npm install && npm run dev # http://localhost:7082
|
||||
|
||||
# Migrations
|
||||
sqlx migrate run # DATABASE_URL doit être défini
|
||||
```
|
||||
|
||||
## Environnement
|
||||
|
||||
```bash
|
||||
cp .env.example .env # puis éditer les valeurs REQUIRED
|
||||
```
|
||||
|
||||
Variables **requises** au démarrage : `DATABASE_URL`, `MEILI_URL`, `MEILI_MASTER_KEY`, `API_BOOTSTRAP_TOKEN`.
|
||||
|
||||
## Gotchas
|
||||
|
||||
- **Dépendances système** : 4 outils requis — `unrar` (CBR listing), `unar` (CBR extraction), `pdfinfo` (PDF page count), `pdftoppm` (PDF rendu). `unrar` ≠ `unar`.
|
||||
- **Port backoffice** : `npm run dev` écoute sur **7082**, pas 3000.
|
||||
- **LIBRARIES_ROOT_PATH** : les chemins en DB commencent par `/libraries/` ; en dev local, définir cette variable pour remapper vers le dossier réel.
|
||||
- **Thumbnails** : stockés dans `THUMBNAIL_DIRECTORY` (défaut `/data/thumbnails`), générés par **l'API** (pas l'indexer) — l'indexer déclenche un checkup via `POST /index/jobs/:id/thumbnails/checkup`.
|
||||
- **Workspace Cargo** : les dépendances externes sont définies dans le `Cargo.toml` racine, pas dans les crates individuels.
|
||||
- **Migrations** : dossier `infra/migrations/`, géré par sqlx. Toujours migrer avant de démarrer les services.
|
||||
|
||||
## Fichiers clés
|
||||
|
||||
| Fichier | Rôle |
|
||||
|---------|------|
|
||||
| `crates/core/src/config.rs` | Config depuis env (API, Indexer, AdminUI) |
|
||||
| `crates/parsers/src/lib.rs` | Détection format, extraction métadonnées |
|
||||
| `apps/api/src/books.rs` | Endpoints CRUD livres |
|
||||
| `apps/api/src/pages.rs` | Rendu pages + cache LRU |
|
||||
| `apps/indexer/src/scanner.rs` | Scan filesystem |
|
||||
| `infra/migrations/*.sql` | Schéma DB |
|
||||
|
||||
> Voir `AGENTS.md` pour les conventions de code détaillées (error handling, patterns sqlx, async/tokio).
|
||||
> Des `AGENTS.md` spécifiques existent dans `apps/api/`, `apps/indexer/`, `apps/backoffice/`, `crates/parsers/`.
|
||||
22
README.md
22
README.md
@@ -38,16 +38,16 @@ docker compose up -d
|
||||
```
|
||||
|
||||
This will start:
|
||||
- PostgreSQL (port 5432)
|
||||
- Meilisearch (port 7700)
|
||||
- API service (port 8080)
|
||||
- Indexer service (port 8081)
|
||||
- Backoffice web UI (port 8082)
|
||||
- PostgreSQL (port 6432)
|
||||
- Meilisearch (port 7700)
|
||||
- API service (port 7080)
|
||||
- Indexer service (port 7081)
|
||||
- Backoffice web UI (port 7082)
|
||||
|
||||
### Accessing the Application
|
||||
|
||||
- **Backoffice**: http://localhost:8082
|
||||
- **API**: http://localhost:8080
|
||||
- **Backoffice**: http://localhost:7082
|
||||
- **API**: http://localhost:7080
|
||||
- **Meilisearch**: http://localhost:7700
|
||||
|
||||
### Default Credentials
|
||||
@@ -113,9 +113,9 @@ The backoffice will be available at http://localhost:3000
|
||||
|
||||
| Variable | Description | Default |
|
||||
|----------|-------------|---------|
|
||||
| `API_LISTEN_ADDR` | API service bind address | `0.0.0.0:8080` |
|
||||
| `INDEXER_LISTEN_ADDR` | Indexer service bind address | `0.0.0.0:8081` |
|
||||
| `BACKOFFICE_PORT` | Backoffice web UI port | `8082` |
|
||||
| `API_LISTEN_ADDR` | API service bind address | `0.0.0.0:7080` |
|
||||
| `INDEXER_LISTEN_ADDR` | Indexer service bind address | `0.0.0.0:7081` |
|
||||
| `BACKOFFICE_PORT` | Backoffice web UI port | `7082` |
|
||||
| `DATABASE_URL` | PostgreSQL connection string | `postgres://stripstream:stripstream@postgres:5432/stripstream` |
|
||||
| `MEILI_URL` | Meilisearch connection URL | `http://meilisearch:7700` |
|
||||
| `MEILI_MASTER_KEY` | Meilisearch master key (required) | - |
|
||||
@@ -128,7 +128,7 @@ The backoffice will be available at http://localhost:3000
|
||||
The API is documented with OpenAPI/Swagger. When running locally, access the docs at:
|
||||
|
||||
```
|
||||
http://localhost:8080/api-docs
|
||||
http://localhost:7080/swagger-ui
|
||||
```
|
||||
|
||||
## Project Structure
|
||||
|
||||
73
apps/api/AGENTS.md
Normal file
73
apps/api/AGENTS.md
Normal file
@@ -0,0 +1,73 @@
|
||||
# apps/api — REST API (axum)
|
||||
|
||||
Service HTTP sur le port **7080**. Voir `AGENTS.md` racine pour les conventions globales.
|
||||
|
||||
## Structure des fichiers
|
||||
|
||||
| Fichier | Rôle |
|
||||
|---------|------|
|
||||
| `main.rs` | Routes, initialisation AppState, Semaphore concurrent_renders |
|
||||
| `state.rs` | `AppState` (pool, caches, métriques), `load_concurrent_renders` |
|
||||
| `auth.rs` | Middlewares `require_admin` / `require_read`, authentification tokens |
|
||||
| `error.rs` | `ApiError` avec constructeurs `bad_request`, `not_found`, `internal`, etc. |
|
||||
| `books.rs` | CRUD livres, thumbnails |
|
||||
| `pages.rs` | Rendu page + double cache (mémoire LRU + disque) |
|
||||
| `libraries.rs` | CRUD bibliothèques, déclenchement scans |
|
||||
| `index_jobs.rs` | Suivi jobs, SSE streaming progression |
|
||||
| `thumbnails.rs` | Rebuild/regénération thumbnails |
|
||||
| `tokens.rs` | Gestion tokens API (create/revoke) |
|
||||
| `settings.rs` | Paramètres applicatifs (stockés en DB, clé `limits`) |
|
||||
| `openapi.rs` | Doc OpenAPI via utoipa, accessible sur `/swagger-ui` |
|
||||
|
||||
## Patterns clés
|
||||
|
||||
### Handler type
|
||||
```rust
|
||||
async fn my_handler(
|
||||
State(state): State<AppState>,
|
||||
Path(id): Path<Uuid>,
|
||||
) -> Result<Json<MyDto>, ApiError> {
|
||||
// ...
|
||||
}
|
||||
```
|
||||
|
||||
### Erreurs API
|
||||
```rust
|
||||
// Constructeurs disponibles dans error.rs
|
||||
ApiError::bad_request("message")
|
||||
ApiError::not_found("resource not found")
|
||||
ApiError::internal("unexpected error")
|
||||
ApiError::unauthorized("missing token")
|
||||
ApiError::forbidden("admin required")
|
||||
|
||||
// Conversion auto depuis sqlx::Error et std::io::Error
|
||||
```
|
||||
|
||||
### Authentification
|
||||
- **Bootstrap token** : comparaison directe (`API_BOOTSTRAP_TOKEN`), scope Admin
|
||||
- **Tokens DB** : format `stl_<prefix>_<secret>`, hash argon2 en DB, scope `admin` ou `read`
|
||||
- Middleware `require_admin` → routes admin ; `require_read` → routes lecture
|
||||
|
||||
### OpenAPI (utoipa)
|
||||
```rust
|
||||
#[utoipa::path(get, path = "/books/{id}", ...)]
|
||||
async fn get_book(...) { }
|
||||
// Ajouter le handler dans openapi.rs (ApiDoc)
|
||||
```
|
||||
|
||||
### Cache pages (`pages.rs`)
|
||||
- **Cache mémoire** : LRU 512 entrées (`AppState.page_cache`)
|
||||
- **Cache disque** : `IMAGE_CACHE_DIR` (défaut `/tmp/stripstream-image-cache`), clé SHA256
|
||||
- Concurrence limitée par `AppState.page_render_limit` (Semaphore, configurable en DB)
|
||||
- `spawn_blocking` pour le rendu image (CPU-bound)
|
||||
|
||||
### Paramètre concurrent_renders
|
||||
Stocké en DB : `SELECT value FROM app_settings WHERE key = 'limits'` → JSON `{"concurrent_renders": N}`.
|
||||
Chargé au démarrage dans `load_concurrent_renders`.
|
||||
|
||||
## Gotchas
|
||||
|
||||
- **LIBRARIES_ROOT_PATH** : les `abs_path` en DB commencent par `/libraries/`. Appeler `remap_libraries_path()` avant tout accès fichier.
|
||||
- **Rate limit lecture** : middleware `read_rate_limit` sur les routes read (100 req/5s par défaut).
|
||||
- **Métriques** : `/metrics` expose `requests_total`, `page_cache_hits`, `page_cache_misses` (atomics dans `AppState.metrics`).
|
||||
- **Swagger** : accessible sur `/swagger-ui`, spec JSON sur `/openapi.json`.
|
||||
@@ -26,5 +26,5 @@ RUN sed -i '/en_US.UTF-8/s/^# //g' /etc/locale.gen && locale-gen
|
||||
ENV LANG=en_US.UTF-8
|
||||
ENV LC_ALL=en_US.UTF-8
|
||||
COPY --from=builder /app/target/release/api /usr/local/bin/api
|
||||
EXPOSE 8080
|
||||
EXPOSE 7080
|
||||
CMD ["/usr/local/bin/api"]
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
API_BASE_URL=http://localhost:8080
|
||||
API_BASE_URL=http://localhost:7080
|
||||
API_BOOTSTRAP_TOKEN=stripstream-dev-bootstrap-token
|
||||
NEXT_PUBLIC_API_BASE_URL=http://localhost:8080
|
||||
NEXT_PUBLIC_API_BASE_URL=http://localhost:7080
|
||||
NEXT_PUBLIC_API_BOOTSTRAP_TOKEN=stripstream-dev-bootstrap-token
|
||||
|
||||
66
apps/backoffice/AGENTS.md
Normal file
66
apps/backoffice/AGENTS.md
Normal file
@@ -0,0 +1,66 @@
|
||||
# apps/backoffice — Interface d'administration (Next.js)
|
||||
|
||||
App Next.js 16 avec React 19, Tailwind CSS v4, TypeScript. Port de dev : **7082** (`npm run dev`).
|
||||
|
||||
## Structure
|
||||
|
||||
```
|
||||
app/
|
||||
├── layout.tsx # Layout global (nav sticky glassmorphism, ThemeProvider)
|
||||
├── page.tsx # Dashboard
|
||||
├── books/ # Liste et détail des livres
|
||||
├── libraries/ # Gestion bibliothèques
|
||||
├── jobs/ # Monitoring jobs
|
||||
├── tokens/ # Tokens API
|
||||
├── settings/ # Paramètres
|
||||
├── components/ # Composants métier
|
||||
│ ├── ui/ # Composants génériques (Button, Card, Badge, Icon, Input, ProgressBar, StatBox...)
|
||||
│ ├── BookCard.tsx
|
||||
│ ├── JobProgress.tsx
|
||||
│ ├── JobsList.tsx
|
||||
│ ├── LibraryForm.tsx
|
||||
│ ├── FolderBrowser.tsx / FolderPicker.tsx
|
||||
│ └── ...
|
||||
└── globals.css # Variables CSS, Tailwind base
|
||||
lib/
|
||||
└── api.ts # Client API : types DTO + fonctions fetch vers l'API Rust
|
||||
```
|
||||
|
||||
## Client API (lib/api.ts)
|
||||
|
||||
Tous les appels vers l'API Rust passent par `lib/api.ts`. Les types DTO sont définis là :
|
||||
- `LibraryDto`, `IndexJobDto`, `BookDto`, `TokenDto`, `FolderItem`
|
||||
|
||||
Ajouter les nouveaux endpoints et types dans ce fichier.
|
||||
|
||||
## Composants UI
|
||||
|
||||
Les composants génériques sont dans `app/components/ui/`. Utiliser ces composants plutôt que des éléments HTML bruts :
|
||||
|
||||
```tsx
|
||||
import { Button, Card, Badge, Icon, Input, ProgressBar, StatBox } from "@/app/components/ui";
|
||||
```
|
||||
|
||||
## Conventions
|
||||
|
||||
- **App Router** : toutes les pages sont des Server Components par défaut. Utiliser `"use client"` seulement pour l'interactivité.
|
||||
- **Tailwind v4** : config dans `postcss.config.js` + `tailwind.config.js`. Variables CSS dans `globals.css`.
|
||||
- **Thème** : `ThemeProvider` + `ThemeToggle` pour dark/light mode via `next-themes`.
|
||||
- **Icônes** : composant `<Icon name="..." size="sm|md|lg" />` dans `ui/Icon.tsx` — pas de librairie externe.
|
||||
- **Navigation** : routes typées dans `layout.tsx` (`"/" | "/books" | "/libraries" | "/jobs" | "/tokens" | "/settings"`).
|
||||
|
||||
## Commandes
|
||||
|
||||
```bash
|
||||
npm install
|
||||
npm run dev # http://localhost:7082
|
||||
npm run build
|
||||
npm run start # Production sur http://localhost:7082
|
||||
```
|
||||
|
||||
## Gotchas
|
||||
|
||||
- **Port 7082** : pas le port Next.js par défaut (3000). Défini dans `package.json` scripts (`-p 7082`).
|
||||
- **API_BASE_URL** : en prod, configuré via env. En dev local, l'API doit tourner sur `http://localhost:7080`.
|
||||
- **React 19 + Next.js 16** : utiliser les nouvelles APIs (actions serveur, `use()` hook) si disponibles.
|
||||
- **Pas de gestion d'état global** : fetch direct depuis les Server Components ou `useState`/`useEffect` dans les Client Components.
|
||||
@@ -12,11 +12,11 @@ RUN npm run build
|
||||
FROM node:22-alpine AS runner
|
||||
WORKDIR /app
|
||||
ENV NODE_ENV=production
|
||||
ENV PORT=8082
|
||||
ENV PORT=7082
|
||||
ENV HOST=0.0.0.0
|
||||
RUN apk add --no-cache wget
|
||||
COPY --from=builder /app/.next/standalone ./
|
||||
COPY --from=builder /app/.next/static ./.next/static
|
||||
COPY --from=builder /app/public ./public
|
||||
EXPOSE 8082
|
||||
EXPOSE 7082
|
||||
CMD ["node", "server.js"]
|
||||
|
||||
@@ -13,7 +13,7 @@ export async function GET(
|
||||
const quality = searchParams.get("quality") || "";
|
||||
|
||||
// Construire l'URL vers l'API backend
|
||||
const apiBaseUrl = process.env.API_BASE_URL || "http://api:8080";
|
||||
const apiBaseUrl = process.env.API_BASE_URL || "http://api:7080";
|
||||
const apiUrl = new URL(`${apiBaseUrl}/books/${bookId}/pages/${pageNum}`);
|
||||
apiUrl.searchParams.set("format", format);
|
||||
if (width) apiUrl.searchParams.set("width", width);
|
||||
|
||||
@@ -6,7 +6,7 @@ export async function GET(
|
||||
) {
|
||||
const { bookId } = await params;
|
||||
|
||||
const apiBaseUrl = process.env.API_BASE_URL || "http://api:8080";
|
||||
const apiBaseUrl = process.env.API_BASE_URL || "http://api:7080";
|
||||
const apiUrl = `${apiBaseUrl}/books/${bookId}/thumbnail`;
|
||||
|
||||
const token = process.env.API_BOOTSTRAP_TOKEN;
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
import { NextRequest, NextResponse } from "next/server";
|
||||
|
||||
export async function GET(request: NextRequest) {
|
||||
const apiBaseUrl = process.env.API_BASE_URL || "http://api:8080";
|
||||
const apiBaseUrl = process.env.API_BASE_URL || "http://api:7080";
|
||||
const apiToken = process.env.API_BOOTSTRAP_TOKEN;
|
||||
|
||||
if (!apiToken) {
|
||||
|
||||
@@ -5,7 +5,7 @@ export async function POST(
|
||||
{ params }: { params: Promise<{ id: string }> }
|
||||
) {
|
||||
const { id } = await params;
|
||||
const apiBaseUrl = process.env.API_BASE_URL || "http://api:8080";
|
||||
const apiBaseUrl = process.env.API_BASE_URL || "http://api:7080";
|
||||
const apiToken = process.env.API_BOOTSTRAP_TOKEN;
|
||||
|
||||
if (!apiToken) {
|
||||
|
||||
@@ -5,7 +5,7 @@ export async function GET(
|
||||
{ params }: { params: Promise<{ id: string }> }
|
||||
) {
|
||||
const { id } = await params;
|
||||
const apiBaseUrl = process.env.API_BASE_URL || "http://api:8080";
|
||||
const apiBaseUrl = process.env.API_BASE_URL || "http://api:7080";
|
||||
const apiToken = process.env.API_BOOTSTRAP_TOKEN;
|
||||
|
||||
if (!apiToken) {
|
||||
|
||||
@@ -5,7 +5,7 @@ export async function GET(
|
||||
{ params }: { params: Promise<{ id: string }> }
|
||||
) {
|
||||
const { id } = await params;
|
||||
const apiBaseUrl = process.env.API_BASE_URL || "http://api:8080";
|
||||
const apiBaseUrl = process.env.API_BASE_URL || "http://api:7080";
|
||||
const apiToken = process.env.API_BOOTSTRAP_TOKEN;
|
||||
|
||||
if (!apiToken) {
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
import { NextRequest, NextResponse } from "next/server";
|
||||
|
||||
export async function GET(request: NextRequest) {
|
||||
const apiBaseUrl = process.env.API_BASE_URL || "http://api:8080";
|
||||
const apiBaseUrl = process.env.API_BASE_URL || "http://api:7080";
|
||||
const apiToken = process.env.API_BOOTSTRAP_TOKEN;
|
||||
|
||||
if (!apiToken) {
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
import { NextRequest } from "next/server";
|
||||
|
||||
export async function GET(request: NextRequest) {
|
||||
const apiBaseUrl = process.env.API_BASE_URL || "http://api:8080";
|
||||
const apiBaseUrl = process.env.API_BASE_URL || "http://api:7080";
|
||||
const apiToken = process.env.API_BOOTSTRAP_TOKEN;
|
||||
|
||||
if (!apiToken) {
|
||||
|
||||
@@ -6,7 +6,7 @@ export async function GET(
|
||||
) {
|
||||
try {
|
||||
const { key } = await params;
|
||||
const baseUrl = process.env.API_BASE_URL || "http://api:8080";
|
||||
const baseUrl = process.env.API_BASE_URL || "http://api:7080";
|
||||
const token = process.env.API_BOOTSTRAP_TOKEN;
|
||||
|
||||
const response = await fetch(`${baseUrl}/settings/${key}`, {
|
||||
@@ -33,7 +33,7 @@ export async function POST(
|
||||
) {
|
||||
try {
|
||||
const { key } = await params;
|
||||
const baseUrl = process.env.API_BASE_URL || "http://api:8080";
|
||||
const baseUrl = process.env.API_BASE_URL || "http://api:7080";
|
||||
const token = process.env.API_BOOTSTRAP_TOKEN;
|
||||
const body = await request.json();
|
||||
|
||||
|
||||
@@ -2,7 +2,7 @@ import { NextRequest, NextResponse } from "next/server";
|
||||
|
||||
export async function POST(request: NextRequest) {
|
||||
try {
|
||||
const baseUrl = process.env.API_BASE_URL || "http://api:8080";
|
||||
const baseUrl = process.env.API_BASE_URL || "http://api:7080";
|
||||
const token = process.env.API_BOOTSTRAP_TOKEN;
|
||||
|
||||
const response = await fetch(`${baseUrl}/settings/cache/clear`, {
|
||||
|
||||
@@ -2,7 +2,7 @@ import { NextRequest, NextResponse } from "next/server";
|
||||
|
||||
export async function GET(request: NextRequest) {
|
||||
try {
|
||||
const baseUrl = process.env.API_BASE_URL || "http://api:8080";
|
||||
const baseUrl = process.env.API_BASE_URL || "http://api:7080";
|
||||
const token = process.env.API_BOOTSTRAP_TOKEN;
|
||||
|
||||
const response = await fetch(`${baseUrl}/settings/cache/stats`, {
|
||||
|
||||
@@ -2,7 +2,7 @@ import { NextRequest, NextResponse } from "next/server";
|
||||
|
||||
export async function GET(request: NextRequest) {
|
||||
try {
|
||||
const baseUrl = process.env.API_BASE_URL || "http://api:8080";
|
||||
const baseUrl = process.env.API_BASE_URL || "http://api:7080";
|
||||
const token = process.env.API_BOOTSTRAP_TOKEN;
|
||||
|
||||
const response = await fetch(`${baseUrl}/settings`, {
|
||||
|
||||
@@ -90,7 +90,7 @@ export type SeriesDto = {
|
||||
};
|
||||
|
||||
function config() {
|
||||
const baseUrl = process.env.API_BASE_URL || "http://api:8080";
|
||||
const baseUrl = process.env.API_BASE_URL || "http://api:7080";
|
||||
const token = process.env.API_BOOTSTRAP_TOKEN;
|
||||
if (!token) {
|
||||
throw new Error("API_BOOTSTRAP_TOKEN is required for backoffice");
|
||||
|
||||
75
apps/indexer/AGENTS.md
Normal file
75
apps/indexer/AGENTS.md
Normal file
@@ -0,0 +1,75 @@
|
||||
# apps/indexer — Service d'indexation
|
||||
|
||||
Service background sur le port **7081**. Voir `AGENTS.md` racine pour les conventions globales.
|
||||
|
||||
## Structure des fichiers
|
||||
|
||||
| Fichier | Rôle |
|
||||
|---------|------|
|
||||
| `main.rs` | Point d'entrée, initialisation, lancement du worker |
|
||||
| `lib.rs` | `AppState` (pool, meili, api_base_url) |
|
||||
| `worker.rs` | Boucle principale : claim job → process → cleanup stale |
|
||||
| `job.rs` | `claim_next_job`, `process_job`, `fail_job`, `cleanup_stale_jobs` |
|
||||
| `scanner.rs` | Scan filesystem, parsing parallèle (rayon), batching DB |
|
||||
| `batch.rs` | `flush_all_batches` avec UNNEST, structures `BookInsert/Update/FileInsert/Update/ErrorInsert` |
|
||||
| `scheduler.rs` | Auto-scan : vérifie toutes les 60s les bibliothèques à monitorer |
|
||||
| `watcher.rs` | File watcher temps réel |
|
||||
| `meili.rs` | Indexation/sync Meilisearch |
|
||||
| `api.rs` | Appels HTTP vers l'API (pour checkup thumbnails) |
|
||||
| `utils.rs` | `remap_libraries_path`, `unmap_libraries_path`, `compute_fingerprint`, `kind_from_format` |
|
||||
|
||||
## Cycle de vie d'un job
|
||||
|
||||
```
|
||||
claim_next_job (UPDATE ... RETURNING, status pending→running)
|
||||
└─ process_job
|
||||
├─ scanner::scan_library (rayon par_iter pour le parsing)
|
||||
│ └─ flush_all_batches toutes les BATCH_SIZE=100 itérations
|
||||
└─ meili sync
|
||||
└─ api checkup thumbnails (POST /index/jobs/:id/thumbnails/checkup)
|
||||
```
|
||||
|
||||
- Annulation : `is_job_cancelled` vérifié toutes les 10 fichiers ou 1s — retourne `Err("Job cancelled")`
|
||||
- Jobs stale (running au redémarrage) → nettoyés par `cleanup_stale_jobs` au boot
|
||||
|
||||
## Pattern batch (batch.rs)
|
||||
|
||||
Toutes les opérations DB massives passent par `flush_all_batches` avec UNNEST :
|
||||
|
||||
```rust
|
||||
// Accumuler dans des Vec<BookInsert>, Vec<FileInsert>, etc.
|
||||
books_to_insert.push(BookInsert { ... });
|
||||
|
||||
// Flush quand plein ou en fin de scan
|
||||
if books_to_insert.len() >= BATCH_SIZE {
|
||||
flush_all_batches(&pool, &mut books_update, &mut files_update,
|
||||
&mut books_insert, &mut files_insert, &mut errors_insert).await?;
|
||||
}
|
||||
```
|
||||
|
||||
Toutes les opérations du flush sont dans une seule transaction.
|
||||
|
||||
## Scan filesystem (scanner.rs)
|
||||
|
||||
Pipeline en 3 étapes :
|
||||
1. **Collect** : WalkDir → filtrer par format (CBZ/CBR/PDF)
|
||||
2. **Parse** : `file_infos.into_par_iter().map(parse_metadata)` (rayon)
|
||||
3. **Process** : séquentiel pour les inserts/updates DB
|
||||
|
||||
Fingerprint = SHA256(taille + mtime) pour détecter les changements sans relire le fichier.
|
||||
|
||||
## Path remapping
|
||||
|
||||
```rust
|
||||
// abs_path en DB = chemin conteneur (/libraries/...)
|
||||
// Sur l'hôte : LIBRARIES_ROOT_PATH remplace /libraries
|
||||
utils::remap_libraries_path(&abs_path) // DB → filesystem local
|
||||
utils::unmap_libraries_path(&local_path) // filesystem local → DB
|
||||
```
|
||||
|
||||
## Gotchas
|
||||
|
||||
- **Thumbnails** : générés par l'API après handoff, pas par l'indexer directement. L'indexer appelle `/index/jobs/:id/thumbnails/checkup` via `api.rs`.
|
||||
- **full_rebuild** : si `true`, ignore les fingerprints → tous les fichiers sont retraités.
|
||||
- **Annulation** : vérifier `is_job_cancelled` régulièrement pour respecter les annulations utilisateur.
|
||||
- **Watcher + scheduler** : tournent en tâches tokio séparées dans `worker.rs`, en parallèle de la boucle principale.
|
||||
@@ -23,5 +23,5 @@ RUN --mount=type=cache,target=/sccache \
|
||||
FROM debian:bookworm-slim
|
||||
RUN apt-get update && apt-get install -y --no-install-recommends ca-certificates wget unrar-free && rm -rf /var/lib/apt/lists/*
|
||||
COPY --from=builder /app/target/release/indexer /usr/local/bin/indexer
|
||||
EXPOSE 8081
|
||||
EXPOSE 7081
|
||||
CMD ["/usr/local/bin/indexer"]
|
||||
|
||||
@@ -13,7 +13,7 @@ impl ApiConfig {
|
||||
pub fn from_env() -> Result<Self> {
|
||||
Ok(Self {
|
||||
listen_addr: std::env::var("API_LISTEN_ADDR")
|
||||
.unwrap_or_else(|_| "0.0.0.0:8080".to_string()),
|
||||
.unwrap_or_else(|_| "0.0.0.0:7080".to_string()),
|
||||
database_url: std::env::var("DATABASE_URL").context("DATABASE_URL is required")?,
|
||||
meili_url: std::env::var("MEILI_URL").context("MEILI_URL is required")?,
|
||||
meili_master_key: std::env::var("MEILI_MASTER_KEY")
|
||||
@@ -32,7 +32,7 @@ pub struct IndexerConfig {
|
||||
pub meili_master_key: String,
|
||||
pub scan_interval_seconds: u64,
|
||||
pub thumbnail_config: ThumbnailConfig,
|
||||
/// API base URL for thumbnail checkup at end of build (e.g. http://api:8080)
|
||||
/// API base URL for thumbnail checkup at end of build (e.g. http://api:7080)
|
||||
pub api_base_url: String,
|
||||
/// Token to call API (e.g. API_BOOTSTRAP_TOKEN)
|
||||
pub api_bootstrap_token: String,
|
||||
@@ -87,7 +87,7 @@ impl IndexerConfig {
|
||||
|
||||
Ok(Self {
|
||||
listen_addr: std::env::var("INDEXER_LISTEN_ADDR")
|
||||
.unwrap_or_else(|_| "0.0.0.0:8081".to_string()),
|
||||
.unwrap_or_else(|_| "0.0.0.0:7081".to_string()),
|
||||
database_url: std::env::var("DATABASE_URL").context("DATABASE_URL is required")?,
|
||||
meili_url: std::env::var("MEILI_URL").context("MEILI_URL is required")?,
|
||||
meili_master_key: std::env::var("MEILI_MASTER_KEY")
|
||||
@@ -98,7 +98,7 @@ impl IndexerConfig {
|
||||
.unwrap_or(5),
|
||||
thumbnail_config,
|
||||
api_base_url: std::env::var("API_BASE_URL")
|
||||
.unwrap_or_else(|_| "http://api:8080".to_string()),
|
||||
.unwrap_or_else(|_| "http://api:7080".to_string()),
|
||||
api_bootstrap_token: std::env::var("API_BOOTSTRAP_TOKEN")
|
||||
.context("API_BOOTSTRAP_TOKEN is required for thumbnail checkup")?,
|
||||
})
|
||||
@@ -116,9 +116,9 @@ impl AdminUiConfig {
|
||||
pub fn from_env() -> Result<Self> {
|
||||
Ok(Self {
|
||||
listen_addr: std::env::var("ADMIN_UI_LISTEN_ADDR")
|
||||
.unwrap_or_else(|_| "0.0.0.0:8082".to_string()),
|
||||
.unwrap_or_else(|_| "0.0.0.0:7082".to_string()),
|
||||
api_base_url: std::env::var("API_BASE_URL")
|
||||
.unwrap_or_else(|_| "http://api:8080".to_string()),
|
||||
.unwrap_or_else(|_| "http://api:7080".to_string()),
|
||||
api_token: std::env::var("API_BOOTSTRAP_TOKEN")
|
||||
.context("API_BOOTSTRAP_TOKEN is required")?,
|
||||
})
|
||||
|
||||
76
crates/parsers/AGENTS.md
Normal file
76
crates/parsers/AGENTS.md
Normal file
@@ -0,0 +1,76 @@
|
||||
# crates/parsers — Parsing de livres (CBZ, CBR, PDF)
|
||||
|
||||
Crate utilitaire sans état, utilisée par `apps/api` et `apps/indexer`.
|
||||
|
||||
## API publique (lib.rs)
|
||||
|
||||
```rust
|
||||
// Détection du format par extension
|
||||
pub fn detect_format(path: &Path) -> Option<BookFormat> // .cbz | .cbr | .pdf
|
||||
|
||||
// Extraction des métadonnées
|
||||
pub fn parse_metadata(path: &Path, format: BookFormat, library_root: &Path) -> Result<ParsedMetadata>
|
||||
|
||||
// Extraction de la première page (pour thumbnails)
|
||||
pub fn extract_first_page(path: &Path, format: BookFormat) -> Result<Vec<u8>>
|
||||
|
||||
pub enum BookFormat { Cbz, Cbr, Pdf }
|
||||
|
||||
pub struct ParsedMetadata {
|
||||
pub title: String, // = nom de fichier (sans extension)
|
||||
pub series: Option<String>, // = premier dossier relatif à library_root
|
||||
pub volume: Option<i32>, // extrait du nom de fichier
|
||||
pub page_count: Option<i32>,
|
||||
}
|
||||
```
|
||||
|
||||
## Logique de parsing
|
||||
|
||||
### Titre
|
||||
Nom de fichier sans extension, conservé tel quel (pas de nettoyage).
|
||||
|
||||
### Série
|
||||
Premier composant du chemin relatif entre `library_root` et le fichier :
|
||||
- `/libraries/One Piece/T01.cbz` → série = `"One Piece"`
|
||||
- `/libraries/one-shot.cbz` → série = `None`
|
||||
|
||||
### Volume (`extract_volume`)
|
||||
Patterns reconnus dans le nom de fichier (dans l'ordre de priorité) :
|
||||
- `T01`, `T1` (manga/comics français)
|
||||
- `Vol. 1`, `Vol 1`, `Volume 1`
|
||||
- `#1`, `#01`
|
||||
- `- 1`, `- 01` (en fin de nom)
|
||||
|
||||
### Nombre de pages
|
||||
| Format | Outil |
|
||||
|--------|-------|
|
||||
| CBZ | `zip::ZipArchive` — compte les entrées image (jpg/jpeg/png/webp/avif) |
|
||||
| CBR | `unrar lb <path>` — liste les fichiers, filtre les images |
|
||||
| PDF | `pdfinfo <path>` — lit la ligne `Pages:` |
|
||||
|
||||
## Dépendances système requises
|
||||
|
||||
| Outil | Utilisé pour | Installation |
|
||||
|-------|-------------|-------------|
|
||||
| `unrar` | CBR page count | `brew install rar` / `apt install unrar` |
|
||||
| `unar` | CBR first page extraction | `brew install unar` / `apt install unar` |
|
||||
| `pdfinfo` | PDF page count | inclus dans `poppler-utils` |
|
||||
| `pdftoppm` | PDF first page render | inclus dans `poppler-utils` |
|
||||
|
||||
**Important** : `unrar` (pour le listing) et `unar` (pour l'extraction) sont deux outils différents.
|
||||
|
||||
## Extraction première page
|
||||
|
||||
- **CBZ** : `zip::ZipArchive`, trie les noms d'images, lit la première
|
||||
- **CBR** : `unar -o <tmp_dir>`, `WalkDir` récursif, trie, lit la première — nettoie `tmp_dir` ensuite
|
||||
- **PDF** : `pdftoppm -f 1 -singlefile -png -scale-to 800` → fichier PNG temporaire — nettoie `tmp_dir` ensuite
|
||||
|
||||
Répertoire temp : `std::env::temp_dir()/stripstream-{cbr|pdf}-thumb-<uuid>`.
|
||||
|
||||
## Gotchas
|
||||
|
||||
- `clean_title()` existe mais est marqué `#[allow(dead_code)]` — le titre n'est **pas** nettoyé (décision volontaire).
|
||||
- Les CBR peuvent avoir des sous-dossiers internes → WalkDir nécessaire (pas de listing plat).
|
||||
- La détection du format est **uniquement par extension** (pas de magic bytes).
|
||||
- `pdfinfo` et `pdftoppm` doivent être du paquet `poppler-utils` (pas `poppler` seul).
|
||||
- En cas d'échec de parsing, l'appelant (indexer/api) stocke `parse_status = 'error'` en DB mais continue.
|
||||
@@ -54,7 +54,7 @@ services:
|
||||
env_file:
|
||||
- .env
|
||||
ports:
|
||||
- "7080:8080"
|
||||
- "7080:7080"
|
||||
volumes:
|
||||
- ${LIBRARIES_HOST_PATH:-./libraries}:/libraries
|
||||
- ${THUMBNAILS_HOST_PATH:-./data/thumbnails}:/data/thumbnails
|
||||
@@ -66,7 +66,7 @@ services:
|
||||
meilisearch:
|
||||
condition: service_healthy
|
||||
healthcheck:
|
||||
test: ["CMD", "wget", "-q", "-O", "-", "http://127.0.0.1:8080/health"]
|
||||
test: ["CMD", "wget", "-q", "-O", "-", "http://127.0.0.1:7080/health"]
|
||||
interval: 10s
|
||||
timeout: 5s
|
||||
retries: 5
|
||||
@@ -78,7 +78,7 @@ services:
|
||||
env_file:
|
||||
- .env
|
||||
ports:
|
||||
- "7081:8081"
|
||||
- "7081:7081"
|
||||
volumes:
|
||||
- ${LIBRARIES_HOST_PATH:-./libraries}:/libraries
|
||||
- ${THUMBNAILS_HOST_PATH:-./data/thumbnails}:/data/thumbnails
|
||||
@@ -90,7 +90,7 @@ services:
|
||||
meilisearch:
|
||||
condition: service_healthy
|
||||
healthcheck:
|
||||
test: ["CMD", "wget", "-q", "-O", "-", "http://127.0.0.1:8081/health"]
|
||||
test: ["CMD", "wget", "-q", "-O", "-", "http://127.0.0.1:7081/health"]
|
||||
interval: 10s
|
||||
timeout: 5s
|
||||
retries: 5
|
||||
@@ -102,15 +102,15 @@ services:
|
||||
env_file:
|
||||
- .env
|
||||
environment:
|
||||
- PORT=8082
|
||||
- PORT=7082
|
||||
- HOST=0.0.0.0
|
||||
ports:
|
||||
- "7082:8082"
|
||||
- "7082:7082"
|
||||
depends_on:
|
||||
api:
|
||||
condition: service_healthy
|
||||
healthcheck:
|
||||
test: ["CMD", "wget", "-q", "-O", "-", "http://127.0.0.1:8082/health"]
|
||||
test: ["CMD", "wget", "-q", "-O", "-", "http://127.0.0.1:7082/health"]
|
||||
interval: 10s
|
||||
timeout: 5s
|
||||
retries: 5
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
BASE_API="${BASE_API:-http://127.0.0.1:8080}"
|
||||
BASE_API="${BASE_API:-http://127.0.0.1:7080}"
|
||||
TOKEN="${API_TOKEN:-stripstream-dev-bootstrap-token}"
|
||||
|
||||
measure() {
|
||||
|
||||
@@ -1,9 +1,9 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
BASE_API="${BASE_API:-http://127.0.0.1:8080}"
|
||||
BASE_INDEXER="${BASE_INDEXER:-http://127.0.0.1:8081}"
|
||||
BASE_BACKOFFICE="${BASE_BACKOFFICE:-${BASE_ADMIN:-http://127.0.0.1:8082}}"
|
||||
BASE_API="${BASE_API:-http://127.0.0.1:7080}"
|
||||
BASE_INDEXER="${BASE_INDEXER:-http://127.0.0.1:7081}"
|
||||
BASE_BACKOFFICE="${BASE_BACKOFFICE:-${BASE_ADMIN:-http://127.0.0.1:7082}}"
|
||||
TOKEN="${API_TOKEN:-stripstream-dev-bootstrap-token}"
|
||||
|
||||
echo "[smoke] health checks"
|
||||
|
||||
Reference in New Issue
Block a user