julienfroidefond/stripstream-librarian
1
0
Fork 0
Code Issues 4 Pull Requests Actions Packages Projects Releases Wiki Activity

perf(indexer): éliminer le pre-count WalkDir en mode incrémental + concurrence adaptative

Browse Source 
- Incremental rebuild: remplace le WalkDir de comptage par un COUNT(*) SQL
  → incrémental 67s → 25s (-62%) sur disque externe
- Full rebuild: conserve le WalkDir (DB vidée avant le comptage)
- Concurrence par défaut: num_cpus/2 clampé [2,8] au lieu de 2 fixe
- Ajoute num_cpus comme dépendance workspace
- Backoffice jobs: un seul formulaire avec formAction par bouton (icônes rétablies)
- infra/perf.sh: corrige l'endpoint /index/jobs/:id (pas /details), exporte BASE_API/TOKEN

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
Julien Froidefond
2026-03-12 22:15:41 +01:00
parent 1d10044d46
commit 358896c7d5
6 changed files with 92 additions and 101 deletions
Download Patch File Download Diff File Expand all files Collapse all files

17
Cargo.lock generated
View File

@@ -863,6 +863,12 @@ version = "0.5.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea"
[[package]]
name = "hermit-abi"
version = "0.5.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "fc0fef456e4baa96da950455cd02c081ca953b141298e41db3fc7e36b1da849c"
[[package]]
name = "hex"
version = "0.4.3"
@@ -1171,6 +1177,7 @@ dependencies = [
"futures",
"image",
"notify",
"num_cpus",
"parsers",
"rand 0.8.5",
"rayon",
@@ -1639,6 +1646,16 @@ dependencies = [
"libm",
]
[[package]]
name = "num_cpus"
version = "1.17.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "91df4bbde75afed763b708b7eee1e8e7651e02d97f6d5dd763e89367e957b23b"
dependencies = [
"hermit-abi",
"libc",
]
[[package]]
name = "once_cell"
version = "1.21.3"

1
Cargo.toml
View File

@@ -33,6 +33,7 @@ tracing = "0.1"
tracing-subscriber = { version = "0.3", features = ["env-filter", "fmt"] }
uuid = { version = "1.12", features = ["serde", "v4"] }
natord = "1.0"
num_cpus = "1.16"
pdfium-render = { version = "0.8", default-features = false, features = ["pdfium_latest", "image_latest", "thread_safe"] }
unrar = "0.5"
walkdir = "2.5"

70
apps/backoffice/app/jobs/page.tsx
View File

@@ -2,7 +2,7 @@ import { revalidatePath } from "next/cache";
import { redirect } from "next/navigation";
import { listJobs, fetchLibraries, rebuildIndex, rebuildThumbnails, regenerateThumbnails, IndexJobDto, LibraryDto } from "../../lib/api";
import { JobsList } from "../components/JobsList";
import { Card, CardHeader, CardTitle, CardDescription, CardContent, Button, FormField, FormSelect, FormRow } from "../components/ui";
import { Card, CardHeader, CardTitle, CardContent, Button, FormField, FormSelect, FormRow } from "../components/ui";
export const dynamic = "force-dynamic";
@@ -61,90 +61,44 @@ export default async function JobsPage({ searchParams }: { searchParams: Promise
<Card className="mb-6">
<CardHeader>
<CardTitle>Queue New Job</CardTitle>
<CardDescription>Rebuild index, full rebuild, generate missing thumbnails, or regenerate all thumbnails</CardDescription>
</CardHeader>
<CardContent className="space-y-4">
<form action={triggerRebuild}>
<CardContent>
<form>
<FormRow>
<FormField className="flex-1">
<FormField className="flex-1 max-w-xs">
<FormSelect name="library_id" defaultValue="">
<option value="">All libraries</option>
{libraries.map((lib) => (
<option key={lib.id} value={lib.id}>
{lib.name}
</option>
<option key={lib.id} value={lib.id}>{lib.name}</option>
))}
</FormSelect>
</FormField>
<Button type="submit">
<div className="flex flex-wrap gap-2">
<Button type="submit" formAction={triggerRebuild}>
<svg className="w-4 h-4 mr-2" fill="none" stroke="currentColor" viewBox="0 0 24 24">
<path strokeLinecap="round" strokeLinejoin="round" strokeWidth={2} d="M4 4v5h.582m15.356 2A8.001 8.001 0 004.582 9m0 0H9m11 11v-5h-.581m0 0a8.003 8.003 0 01-15.357-2m15.357 2H15" />
</svg>
Queue Rebuild
Rebuild
</Button>
</FormRow>
</form>
<form action={triggerFullRebuild}>
<FormRow>
<FormField className="flex-1">
<FormSelect name="library_id" defaultValue="">
<option value="">All libraries</option>
{libraries.map((lib) => (
<option key={lib.id} value={lib.id}>
{lib.name}
</option>
))}
</FormSelect>
</FormField>
<Button type="submit" variant="warning">
<Button type="submit" formAction={triggerFullRebuild} variant="warning">
<svg className="w-4 h-4 mr-2" fill="none" stroke="currentColor" viewBox="0 0 24 24">
<path strokeLinecap="round" strokeLinejoin="round" strokeWidth={2} d="M19 7l-.867 12.142A2 2 0 0116.138 21H7.862a2 2 0 01-1.995-1.858L5 7m5 4v6m4-6v6m1-10V4a1 1 0 00-1-1h-4a1 1 0 00-1 1v3M4 7h16" />
</svg>
Full Rebuild
</Button>
</FormRow>
</form>
<form action={triggerThumbnailsRebuild}>
<FormRow>
<FormField className="flex-1">
<FormSelect name="library_id" defaultValue="">
<option value="">All libraries</option>
{libraries.map((lib) => (
<option key={lib.id} value={lib.id}>
{lib.name}
</option>
))}
</FormSelect>
</FormField>
<Button type="submit" variant="secondary">
<Button type="submit" formAction={triggerThumbnailsRebuild} variant="secondary">
<svg className="w-4 h-4 mr-2" fill="none" stroke="currentColor" viewBox="0 0 24 24">
<path strokeLinecap="round" strokeLinejoin="round" strokeWidth={2} d="M4 16l4.586-4.586a2 2 0 012.828 0L16 16m-2-2l1.586-1.586a2 2 0 012.828 0L20 14m-6-6h.01M6 20h12a2 2 0 002-2V6a2 2 0 00-2-2H6a2 2 0 00-2 2v12a2 2 0 002 2z" />
</svg>
Generate thumbnails
</Button>
</FormRow>
</form>
<form action={triggerThumbnailsRegenerate}>
<FormRow>
<FormField className="flex-1">
<FormSelect name="library_id" defaultValue="">
<option value="">All libraries</option>
{libraries.map((lib) => (
<option key={lib.id} value={lib.id}>
{lib.name}
</option>
))}
</FormSelect>
</FormField>
<Button type="submit" variant="warning">
<Button type="submit" formAction={triggerThumbnailsRegenerate} variant="warning">
<svg className="w-4 h-4 mr-2" fill="none" stroke="currentColor" viewBox="0 0 24 24">
<path strokeLinecap="round" strokeLinejoin="round" strokeWidth={2} d="M4 4v5h.582m15.356 2A8.001 8.001 0 004.582 9m0 0H9m11 11v-5h-.581m0 0a8.003 8.003 0 01-15.357-2m15.357 2H15" />
</svg>
Regenerate thumbnails
</Button>
</div>
</FormRow>
</form>
</CardContent>

1
apps/indexer/Cargo.toml
View File

@@ -13,6 +13,7 @@ chrono.workspace = true
futures = "0.3"
image.workspace = true
notify = "6.1"
num_cpus.workspace = true
parsers = { path = "../../crates/parsers" }
rand.workspace = true
rayon.workspace = true

5
apps/indexer/src/analyzer.rs
View File

@@ -67,7 +67,10 @@ async fn load_thumbnail_config(pool: &sqlx::PgPool) -> ThumbnailConfig {
}
async fn load_thumbnail_concurrency(pool: &sqlx::PgPool) -> usize {
let default_concurrency = 2;
// Default: half the logical CPUs, clamped between 2 and 8.
// Archive extraction is I/O bound but benefits from moderate parallelism.
let cpus = num_cpus::get();
let default_concurrency = (cpus / 2).clamp(2, 8);
let row = sqlx::query(r#"SELECT value FROM app_settings WHERE key = 'limits'"#)
.fetch_optional(pool)
.await;

23
apps/indexer/src/job.rs
View File

@@ -238,15 +238,29 @@ pub async fn process_job(
.await?
};
// Count total files for progress estimation
// Count total files for progress estimation.
// For incremental rebuilds, use the DB count (instant) — the filesystem will be walked
// once during discovery anyway, no need for a second full WalkDir pass.
// For full rebuilds, the DB is already cleared, so we must walk the filesystem.
let library_ids: Vec<uuid::Uuid> = libraries.iter().map(|r| r.get("id")).collect();
let total_files: usize = if !is_full_rebuild {
let count: i64 = sqlx::query_scalar(
"SELECT COUNT(*) FROM book_files bf JOIN books b ON b.id = bf.book_id WHERE b.library_id = ANY($1)"
)
.bind(&library_ids)
.fetch_one(&state.pool)
.await
.unwrap_or(0);
count as usize
} else {
let library_paths: Vec<String> = libraries
.iter()
.map(|library| {
crate::utils::remap_libraries_path(&library.get::<String, _>("root_path"))
})
.collect();
let total_files: usize = library_paths
library_paths
.par_iter()
.map(|root_path| {
walkdir::WalkDir::new(root_path)
@@ -258,7 +272,8 @@ pub async fn process_job(
})
.count()
})
.sum();
.sum()
};
info!(
"[JOB] Found {} libraries, {} total files to index",