perf(indexer): éliminer le pre-count WalkDir en mode incrémental + concurrence adaptative

- Incremental rebuild: remplace le WalkDir de comptage par un COUNT(*) SQL
  → incrémental 67s → 25s (-62%) sur disque externe
- Full rebuild: conserve le WalkDir (DB vidée avant le comptage)
- Concurrence par défaut: num_cpus/2 clampé [2,8] au lieu de 2 fixe
- Ajoute num_cpus comme dépendance workspace
- Backoffice jobs: un seul formulaire avec formAction par bouton (icônes rétablies)
- infra/perf.sh: corrige l'endpoint /index/jobs/:id (pas /details), exporte BASE_API/TOKEN

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
2026-03-12 22:15:41 +01:00
parent 1d10044d46
commit 358896c7d5
6 changed files with 92 additions and 101 deletions

View File

@@ -13,6 +13,7 @@ chrono.workspace = true
futures = "0.3"
image.workspace = true
notify = "6.1"
num_cpus.workspace = true
parsers = { path = "../../crates/parsers" }
rand.workspace = true
rayon.workspace = true

View File

@@ -67,7 +67,10 @@ async fn load_thumbnail_config(pool: &sqlx::PgPool) -> ThumbnailConfig {
}
async fn load_thumbnail_concurrency(pool: &sqlx::PgPool) -> usize {
let default_concurrency = 2;
// Default: half the logical CPUs, clamped between 2 and 8.
// Archive extraction is I/O bound but benefits from moderate parallelism.
let cpus = num_cpus::get();
let default_concurrency = (cpus / 2).clamp(2, 8);
let row = sqlx::query(r#"SELECT value FROM app_settings WHERE key = 'limits'"#)
.fetch_optional(pool)
.await;

View File

@@ -238,27 +238,42 @@ pub async fn process_job(
.await?
};
// Count total files for progress estimation
let library_paths: Vec<String> = libraries
.iter()
.map(|library| {
crate::utils::remap_libraries_path(&library.get::<String, _>("root_path"))
})
.collect();
// Count total files for progress estimation.
// For incremental rebuilds, use the DB count (instant) — the filesystem will be walked
// once during discovery anyway, no need for a second full WalkDir pass.
// For full rebuilds, the DB is already cleared, so we must walk the filesystem.
let library_ids: Vec<uuid::Uuid> = libraries.iter().map(|r| r.get("id")).collect();
let total_files: usize = library_paths
.par_iter()
.map(|root_path| {
walkdir::WalkDir::new(root_path)
.into_iter()
.filter_map(Result::ok)
.filter(|entry| {
entry.file_type().is_file()
&& parsers::detect_format(entry.path()).is_some()
})
.count()
})
.sum();
let total_files: usize = if !is_full_rebuild {
let count: i64 = sqlx::query_scalar(
"SELECT COUNT(*) FROM book_files bf JOIN books b ON b.id = bf.book_id WHERE b.library_id = ANY($1)"
)
.bind(&library_ids)
.fetch_one(&state.pool)
.await
.unwrap_or(0);
count as usize
} else {
let library_paths: Vec<String> = libraries
.iter()
.map(|library| {
crate::utils::remap_libraries_path(&library.get::<String, _>("root_path"))
})
.collect();
library_paths
.par_iter()
.map(|root_path| {
walkdir::WalkDir::new(root_path)
.into_iter()
.filter_map(Result::ok)
.filter(|entry| {
entry.file_type().is_file()
&& parsers::detect_format(entry.path()).is_some()
})
.count()
})
.sum()
};
info!(
"[JOB] Found {} libraries, {} total files to index",