perf(indexer): éliminer le pre-count WalkDir en mode incrémental + concurrence adaptative
- Incremental rebuild: remplace le WalkDir de comptage par un COUNT(*) SQL → incrémental 67s → 25s (-62%) sur disque externe - Full rebuild: conserve le WalkDir (DB vidée avant le comptage) - Concurrence par défaut: num_cpus/2 clampé [2,8] au lieu de 2 fixe - Ajoute num_cpus comme dépendance workspace - Backoffice jobs: un seul formulaire avec formAction par bouton (icônes rétablies) - infra/perf.sh: corrige l'endpoint /index/jobs/:id (pas /details), exporte BASE_API/TOKEN Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
@@ -67,7 +67,10 @@ async fn load_thumbnail_config(pool: &sqlx::PgPool) -> ThumbnailConfig {
|
||||
}
|
||||
|
||||
async fn load_thumbnail_concurrency(pool: &sqlx::PgPool) -> usize {
|
||||
let default_concurrency = 2;
|
||||
// Default: half the logical CPUs, clamped between 2 and 8.
|
||||
// Archive extraction is I/O bound but benefits from moderate parallelism.
|
||||
let cpus = num_cpus::get();
|
||||
let default_concurrency = (cpus / 2).clamp(2, 8);
|
||||
let row = sqlx::query(r#"SELECT value FROM app_settings WHERE key = 'limits'"#)
|
||||
.fetch_optional(pool)
|
||||
.await;
|
||||
|
||||
@@ -238,27 +238,42 @@ pub async fn process_job(
|
||||
.await?
|
||||
};
|
||||
|
||||
// Count total files for progress estimation
|
||||
let library_paths: Vec<String> = libraries
|
||||
.iter()
|
||||
.map(|library| {
|
||||
crate::utils::remap_libraries_path(&library.get::<String, _>("root_path"))
|
||||
})
|
||||
.collect();
|
||||
// Count total files for progress estimation.
|
||||
// For incremental rebuilds, use the DB count (instant) — the filesystem will be walked
|
||||
// once during discovery anyway, no need for a second full WalkDir pass.
|
||||
// For full rebuilds, the DB is already cleared, so we must walk the filesystem.
|
||||
let library_ids: Vec<uuid::Uuid> = libraries.iter().map(|r| r.get("id")).collect();
|
||||
|
||||
let total_files: usize = library_paths
|
||||
.par_iter()
|
||||
.map(|root_path| {
|
||||
walkdir::WalkDir::new(root_path)
|
||||
.into_iter()
|
||||
.filter_map(Result::ok)
|
||||
.filter(|entry| {
|
||||
entry.file_type().is_file()
|
||||
&& parsers::detect_format(entry.path()).is_some()
|
||||
})
|
||||
.count()
|
||||
})
|
||||
.sum();
|
||||
let total_files: usize = if !is_full_rebuild {
|
||||
let count: i64 = sqlx::query_scalar(
|
||||
"SELECT COUNT(*) FROM book_files bf JOIN books b ON b.id = bf.book_id WHERE b.library_id = ANY($1)"
|
||||
)
|
||||
.bind(&library_ids)
|
||||
.fetch_one(&state.pool)
|
||||
.await
|
||||
.unwrap_or(0);
|
||||
count as usize
|
||||
} else {
|
||||
let library_paths: Vec<String> = libraries
|
||||
.iter()
|
||||
.map(|library| {
|
||||
crate::utils::remap_libraries_path(&library.get::<String, _>("root_path"))
|
||||
})
|
||||
.collect();
|
||||
library_paths
|
||||
.par_iter()
|
||||
.map(|root_path| {
|
||||
walkdir::WalkDir::new(root_path)
|
||||
.into_iter()
|
||||
.filter_map(Result::ok)
|
||||
.filter(|entry| {
|
||||
entry.file_type().is_file()
|
||||
&& parsers::detect_format(entry.path()).is_some()
|
||||
})
|
||||
.count()
|
||||
})
|
||||
.sum()
|
||||
};
|
||||
|
||||
info!(
|
||||
"[JOB] Found {} libraries, {} total files to index",
|
||||
|
||||
Reference in New Issue
Block a user