feat(indexer,backoffice): logs par domaine, réduction fd, UI mobile
- Ajout de targets de log par domaine (scan, extraction, thumbnail, watcher) contrôlables via RUST_LOG pour activer/désactiver les logs granulaires - Ajout de logs détaillés dans extracting_pages (per-book timing en debug, progression toutes les 25 books en info) - Réduction de la consommation de fd: walkdir max_open(20/10), comptage séquentiel au lieu de par_iter parallèle, suppression de rayon - Détection ENFILE dans le scanner: abort après 10 erreurs IO consécutives - Backoffice: settings dans le burger mobile, masquer "backoffice" et icône settings en mobile Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
18
.env.example
18
.env.example
@@ -34,6 +34,24 @@ MEILI_URL=http://meilisearch:7700
|
|||||||
# PostgreSQL Database
|
# PostgreSQL Database
|
||||||
DATABASE_URL=postgres://stripstream:stripstream@postgres:5432/stripstream
|
DATABASE_URL=postgres://stripstream:stripstream@postgres:5432/stripstream
|
||||||
|
|
||||||
|
# =============================================================================
|
||||||
|
# Logging
|
||||||
|
# =============================================================================
|
||||||
|
# Log levels per domain. Default: indexer=info,scan=info,extraction=info,thumbnail=warn,watcher=info
|
||||||
|
# Domains:
|
||||||
|
# scan — filesystem scan (discovery phase)
|
||||||
|
# extraction — page extraction from archives (extracting_pages phase)
|
||||||
|
# thumbnail — thumbnail generation (resize/encode)
|
||||||
|
# watcher — file watcher polling
|
||||||
|
# indexer — general indexer logs
|
||||||
|
# Levels: error, warn, info, debug, trace
|
||||||
|
# Examples:
|
||||||
|
# RUST_LOG=indexer=info # default, quiet thumbnails
|
||||||
|
# RUST_LOG=indexer=info,thumbnail=debug # enable thumbnail timing logs
|
||||||
|
# RUST_LOG=indexer=info,extraction=debug # per-book extraction details
|
||||||
|
# RUST_LOG=indexer=debug,scan=debug,extraction=debug,thumbnail=debug,watcher=debug # tout voir
|
||||||
|
# RUST_LOG=indexer=info,scan=info,extraction=info,thumbnail=warn,watcher=info
|
||||||
|
|
||||||
# =============================================================================
|
# =============================================================================
|
||||||
# Storage Configuration
|
# Storage Configuration
|
||||||
# =============================================================================
|
# =============================================================================
|
||||||
|
|||||||
1
Cargo.lock
generated
1
Cargo.lock
generated
@@ -1132,7 +1132,6 @@ dependencies = [
|
|||||||
"jpeg-decoder",
|
"jpeg-decoder",
|
||||||
"num_cpus",
|
"num_cpus",
|
||||||
"parsers",
|
"parsers",
|
||||||
"rayon",
|
|
||||||
"reqwest",
|
"reqwest",
|
||||||
"serde",
|
"serde",
|
||||||
"serde_json",
|
"serde_json",
|
||||||
|
|||||||
@@ -68,6 +68,17 @@ export function MobileNav({ navItems }: { navItems: NavItem[] }) {
|
|||||||
<span className="font-medium">{item.label}</span>
|
<span className="font-medium">{item.label}</span>
|
||||||
</Link>
|
</Link>
|
||||||
))}
|
))}
|
||||||
|
|
||||||
|
<div className="border-t border-border/40 mt-2 pt-2">
|
||||||
|
<Link
|
||||||
|
href="/settings"
|
||||||
|
className="flex items-center gap-3 px-3 py-3 rounded-lg text-muted-foreground hover:text-foreground hover:bg-accent transition-colors duration-200 active:scale-[0.98]"
|
||||||
|
onClick={() => setIsOpen(false)}
|
||||||
|
>
|
||||||
|
<NavIcon name="settings" />
|
||||||
|
<span className="font-medium">Settings</span>
|
||||||
|
</Link>
|
||||||
|
</div>
|
||||||
</nav>
|
</nav>
|
||||||
</div>
|
</div>
|
||||||
</>
|
</>
|
||||||
|
|||||||
@@ -52,7 +52,7 @@ export default function RootLayout({ children }: { children: ReactNode }) {
|
|||||||
<span className="text-xl font-bold tracking-tight text-foreground">
|
<span className="text-xl font-bold tracking-tight text-foreground">
|
||||||
StripStream
|
StripStream
|
||||||
</span>
|
</span>
|
||||||
<span className="text-sm text-muted-foreground font-medium">
|
<span className="text-sm text-muted-foreground font-medium hidden md:inline">
|
||||||
backoffice
|
backoffice
|
||||||
</span>
|
</span>
|
||||||
</div>
|
</div>
|
||||||
@@ -74,7 +74,7 @@ export default function RootLayout({ children }: { children: ReactNode }) {
|
|||||||
<JobsIndicator />
|
<JobsIndicator />
|
||||||
<Link
|
<Link
|
||||||
href="/settings"
|
href="/settings"
|
||||||
className="p-2 rounded-lg text-muted-foreground hover:text-foreground hover:bg-accent transition-colors"
|
className="hidden md:flex p-2 rounded-lg text-muted-foreground hover:text-foreground hover:bg-accent transition-colors"
|
||||||
title="Settings"
|
title="Settings"
|
||||||
>
|
>
|
||||||
<Icon name="settings" size="md" />
|
<Icon name="settings" size="md" />
|
||||||
|
|||||||
@@ -15,7 +15,6 @@ image.workspace = true
|
|||||||
jpeg-decoder.workspace = true
|
jpeg-decoder.workspace = true
|
||||||
num_cpus.workspace = true
|
num_cpus.workspace = true
|
||||||
parsers = { path = "../../crates/parsers" }
|
parsers = { path = "../../crates/parsers" }
|
||||||
rayon.workspace = true
|
|
||||||
reqwest.workspace = true
|
reqwest.workspace = true
|
||||||
serde.workspace = true
|
serde.workspace = true
|
||||||
serde_json.workspace = true
|
serde_json.workspace = true
|
||||||
|
|||||||
@@ -6,7 +6,7 @@ use sqlx::Row;
|
|||||||
use std::path::Path;
|
use std::path::Path;
|
||||||
use std::sync::atomic::{AtomicBool, AtomicI32, Ordering};
|
use std::sync::atomic::{AtomicBool, AtomicI32, Ordering};
|
||||||
use std::sync::Arc;
|
use std::sync::Arc;
|
||||||
use tracing::{info, warn};
|
use tracing::{debug, info, warn};
|
||||||
use uuid::Uuid;
|
use uuid::Uuid;
|
||||||
|
|
||||||
use crate::{job::is_job_cancelled, utils, AppState};
|
use crate::{job::is_job_cancelled, utils, AppState};
|
||||||
@@ -179,7 +179,8 @@ fn generate_thumbnail(image_bytes: &[u8], config: &ThumbnailConfig) -> anyhow::R
|
|||||||
let t_resize = t1.elapsed();
|
let t_resize = t1.elapsed();
|
||||||
|
|
||||||
let format = config.format.as_deref().unwrap_or("webp");
|
let format = config.format.as_deref().unwrap_or("webp");
|
||||||
info!(
|
debug!(
|
||||||
|
target: "thumbnail",
|
||||||
"[THUMBNAIL] {}x{} -> {}x{} decode={:.0}ms resize={:.0}ms encode_format={}",
|
"[THUMBNAIL] {}x{} -> {}x{} decode={:.0}ms resize={:.0}ms encode_format={}",
|
||||||
orig_w, orig_h, w, h,
|
orig_w, orig_h, w, h,
|
||||||
t_decode.as_secs_f64() * 1000.0,
|
t_decode.as_secs_f64() * 1000.0,
|
||||||
@@ -237,7 +238,8 @@ fn generate_thumbnail(image_bytes: &[u8], config: &ThumbnailConfig) -> anyhow::R
|
|||||||
}
|
}
|
||||||
};
|
};
|
||||||
let t_encode = t2.elapsed();
|
let t_encode = t2.elapsed();
|
||||||
info!(
|
debug!(
|
||||||
|
target: "thumbnail",
|
||||||
"[THUMBNAIL] encode={:.0}ms total={:.0}ms output_size={}KB",
|
"[THUMBNAIL] encode={:.0}ms total={:.0}ms output_size={}KB",
|
||||||
t_encode.as_secs_f64() * 1000.0,
|
t_encode.as_secs_f64() * 1000.0,
|
||||||
t0.elapsed().as_secs_f64() * 1000.0,
|
t0.elapsed().as_secs_f64() * 1000.0,
|
||||||
@@ -263,7 +265,7 @@ fn resize_raw_to_thumbnail(
|
|||||||
) -> anyhow::Result<String> {
|
) -> anyhow::Result<String> {
|
||||||
let raw_bytes = std::fs::read(raw_path)
|
let raw_bytes = std::fs::read(raw_path)
|
||||||
.map_err(|e| anyhow::anyhow!("failed to read raw image {}: {}", raw_path, e))?;
|
.map_err(|e| anyhow::anyhow!("failed to read raw image {}: {}", raw_path, e))?;
|
||||||
info!("[THUMBNAIL] book={} raw_size={}KB", book_id, raw_bytes.len() / 1024);
|
debug!(target: "thumbnail", "[THUMBNAIL] book={} raw_size={}KB", book_id, raw_bytes.len() / 1024);
|
||||||
let thumb_bytes = generate_thumbnail(&raw_bytes, config)?;
|
let thumb_bytes = generate_thumbnail(&raw_bytes, config)?;
|
||||||
|
|
||||||
let format = config.format.as_deref().unwrap_or("webp");
|
let format = config.format.as_deref().unwrap_or("webp");
|
||||||
@@ -449,6 +451,13 @@ pub async fn analyze_library_books(
|
|||||||
let pdf_scale = config.width.max(config.height);
|
let pdf_scale = config.width.max(config.height);
|
||||||
let path_owned = path.to_path_buf();
|
let path_owned = path.to_path_buf();
|
||||||
let timeout_secs = config.timeout_secs;
|
let timeout_secs = config.timeout_secs;
|
||||||
|
let file_name = path.file_name()
|
||||||
|
.map(|n| n.to_string_lossy().to_string())
|
||||||
|
.unwrap_or_else(|| local_path.clone());
|
||||||
|
|
||||||
|
debug!(target: "extraction", "[EXTRACTION] Starting: {} ({})", file_name, task.format);
|
||||||
|
let extract_start = std::time::Instant::now();
|
||||||
|
|
||||||
let analyze_result = tokio::time::timeout(
|
let analyze_result = tokio::time::timeout(
|
||||||
std::time::Duration::from_secs(timeout_secs),
|
std::time::Duration::from_secs(timeout_secs),
|
||||||
tokio::task::spawn_blocking(move || analyze_book(&path_owned, format, pdf_scale)),
|
tokio::task::spawn_blocking(move || analyze_book(&path_owned, format, pdf_scale)),
|
||||||
@@ -458,7 +467,7 @@ pub async fn analyze_library_books(
|
|||||||
let (page_count, raw_bytes) = match analyze_result {
|
let (page_count, raw_bytes) = match analyze_result {
|
||||||
Ok(Ok(Ok(result))) => result,
|
Ok(Ok(Ok(result))) => result,
|
||||||
Ok(Ok(Err(e))) => {
|
Ok(Ok(Err(e))) => {
|
||||||
warn!("[ANALYZER] analyze_book failed for book {}: {}", book_id, e);
|
warn!(target: "extraction", "[EXTRACTION] Failed: {} — {}", file_name, e);
|
||||||
let _ = sqlx::query(
|
let _ = sqlx::query(
|
||||||
"UPDATE book_files SET parse_status = 'error', parse_error_opt = $2 WHERE book_id = $1",
|
"UPDATE book_files SET parse_status = 'error', parse_error_opt = $2 WHERE book_id = $1",
|
||||||
)
|
)
|
||||||
@@ -469,11 +478,11 @@ pub async fn analyze_library_books(
|
|||||||
return None;
|
return None;
|
||||||
}
|
}
|
||||||
Ok(Err(e)) => {
|
Ok(Err(e)) => {
|
||||||
warn!("[ANALYZER] spawn_blocking error for book {}: {}", book_id, e);
|
warn!(target: "extraction", "[EXTRACTION] spawn error: {} — {}", file_name, e);
|
||||||
return None;
|
return None;
|
||||||
}
|
}
|
||||||
Err(_) => {
|
Err(_) => {
|
||||||
warn!("[ANALYZER] analyze_book timed out after {}s for book {}: {}", timeout_secs, book_id, local_path);
|
warn!(target: "extraction", "[EXTRACTION] Timeout ({}s): {}", timeout_secs, file_name);
|
||||||
let _ = sqlx::query(
|
let _ = sqlx::query(
|
||||||
"UPDATE book_files SET parse_status = 'error', parse_error_opt = $2 WHERE book_id = $1",
|
"UPDATE book_files SET parse_status = 'error', parse_error_opt = $2 WHERE book_id = $1",
|
||||||
)
|
)
|
||||||
@@ -485,15 +494,24 @@ pub async fn analyze_library_books(
|
|||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
|
||||||
|
let extract_elapsed = extract_start.elapsed();
|
||||||
|
debug!(
|
||||||
|
target: "extraction",
|
||||||
|
"[EXTRACTION] Done: {} — {} pages, image={}KB in {:.0}ms",
|
||||||
|
file_name, page_count, raw_bytes.len() / 1024,
|
||||||
|
extract_elapsed.as_secs_f64() * 1000.0,
|
||||||
|
);
|
||||||
|
|
||||||
// If thumbnail already exists, just update page_count and skip thumbnail generation
|
// If thumbnail already exists, just update page_count and skip thumbnail generation
|
||||||
if !needs_thumbnail {
|
if !needs_thumbnail {
|
||||||
|
debug!(target: "extraction", "[EXTRACTION] Page count only: {} — {} pages", file_name, page_count);
|
||||||
if let Err(e) = sqlx::query("UPDATE books SET page_count = $1 WHERE id = $2")
|
if let Err(e) = sqlx::query("UPDATE books SET page_count = $1 WHERE id = $2")
|
||||||
.bind(page_count)
|
.bind(page_count)
|
||||||
.bind(book_id)
|
.bind(book_id)
|
||||||
.execute(&pool)
|
.execute(&pool)
|
||||||
.await
|
.await
|
||||||
{
|
{
|
||||||
warn!("[ANALYZER] DB page_count update failed for book {}: {}", book_id, e);
|
warn!(target: "extraction", "[EXTRACTION] DB page_count update failed for {}: {}", file_name, e);
|
||||||
}
|
}
|
||||||
let processed = extracted_count.fetch_add(1, Ordering::Relaxed) + 1;
|
let processed = extracted_count.fetch_add(1, Ordering::Relaxed) + 1;
|
||||||
let percent = (processed as f64 / total as f64 * 50.0) as i32;
|
let percent = (processed as f64 / total as f64 * 50.0) as i32;
|
||||||
@@ -505,6 +523,14 @@ pub async fn analyze_library_books(
|
|||||||
.bind(percent)
|
.bind(percent)
|
||||||
.execute(&pool)
|
.execute(&pool)
|
||||||
.await;
|
.await;
|
||||||
|
|
||||||
|
if processed % 25 == 0 || processed == total {
|
||||||
|
info!(
|
||||||
|
target: "extraction",
|
||||||
|
"[EXTRACTION] Progress: {}/{} books extracted ({}%)",
|
||||||
|
processed, total, percent
|
||||||
|
);
|
||||||
|
}
|
||||||
return None; // don't enqueue for thumbnail sub-phase
|
return None; // don't enqueue for thumbnail sub-phase
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -549,6 +575,14 @@ pub async fn analyze_library_books(
|
|||||||
.execute(&pool)
|
.execute(&pool)
|
||||||
.await;
|
.await;
|
||||||
|
|
||||||
|
if processed % 25 == 0 || processed == total {
|
||||||
|
info!(
|
||||||
|
target: "extraction",
|
||||||
|
"[EXTRACTION] Progress: {}/{} books extracted ({}%)",
|
||||||
|
processed, total, percent
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
Some((book_id, raw_path, page_count))
|
Some((book_id, raw_path, page_count))
|
||||||
}
|
}
|
||||||
})
|
})
|
||||||
@@ -643,6 +677,14 @@ pub async fn analyze_library_books(
|
|||||||
.bind(percent)
|
.bind(percent)
|
||||||
.execute(&pool)
|
.execute(&pool)
|
||||||
.await;
|
.await;
|
||||||
|
|
||||||
|
if processed % 25 == 0 || processed == extracted_total {
|
||||||
|
info!(
|
||||||
|
target: "thumbnail",
|
||||||
|
"[THUMBNAIL] Progress: {}/{} thumbnails generated ({}%)",
|
||||||
|
processed, extracted_total, percent
|
||||||
|
);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
})
|
})
|
||||||
.await;
|
.await;
|
||||||
|
|||||||
@@ -1,5 +1,4 @@
|
|||||||
use anyhow::Result;
|
use anyhow::Result;
|
||||||
use rayon::prelude::*;
|
|
||||||
use sqlx::{PgPool, Row};
|
use sqlx::{PgPool, Row};
|
||||||
use tracing::{error, info};
|
use tracing::{error, info};
|
||||||
use uuid::Uuid;
|
use uuid::Uuid;
|
||||||
@@ -270,10 +269,12 @@ pub async fn process_job(
|
|||||||
crate::utils::remap_libraries_path(&library.get::<String, _>("root_path"))
|
crate::utils::remap_libraries_path(&library.get::<String, _>("root_path"))
|
||||||
})
|
})
|
||||||
.collect();
|
.collect();
|
||||||
|
// Count sequentially with limited open fds to avoid ENFILE exhaustion
|
||||||
library_paths
|
library_paths
|
||||||
.par_iter()
|
.iter()
|
||||||
.map(|root_path| {
|
.map(|root_path| {
|
||||||
walkdir::WalkDir::new(root_path)
|
walkdir::WalkDir::new(root_path)
|
||||||
|
.max_open(20)
|
||||||
.into_iter()
|
.into_iter()
|
||||||
.filter_map(Result::ok)
|
.filter_map(Result::ok)
|
||||||
.filter(|entry| {
|
.filter(|entry| {
|
||||||
|
|||||||
@@ -8,7 +8,9 @@ use tracing::info;
|
|||||||
async fn main() -> anyhow::Result<()> {
|
async fn main() -> anyhow::Result<()> {
|
||||||
tracing_subscriber::fmt()
|
tracing_subscriber::fmt()
|
||||||
.with_env_filter(
|
.with_env_filter(
|
||||||
std::env::var("RUST_LOG").unwrap_or_else(|_| "indexer=info,axum=info".to_string()),
|
std::env::var("RUST_LOG").unwrap_or_else(|_| {
|
||||||
|
"indexer=info,axum=info,scan=info,extraction=info,thumbnail=warn,watcher=info".to_string()
|
||||||
|
}),
|
||||||
)
|
)
|
||||||
.init();
|
.init();
|
||||||
|
|
||||||
|
|||||||
@@ -4,7 +4,7 @@ use parsers::{detect_format, parse_metadata_fast};
|
|||||||
use serde::Serialize;
|
use serde::Serialize;
|
||||||
use sqlx::Row;
|
use sqlx::Row;
|
||||||
use std::{collections::HashMap, path::Path, time::Duration};
|
use std::{collections::HashMap, path::Path, time::Duration};
|
||||||
use tracing::{error, info, trace, warn};
|
use tracing::{debug, error, info, trace, warn};
|
||||||
use uuid::Uuid;
|
use uuid::Uuid;
|
||||||
use walkdir::WalkDir;
|
use walkdir::WalkDir;
|
||||||
|
|
||||||
@@ -124,7 +124,37 @@ pub async fn scan_library_discovery(
|
|||||||
// Files under these prefixes are added to `seen` but not reprocessed.
|
// Files under these prefixes are added to `seen` but not reprocessed.
|
||||||
let mut skipped_dir_prefixes: Vec<String> = Vec::new();
|
let mut skipped_dir_prefixes: Vec<String> = Vec::new();
|
||||||
|
|
||||||
for entry in WalkDir::new(root).into_iter().filter_map(Result::ok) {
|
// Track consecutive IO errors to detect fd exhaustion (ENFILE)
|
||||||
|
let mut consecutive_io_errors: usize = 0;
|
||||||
|
const MAX_CONSECUTIVE_IO_ERRORS: usize = 10;
|
||||||
|
|
||||||
|
for result in WalkDir::new(root).max_open(20).into_iter() {
|
||||||
|
let entry = match result {
|
||||||
|
Ok(e) => {
|
||||||
|
consecutive_io_errors = 0;
|
||||||
|
e
|
||||||
|
}
|
||||||
|
Err(e) => {
|
||||||
|
consecutive_io_errors += 1;
|
||||||
|
let is_enfile = e
|
||||||
|
.io_error()
|
||||||
|
.map(|io| io.raw_os_error() == Some(23) || io.raw_os_error() == Some(24))
|
||||||
|
.unwrap_or(false);
|
||||||
|
if is_enfile || consecutive_io_errors >= MAX_CONSECUTIVE_IO_ERRORS {
|
||||||
|
error!(
|
||||||
|
"[SCAN] Too many IO errors ({} consecutive) scanning library {} — \
|
||||||
|
fd limit likely exhausted. Aborting scan for this library.",
|
||||||
|
consecutive_io_errors, library_id
|
||||||
|
);
|
||||||
|
stats.warnings += 1;
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
warn!("[SCAN] walkdir error: {}", e);
|
||||||
|
stats.warnings += 1;
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
let path = entry.path().to_path_buf();
|
let path = entry.path().to_path_buf();
|
||||||
let local_path = path.to_string_lossy().to_string();
|
let local_path = path.to_string_lossy().to_string();
|
||||||
|
|
||||||
@@ -192,7 +222,8 @@ pub async fn scan_library_discovery(
|
|||||||
continue;
|
continue;
|
||||||
};
|
};
|
||||||
|
|
||||||
info!(
|
debug!(
|
||||||
|
target: "scan",
|
||||||
"[SCAN] Found book file: {} (format: {:?})",
|
"[SCAN] Found book file: {} (format: {:?})",
|
||||||
path.display(),
|
path.display(),
|
||||||
format
|
format
|
||||||
@@ -209,6 +240,17 @@ pub async fn scan_library_discovery(
|
|||||||
let metadata = match std::fs::metadata(&path) {
|
let metadata = match std::fs::metadata(&path) {
|
||||||
Ok(m) => m,
|
Ok(m) => m,
|
||||||
Err(e) => {
|
Err(e) => {
|
||||||
|
let is_enfile = e.raw_os_error() == Some(23) || e.raw_os_error() == Some(24);
|
||||||
|
if is_enfile {
|
||||||
|
consecutive_io_errors += 1;
|
||||||
|
}
|
||||||
|
if consecutive_io_errors >= MAX_CONSECUTIVE_IO_ERRORS {
|
||||||
|
error!(
|
||||||
|
"[SCAN] fd limit exhausted while stat'ing files in library {}. Aborting.",
|
||||||
|
library_id
|
||||||
|
);
|
||||||
|
break;
|
||||||
|
}
|
||||||
warn!("[SCAN] cannot stat {}, skipping: {}", path.display(), e);
|
warn!("[SCAN] cannot stat {}, skipping: {}", path.display(), e);
|
||||||
stats.warnings += 1;
|
stats.warnings += 1;
|
||||||
continue;
|
continue;
|
||||||
@@ -278,8 +320,9 @@ pub async fn scan_library_discovery(
|
|||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
|
|
||||||
info!(
|
debug!(
|
||||||
"[PROCESS] Updating existing file: {} (fingerprint_changed={})",
|
target: "scan",
|
||||||
|
"[SCAN] Updating: {} (fingerprint_changed={})",
|
||||||
file_name,
|
file_name,
|
||||||
old_fingerprint != fingerprint
|
old_fingerprint != fingerprint
|
||||||
);
|
);
|
||||||
@@ -335,7 +378,7 @@ pub async fn scan_library_discovery(
|
|||||||
}
|
}
|
||||||
|
|
||||||
// New file — insert with page_count = NULL (analyzer fills it in)
|
// New file — insert with page_count = NULL (analyzer fills it in)
|
||||||
info!("[PROCESS] Inserting new file: {}", file_name);
|
debug!(target: "scan", "[SCAN] Inserting: {}", file_name);
|
||||||
let book_id = Uuid::new_v4();
|
let book_id = Uuid::new_v4();
|
||||||
let file_id = Uuid::new_v4();
|
let file_id = Uuid::new_v4();
|
||||||
|
|
||||||
@@ -401,7 +444,28 @@ pub async fn scan_library_discovery(
|
|||||||
library_id, library_processed_count, stats.indexed_files, stats.errors
|
library_id, library_processed_count, stats.indexed_files, stats.errors
|
||||||
);
|
);
|
||||||
|
|
||||||
// Handle deletions
|
// Handle deletions — with safety check against volume mount failures
|
||||||
|
let existing_count = existing.len();
|
||||||
|
let seen_count = seen.len();
|
||||||
|
let stale_count = existing.iter().filter(|(p, _)| !seen.contains_key(p.as_str())).count();
|
||||||
|
|
||||||
|
// Safety: if the library root is not accessible, or if we found zero files
|
||||||
|
// but the DB had many, the volume is probably not mounted correctly.
|
||||||
|
// Do NOT delete anything in that case.
|
||||||
|
let root_accessible = root.is_dir() && std::fs::read_dir(root).is_ok();
|
||||||
|
let skip_deletions = !root_accessible
|
||||||
|
|| (seen_count == 0 && existing_count > 0)
|
||||||
|
|| (stale_count > 0 && stale_count == existing_count);
|
||||||
|
|
||||||
|
if skip_deletions && stale_count > 0 {
|
||||||
|
warn!(
|
||||||
|
"[SCAN] Skipping deletion of {} stale files for library {} — \
|
||||||
|
root accessible={}, seen={}, existing={}. \
|
||||||
|
Volume may not be mounted correctly.",
|
||||||
|
stale_count, library_id, root_accessible, seen_count, existing_count
|
||||||
|
);
|
||||||
|
stats.warnings += stale_count;
|
||||||
|
} else {
|
||||||
let mut removed_count = 0usize;
|
let mut removed_count = 0usize;
|
||||||
for (abs_path, (file_id, book_id, _)) in &existing {
|
for (abs_path, (file_id, book_id, _)) in &existing {
|
||||||
if seen.contains_key(abs_path) {
|
if seen.contains_key(abs_path) {
|
||||||
@@ -427,6 +491,7 @@ pub async fn scan_library_discovery(
|
|||||||
removed_count
|
removed_count
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
}
|
||||||
|
|
||||||
// Upsert directory mtimes for next incremental scan
|
// Upsert directory mtimes for next incremental scan
|
||||||
if !new_dir_mtimes.is_empty() {
|
if !new_dir_mtimes.is_empty() {
|
||||||
|
|||||||
@@ -3,7 +3,7 @@ use sqlx::Row;
|
|||||||
use std::collections::{HashMap, HashSet};
|
use std::collections::{HashMap, HashSet};
|
||||||
use std::path::Path;
|
use std::path::Path;
|
||||||
use std::time::Duration;
|
use std::time::Duration;
|
||||||
use tracing::{error, info, trace, warn};
|
use tracing::{debug, error, info, warn};
|
||||||
use uuid::Uuid;
|
use uuid::Uuid;
|
||||||
use walkdir::WalkDir;
|
use walkdir::WalkDir;
|
||||||
|
|
||||||
@@ -29,6 +29,7 @@ fn snapshot_library(root_path: &str) -> LibrarySnapshot {
|
|||||||
let mut files = HashSet::new();
|
let mut files = HashSet::new();
|
||||||
let walker = WalkDir::new(root_path)
|
let walker = WalkDir::new(root_path)
|
||||||
.follow_links(true)
|
.follow_links(true)
|
||||||
|
.max_open(10)
|
||||||
.into_iter()
|
.into_iter()
|
||||||
.filter_map(|e| e.ok());
|
.filter_map(|e| e.ok());
|
||||||
|
|
||||||
@@ -54,7 +55,7 @@ pub async fn run_file_watcher(state: AppState) -> Result<()> {
|
|||||||
|
|
||||||
// Skip if any job is active — avoid competing for file descriptors
|
// Skip if any job is active — avoid competing for file descriptors
|
||||||
if has_active_jobs(&pool).await {
|
if has_active_jobs(&pool).await {
|
||||||
trace!("[WATCHER] Skipping poll — job active");
|
debug!(target: "watcher", "[WATCHER] Skipping poll — job active");
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -113,7 +114,7 @@ pub async fn run_file_watcher(state: AppState) -> Result<()> {
|
|||||||
|
|
||||||
// Re-check between libraries in case a job was created
|
// Re-check between libraries in case a job was created
|
||||||
if has_active_jobs(&pool).await {
|
if has_active_jobs(&pool).await {
|
||||||
trace!("[WATCHER] Job became active during poll, stopping");
|
debug!(target: "watcher", "[WATCHER] Job became active during poll, stopping");
|
||||||
break;
|
break;
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -126,7 +127,8 @@ pub async fn run_file_watcher(state: AppState) -> Result<()> {
|
|||||||
Some(old_snapshot) => *old_snapshot != new_snapshot,
|
Some(old_snapshot) => *old_snapshot != new_snapshot,
|
||||||
None => {
|
None => {
|
||||||
// First scan — store baseline, don't trigger a job
|
// First scan — store baseline, don't trigger a job
|
||||||
trace!(
|
debug!(
|
||||||
|
target: "watcher",
|
||||||
"[WATCHER] Initial snapshot for library {}: {} files",
|
"[WATCHER] Initial snapshot for library {}: {} files",
|
||||||
library_id,
|
library_id,
|
||||||
new_snapshot.len()
|
new_snapshot.len()
|
||||||
@@ -168,7 +170,7 @@ pub async fn run_file_watcher(state: AppState) -> Result<()> {
|
|||||||
Err(err) => error!("[WATCHER] Failed to create job: {}", err),
|
Err(err) => error!("[WATCHER] Failed to create job: {}", err),
|
||||||
}
|
}
|
||||||
} else {
|
} else {
|
||||||
trace!("[WATCHER] Job already active for library {}, skipping", library_id);
|
debug!(target: "watcher", "[WATCHER] Job already active for library {}, skipping", library_id);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
Reference in New Issue
Block a user