feat(indexing): Lot 4 - Progression temps reel, Full Rebuild, Optimisations
- Ajout migrations DB: index_job_errors, library_monitoring, full_rebuild_type - API: endpoints progression temps reel (/jobs/:id/stream), active jobs, details - API: support full_rebuild avec suppression donnees existantes - Indexer: logs detailles avec timing [SCAN][META][PARSER][BDD] - Indexer: optimisation parsing PDF (lopdf -> pdfinfo) 235x plus rapide - Indexer: corrections chemins LIBRARIES_ROOT_PATH pour dev local - Backoffice: composants JobProgress, JobsIndicator (header), JobsList - Backoffice: SSE streaming pour progression temps reel - Backoffice: boutons Index/Index Full sur page libraries - Backoffice: highlight job apres creation avec redirection - Fix: parsing volume type i32, sync meilisearch cleanup Perf: parsing PDF passe de 8.7s a 37ms Perf: indexation 45 fichiers en ~15s vs plusieurs minutes avant
This commit is contained in:
26
infra/migrations/0003_index_job_errors.sql
Normal file
26
infra/migrations/0003_index_job_errors.sql
Normal file
@@ -0,0 +1,26 @@
|
||||
-- Migration: Progression temps reel et erreurs d'indexation (T19 - Option 2)
|
||||
|
||||
-- Colonnes pour la progression temps reel dans index_jobs
|
||||
ALTER TABLE index_jobs
|
||||
ADD COLUMN IF NOT EXISTS current_file TEXT,
|
||||
ADD COLUMN IF NOT EXISTS progress_percent INTEGER CHECK (progress_percent >= 0 AND progress_percent <= 100),
|
||||
ADD COLUMN IF NOT EXISTS total_files INTEGER,
|
||||
ADD COLUMN IF NOT EXISTS processed_files INTEGER DEFAULT 0;
|
||||
|
||||
-- Table pour stocker les erreurs d'indexation
|
||||
CREATE TABLE IF NOT EXISTS index_job_errors (
|
||||
id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
|
||||
job_id UUID NOT NULL REFERENCES index_jobs(id) ON DELETE CASCADE,
|
||||
file_path TEXT NOT NULL,
|
||||
error_message TEXT NOT NULL,
|
||||
created_at TIMESTAMPTZ NOT NULL DEFAULT NOW()
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_index_job_errors_job_id ON index_job_errors(job_id);
|
||||
CREATE INDEX IF NOT EXISTS idx_index_job_errors_created_at ON index_job_errors(created_at);
|
||||
|
||||
COMMENT ON TABLE index_job_errors IS 'Stocke uniquement les erreurs d indexation pour debug';
|
||||
COMMENT ON COLUMN index_jobs.current_file IS 'Fichier en cours de traitement (pour affichage temps reel)';
|
||||
COMMENT ON COLUMN index_jobs.progress_percent IS 'Pourcentage de completion estime';
|
||||
COMMENT ON COLUMN index_jobs.total_files IS 'Nombre total de fichiers a traiter (estimation)';
|
||||
COMMENT ON COLUMN index_jobs.processed_files IS 'Nombre de fichiers deja traites';
|
||||
17
infra/migrations/0004_library_monitoring.sql
Normal file
17
infra/migrations/0004_library_monitoring.sql
Normal file
@@ -0,0 +1,17 @@
|
||||
-- Migration: Surveillance automatique des libraries (T23)
|
||||
-- Ajout des colonnes pour la configuration du scan automatique
|
||||
|
||||
ALTER TABLE libraries
|
||||
ADD COLUMN IF NOT EXISTS monitor_enabled BOOLEAN NOT NULL DEFAULT FALSE,
|
||||
ADD COLUMN IF NOT EXISTS scan_mode TEXT NOT NULL DEFAULT 'manual' CHECK (scan_mode IN ('manual', 'hourly', 'daily', 'weekly')),
|
||||
ADD COLUMN IF NOT EXISTS last_scan_at TIMESTAMPTZ,
|
||||
ADD COLUMN IF NOT EXISTS next_scan_at TIMESTAMPTZ;
|
||||
|
||||
-- Index pour trouver rapidement les libraries à scanner
|
||||
CREATE INDEX IF NOT EXISTS idx_libraries_monitor_enabled ON libraries(monitor_enabled);
|
||||
CREATE INDEX IF NOT EXISTS idx_libraries_next_scan_at ON libraries(next_scan_at) WHERE monitor_enabled = TRUE;
|
||||
|
||||
COMMENT ON COLUMN libraries.monitor_enabled IS 'Active la surveillance automatique de la library';
|
||||
COMMENT ON COLUMN libraries.scan_mode IS 'Mode de scan: manual, hourly (60min), daily (1440min), weekly (10080min)';
|
||||
COMMENT ON COLUMN libraries.last_scan_at IS 'Date du dernier scan (manuel ou automatique)';
|
||||
COMMENT ON COLUMN libraries.next_scan_at IS 'Date du prochain scan automatique prévu';
|
||||
6
infra/migrations/0005_full_rebuild_type.sql
Normal file
6
infra/migrations/0005_full_rebuild_type.sql
Normal file
@@ -0,0 +1,6 @@
|
||||
-- Migration: Ajout du type de job "full_rebuild" pour réindexation complète
|
||||
|
||||
ALTER TABLE index_jobs
|
||||
DROP CONSTRAINT IF EXISTS index_jobs_type_check,
|
||||
ADD CONSTRAINT index_jobs_type_check
|
||||
CHECK (type IN ('scan', 'rebuild', 'full_rebuild'));
|
||||
Reference in New Issue
Block a user