feat(indexing): Lot 4 - Progression temps reel, Full Rebuild, Optimisations
- Ajout migrations DB: index_job_errors, library_monitoring, full_rebuild_type - API: endpoints progression temps reel (/jobs/:id/stream), active jobs, details - API: support full_rebuild avec suppression donnees existantes - Indexer: logs detailles avec timing [SCAN][META][PARSER][BDD] - Indexer: optimisation parsing PDF (lopdf -> pdfinfo) 235x plus rapide - Indexer: corrections chemins LIBRARIES_ROOT_PATH pour dev local - Backoffice: composants JobProgress, JobsIndicator (header), JobsList - Backoffice: SSE streaming pour progression temps reel - Backoffice: boutons Index/Index Full sur page libraries - Backoffice: highlight job apres creation avec redirection - Fix: parsing volume type i32, sync meilisearch cleanup Perf: parsing PDF passe de 8.7s a 37ms Perf: indexation 45 fichiers en ~15s vs plusieurs minutes avant
This commit is contained in:
123
apps/backoffice/app/components/JobProgress.tsx
Normal file
123
apps/backoffice/app/components/JobProgress.tsx
Normal file
@@ -0,0 +1,123 @@
|
||||
"use client";
|
||||
|
||||
import { useEffect, useState } from "react";
|
||||
|
||||
interface ProgressEvent {
|
||||
job_id: string;
|
||||
status: string;
|
||||
current_file: string | null;
|
||||
progress_percent: number | null;
|
||||
processed_files: number | null;
|
||||
total_files: number | null;
|
||||
stats_json: {
|
||||
scanned_files: number;
|
||||
indexed_files: number;
|
||||
removed_files: number;
|
||||
errors: number;
|
||||
} | null;
|
||||
}
|
||||
|
||||
interface JobProgressProps {
|
||||
jobId: string;
|
||||
onComplete?: () => void;
|
||||
}
|
||||
|
||||
export function JobProgress({ jobId, onComplete }: JobProgressProps) {
|
||||
const [progress, setProgress] = useState<ProgressEvent | null>(null);
|
||||
const [error, setError] = useState<string | null>(null);
|
||||
const [isComplete, setIsComplete] = useState(false);
|
||||
|
||||
useEffect(() => {
|
||||
// Use SSE via local proxy
|
||||
const eventSource = new EventSource(`/api/jobs/${jobId}/stream`);
|
||||
|
||||
eventSource.onmessage = (event) => {
|
||||
try {
|
||||
const data = JSON.parse(event.data);
|
||||
|
||||
const progressData: ProgressEvent = {
|
||||
job_id: data.id,
|
||||
status: data.status,
|
||||
current_file: data.current_file,
|
||||
progress_percent: data.progress_percent,
|
||||
processed_files: data.processed_files,
|
||||
total_files: data.total_files,
|
||||
stats_json: data.stats_json,
|
||||
};
|
||||
|
||||
setProgress(progressData);
|
||||
|
||||
if (data.status === "success" || data.status === "failed" || data.status === "cancelled") {
|
||||
setIsComplete(true);
|
||||
eventSource.close();
|
||||
onComplete?.();
|
||||
}
|
||||
} catch (err) {
|
||||
setError("Failed to parse SSE data");
|
||||
}
|
||||
};
|
||||
|
||||
eventSource.onerror = (err) => {
|
||||
console.error("SSE error:", err);
|
||||
eventSource.close();
|
||||
setError("Connection lost");
|
||||
};
|
||||
|
||||
return () => {
|
||||
eventSource.close();
|
||||
};
|
||||
}, [jobId, onComplete]);
|
||||
|
||||
if (error) {
|
||||
return <div className="progress-error">Error: {error}</div>;
|
||||
}
|
||||
|
||||
if (!progress) {
|
||||
return <div className="progress-loading">Loading progress...</div>;
|
||||
}
|
||||
|
||||
const percent = progress.progress_percent ?? 0;
|
||||
const processed = progress.processed_files ?? 0;
|
||||
const total = progress.total_files ?? 0;
|
||||
|
||||
return (
|
||||
<div className="job-progress">
|
||||
<div className="progress-header">
|
||||
<span className={`status-badge status-${progress.status}`}>
|
||||
{progress.status}
|
||||
</span>
|
||||
{isComplete && <span className="complete-badge">Complete</span>}
|
||||
</div>
|
||||
|
||||
<div className="progress-bar-container">
|
||||
<div
|
||||
className="progress-bar-fill"
|
||||
style={{ width: `${percent}%` }}
|
||||
/>
|
||||
<span className="progress-percent">{percent}%</span>
|
||||
</div>
|
||||
|
||||
<div className="progress-stats">
|
||||
<span>{processed} / {total} files</span>
|
||||
{progress.current_file && (
|
||||
<span className="current-file" title={progress.current_file}>
|
||||
Current: {progress.current_file.length > 40
|
||||
? progress.current_file.substring(0, 40) + "..."
|
||||
: progress.current_file}
|
||||
</span>
|
||||
)}
|
||||
</div>
|
||||
|
||||
{progress.stats_json && (
|
||||
<div className="progress-detailed-stats">
|
||||
<span>Scanned: {progress.stats_json.scanned_files}</span>
|
||||
<span>Indexed: {progress.stats_json.indexed_files}</span>
|
||||
<span>Removed: {progress.stats_json.removed_files}</span>
|
||||
{progress.stats_json.errors > 0 && (
|
||||
<span className="error-count">Errors: {progress.stats_json.errors}</span>
|
||||
)}
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
);
|
||||
}
|
||||
75
apps/backoffice/app/components/JobRow.tsx
Normal file
75
apps/backoffice/app/components/JobRow.tsx
Normal file
@@ -0,0 +1,75 @@
|
||||
"use client";
|
||||
|
||||
import { useState } from "react";
|
||||
import { JobProgress } from "./JobProgress";
|
||||
|
||||
interface JobRowProps {
|
||||
job: {
|
||||
id: string;
|
||||
library_id: string | null;
|
||||
type: string;
|
||||
status: string;
|
||||
created_at: string;
|
||||
error_opt: string | null;
|
||||
};
|
||||
libraryName: string | undefined;
|
||||
highlighted?: boolean;
|
||||
onCancel: (id: string) => void;
|
||||
}
|
||||
|
||||
export function JobRow({ job, libraryName, highlighted, onCancel }: JobRowProps) {
|
||||
const [showProgress, setShowProgress] = useState(
|
||||
highlighted || job.status === "running" || job.status === "pending"
|
||||
);
|
||||
|
||||
const handleComplete = () => {
|
||||
setShowProgress(false);
|
||||
// Trigger a page refresh to update the job status
|
||||
window.location.reload();
|
||||
};
|
||||
|
||||
return (
|
||||
<>
|
||||
<tr className={highlighted ? "job-highlighted" : undefined}>
|
||||
<td>
|
||||
<code>{job.id.slice(0, 8)}</code>
|
||||
</td>
|
||||
<td>{job.library_id ? libraryName || job.library_id.slice(0, 8) : "—"}</td>
|
||||
<td>{job.type}</td>
|
||||
<td>
|
||||
<span className={`status-${job.status}`}>{job.status}</span>
|
||||
{job.error_opt && <span className="error-hint" title={job.error_opt}>!</span>}
|
||||
{job.status === "running" && (
|
||||
<button
|
||||
className="toggle-progress-btn"
|
||||
onClick={() => setShowProgress(!showProgress)}
|
||||
>
|
||||
{showProgress ? "Hide" : "Show"} progress
|
||||
</button>
|
||||
)}
|
||||
</td>
|
||||
<td>{new Date(job.created_at).toLocaleString()}</td>
|
||||
<td>
|
||||
{job.status === "pending" || job.status === "running" ? (
|
||||
<button
|
||||
className="cancel-btn"
|
||||
onClick={() => onCancel(job.id)}
|
||||
>
|
||||
Cancel
|
||||
</button>
|
||||
) : null}
|
||||
</td>
|
||||
</tr>
|
||||
{showProgress && (job.status === "running" || job.status === "pending") && (
|
||||
<tr className="progress-row">
|
||||
<td colSpan={6}>
|
||||
<JobProgress
|
||||
jobId={job.id}
|
||||
onComplete={handleComplete}
|
||||
/>
|
||||
</td>
|
||||
</tr>
|
||||
)}
|
||||
</>
|
||||
);
|
||||
}
|
||||
144
apps/backoffice/app/components/JobsIndicator.tsx
Normal file
144
apps/backoffice/app/components/JobsIndicator.tsx
Normal file
@@ -0,0 +1,144 @@
|
||||
"use client";
|
||||
|
||||
import { useEffect, useState } from "react";
|
||||
import Link from "next/link";
|
||||
|
||||
interface Job {
|
||||
id: string;
|
||||
status: string;
|
||||
current_file: string | null;
|
||||
progress_percent: number | null;
|
||||
}
|
||||
|
||||
interface JobsIndicatorProps {
|
||||
apiBaseUrl: string;
|
||||
apiToken: string;
|
||||
}
|
||||
|
||||
export function JobsIndicator({ apiBaseUrl, apiToken }: JobsIndicatorProps) {
|
||||
const [activeJobs, setActiveJobs] = useState<Job[]>([]);
|
||||
const [isOpen, setIsOpen] = useState(false);
|
||||
|
||||
useEffect(() => {
|
||||
const fetchActiveJobs = async () => {
|
||||
try {
|
||||
const response = await fetch(`${apiBaseUrl}/index/jobs/active`, {
|
||||
headers: {
|
||||
"Authorization": `Bearer ${apiToken}`,
|
||||
},
|
||||
});
|
||||
|
||||
if (response.ok) {
|
||||
const jobs = await response.json();
|
||||
// Enrich with details for running jobs
|
||||
const jobsWithDetails = await Promise.all(
|
||||
jobs.map(async (job: Job) => {
|
||||
if (job.status === "running") {
|
||||
try {
|
||||
const detailRes = await fetch(`${apiBaseUrl}/index/jobs/${job.id}`, {
|
||||
headers: { "Authorization": `Bearer ${apiToken}` },
|
||||
});
|
||||
if (detailRes.ok) {
|
||||
const detail = await detailRes.json();
|
||||
return { ...job, ...detail };
|
||||
}
|
||||
} catch {
|
||||
// ignore detail fetch errors
|
||||
}
|
||||
}
|
||||
return job;
|
||||
})
|
||||
);
|
||||
setActiveJobs(jobsWithDetails);
|
||||
}
|
||||
} catch {
|
||||
// Silently fail
|
||||
}
|
||||
};
|
||||
|
||||
fetchActiveJobs();
|
||||
const interval = setInterval(fetchActiveJobs, 5000);
|
||||
|
||||
return () => clearInterval(interval);
|
||||
}, [apiBaseUrl, apiToken]);
|
||||
|
||||
const pendingCount = activeJobs.filter(j => j.status === "pending").length;
|
||||
const runningCount = activeJobs.filter(j => j.status === "running").length;
|
||||
const totalCount = activeJobs.length;
|
||||
|
||||
if (totalCount === 0) {
|
||||
return (
|
||||
<Link href="/jobs" className="jobs-indicator empty">
|
||||
<svg width="20" height="20" viewBox="0 0 24 24" fill="none" stroke="currentColor" strokeWidth="2">
|
||||
<rect x="4" y="4" width="16" height="16" rx="2" />
|
||||
<path d="M8 12h8M12 8v8" />
|
||||
</svg>
|
||||
</Link>
|
||||
);
|
||||
}
|
||||
|
||||
return (
|
||||
<div className="jobs-indicator-container">
|
||||
<button
|
||||
className="jobs-indicator active"
|
||||
onClick={() => setIsOpen(!isOpen)}
|
||||
title={`${totalCount} active job${totalCount !== 1 ? 's' : ''}`}
|
||||
>
|
||||
<svg width="20" height="20" viewBox="0 0 24 24" fill="none" stroke="currentColor" strokeWidth="2">
|
||||
<rect x="4" y="4" width="16" height="16" rx="2" />
|
||||
<path d="M8 12h8M12 8v8" />
|
||||
</svg>
|
||||
{totalCount > 0 && (
|
||||
<span className="jobs-badge">
|
||||
{totalCount > 9 ? "9+" : totalCount}
|
||||
</span>
|
||||
)}
|
||||
{runningCount > 0 && (
|
||||
<span className="jobs-pulse" />
|
||||
)}
|
||||
</button>
|
||||
|
||||
{isOpen && (
|
||||
<div className="jobs-dropdown">
|
||||
<div className="jobs-dropdown-header">
|
||||
<strong>Active Jobs</strong>
|
||||
<Link href="/jobs" onClick={() => setIsOpen(false)}>View all</Link>
|
||||
</div>
|
||||
|
||||
{activeJobs.length === 0 ? (
|
||||
<p className="jobs-empty">No active jobs</p>
|
||||
) : (
|
||||
<ul className="jobs-list">
|
||||
{activeJobs.map(job => (
|
||||
<li key={job.id} className={`job-item job-${job.status}`}>
|
||||
<div className="job-header">
|
||||
<span className={`job-status status-${job.status}`}>
|
||||
{job.status}
|
||||
</span>
|
||||
<code className="job-id">{job.id.slice(0, 8)}</code>
|
||||
</div>
|
||||
{job.status === "running" && job.progress_percent !== null && (
|
||||
<div className="job-mini-progress">
|
||||
<div
|
||||
className="job-progress-bar"
|
||||
style={{ width: `${job.progress_percent}%` }}
|
||||
/>
|
||||
<span>{job.progress_percent}%</span>
|
||||
</div>
|
||||
)}
|
||||
{job.current_file && (
|
||||
<p className="job-file" title={job.current_file}>
|
||||
{job.current_file.length > 30
|
||||
? job.current_file.substring(0, 30) + "..."
|
||||
: job.current_file}
|
||||
</p>
|
||||
)}
|
||||
</li>
|
||||
))}
|
||||
</ul>
|
||||
)}
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
);
|
||||
}
|
||||
12
apps/backoffice/app/components/JobsIndicatorWrapper.tsx
Normal file
12
apps/backoffice/app/components/JobsIndicatorWrapper.tsx
Normal file
@@ -0,0 +1,12 @@
|
||||
"use client";
|
||||
|
||||
import { JobsIndicator } from "./JobsIndicator";
|
||||
|
||||
interface JobsIndicatorWrapperProps {
|
||||
apiBaseUrl: string;
|
||||
apiToken: string;
|
||||
}
|
||||
|
||||
export function JobsIndicatorWrapper({ apiBaseUrl, apiToken }: JobsIndicatorWrapperProps) {
|
||||
return <JobsIndicator apiBaseUrl={apiBaseUrl} apiToken={apiToken} />;
|
||||
}
|
||||
91
apps/backoffice/app/components/JobsList.tsx
Normal file
91
apps/backoffice/app/components/JobsList.tsx
Normal file
@@ -0,0 +1,91 @@
|
||||
"use client";
|
||||
|
||||
import { useState, useEffect } from "react";
|
||||
import { JobRow } from "./JobRow";
|
||||
|
||||
interface Job {
|
||||
id: string;
|
||||
library_id: string | null;
|
||||
type: string;
|
||||
status: string;
|
||||
created_at: string;
|
||||
error_opt: string | null;
|
||||
}
|
||||
|
||||
interface JobsListProps {
|
||||
initialJobs: Job[];
|
||||
libraries: Map<string, string>;
|
||||
highlightJobId?: string;
|
||||
}
|
||||
|
||||
export function JobsList({ initialJobs, libraries, highlightJobId }: JobsListProps) {
|
||||
const [jobs, setJobs] = useState(initialJobs);
|
||||
|
||||
// Refresh jobs list via SSE
|
||||
useEffect(() => {
|
||||
const eventSource = new EventSource("/api/jobs/stream");
|
||||
|
||||
eventSource.onmessage = (event) => {
|
||||
try {
|
||||
const data = JSON.parse(event.data);
|
||||
if (Array.isArray(data)) {
|
||||
setJobs(data);
|
||||
}
|
||||
} catch (error) {
|
||||
console.error("Failed to parse SSE data:", error);
|
||||
}
|
||||
};
|
||||
|
||||
eventSource.onerror = (err) => {
|
||||
console.error("SSE error:", err);
|
||||
eventSource.close();
|
||||
};
|
||||
|
||||
return () => {
|
||||
eventSource.close();
|
||||
};
|
||||
}, []);
|
||||
|
||||
const handleCancel = async (id: string) => {
|
||||
try {
|
||||
const response = await fetch(`/api/jobs/${id}/cancel`, {
|
||||
method: "POST",
|
||||
});
|
||||
|
||||
if (response.ok) {
|
||||
// Update local state to reflect cancellation
|
||||
setJobs(jobs.map(job =>
|
||||
job.id === id ? { ...job, status: "cancelled" } : job
|
||||
));
|
||||
}
|
||||
} catch (error) {
|
||||
console.error("Failed to cancel job:", error);
|
||||
}
|
||||
};
|
||||
|
||||
return (
|
||||
<table>
|
||||
<thead>
|
||||
<tr>
|
||||
<th>ID</th>
|
||||
<th>Library</th>
|
||||
<th>Type</th>
|
||||
<th>Status</th>
|
||||
<th>Created</th>
|
||||
<th>Actions</th>
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody>
|
||||
{jobs.map((job) => (
|
||||
<JobRow
|
||||
key={job.id}
|
||||
job={job}
|
||||
libraryName={job.library_id ? libraries.get(job.library_id) : undefined}
|
||||
highlighted={job.id === highlightJobId}
|
||||
onCancel={handleCancel}
|
||||
/>
|
||||
))}
|
||||
</tbody>
|
||||
</table>
|
||||
);
|
||||
}
|
||||
Reference in New Issue
Block a user