julienfroidefond/stripstream-librarian
1
0
Fork 0
Code Issues 4 Pull Requests Actions Packages Projects Releases Wiki Activity

feat: add scheduled metadata refresh for libraries

Browse Source 
Add metadata_refresh_mode (manual/hourly/daily/weekly) to libraries,
with automatic scheduling via the indexer. Includes API support,
backoffice UI controls, i18n translations, and DB migration.

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
Julien Froidefond
2026-03-20 10:51:52 +01:00
parent 5a224c48c0
commit fe5de3d5c1
8 changed files with 143 additions and 10 deletions
Download Patch File Download Diff File Expand all files Collapse all files

40
apps/api/src/libraries.rs
View File

@@ -23,6 +23,9 @@ pub struct LibraryResponse {
pub watcher_enabled: bool,
pub metadata_provider: Option<String>,
pub fallback_metadata_provider: Option<String>,
pub metadata_refresh_mode: String,
#[schema(value_type = Option<String>)]
pub next_metadata_refresh_at: Option<chrono::DateTime<chrono::Utc>>,
}
#[derive(Deserialize, ToSchema)]
@@ -47,7 +50,7 @@ pub struct CreateLibraryRequest {
)]
pub async fn list_libraries(State(state): State<AppState>) -> Result<Json<Vec<LibraryResponse>>, ApiError> {
let rows = sqlx::query(
"SELECT l.id, l.name, l.root_path, l.enabled, l.monitor_enabled, l.scan_mode, l.next_scan_at, l.watcher_enabled, l.metadata_provider, l.fallback_metadata_provider,
"SELECT l.id, l.name, l.root_path, l.enabled, l.monitor_enabled, l.scan_mode, l.next_scan_at, l.watcher_enabled, l.metadata_provider, l.fallback_metadata_provider, l.metadata_refresh_mode, l.next_metadata_refresh_at,
(SELECT COUNT(*) FROM books b WHERE b.library_id = l.id) as book_count
FROM libraries l ORDER BY l.created_at DESC"
)
@@ -68,6 +71,8 @@ pub async fn list_libraries(State(state): State<AppState>) -> Result<Json<Vec<Li
watcher_enabled: row.get("watcher_enabled"),
metadata_provider: row.get("metadata_provider"),
fallback_metadata_provider: row.get("fallback_metadata_provider"),
metadata_refresh_mode: row.get("metadata_refresh_mode"),
next_metadata_refresh_at: row.get("next_metadata_refresh_at"),
})
.collect();
@@ -121,6 +126,8 @@ pub async fn create_library(
watcher_enabled: false,
metadata_provider: None,
fallback_metadata_provider: None,
metadata_refresh_mode: "manual".to_string(),
next_metadata_refresh_at: None,
}))
}
@@ -241,6 +248,8 @@ pub struct UpdateMonitoringRequest {
#[schema(value_type = String, example = "hourly")]
pub scan_mode: String, // 'manual', 'hourly', 'daily', 'weekly'
pub watcher_enabled: Option<bool>,
#[schema(value_type = Option<String>, example = "daily")]
pub metadata_refresh_mode: Option<String>, // 'manual', 'hourly', 'daily', 'weekly'
}
/// Update monitoring settings for a library
@@ -271,6 +280,12 @@ pub async fn update_monitoring(
return Err(ApiError::bad_request("scan_mode must be one of: manual, hourly, daily, weekly"));
}
// Validate metadata_refresh_mode
let metadata_refresh_mode = input.metadata_refresh_mode.as_deref().unwrap_or("manual");
if !valid_modes.contains(&metadata_refresh_mode) {
return Err(ApiError::bad_request("metadata_refresh_mode must be one of: manual, hourly, daily, weekly"));
}
// Calculate next_scan_at if monitoring is enabled
let next_scan_at = if input.monitor_enabled {
let interval_minutes = match input.scan_mode.as_str() {
@@ -284,16 +299,31 @@ pub async fn update_monitoring(
None
};
// Calculate next_metadata_refresh_at
let next_metadata_refresh_at = if metadata_refresh_mode != "manual" {
let interval_minutes = match metadata_refresh_mode {
"hourly" => 60,
"daily" => 1440,
"weekly" => 10080,
_ => 1440,
};
Some(chrono::Utc::now() + chrono::Duration::minutes(interval_minutes))
} else {
None
};
let watcher_enabled = input.watcher_enabled.unwrap_or(false);
let result = sqlx::query(
"UPDATE libraries SET monitor_enabled = $2, scan_mode = $3, next_scan_at = $4, watcher_enabled = $5 WHERE id = $1 RETURNING id, name, root_path, enabled, monitor_enabled, scan_mode, next_scan_at, watcher_enabled, metadata_provider, fallback_metadata_provider"
"UPDATE libraries SET monitor_enabled = $2, scan_mode = $3, next_scan_at = $4, watcher_enabled = $5, metadata_refresh_mode = $6, next_metadata_refresh_at = $7 WHERE id = $1 RETURNING id, name, root_path, enabled, monitor_enabled, scan_mode, next_scan_at, watcher_enabled, metadata_provider, fallback_metadata_provider, metadata_refresh_mode, next_metadata_refresh_at"
)
.bind(library_id)
.bind(input.monitor_enabled)
.bind(input.scan_mode)
.bind(next_scan_at)
.bind(watcher_enabled)
.bind(metadata_refresh_mode)
.bind(next_metadata_refresh_at)
.fetch_optional(&state.pool)
.await?;
@@ -318,6 +348,8 @@ pub async fn update_monitoring(
watcher_enabled: row.get("watcher_enabled"),
metadata_provider: row.get("metadata_provider"),
fallback_metadata_provider: row.get("fallback_metadata_provider"),
metadata_refresh_mode: row.get("metadata_refresh_mode"),
next_metadata_refresh_at: row.get("next_metadata_refresh_at"),
}))
}
@@ -353,7 +385,7 @@ pub async fn update_metadata_provider(
let fallback = input.fallback_metadata_provider.as_deref().filter(|s| !s.is_empty());
let result = sqlx::query(
"UPDATE libraries SET metadata_provider = $2, fallback_metadata_provider = $3 WHERE id = $1 RETURNING id, name, root_path, enabled, monitor_enabled, scan_mode, next_scan_at, watcher_enabled, metadata_provider, fallback_metadata_provider"
"UPDATE libraries SET metadata_provider = $2, fallback_metadata_provider = $3 WHERE id = $1 RETURNING id, name, root_path, enabled, monitor_enabled, scan_mode, next_scan_at, watcher_enabled, metadata_provider, fallback_metadata_provider, metadata_refresh_mode, next_metadata_refresh_at"
)
.bind(library_id)
.bind(provider)
@@ -382,5 +414,7 @@ pub async fn update_metadata_provider(
watcher_enabled: row.get("watcher_enabled"),
metadata_provider: row.get("metadata_provider"),
fallback_metadata_provider: row.get("fallback_metadata_provider"),
metadata_refresh_mode: row.get("metadata_refresh_mode"),
next_metadata_refresh_at: row.get("next_metadata_refresh_at"),
}))
}

18
apps/backoffice/app/components/LibraryActions.tsx
View File

@@ -12,6 +12,7 @@ interface LibraryActionsProps {
watcherEnabled: boolean;
metadataProvider: string | null;
fallbackMetadataProvider: string | null;
metadataRefreshMode: string;
onUpdate?: () => void;
}
@@ -22,6 +23,7 @@ export function LibraryActions({
watcherEnabled,
metadataProvider,
fallbackMetadataProvider,
metadataRefreshMode,
onUpdate
}: LibraryActionsProps) {
const { t } = useTranslation();
@@ -48,6 +50,7 @@ export function LibraryActions({
const scanMode = formData.get("scan_mode") as string;
const newMetadataProvider = (formData.get("metadata_provider") as string) || null;
const newFallbackProvider = (formData.get("fallback_metadata_provider") as string) || null;
const newMetadataRefreshMode = formData.get("metadata_refresh_mode") as string;
try {
const [response] = await Promise.all([
@@ -58,6 +61,7 @@ export function LibraryActions({
monitor_enabled: monitorEnabled,
scan_mode: scanMode,
watcher_enabled: watcherEnabled,
metadata_refresh_mode: newMetadataRefreshMode,
}),
}),
fetch(`/api/libraries/${libraryId}/metadata-provider`, {
@@ -181,6 +185,20 @@ export function LibraryActions({
</select>
</div>
<div className="flex items-center justify-between">
<label className="text-sm font-medium text-foreground">{t("libraryActions.metadataRefreshSchedule")}</label>
<select
name="metadata_refresh_mode"
defaultValue={metadataRefreshMode}
className="text-sm border border-border rounded-lg px-2 py-1 bg-background"
>
<option value="manual">{t("monitoring.manual")}</option>
<option value="hourly">{t("monitoring.hourly")}</option>
<option value="daily">{t("monitoring.daily")}</option>
<option value="weekly">{t("monitoring.weekly")}</option>
</select>
</div>
{saveError && (
<p className="text-xs text-destructive bg-destructive/10 px-2 py-1.5 rounded-lg break-all">
{saveError}

6
apps/backoffice/app/libraries/page.tsx
View File

@@ -131,6 +131,7 @@ export default async function LibrariesPage() {
watcherEnabled={lib.watcher_enabled}
metadataProvider={lib.metadata_provider}
fallbackMetadataProvider={lib.fallback_metadata_provider}
metadataRefreshMode={lib.metadata_refresh_mode}
/>
</div>
</CardHeader>
@@ -169,6 +170,11 @@ export default async function LibrariesPage() {
{t("libraries.nextScan", { time: formatNextScan(lib.next_scan_at, t("libraries.imminent")) })}
</span>
)}
{lib.metadata_refresh_mode !== "manual" && lib.next_metadata_refresh_at && (
<span className="text-xs text-muted-foreground ml-auto" title={t("libraries.nextMetadataRefresh", { time: formatNextScan(lib.next_metadata_refresh_at, t("libraries.imminent")) })}>
{t("libraries.nextMetadataRefreshShort", { time: formatNextScan(lib.next_metadata_refresh_at, t("libraries.imminent")) })}
</span>
)}
</div>
{/* Actions */}

3
apps/backoffice/lib/i18n/en.ts
View File

@@ -124,6 +124,8 @@ const en: Record<TranslationKey, string> = {
"libraries.manual": "Manual",
"libraries.nextScan": "Next: {{time}}",
"libraries.imminent": "Imminent",
"libraries.nextMetadataRefresh": "Next metadata refresh: {{time}}",
"libraries.nextMetadataRefreshShort": "Meta.: {{time}}",
"libraries.index": "Index",
"libraries.fullIndex": "Full",
"libraries.batchMetadata": "Batch metadata",
@@ -148,6 +150,7 @@ const en: Record<TranslationKey, string> = {
"libraryActions.fallback": "Fallback",
"libraryActions.default": "Default",
"libraryActions.none": "None",
"libraryActions.metadataRefreshSchedule": "Refresh meta.",
"libraryActions.saving": "Saving...",
// Library sub-page header

3
apps/backoffice/lib/i18n/fr.ts
View File

@@ -122,6 +122,8 @@ const fr = {
"libraries.manual": "Manuel",
"libraries.nextScan": "Prochain : {{time}}",
"libraries.imminent": "Imminent",
"libraries.nextMetadataRefresh": "Prochain rafraîchissement méta. : {{time}}",
"libraries.nextMetadataRefreshShort": "Méta. : {{time}}",
"libraries.index": "Indexer",
"libraries.fullIndex": "Complet",
"libraries.batchMetadata": "Métadonnées en lot",
@@ -146,6 +148,7 @@ const fr = {
"libraryActions.fallback": "Secours",
"libraryActions.default": "Par défaut",
"libraryActions.none": "Aucun",
"libraryActions.metadataRefreshSchedule": "Rafraîchir méta.",
"libraryActions.saving": "Enregistrement...",
// Library sub-page header

76
apps/indexer/src/scheduler.rs
View File

@@ -26,15 +26,15 @@ pub async fn check_and_schedule_auto_scans(pool: &PgPool) -> Result<()> {
for row in libraries {
let library_id: Uuid = row.get("id");
let scan_mode: String = row.get("scan_mode");
info!("[SCHEDULER] Auto-scanning library {} (mode: {})", library_id, scan_mode);
let job_id = Uuid::new_v4();
let job_type = match scan_mode.as_str() {
"full" => "full_rebuild",
_ => "rebuild",
};
sqlx::query(
"INSERT INTO index_jobs (id, library_id, type, status) VALUES ($1, $2, $3, 'pending')"
)
@@ -43,7 +43,7 @@ pub async fn check_and_schedule_auto_scans(pool: &PgPool) -> Result<()> {
.bind(job_type)
.execute(pool)
.await?;
// Update next_scan_at
let interval_minutes = match scan_mode.as_str() {
"hourly" => 60,
@@ -51,7 +51,7 @@ pub async fn check_and_schedule_auto_scans(pool: &PgPool) -> Result<()> {
"weekly" => 10080,
_ => 1440, // default daily
};
sqlx::query(
"UPDATE libraries SET last_scan_at = NOW(), next_scan_at = NOW() + INTERVAL '1 minute' * $2 WHERE id = $1"
)
@@ -59,9 +59,71 @@ pub async fn check_and_schedule_auto_scans(pool: &PgPool) -> Result<()> {
.bind(interval_minutes)
.execute(pool)
.await?;
info!("[SCHEDULER] Created job {} for library {}", job_id, library_id);
}
Ok(())
}
pub async fn check_and_schedule_metadata_refreshes(pool: &PgPool) -> Result<()> {
let libraries = sqlx::query(
r#"
SELECT id, metadata_refresh_mode
FROM libraries
WHERE metadata_refresh_mode != 'manual'
AND (
next_metadata_refresh_at IS NULL
OR next_metadata_refresh_at <= NOW()
)
AND NOT EXISTS (
SELECT 1 FROM index_jobs
WHERE library_id = libraries.id
AND type = 'metadata_refresh'
AND status IN ('pending', 'running')
)
AND EXISTS (
SELECT 1 FROM external_metadata_links
WHERE library_id = libraries.id
AND status = 'approved'
)
"#
)
.fetch_all(pool)
.await?;
for row in libraries {
let library_id: Uuid = row.get("id");
let refresh_mode: String = row.get("metadata_refresh_mode");
info!("[SCHEDULER] Auto-refreshing metadata for library {} (mode: {})", library_id, refresh_mode);
let job_id = Uuid::new_v4();
sqlx::query(
"INSERT INTO index_jobs (id, library_id, type, status) VALUES ($1, $2, 'metadata_refresh', 'pending')"
)
.bind(job_id)
.bind(library_id)
.execute(pool)
.await?;
let interval_minutes = match refresh_mode.as_str() {
"hourly" => 60,
"daily" => 1440,
"weekly" => 10080,
_ => 1440,
};
sqlx::query(
"UPDATE libraries SET last_metadata_refresh_at = NOW(), next_metadata_refresh_at = NOW() + INTERVAL '1 minute' * $2 WHERE id = $1"
)
.bind(library_id)
.bind(interval_minutes)
.execute(pool)
.await?;
info!("[SCHEDULER] Created metadata_refresh job {} for library {}", job_id, library_id);
}
Ok(())
}

3
apps/indexer/src/worker.rs
View File

@@ -27,6 +27,9 @@ pub async fn run_worker(state: AppState, interval_seconds: u64) {
if let Err(err) = scheduler::check_and_schedule_auto_scans(&scheduler_state.pool).await {
error!("[SCHEDULER] Error: {}", err);
}
if let Err(err) = scheduler::check_and_schedule_metadata_refreshes(&scheduler_state.pool).await {
error!("[SCHEDULER] Metadata refresh error: {}", err);
}
tokio::time::sleep(scheduler_wait).await;
}
});

4
infra/migrations/0045_add_metadata_refresh_schedule.sql Normal file
View File

@@ -0,0 +1,4 @@
ALTER TABLE libraries
ADD COLUMN metadata_refresh_mode TEXT NOT NULL DEFAULT 'manual',
ADD COLUMN last_metadata_refresh_at TIMESTAMPTZ,
ADD COLUMN next_metadata_refresh_at TIMESTAMPTZ;