import path from 'path' import type Database from 'better-sqlite3' import type { Library, Movie, TvSeries, TvSeason, TvEpisode, Game, GameSeries } from '@/types' import { getDb } from './db' import { getLibraries, resolveLibraryRoot } from './libraries' import { setScanLastRan } from './app-settings' import { scanMoviesLibrary } from './movies' import { scanTvLibrary, scanTvSeasons, scanTvEpisodes } from './tv' import { scanGamesLibrary } from './games' import { getThumbnailPath } from './thumbnails' import { computeFingerprint } from './fingerprint' import { reKeyMediaItem } from './tags' let scanRunning = false export function isScanRunning(): boolean { return scanRunning } export async function runFullScan(): Promise { if (scanRunning) return scanRunning = true console.log('[scanner] Starting full library scan') try { const libraries = getLibraries() for (const library of libraries) { try { await runLibraryScan(library) } catch (err) { console.error(`[scanner] Error scanning library "${library.name}":`, err) } } const now = Date.now() setScanLastRan(now) console.log('[scanner] Full scan complete') } finally { scanRunning = false } } export async function runLibraryScan(library: Library): Promise { const libraryRoot = resolveLibraryRoot(library) console.log(`[scanner] Scanning library "${library.name}" (${library.type}) at ${libraryRoot}`) switch (library.type) { case 'movies': await scanMovies(library, libraryRoot) break case 'tv': await scanTv(library, libraryRoot) break case 'games': await scanGames(library, libraryRoot) break case 'mixed': await scanMixed(library, libraryRoot) break } } // --------------------------------------------------------------------------- // Movies // --------------------------------------------------------------------------- async function scanMovies(library: Library, libraryRoot: string): Promise { const movies = scanMoviesLibrary(libraryRoot, library.id) const db = getDb() const now = Date.now() // Load existing fingerprints for incremental hashing (skip re-reading unchanged files) const existingFps = db .prepare('SELECT item_key, fingerprint FROM media_items WHERE library_id = ? AND fingerprint IS NOT NULL') .all(library.id) as Array<{ item_key: string; fingerprint: string }> const existingFpMap = new Map(existingFps.map((r) => [r.item_key, r.fingerprint])) // Build new items map: item_key → { fingerprint, movie } type MovieEntry = { fingerprint: string | null; movie: Movie } const newItems = new Map() for (const movie of movies) { const itemKey = `${library.id}:movie:${movie.id}` const fingerprint = existingFpMap.get(itemKey) ?? (movie.videoPath ? computeFingerprint(path.join(libraryRoot, movie.videoPath)) : null) newItems.set(itemKey, { fingerprint, movie }) } // Detect moves using fingerprints const moves = detectMoves(db, library.id, newItems) // Apply renames + prune stale rows reconcileAndPrune(db, library.id, new Set(newItems.keys()), moves) const upsert = db.prepare(` INSERT INTO media_items (library_id, item_key, item_type, title, year, plot, genres, metadata, file_path, fingerprint, scanned_at) VALUES (@library_id, @item_key, @item_type, @title, @year, @plot, @genres, @metadata, @file_path, @fingerprint, @scanned_at) ON CONFLICT(item_key) DO UPDATE SET title = excluded.title, year = excluded.year, plot = excluded.plot, genres = excluded.genres, metadata = excluded.metadata, file_path = excluded.file_path, fingerprint = excluded.fingerprint, scanned_at = excluded.scanned_at `) db.transaction(() => { for (const [itemKey, { fingerprint, movie }] of newItems) { upsert.run({ library_id: library.id, item_key: itemKey, item_type: 'movie', title: movie.title, year: movie.year ?? null, plot: movie.plot ?? null, genres: JSON.stringify(movie.genres), metadata: JSON.stringify({ rating: movie.rating, runtime: movie.runtime, posterUrl: movie.posterUrl, backdropUrl: movie.backdropUrl, }), file_path: movie.videoPath, fingerprint, scanned_at: now, }) } })() // Prewarm poster thumbnails after the transaction (bounded by number of movies) for (const [, { movie }] of newItems) { if (movie.posterUrl) { await prewarmThumbnailFromUrl(movie.posterUrl, library.id, libraryRoot, 'image') } } console.log(`[scanner] movies: indexed ${movies.length} items`) } // --------------------------------------------------------------------------- // TV // --------------------------------------------------------------------------- async function scanTv(library: Library, libraryRoot: string): Promise { const db = getDb() const now = Date.now() // Single filesystem pass — collect everything before touching the DB type SeasonRow = { season: TvSeason; seasonKey: string; episodes: EpisodeRow[] } type EpisodeRow = { episode: TvEpisode; episodeKey: string; fingerprint: string | null } type SeriesRow = { show: TvSeries; seriesKey: string; seasons: SeasonRow[] } // Load existing episode fingerprints for incremental hashing const existingEpFps = db .prepare('SELECT item_key, fingerprint FROM media_items WHERE library_id = ? AND item_type = ? AND fingerprint IS NOT NULL') .all(library.id, 'tv_episode') as Array<{ item_key: string; fingerprint: string }> const existingEpFpMap = new Map(existingEpFps.map((r) => [r.item_key, r.fingerprint])) const allSeries: SeriesRow[] = [] const newKeys = new Set() const newEpisodes = new Map() for (const show of scanTvLibrary(libraryRoot, library.id)) { const seriesKey = `${library.id}:tv_series:${show.id}` newKeys.add(seriesKey) const seasonRows: SeasonRow[] = [] for (const season of scanTvSeasons(libraryRoot, library.id, show.id)) { const seasonKey = `${library.id}:tv_season:${show.id}:${season.id}` newKeys.add(seasonKey) const episodeRows: EpisodeRow[] = [] for (const episode of scanTvEpisodes(libraryRoot, library.id, show.id, season.id)) { const episodeKey = `${library.id}:tv_episode:${show.id}:${season.id}:${episode.id}` newKeys.add(episodeKey) const fingerprint = existingEpFpMap.get(episodeKey) ?? (episode.videoPath ? computeFingerprint(path.join(libraryRoot, episode.videoPath)) : null) episodeRows.push({ episode, episodeKey, fingerprint }) newEpisodes.set(episodeKey, { fingerprint }) } seasonRows.push({ season, seasonKey, episodes: episodeRows }) } allSeries.push({ show, seriesKey, seasons: seasonRows }) } // Detect moves among episodes (only episodes have fingerprints) const moves = detectMoves(db, library.id, newEpisodes) // Apply renames + prune stale rows (series, seasons, and episodes) reconcileAndPrune(db, library.id, newKeys, moves) const upsertSeries = db.prepare(` INSERT INTO media_items (library_id, item_key, item_type, title, year, plot, genres, metadata, file_path, fingerprint, scanned_at) VALUES (@library_id, @item_key, @item_type, @title, @year, @plot, @genres, @metadata, @file_path, @fingerprint, @scanned_at) ON CONFLICT(item_key) DO UPDATE SET title = excluded.title, year = excluded.year, plot = excluded.plot, genres = excluded.genres, metadata = excluded.metadata, file_path = excluded.file_path, fingerprint = excluded.fingerprint, scanned_at = excluded.scanned_at `) const upsertChild = db.prepare(` INSERT INTO media_items (library_id, item_key, item_type, parent_key, title, year, plot, genres, metadata, file_path, fingerprint, scanned_at) VALUES (@library_id, @item_key, @item_type, @parent_key, @title, @year, @plot, @genres, @metadata, @file_path, @fingerprint, @scanned_at) ON CONFLICT(item_key) DO UPDATE SET parent_key = excluded.parent_key, title = excluded.title, year = excluded.year, plot = excluded.plot, genres = excluded.genres, metadata = excluded.metadata, file_path = excluded.file_path, fingerprint = excluded.fingerprint, scanned_at = excluded.scanned_at `) let episodeCount = 0 // Phase 1: all DB writes in a single transaction db.transaction(() => { for (const { show, seriesKey, seasons } of allSeries) { upsertSeries.run({ library_id: library.id, item_key: seriesKey, item_type: 'tv_series', title: show.title, year: show.year ?? null, plot: show.plot ?? null, genres: JSON.stringify(show.genres), metadata: JSON.stringify({ status: show.status, seasonCount: show.seasonCount, posterUrl: show.posterUrl, backdropUrl: show.backdropUrl, }), file_path: null, fingerprint: null, scanned_at: now, }) for (const { season, seasonKey, episodes } of seasons) { upsertChild.run({ library_id: library.id, item_key: seasonKey, item_type: 'tv_season', parent_key: seriesKey, title: season.title, year: null, plot: null, genres: JSON.stringify([]), metadata: JSON.stringify({ seasonNumber: season.seasonNumber, episodeCount: season.episodeCount, posterUrl: season.posterUrl, }), file_path: null, fingerprint: null, scanned_at: now, }) for (const { episode, episodeKey, fingerprint } of episodes) { upsertChild.run({ library_id: library.id, item_key: episodeKey, item_type: 'tv_episode', parent_key: seasonKey, title: episode.title, year: null, plot: episode.plot ?? null, genres: JSON.stringify([]), metadata: JSON.stringify({ episodeNumber: episode.episodeNumber, seasonNumber: episode.seasonNumber, aired: episode.aired, rating: episode.rating, thumbnailUrl: episode.thumbnailUrl, }), file_path: episode.videoPath, fingerprint, scanned_at: now, }) episodeCount++ } } } })() // Phase 2: async thumbnail generation (bounded — one at a time, awaited) for (const { show, seasons } of allSeries) { if (show.posterUrl) { await prewarmThumbnailFromUrl(show.posterUrl, library.id, libraryRoot, 'image') } for (const { season, episodes } of seasons) { if (season.posterUrl) { await prewarmThumbnailFromUrl(season.posterUrl, library.id, libraryRoot, 'image') } for (const { episode } of episodes) { const videoAbsPath = path.join(libraryRoot, episode.videoPath) try { await getThumbnailPath(videoAbsPath, library.id, 'video') } catch (err) { console.warn(`[scanner] Could not generate thumbnail for ${episode.videoPath}:`, err instanceof Error ? err.message : err) } } } } console.log(`[scanner] tv: indexed ${allSeries.length} series, ${episodeCount} episodes`) } // --------------------------------------------------------------------------- // Games (v1: no fingerprinting — clear+upsert pattern retained) // --------------------------------------------------------------------------- async function scanGames(library: Library, libraryRoot: string): Promise { const items = scanGamesLibrary(libraryRoot, library.id) const db = getDb() const now = Date.now() clearLibraryItems(db, library.id) const upsertGame = db.prepare(` INSERT INTO media_items (library_id, item_key, item_type, title, metadata, file_path, fingerprint, scanned_at) VALUES (@library_id, @item_key, @item_type, @title, @metadata, @file_path, @fingerprint, @scanned_at) ON CONFLICT(item_key) DO UPDATE SET title = excluded.title, metadata = excluded.metadata, file_path = excluded.file_path, fingerprint = excluded.fingerprint, scanned_at = excluded.scanned_at `) const upsertChildGame = db.prepare(` INSERT INTO media_items (library_id, item_key, item_type, parent_key, title, metadata, file_path, fingerprint, scanned_at) VALUES (@library_id, @item_key, @item_type, @parent_key, @title, @metadata, @file_path, @fingerprint, @scanned_at) ON CONFLICT(item_key) DO UPDATE SET parent_key = excluded.parent_key, title = excluded.title, metadata = excluded.metadata, file_path = excluded.file_path, fingerprint = excluded.fingerprint, scanned_at = excluded.scanned_at `) let gameCount = 0 for (const item of items) { if ('games' in item) { const series = item as GameSeries const seriesKey = `${library.id}:game_series:${series.id}` upsertGame.run({ library_id: library.id, item_key: seriesKey, item_type: 'game_series', title: series.title, metadata: JSON.stringify({ gameCount: series.games.length, coverUrl: series.coverUrl, wideCoverUrl: series.wideCoverUrl, }), file_path: null, fingerprint: null, scanned_at: now, }) if (series.coverUrl) { await prewarmThumbnailFromUrl(series.coverUrl, library.id, libraryRoot, 'image') } for (const game of series.games) { const gameKey = `${library.id}:game:${game.id}` upsertChildGame.run({ library_id: library.id, item_key: gameKey, item_type: 'game', parent_key: seriesKey, title: game.title, metadata: JSON.stringify({ zipFiles: game.zipFiles, coverUrl: game.coverUrl, wideCoverUrl: game.wideCoverUrl, }), file_path: null, fingerprint: null, scanned_at: now, }) if (game.coverUrl) { await prewarmThumbnailFromUrl(game.coverUrl, library.id, libraryRoot, 'image') } gameCount++ } } else { const game = item as Game const gameKey = `${library.id}:game:${game.id}` upsertGame.run({ library_id: library.id, item_key: gameKey, item_type: 'game', title: game.title, metadata: JSON.stringify({ zipFiles: game.zipFiles, coverUrl: game.coverUrl, wideCoverUrl: game.wideCoverUrl, }), file_path: null, fingerprint: null, scanned_at: now, }) if (game.coverUrl) { await prewarmThumbnailFromUrl(game.coverUrl, library.id, libraryRoot, 'image') } gameCount++ } } console.log(`[scanner] games: indexed ${gameCount} games`) } // --------------------------------------------------------------------------- // Mixed // --------------------------------------------------------------------------- async function scanMixed(library: Library, libraryRoot: string): Promise { const fsSync = await import('fs') as typeof import('fs') const db = getDb() const now = Date.now() // Load existing fingerprints for incremental hashing (skip re-reading unchanged files) const existingMixedFps = db .prepare('SELECT item_key, fingerprint FROM media_items WHERE library_id = ? AND item_type = ? AND fingerprint IS NOT NULL') .all(library.id, 'mixed_file') as Array<{ item_key: string; fingerprint: string }> const existingMixedFpMap = new Map(existingMixedFps.map((r) => [r.item_key, r.fingerprint])) // Collect all new items with fingerprints type MixedEntry = { fingerprint: string | null; relPath: string; title: string } const newItems = new Map() function walk(absDir: string, relDir: string): void { let dirents: import('fs').Dirent[] try { dirents = fsSync.readdirSync(absDir, { withFileTypes: true, encoding: 'utf-8' }) as import('fs').Dirent[] } catch { return } for (const d of dirents) { const name = d.name as string if (name.startsWith('.')) continue const relPath = relDir ? path.join(relDir, name) : name if (d.isDirectory()) { walk(path.join(absDir, name), relPath) } else { const itemKey = `${library.id}:mixed_file:${encodeURIComponent(relPath)}` // Reuse stored fingerprint if the path is unchanged; only read for new/unknown files const fingerprint = existingMixedFpMap.get(itemKey) ?? computeFingerprint(path.join(absDir, name)) newItems.set(itemKey, { fingerprint, relPath, title: path.basename(name, path.extname(name)), }) } } } walk(libraryRoot, '') // Detect moves + reconcile const moves = detectMoves(db, library.id, newItems) reconcileAndPrune(db, library.id, new Set(newItems.keys()), moves) const upsert = db.prepare(` INSERT INTO media_items (library_id, item_key, item_type, title, file_path, fingerprint, scanned_at) VALUES (@library_id, @item_key, @item_type, @title, @file_path, @fingerprint, @scanned_at) ON CONFLICT(item_key) DO UPDATE SET title = excluded.title, file_path = excluded.file_path, fingerprint = excluded.fingerprint, scanned_at = excluded.scanned_at `) // All upserts in a single transaction — critical for large libraries (48k+ files) db.transaction(() => { for (const [itemKey, { fingerprint, relPath, title }] of newItems) { upsert.run({ library_id: library.id, item_key: itemKey, item_type: 'mixed_file', title, file_path: relPath, fingerprint, scanned_at: now, }) } })() // Thumbnails for mixed libraries are generated on-demand by /api/thumbnail. // Pre-warming 48k+ files simultaneously was the cause of the post-scan CPU spike. console.log(`[scanner] mixed: indexed ${newItems.size} files`) } // --------------------------------------------------------------------------- // Helpers // --------------------------------------------------------------------------- function clearLibraryItems(db: Database.Database, libraryId: string): void { db.prepare('DELETE FROM media_items WHERE library_id = ?').run(libraryId) } /** * Given a map of new items (item_key → { fingerprint }), compare against * existing DB rows for this library to find items that moved (same fingerprint, * different item_key). Returns an array of { oldKey, newKey } pairs. * * Only items that have a non-null fingerprint and whose old key is NOT already * present in the new scan (i.e. the file truly moved, not a hash collision) * are treated as moves. */ function detectMoves( db: Database.Database, libraryId: string, newItems: Map ): Array<{ oldKey: string; newKey: string }> { const existing = db .prepare('SELECT item_key, fingerprint FROM media_items WHERE library_id = ? AND fingerprint IS NOT NULL') .all(libraryId) as Array<{ item_key: string; fingerprint: string }> const fingerprintToOldKey = new Map() for (const row of existing) { fingerprintToOldKey.set(row.fingerprint, row.item_key) } const moves: Array<{ oldKey: string; newKey: string }> = [] for (const [newKey, { fingerprint }] of newItems) { if (!fingerprint) continue const oldKey = fingerprintToOldKey.get(fingerprint) if (oldKey && oldKey !== newKey && !newItems.has(oldKey)) { // File moved: same fingerprint, different key, old key is no longer present moves.push({ oldKey, newKey }) } } return moves } /** * Applies detected moves to the DB (renames item_key and updates media_tags), * then deletes any rows for this library whose item_key is not in newKeys. * Tags on deleted items are intentionally left as orphans — harmless and * recoverable if the file reappears. */ /** * Converts an item_key (used in media_items) to the media_key format used in * media_tags. The UI constructs media_keys as `${libraryId}:${shortId}` where * shortId is only the terminal path segment — e.g.: * "lib1:movie:Inception%20(2010)" → "lib1:Inception%20(2010)" * "lib1:tv_episode:Show:S1:ep.mkv" → "lib1:ep.mkv" * "lib1:mixed_file:dir%2Ffile.mp4" → "lib1:dir%2Ffile.mp4" */ function itemKeyToMediaKey(itemKey: string): string { const firstColon = itemKey.indexOf(':') const lastColon = itemKey.lastIndexOf(':') const libraryId = itemKey.slice(0, firstColon) const shortId = itemKey.slice(lastColon + 1) return `${libraryId}:${shortId}` } function reconcileAndPrune( db: Database.Database, libraryId: string, newKeys: Set, moves: Array<{ oldKey: string; newKey: string }> ): void { const renameItem = db.prepare('UPDATE media_items SET item_key = ? WHERE item_key = ?') // Apply moves first (outside transaction so console.log is visible as they happen) for (const { oldKey, newKey } of moves) { renameItem.run(newKey, oldKey) // Convert item_keys to the media_key format actually used in media_tags const oldMediaKey = itemKeyToMediaKey(oldKey) const newMediaKey = itemKeyToMediaKey(newKey) if (oldMediaKey !== newMediaKey) { reKeyMediaItem(oldMediaKey, newMediaKey) } console.log(`[scanner] fingerprint match: renamed "${oldKey}" → "${newKey}"`) } const existing = db .prepare('SELECT item_key FROM media_items WHERE library_id = ?') .all(libraryId) as Array<{ item_key: string }> // Batch all deletes in a single transaction const deleteItem = db.prepare('DELETE FROM media_items WHERE item_key = ?') db.transaction(() => { for (const { item_key } of existing) { if (!newKeys.has(item_key)) { deleteItem.run(item_key) } } })() } /** * Extract the `path` query param from an /api/thumbnail URL and pre-warm * the thumbnail cache for that file. */ async function prewarmThumbnailFromUrl( apiUrl: string, libraryId: string, libraryRoot: string, mediaType: 'image' | 'video' ): Promise { try { const relPath = decodeURIComponent( new URL(apiUrl, 'http://localhost').searchParams.get('path') ?? '' ) if (!relPath) return const absPath = path.join(libraryRoot, relPath) await getThumbnailPath(absPath, libraryId, mediaType) } catch (err) { console.warn(`[scanner] Could not prewarm thumbnail for ${apiUrl}:`, err instanceof Error ? err.message : err) } }