gpatti/MediaLore
Archived
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Fix post-scan CPU spike and improve scan performance at scale

Browse Source 
- Remove fire-and-forget thumbnail pre-warming from scanMixed(): firing
  48k+ simultaneous unresolved getThumbnailPath() promises was saturating
  sharp and ffmpeg after scan completion, keeping CPU pegged. Mixed-library
  thumbnails are now generated on-demand by /api/thumbnail as before.
- Add incremental fingerprinting: load existing (item_key → fingerprint)
  map from DB before each walk; reuse stored fingerprint for unchanged paths
  instead of re-reading 64 KB per file. Stable re-scans now do ~0 bytes of
  fingerprint I/O.
- Wrap all bulk DB upsert and delete loops in db.transaction() in
  scanMovies(), scanTv(), scanMixed(), and reconcileAndPrune(). Reduces
  N auto-committed WAL writes to a single batch commit per scan phase.

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
Garret Patti
2026-04-06 19:58:05 -04:00
parent e5953100d6
commit 4d75e74cab
1 changed files with 138 additions and 109 deletions
Download Patch File Download Diff File Expand all files Collapse all files

247
src/lib/scanner.ts
View File

@@ -10,9 +10,6 @@ import { scanGamesLibrary } from './games'
import { getThumbnailPath } from './thumbnails' import { getThumbnailPath } from './thumbnails'
import { computeFingerprint } from './fingerprint' import { computeFingerprint } from './fingerprint'
import { reKeyMediaItem } from './tags' import { reKeyMediaItem } from './tags'
import { VIDEO_EXTENSIONS } from './media-utils'
const IMAGE_EXTENSIONS = new Set(['.jpg', '.jpeg', '.png', '.gif', '.webp', '.bmp'])
let scanRunning = false let scanRunning = false
@@ -70,14 +67,20 @@ async function scanMovies(library: Library, libraryRoot: string): Promise<void>
const db = getDb() const db = getDb()
const now = Date.now() const now = Date.now()
// Load existing fingerprints for incremental hashing (skip re-reading unchanged files)
const existingFps = db
.prepare('SELECT item_key, fingerprint FROM media_items WHERE library_id = ? AND fingerprint IS NOT NULL')
.all(library.id) as Array<{ item_key: string; fingerprint: string }>
const existingFpMap = new Map(existingFps.map((r) => [r.item_key, r.fingerprint]))
// Build new items map: item_key → { fingerprint, movie } // Build new items map: item_key → { fingerprint, movie }
type MovieEntry = { fingerprint: string | null; movie: Movie } type MovieEntry = { fingerprint: string | null; movie: Movie }
const newItems = new Map<string, MovieEntry>() const newItems = new Map<string, MovieEntry>()
for (const movie of movies) { for (const movie of movies) {
const itemKey = `${library.id}:movie:${movie.id}` const itemKey = `${library.id}:movie:${movie.id}`
const fingerprint = movie.videoPath const fingerprint =
? computeFingerprint(path.join(libraryRoot, movie.videoPath)) existingFpMap.get(itemKey) ??
: null (movie.videoPath ? computeFingerprint(path.join(libraryRoot, movie.videoPath)) : null)
newItems.set(itemKey, { fingerprint, movie }) newItems.set(itemKey, { fingerprint, movie })
} }
@@ -101,26 +104,31 @@ async function scanMovies(library: Library, libraryRoot: string): Promise<void>
scanned_at = excluded.scanned_at scanned_at = excluded.scanned_at
`) `)
for (const [itemKey, { fingerprint, movie }] of newItems) { db.transaction(() => {
upsert.run({ for (const [itemKey, { fingerprint, movie }] of newItems) {
library_id: library.id, upsert.run({
item_key: itemKey, library_id: library.id,
item_type: 'movie', item_key: itemKey,
title: movie.title, item_type: 'movie',
year: movie.year ?? null, title: movie.title,
plot: movie.plot ?? null, year: movie.year ?? null,
genres: JSON.stringify(movie.genres), plot: movie.plot ?? null,
metadata: JSON.stringify({ genres: JSON.stringify(movie.genres),
rating: movie.rating, metadata: JSON.stringify({
runtime: movie.runtime, rating: movie.rating,
posterUrl: movie.posterUrl, runtime: movie.runtime,
backdropUrl: movie.backdropUrl, posterUrl: movie.posterUrl,
}), backdropUrl: movie.backdropUrl,
file_path: movie.videoPath, }),
fingerprint, file_path: movie.videoPath,
scanned_at: now, fingerprint,
}) scanned_at: now,
})
}
})()
// Prewarm poster thumbnails after the transaction (bounded by number of movies)
for (const [, { movie }] of newItems) {
if (movie.posterUrl) { if (movie.posterUrl) {
await prewarmThumbnailFromUrl(movie.posterUrl, library.id, libraryRoot, 'image') await prewarmThumbnailFromUrl(movie.posterUrl, library.id, libraryRoot, 'image')
} }
@@ -142,6 +150,12 @@ async function scanTv(library: Library, libraryRoot: string): Promise<void> {
type EpisodeRow = { episode: TvEpisode; episodeKey: string; fingerprint: string | null } type EpisodeRow = { episode: TvEpisode; episodeKey: string; fingerprint: string | null }
type SeriesRow = { show: TvSeries; seriesKey: string; seasons: SeasonRow[] } type SeriesRow = { show: TvSeries; seriesKey: string; seasons: SeasonRow[] }
// Load existing episode fingerprints for incremental hashing
const existingEpFps = db
.prepare('SELECT item_key, fingerprint FROM media_items WHERE library_id = ? AND item_type = ? AND fingerprint IS NOT NULL')
.all(library.id, 'tv_episode') as Array<{ item_key: string; fingerprint: string }>
const existingEpFpMap = new Map(existingEpFps.map((r) => [r.item_key, r.fingerprint]))
const allSeries: SeriesRow[] = [] const allSeries: SeriesRow[] = []
const newKeys = new Set<string>() const newKeys = new Set<string>()
const newEpisodes = new Map<string, { fingerprint: string | null }>() const newEpisodes = new Map<string, { fingerprint: string | null }>()
@@ -159,9 +173,9 @@ async function scanTv(library: Library, libraryRoot: string): Promise<void> {
for (const episode of scanTvEpisodes(libraryRoot, library.id, show.id, season.id)) { for (const episode of scanTvEpisodes(libraryRoot, library.id, show.id, season.id)) {
const episodeKey = `${library.id}:tv_episode:${show.id}:${season.id}:${episode.id}` const episodeKey = `${library.id}:tv_episode:${show.id}:${season.id}:${episode.id}`
newKeys.add(episodeKey) newKeys.add(episodeKey)
const fingerprint = episode.videoPath const fingerprint =
? computeFingerprint(path.join(libraryRoot, episode.videoPath)) existingEpFpMap.get(episodeKey) ??
: null (episode.videoPath ? computeFingerprint(path.join(libraryRoot, episode.videoPath)) : null)
episodeRows.push({ episode, episodeKey, fingerprint }) episodeRows.push({ episode, episodeKey, fingerprint })
newEpisodes.set(episodeKey, { fingerprint }) newEpisodes.set(episodeKey, { fingerprint })
} }
@@ -207,83 +221,91 @@ async function scanTv(library: Library, libraryRoot: string): Promise<void> {
let episodeCount = 0 let episodeCount = 0
for (const { show, seriesKey, seasons } of allSeries) { // Phase 1: all DB writes in a single transaction
upsertSeries.run({ db.transaction(() => {
library_id: library.id, for (const { show, seriesKey, seasons } of allSeries) {
item_key: seriesKey, upsertSeries.run({
item_type: 'tv_series',
title: show.title,
year: show.year ?? null,
plot: show.plot ?? null,
genres: JSON.stringify(show.genres),
metadata: JSON.stringify({
status: show.status,
seasonCount: show.seasonCount,
posterUrl: show.posterUrl,
backdropUrl: show.backdropUrl,
}),
file_path: null,
fingerprint: null,
scanned_at: now,
})
if (show.posterUrl) {
await prewarmThumbnailFromUrl(show.posterUrl, library.id, libraryRoot, 'image')
}
for (const { season, seasonKey, episodes } of seasons) {
upsertChild.run({
library_id: library.id, library_id: library.id,
item_key: seasonKey, item_key: seriesKey,
item_type: 'tv_season', item_type: 'tv_series',
parent_key: seriesKey, title: show.title,
title: season.title, year: show.year ?? null,
year: null, plot: show.plot ?? null,
plot: null, genres: JSON.stringify(show.genres),
genres: JSON.stringify([]),
metadata: JSON.stringify({ metadata: JSON.stringify({
seasonNumber: season.seasonNumber, status: show.status,
episodeCount: season.episodeCount, seasonCount: show.seasonCount,
posterUrl: season.posterUrl, posterUrl: show.posterUrl,
backdropUrl: show.backdropUrl,
}), }),
file_path: null, file_path: null,
fingerprint: null, fingerprint: null,
scanned_at: now, scanned_at: now,
}) })
if (season.posterUrl) { for (const { season, seasonKey, episodes } of seasons) {
await prewarmThumbnailFromUrl(season.posterUrl, library.id, libraryRoot, 'image')
}
for (const { episode, episodeKey, fingerprint } of episodes) {
upsertChild.run({ upsertChild.run({
library_id: library.id, library_id: library.id,
item_key: episodeKey, item_key: seasonKey,
item_type: 'tv_episode', item_type: 'tv_season',
parent_key: seasonKey, parent_key: seriesKey,
title: episode.title, title: season.title,
year: null, year: null,
plot: episode.plot ?? null, plot: null,
genres: JSON.stringify([]), genres: JSON.stringify([]),
metadata: JSON.stringify({ metadata: JSON.stringify({
episodeNumber: episode.episodeNumber, seasonNumber: season.seasonNumber,
seasonNumber: episode.seasonNumber, episodeCount: season.episodeCount,
aired: episode.aired, posterUrl: season.posterUrl,
rating: episode.rating,
thumbnailUrl: episode.thumbnailUrl,
}), }),
file_path: episode.videoPath, file_path: null,
fingerprint, fingerprint: null,
scanned_at: now, scanned_at: now,
}) })
for (const { episode, episodeKey, fingerprint } of episodes) {
upsertChild.run({
library_id: library.id,
item_key: episodeKey,
item_type: 'tv_episode',
parent_key: seasonKey,
title: episode.title,
year: null,
plot: episode.plot ?? null,
genres: JSON.stringify([]),
metadata: JSON.stringify({
episodeNumber: episode.episodeNumber,
seasonNumber: episode.seasonNumber,
aired: episode.aired,
rating: episode.rating,
thumbnailUrl: episode.thumbnailUrl,
}),
file_path: episode.videoPath,
fingerprint,
scanned_at: now,
})
episodeCount++
}
}
}
})()
// Phase 2: async thumbnail generation (bounded — one at a time, awaited)
for (const { show, seasons } of allSeries) {
if (show.posterUrl) {
await prewarmThumbnailFromUrl(show.posterUrl, library.id, libraryRoot, 'image')
}
for (const { season, episodes } of seasons) {
if (season.posterUrl) {
await prewarmThumbnailFromUrl(season.posterUrl, library.id, libraryRoot, 'image')
}
for (const { episode } of episodes) {
const videoAbsPath = path.join(libraryRoot, episode.videoPath) const videoAbsPath = path.join(libraryRoot, episode.videoPath)
try { try {
await getThumbnailPath(videoAbsPath, library.id, 'video') await getThumbnailPath(videoAbsPath, library.id, 'video')
} catch (err) { } catch (err) {
console.warn(`[scanner] Could not generate thumbnail for ${episode.videoPath}:`, err instanceof Error ? err.message : err) console.warn(`[scanner] Could not generate thumbnail for ${episode.videoPath}:`, err instanceof Error ? err.message : err)
} }
episodeCount++
} }
} }
} }
@@ -410,6 +432,12 @@ async function scanMixed(library: Library, libraryRoot: string): Promise<void> {
const db = getDb() const db = getDb()
const now = Date.now() const now = Date.now()
// Load existing fingerprints for incremental hashing (skip re-reading unchanged files)
const existingMixedFps = db
.prepare('SELECT item_key, fingerprint FROM media_items WHERE library_id = ? AND item_type = ? AND fingerprint IS NOT NULL')
.all(library.id, 'mixed_file') as Array<{ item_key: string; fingerprint: string }>
const existingMixedFpMap = new Map(existingMixedFps.map((r) => [r.item_key, r.fingerprint]))
// Collect all new items with fingerprints // Collect all new items with fingerprints
type MixedEntry = { fingerprint: string | null; relPath: string; title: string } type MixedEntry = { fingerprint: string | null; relPath: string; title: string }
const newItems = new Map<string, MixedEntry>() const newItems = new Map<string, MixedEntry>()
@@ -429,8 +457,10 @@ async function scanMixed(library: Library, libraryRoot: string): Promise<void> {
walk(path.join(absDir, name), relPath) walk(path.join(absDir, name), relPath)
} else { } else {
const itemKey = `${library.id}:mixed_file:${encodeURIComponent(relPath)}` const itemKey = `${library.id}:mixed_file:${encodeURIComponent(relPath)}`
const absPath = path.join(absDir, name) // Reuse stored fingerprint if the path is unchanged; only read for new/unknown files
const fingerprint = computeFingerprint(absPath) const fingerprint =
existingMixedFpMap.get(itemKey) ??
computeFingerprint(path.join(absDir, name))
newItems.set(itemKey, { newItems.set(itemKey, {
fingerprint, fingerprint,
relPath, relPath,
@@ -456,30 +486,25 @@ async function scanMixed(library: Library, libraryRoot: string): Promise<void> {
scanned_at = excluded.scanned_at scanned_at = excluded.scanned_at
`) `)
for (const [itemKey, { fingerprint, relPath, title }] of newItems) { // All upserts in a single transaction — critical for large libraries (48k+ files)
upsert.run({ db.transaction(() => {
library_id: library.id, for (const [itemKey, { fingerprint, relPath, title }] of newItems) {
item_key: itemKey, upsert.run({
item_type: 'mixed_file', library_id: library.id,
title, item_key: itemKey,
file_path: relPath, item_type: 'mixed_file',
fingerprint, title,
scanned_at: now, file_path: relPath,
}) fingerprint,
scanned_at: now,
const ext = path.extname(relPath).toLowerCase()
let mediaType: 'image' | 'video' | null = null
if (IMAGE_EXTENSIONS.has(ext)) mediaType = 'image'
else if (VIDEO_EXTENSIONS.has(ext)) mediaType = 'video'
if (mediaType) {
const absPath = path.join(libraryRoot, relPath)
getThumbnailPath(absPath, library.id, mediaType).catch((err) => {
console.warn(`[scanner] Could not generate thumbnail for ${relPath}:`, err instanceof Error ? err.message : err)
}) })
} }
} })()
console.log(`[scanner] mixed: indexed ${newItems.size} files, pre-generating thumbnails`) // Thumbnails for mixed libraries are generated on-demand by /api/thumbnail.
// Pre-warming 48k+ files simultaneously was the cause of the post-scan CPU spike.
console.log(`[scanner] mixed: indexed ${newItems.size} files`)
} }
// --------------------------------------------------------------------------- // ---------------------------------------------------------------------------
@@ -555,6 +580,7 @@ function reconcileAndPrune(
): void { ): void {
const renameItem = db.prepare('UPDATE media_items SET item_key = ? WHERE item_key = ?') const renameItem = db.prepare('UPDATE media_items SET item_key = ? WHERE item_key = ?')
// Apply moves first (outside transaction so console.log is visible as they happen)
for (const { oldKey, newKey } of moves) { for (const { oldKey, newKey } of moves) {
renameItem.run(newKey, oldKey) renameItem.run(newKey, oldKey)
// Convert item_keys to the media_key format actually used in media_tags // Convert item_keys to the media_key format actually used in media_tags
@@ -570,12 +596,15 @@ function reconcileAndPrune(
.prepare('SELECT item_key FROM media_items WHERE library_id = ?') .prepare('SELECT item_key FROM media_items WHERE library_id = ?')
.all(libraryId) as Array<{ item_key: string }> .all(libraryId) as Array<{ item_key: string }>
// Batch all deletes in a single transaction
const deleteItem = db.prepare('DELETE FROM media_items WHERE item_key = ?') const deleteItem = db.prepare('DELETE FROM media_items WHERE item_key = ?')
for (const { item_key } of existing) { db.transaction(() => {
if (!newKeys.has(item_key)) { for (const { item_key } of existing) {
deleteItem.run(item_key) if (!newKeys.has(item_key)) {
deleteItem.run(item_key)
}
} }
} })()
} }
/** /**