From a6d657d87d46d71609b7d45cc8b8ccf63e66ffda Mon Sep 17 00:00:00 2001 From: Garret Patti <42485635+garretpatti@users.noreply.github.com> Date: Mon, 20 Apr 2026 08:28:43 -0400 Subject: [PATCH] don't block during scan --- src/app/api/comics/route.ts | 6 +- src/components/comics/ComicsView.tsx | 128 ++++++++++++++++++++------- src/lib/comics.ts | 108 ++++++++++++---------- src/lib/db.ts | 8 ++ src/lib/scanner.ts | 85 +++++++++++------- 5 files changed, 223 insertions(+), 112 deletions(-) diff --git a/src/app/api/comics/route.ts b/src/app/api/comics/route.ts index 9fc0ec9..b4a4952 100644 --- a/src/app/api/comics/route.ts +++ b/src/app/api/comics/route.ts @@ -31,7 +31,11 @@ export async function GET(request: NextRequest) { return NextResponse.json(comicIssuesFromDb(libraryId, seriesId)) } - return NextResponse.json(comicsFromDb(libraryId)) + const page = Math.max(1, parseInt(searchParams.get('page') ?? '1', 10) || 1) + const pageSize = Math.min(500, Math.max(1, parseInt(searchParams.get('pageSize') ?? '200', 10) || 200)) + const search = (searchParams.get('search') ?? '').trim() || undefined + + return NextResponse.json(comicsFromDb(libraryId, { page, pageSize, search })) } export async function DELETE(request: NextRequest) { diff --git a/src/components/comics/ComicsView.tsx b/src/components/comics/ComicsView.tsx index 4e6b705..08db674 100644 --- a/src/components/comics/ComicsView.tsx +++ b/src/components/comics/ComicsView.tsx @@ -1,6 +1,6 @@ 'use client' -import { useCallback, useEffect, useState } from 'react' +import { useCallback, useEffect, useRef, useState } from 'react' import type { ComicIssue, ComicSeries } from '@/types' import ComicSeriesView from './ComicSeriesView' import ComicIssueView from './ComicIssueView' @@ -12,10 +12,15 @@ interface Props { readOnly?: boolean } +const PAGE_SIZE = 200 + export default function ComicsView({ libraryId, readOnly }: Props) { const [items, setItems] = useState<(ComicIssue | ComicSeries)[]>([]) const [loading, setLoading] = useState(true) + const [loadingMore, setLoadingMore] = useState(false) const [error, setError] = useState(null) + const [page, setPage] = useState(1) + const [total, setTotal] = useState(0) const [selectedSeries, setSelectedSeries] = useState(null) const [selectedIssue, setSelectedIssue] = useState(null) const [tagPanel, setTagPanel] = useState<{ itemKey: string; title: string } | null>(null) @@ -29,6 +34,8 @@ export default function ComicsView({ libraryId, readOnly }: Props) { const [showFilters, setShowFilters] = useState( () => typeof window !== 'undefined' && window.innerWidth >= 768 ) + const debounceRef = useRef | null>(null) + const sentinelRef = useRef(null) const toggleTag = (tagId: string) => setSelectedTagIds((prev) => { @@ -37,12 +44,22 @@ export default function ComicsView({ libraryId, readOnly }: Props) { return next }) - const fetchItems = useCallback(() => { - fetch(`/api/comics?libraryId=${encodeURIComponent(libraryId)}`) + const fetchItems = useCallback((pageNum: number, searchVal: string, replace: boolean) => { + const params = new URLSearchParams({ + libraryId, + page: String(pageNum), + pageSize: String(PAGE_SIZE), + }) + if (searchVal) params.set('search', searchVal) + if (pageNum === 1) setLoading(true) + else setLoadingMore(true) + fetch(`/api/comics?${params}`) .then((r) => r.json()) - .then((data: (ComicIssue | ComicSeries)[]) => { - setItems(data) - setLoading(false) + .then((data: { items: (ComicIssue | ComicSeries)[]; total: number }) => { + setItems((prev) => (replace ? data.items : [...prev, ...data.items])) + setTotal(data.total) + if (pageNum === 1) setLoading(false) + else setLoadingMore(false) }) .catch(() => { setError('Failed to load comics') @@ -50,7 +67,34 @@ export default function ComicsView({ libraryId, readOnly }: Props) { }) }, [libraryId]) - useEffect(() => { fetchItems() }, [fetchItems]) + useEffect(() => { fetchItems(1, '', true) }, [fetchItems]) + + // IntersectionObserver: load next page when sentinel scrolls into view + useEffect(() => { + const sentinel = sentinelRef.current + if (!sentinel || items.length >= total || total === 0) return + const observer = new IntersectionObserver( + ([entry]) => { + if (entry.isIntersecting && !loadingMore) { + const next = page + 1 + setPage(next) + fetchItems(next, search, false) + } + }, + { rootMargin: '400px' } + ) + observer.observe(sentinel) + return () => observer.disconnect() + }, [items.length, total, loadingMore, page, search, fetchItems]) + + const handleSearchChange = useCallback((val: string) => { + setSearch(val) + if (debounceRef.current) clearTimeout(debounceRef.current) + debounceRef.current = setTimeout(() => { + setPage(1) + fetchItems(1, val, true) + }, 300) + }, [fetchItems]) const fetchAssignments = useCallback(() => { fetch(`/api/tags/library-assignments?libraryId=${encodeURIComponent(libraryId)}`) @@ -133,7 +177,7 @@ export default function ComicsView({ libraryId, readOnly }: Props) { libraryId={libraryId} assignments={assignments} search={search} - onSearchChange={setSearch} + onSearchChange={handleSearchChange} selectedTagIds={selectedTagIds} onTagToggle={toggleTag} refreshKey={filterRefreshKey} @@ -160,31 +204,40 @@ export default function ComicsView({ libraryId, readOnly }: Props) {

Add .cbz files or folders of .cbz files to this library and scan.

) : ( -
- {filtered.map((item) => - 'issueCount' in item ? ( - setSelectedSeries(item as ComicSeries)} - onTagClick={(item as ComicSeries).item_key && !readOnly - ? () => setTagPanel({ itemKey: (item as ComicSeries).item_key!, title: item.title }) - : undefined} - /> - ) : ( - setSelectedIssue(item as ComicIssue)} - onTagClick={(item as ComicIssue).item_key && !readOnly - ? () => setTagPanel({ itemKey: (item as ComicIssue).item_key!, title: item.title }) - : undefined} - /> - ) + <> + {total > PAGE_SIZE && ( +

+ Showing {filtered.length.toLocaleString()} of {total.toLocaleString()} +

)} -
+
+ {filtered.map((item) => + 'issueCount' in item ? ( + setSelectedSeries(item as ComicSeries)} + onTagClick={(item as ComicSeries).item_key && !readOnly + ? () => setTagPanel({ itemKey: (item as ComicSeries).item_key!, title: item.title }) + : undefined} + /> + ) : ( + setSelectedIssue(item as ComicIssue)} + onTagClick={(item as ComicIssue).item_key && !readOnly + ? () => setTagPanel({ itemKey: (item as ComicIssue).item_key!, title: item.title }) + : undefined} + /> + ) + )} +
+
+ {loadingMore && } + )}
@@ -373,6 +426,17 @@ function IssueCard({ ) } +function LoadingMore() { + return ( +
+
+
+ ) +} + function LoadingGrid() { return (
diff --git a/src/lib/comics.ts b/src/lib/comics.ts index 6a37bc3..52c5437 100644 --- a/src/lib/comics.ts +++ b/src/lib/comics.ts @@ -126,10 +126,18 @@ export function scanComicsLibrary( return results.sort((a, b) => naturalCompare(a.title, b.title)) } -// comicsFromDb returns series + standalone issues for the top-level grid. +function escapeLike(s: string): string { + return `%${s.replace(/%/g, '\\%').replace(/_/g, '\\_')}%` +} + +// comicsFromDb returns series + standalone issues for the top-level grid, paginated. // Series issues are retrieved separately via comicIssuesFromDb. -export function comicsFromDb(libraryId: string): (ComicIssue | ComicSeries)[] { +export function comicsFromDb( + libraryId: string, + opts: { page: number; pageSize: number; search?: string } +): { items: (ComicIssue | ComicSeries)[]; total: number } { const db = getDb() + const offset = (opts.page - 1) * opts.pageSize type DbRow = { item_key: string @@ -140,59 +148,63 @@ export function comicsFromDb(libraryId: string): (ComicIssue | ComicSeries)[] { file_path: string | null } - const allRows = db - .prepare( - `SELECT item_key, item_type, parent_key, title, metadata, file_path - FROM media_items - WHERE library_id = ? AND item_type IN ('comic_series','comic_issue') - ORDER BY title` - ) - .all(libraryId) as DbRow[] + const baseWhere = ` + WHERE library_id = ? + AND (item_type = 'comic_series' OR (item_type = 'comic_issue' AND parent_key IS NULL)) + ` - const seriesMap = new Map() - const standaloneIssues: ComicIssue[] = [] + const total: number = opts.search + ? (db + .prepare(`SELECT COUNT(*) as cnt FROM media_items ${baseWhere} AND title LIKE ? ESCAPE '\\'`) + .get(libraryId, escapeLike(opts.search)) as { cnt: number }).cnt + : (db + .prepare(`SELECT COUNT(*) as cnt FROM media_items ${baseWhere}`) + .get(libraryId) as { cnt: number }).cnt - for (const row of allRows) { - if (row.item_type !== 'comic_series') continue + const rows: DbRow[] = opts.search + ? db + .prepare( + `SELECT item_key, item_type, parent_key, title, metadata, file_path + FROM media_items ${baseWhere} AND title LIKE ? ESCAPE '\\' + ORDER BY title LIMIT ? OFFSET ?` + ) + .all(libraryId, escapeLike(opts.search), opts.pageSize, offset) as DbRow[] + : db + .prepare( + `SELECT item_key, item_type, parent_key, title, metadata, file_path + FROM media_items ${baseWhere} + ORDER BY title LIMIT ? OFFSET ?` + ) + .all(libraryId, opts.pageSize, offset) as DbRow[] + + const items: (ComicIssue | ComicSeries)[] = [] + for (const row of rows) { const meta = row.metadata ? JSON.parse(row.metadata) : {} - const idPart = row.item_key.split(':comic_series:')[1] ?? row.item_key - seriesMap.set(row.item_key, { - id: idPart, - item_key: row.item_key, - title: row.title ?? decodeURIComponent(idPart), - coverUrl: meta.coverUrl ?? null, - issueCount: meta.issueCount ?? 0, - }) - } - - for (const row of allRows) { - if (row.item_type !== 'comic_issue') continue - const meta = row.metadata ? JSON.parse(row.metadata) : {} - const idPart = row.item_key.split(':comic_issue:')[1] ?? row.item_key - const issue: ComicIssue = { - id: idPart, - item_key: row.item_key, - title: row.title ?? decodeURIComponent(idPart.split(':').pop() ?? idPart), - issueNumber: meta.issueNumber ?? null, - pageCount: meta.pageCount ?? 0, - coverUrl: meta.coverUrl ?? null, - filePath: row.file_path ?? '', - isStandalone: meta.isStandalone ?? false, - } - - if (row.parent_key && seriesMap.has(row.parent_key)) { - // Series issues are not included in the top-level grid — series card represents them - // We only include series cards + standalone issues in the grid + if (row.item_type === 'comic_series') { + const idPart = row.item_key.split(':comic_series:')[1] ?? row.item_key + items.push({ + id: idPart, + item_key: row.item_key, + title: row.title ?? decodeURIComponent(idPart), + coverUrl: meta.coverUrl ?? null, + issueCount: meta.issueCount ?? 0, + } as ComicSeries) } else { - standaloneIssues.push(issue) + const idPart = row.item_key.split(':comic_issue:')[1] ?? row.item_key + items.push({ + id: idPart, + item_key: row.item_key, + title: row.title ?? decodeURIComponent(idPart.split(':').pop() ?? idPart), + issueNumber: meta.issueNumber ?? null, + pageCount: meta.pageCount ?? 0, + coverUrl: meta.coverUrl ?? null, + filePath: row.file_path ?? '', + isStandalone: meta.isStandalone ?? true, + } as ComicIssue) } } - const results: (ComicIssue | ComicSeries)[] = [ - ...Array.from(seriesMap.values()), - ...standaloneIssues, - ] - return results.sort((a, b) => naturalCompare(a.title, b.title)) + return { items, total } } export function comicIssuesFromDb(libraryId: string, seriesId: string): ComicIssue[] { diff --git a/src/lib/db.ts b/src/lib/db.ts index ca4f516..8a0970c 100644 --- a/src/lib/db.ts +++ b/src/lib/db.ts @@ -110,6 +110,7 @@ function initDb(db: Database.Database): void { migrateLibrariesAddComics(db) migrateComicItemTypes(db) migrateImportedTags(db) + migrateComicsIndex(db) seedAppSettings(db) } @@ -447,3 +448,10 @@ function migrateImportedTags(db: Database.Database): void { ); `) } + +function migrateComicsIndex(db: Database.Database): void { + db.exec(` + CREATE INDEX IF NOT EXISTS media_items_library_type_title + ON media_items(library_id, item_type, title); + `) +} diff --git a/src/lib/scanner.ts b/src/lib/scanner.ts index 71c411c..8cf71a6 100644 --- a/src/lib/scanner.ts +++ b/src/lib/scanner.ts @@ -573,29 +573,37 @@ async function scanComics(library: Library, libraryRoot: string): Promise scanned_at = excluded.scanned_at `) + type SeriesRec = Parameters[0] + type IssueRec = Parameters[0] + type BatchEntry = { type: 'series'; rec: SeriesRec } | { type: 'issue'; rec: IssueRec } + + // Collect all records before touching the DB so we can batch-insert with event-loop yields. + // Note: between clearLibraryItems and the final batch, the library will appear partially + // populated — acceptable for a background scan. + const allRecords: BatchEntry[] = [] let issueCount = 0 - db.transaction(() => { - for (const item of items) { - if ('issues' in item) { - const series = item as ScannedComicSeries - const seriesKey = `${library.id}:comic_series:${series.id}` - upsertSeries.run({ + for (const item of items) { + if ('issues' in item) { + const series = item as ScannedComicSeries + const seriesKey = `${library.id}:comic_series:${series.id}` + allRecords.push({ + type: 'series', + rec: { library_id: library.id, item_key: seriesKey, item_type: 'comic_series', title: series.title, - metadata: JSON.stringify({ - issueCount: series.issueCount, - coverUrl: series.coverUrl, - }), + metadata: JSON.stringify({ issueCount: series.issueCount, coverUrl: series.coverUrl }), file_path: null, scanned_at: now, - }) - - for (const issue of series.issues) { - const issueKey = `${library.id}:comic_issue:${issue.id}` - upsertIssue.run({ + }, + }) + for (const issue of series.issues) { + const issueKey = `${library.id}:comic_issue:${issue.id}` + allRecords.push({ + type: 'issue', + rec: { library_id: library.id, item_key: issueKey, item_type: 'comic_issue', @@ -609,13 +617,16 @@ async function scanComics(library: Library, libraryRoot: string): Promise }), file_path: issue.filePath, scanned_at: now, - }) - issueCount++ - } - } else { - const issue = item as ComicIssue - const issueKey = `${library.id}:comic_issue:${issue.id}` - upsertIssue.run({ + }, + }) + issueCount++ + } + } else { + const issue = item as ComicIssue + const issueKey = `${library.id}:comic_issue:${issue.id}` + allRecords.push({ + type: 'issue', + rec: { library_id: library.id, item_key: issueKey, item_type: 'comic_issue', @@ -629,13 +640,27 @@ async function scanComics(library: Library, libraryRoot: string): Promise }), file_path: issue.filePath, scanned_at: now, - }) - issueCount++ - } + }, + }) + issueCount++ } - })() + } - // Prewarm CBZ cover thumbnails + // Insert in batches of 500, yielding the event loop between batches so the app + // remains responsive to HTTP requests during a large scan. + const BATCH_SIZE = 500 + for (let i = 0; i < allRecords.length; i += BATCH_SIZE) { + const batch = allRecords.slice(i, i + BATCH_SIZE) + db.transaction(() => { + for (const entry of batch) { + if (entry.type === 'series') upsertSeries.run(entry.rec) + else upsertIssue.run(entry.rec) + } + })() + await new Promise((r) => setImmediate(r)) + } + + // Prewarm CBZ cover thumbnails — fire-and-forget so they don't block scan completion. for (const item of items) { const issuesToWarm: ComicIssue[] = 'issues' in item ? (item as ScannedComicSeries).issues.slice(0, 1) @@ -643,11 +668,9 @@ async function scanComics(library: Library, libraryRoot: string): Promise for (const issue of issuesToWarm) { const absPath = path.join(libraryRoot, issue.filePath) - try { - await getCbzThumbnailPath(absPath, library.id) - } catch (err) { + getCbzThumbnailPath(absPath, library.id).catch((err) => { console.warn(`[scanner] Could not generate CBZ thumbnail for ${issue.filePath}:`, err instanceof Error ? err.message : err) - } + }) } } -- 2.49.1