Merge pull request 'don't block during scan' (#33) from large-library-fix into main

Reviewed-on: http://gitea.lan/gpatti/MediaLore/pulls/33

src/app/api/comics/route.ts

@@ -31,7 +31,11 @@ export async function GET(request: NextRequest) {
     return NextResponse.json(comicIssuesFromDb(libraryId, seriesId))
   }
 
-  return NextResponse.json(comicsFromDb(libraryId))
+  const page     = Math.max(1, parseInt(searchParams.get('page')     ?? '1',   10) || 1)
+  const pageSize = Math.min(500, Math.max(1, parseInt(searchParams.get('pageSize') ?? '200', 10) || 200))
+  const search   = (searchParams.get('search') ?? '').trim() || undefined
+
+  return NextResponse.json(comicsFromDb(libraryId, { page, pageSize, search }))
 }
 
 export async function DELETE(request: NextRequest) {

src/components/comics/ComicsView.tsx

@@ -1,6 +1,6 @@
 'use client'
 
-import { useCallback, useEffect, useState } from 'react'
+import { useCallback, useEffect, useRef, useState } from 'react'
 import type { ComicIssue, ComicSeries } from '@/types'
 import ComicSeriesView from './ComicSeriesView'
 import ComicIssueView from './ComicIssueView'
@@ -12,10 +12,15 @@ interface Props {
   readOnly?: boolean
 }
 
+const PAGE_SIZE = 200
+
 export default function ComicsView({ libraryId, readOnly }: Props) {
   const [items, setItems] = useState<(ComicIssue | ComicSeries)[]>([])
   const [loading, setLoading] = useState(true)
+  const [loadingMore, setLoadingMore] = useState(false)
   const [error, setError] = useState<string | null>(null)
+  const [page, setPage] = useState(1)
+  const [total, setTotal] = useState(0)
   const [selectedSeries, setSelectedSeries] = useState<ComicSeries | null>(null)
   const [selectedIssue, setSelectedIssue] = useState<ComicIssue | null>(null)
   const [tagPanel, setTagPanel] = useState<{ itemKey: string; title: string } | null>(null)
@@ -29,6 +34,8 @@ export default function ComicsView({ libraryId, readOnly }: Props) {
   const [showFilters, setShowFilters] = useState(
     () => typeof window !== 'undefined' && window.innerWidth >= 768
   )
+  const debounceRef = useRef<ReturnType<typeof setTimeout> | null>(null)
+  const sentinelRef = useRef<HTMLDivElement | null>(null)
 
   const toggleTag = (tagId: string) =>
     setSelectedTagIds((prev) => {
@@ -37,12 +44,22 @@ export default function ComicsView({ libraryId, readOnly }: Props) {
       return next
     })
 
-  const fetchItems = useCallback(() => {
+  const fetchItems = useCallback((pageNum: number, searchVal: string, replace: boolean) => {
-    fetch(`/api/comics?libraryId=${encodeURIComponent(libraryId)}`)
+    const params = new URLSearchParams({
+      libraryId,
+      page: String(pageNum),
+      pageSize: String(PAGE_SIZE),
+    })
+    if (searchVal) params.set('search', searchVal)
+    if (pageNum === 1) setLoading(true)
+    else setLoadingMore(true)
+    fetch(`/api/comics?${params}`)
       .then((r) => r.json())
-      .then((data: (ComicIssue | ComicSeries)[]) => {
+      .then((data: { items: (ComicIssue | ComicSeries)[]; total: number }) => {
-        setItems(data)
+        setItems((prev) => (replace ? data.items : [...prev, ...data.items]))
-        setLoading(false)
+        setTotal(data.total)
+        if (pageNum === 1) setLoading(false)
+        else setLoadingMore(false)
       })
       .catch(() => {
         setError('Failed to load comics')
@@ -50,7 +67,34 @@ export default function ComicsView({ libraryId, readOnly }: Props) {
       })
   }, [libraryId])
 
-  useEffect(() => { fetchItems() }, [fetchItems])
+  useEffect(() => { fetchItems(1, '', true) }, [fetchItems])
+
+  // IntersectionObserver: load next page when sentinel scrolls into view
+  useEffect(() => {
+    const sentinel = sentinelRef.current
+    if (!sentinel || items.length >= total || total === 0) return
+    const observer = new IntersectionObserver(
+      ([entry]) => {
+        if (entry.isIntersecting && !loadingMore) {
+          const next = page + 1
+          setPage(next)
+          fetchItems(next, search, false)
+        }
+      },
+      { rootMargin: '400px' }
+    )
+    observer.observe(sentinel)
+    return () => observer.disconnect()
+  }, [items.length, total, loadingMore, page, search, fetchItems])
+
+  const handleSearchChange = useCallback((val: string) => {
+    setSearch(val)
+    if (debounceRef.current) clearTimeout(debounceRef.current)
+    debounceRef.current = setTimeout(() => {
+      setPage(1)
+      fetchItems(1, val, true)
+    }, 300)
+  }, [fetchItems])
 
   const fetchAssignments = useCallback(() => {
     fetch(`/api/tags/library-assignments?libraryId=${encodeURIComponent(libraryId)}`)
@@ -133,7 +177,7 @@ export default function ComicsView({ libraryId, readOnly }: Props) {
               libraryId={libraryId}
               assignments={assignments}
               search={search}
-              onSearchChange={setSearch}
+              onSearchChange={handleSearchChange}
               selectedTagIds={selectedTagIds}
               onTagToggle={toggleTag}
               refreshKey={filterRefreshKey}
@@ -160,31 +204,40 @@ export default function ComicsView({ libraryId, readOnly }: Props) {
               <p className="text-sm">Add .cbz files or folders of .cbz files to this library and scan.</p>
             </div>
           ) : (
-            <div className="grid gap-4 grid-cols-2 sm:grid-cols-3 md:grid-cols-4 lg:grid-cols-5 xl:grid-cols-6">
+            <>
-              {filtered.map((item) =>
+              {total > PAGE_SIZE && (
-                'issueCount' in item ? (
+                <p className="text-xs mb-3" style={{ color: 'var(--text-secondary)' }}>
-                  <SeriesCard
+                  Showing {filtered.length.toLocaleString()} of {total.toLocaleString()}
-                    key={item.id}
+                </p>
-                    series={item as ComicSeries}
-                    readOnly={readOnly}
-                    onClick={() => setSelectedSeries(item as ComicSeries)}
-                    onTagClick={(item as ComicSeries).item_key && !readOnly
-                      ? () => setTagPanel({ itemKey: (item as ComicSeries).item_key!, title: item.title })
-                      : undefined}
-                  />
-                ) : (
-                  <IssueCard
-                    key={item.id}
-                    issue={item as ComicIssue}
-                    readOnly={readOnly}
-                    onClick={() => setSelectedIssue(item as ComicIssue)}
-                    onTagClick={(item as ComicIssue).item_key && !readOnly
-                      ? () => setTagPanel({ itemKey: (item as ComicIssue).item_key!, title: item.title })
-                      : undefined}
-                  />
-                )
               )}
-            </div>
+              <div className="grid gap-4 grid-cols-2 sm:grid-cols-3 md:grid-cols-4 lg:grid-cols-5 xl:grid-cols-6">
+                {filtered.map((item) =>
+                  'issueCount' in item ? (
+                    <SeriesCard
+                      key={item.id}
+                      series={item as ComicSeries}
+                      readOnly={readOnly}
+                      onClick={() => setSelectedSeries(item as ComicSeries)}
+                      onTagClick={(item as ComicSeries).item_key && !readOnly
+                        ? () => setTagPanel({ itemKey: (item as ComicSeries).item_key!, title: item.title })
+                        : undefined}
+                    />
+                  ) : (
+                    <IssueCard
+                      key={item.id}
+                      issue={item as ComicIssue}
+                      readOnly={readOnly}
+                      onClick={() => setSelectedIssue(item as ComicIssue)}
+                      onTagClick={(item as ComicIssue).item_key && !readOnly
+                        ? () => setTagPanel({ itemKey: (item as ComicIssue).item_key!, title: item.title })
+                        : undefined}
+                    />
+                  )
+                )}
+              </div>
+              <div ref={sentinelRef} style={{ height: 1 }} aria-hidden />
+              {loadingMore && <LoadingMore />}
+            </>
           )}
         </div>
       </div>
@@ -373,6 +426,17 @@ function IssueCard({
   )
 }
 
+function LoadingMore() {
+  return (
+    <div className="flex justify-center py-6">
+      <div
+        className="w-6 h-6 rounded-full border-2 animate-spin"
+        style={{ borderColor: 'var(--border)', borderTopColor: 'var(--accent)' }}
+      />
+    </div>
+  )
+}
+
 function LoadingGrid() {
   return (
     <div className="grid gap-4 grid-cols-2 sm:grid-cols-3 md:grid-cols-4 lg:grid-cols-5 xl:grid-cols-6">

src/lib/comics.ts

@@ -126,10 +126,18 @@ export function scanComicsLibrary(
   return results.sort((a, b) => naturalCompare(a.title, b.title))
 }
 
-// comicsFromDb returns series + standalone issues for the top-level grid.
+function escapeLike(s: string): string {
+  return `%${s.replace(/%/g, '\\%').replace(/_/g, '\\_')}%`
+}
+
+// comicsFromDb returns series + standalone issues for the top-level grid, paginated.
 // Series issues are retrieved separately via comicIssuesFromDb.
-export function comicsFromDb(libraryId: string): (ComicIssue | ComicSeries)[] {
+export function comicsFromDb(
+  libraryId: string,
+  opts: { page: number; pageSize: number; search?: string }
+): { items: (ComicIssue | ComicSeries)[]; total: number } {
   const db = getDb()
+  const offset = (opts.page - 1) * opts.pageSize
 
   type DbRow = {
     item_key: string
@@ -140,59 +148,63 @@ export function comicsFromDb(libraryId: string): (ComicIssue | ComicSeries)[] {
     file_path: string | null
   }
 
-  const allRows = db
+  const baseWhere = `
-    .prepare(
+    WHERE library_id = ?
-      `SELECT item_key, item_type, parent_key, title, metadata, file_path
+      AND (item_type = 'comic_series' OR (item_type = 'comic_issue' AND parent_key IS NULL))
-       FROM media_items
+  `
-       WHERE library_id = ? AND item_type IN ('comic_series','comic_issue')
-       ORDER BY title`
-    )
-    .all(libraryId) as DbRow[]
 
-  const seriesMap = new Map<string, ComicSeries>()
+  const total: number = opts.search
-  const standaloneIssues: ComicIssue[] = []
+    ? (db
+        .prepare(`SELECT COUNT(*) as cnt FROM media_items ${baseWhere} AND title LIKE ? ESCAPE '\\'`)
+        .get(libraryId, escapeLike(opts.search)) as { cnt: number }).cnt
+    : (db
+        .prepare(`SELECT COUNT(*) as cnt FROM media_items ${baseWhere}`)
+        .get(libraryId) as { cnt: number }).cnt
 
-  for (const row of allRows) {
+  const rows: DbRow[] = opts.search
-    if (row.item_type !== 'comic_series') continue
+    ? db
+        .prepare(
+          `SELECT item_key, item_type, parent_key, title, metadata, file_path
+           FROM media_items ${baseWhere} AND title LIKE ? ESCAPE '\\'
+           ORDER BY title LIMIT ? OFFSET ?`
+        )
+        .all(libraryId, escapeLike(opts.search), opts.pageSize, offset) as DbRow[]
+    : db
+        .prepare(
+          `SELECT item_key, item_type, parent_key, title, metadata, file_path
+           FROM media_items ${baseWhere}
+           ORDER BY title LIMIT ? OFFSET ?`
+        )
+        .all(libraryId, opts.pageSize, offset) as DbRow[]
+
+  const items: (ComicIssue | ComicSeries)[] = []
+  for (const row of rows) {
     const meta = row.metadata ? JSON.parse(row.metadata) : {}
-    const idPart = row.item_key.split(':comic_series:')[1] ?? row.item_key
+    if (row.item_type === 'comic_series') {
-    seriesMap.set(row.item_key, {
+      const idPart = row.item_key.split(':comic_series:')[1] ?? row.item_key
-      id: idPart,
+      items.push({
-      item_key: row.item_key,
+        id: idPart,
-      title: row.title ?? decodeURIComponent(idPart),
+        item_key: row.item_key,
-      coverUrl: meta.coverUrl ?? null,
+        title: row.title ?? decodeURIComponent(idPart),
-      issueCount: meta.issueCount ?? 0,
+        coverUrl: meta.coverUrl ?? null,
-    })
+        issueCount: meta.issueCount ?? 0,
-  }
+      } as ComicSeries)
-
-  for (const row of allRows) {
-    if (row.item_type !== 'comic_issue') continue
-    const meta = row.metadata ? JSON.parse(row.metadata) : {}
-    const idPart = row.item_key.split(':comic_issue:')[1] ?? row.item_key
-    const issue: ComicIssue = {
-      id: idPart,
-      item_key: row.item_key,
-      title: row.title ?? decodeURIComponent(idPart.split(':').pop() ?? idPart),
-      issueNumber: meta.issueNumber ?? null,
-      pageCount: meta.pageCount ?? 0,
-      coverUrl: meta.coverUrl ?? null,
-      filePath: row.file_path ?? '',
-      isStandalone: meta.isStandalone ?? false,
-    }
-
-    if (row.parent_key && seriesMap.has(row.parent_key)) {
-      // Series issues are not included in the top-level grid — series card represents them
-      // We only include series cards + standalone issues in the grid
     } else {
-      standaloneIssues.push(issue)
+      const idPart = row.item_key.split(':comic_issue:')[1] ?? row.item_key
+      items.push({
+        id: idPart,
+        item_key: row.item_key,
+        title: row.title ?? decodeURIComponent(idPart.split(':').pop() ?? idPart),
+        issueNumber: meta.issueNumber ?? null,
+        pageCount: meta.pageCount ?? 0,
+        coverUrl: meta.coverUrl ?? null,
+        filePath: row.file_path ?? '',
+        isStandalone: meta.isStandalone ?? true,
+      } as ComicIssue)
     }
   }
 
-  const results: (ComicIssue | ComicSeries)[] = [
+  return { items, total }
-    ...Array.from(seriesMap.values()),
-    ...standaloneIssues,
-  ]
-  return results.sort((a, b) => naturalCompare(a.title, b.title))
 }
 
 export function comicIssuesFromDb(libraryId: string, seriesId: string): ComicIssue[] {

src/lib/db.ts

@@ -110,6 +110,7 @@ function initDb(db: Database.Database): void {
   migrateLibrariesAddComics(db)
   migrateComicItemTypes(db)
   migrateImportedTags(db)
+  migrateComicsIndex(db)
   seedAppSettings(db)
 }
 
@@ -447,3 +448,10 @@ function migrateImportedTags(db: Database.Database): void {
     );
   `)
 }
+
+function migrateComicsIndex(db: Database.Database): void {
+  db.exec(`
+    CREATE INDEX IF NOT EXISTS media_items_library_type_title
+      ON media_items(library_id, item_type, title);
+  `)
+}

src/lib/scanner.ts

@@ -573,29 +573,37 @@ async function scanComics(library: Library, libraryRoot: string): Promise<void>
       scanned_at = excluded.scanned_at
   `)
 
+  type SeriesRec = Parameters<typeof upsertSeries.run>[0]
+  type IssueRec  = Parameters<typeof upsertIssue.run>[0]
+  type BatchEntry = { type: 'series'; rec: SeriesRec } | { type: 'issue'; rec: IssueRec }
+
+  // Collect all records before touching the DB so we can batch-insert with event-loop yields.
+  // Note: between clearLibraryItems and the final batch, the library will appear partially
+  // populated — acceptable for a background scan.
+  const allRecords: BatchEntry[] = []
   let issueCount = 0
 
-  db.transaction(() => {
+  for (const item of items) {
-    for (const item of items) {
+    if ('issues' in item) {
-      if ('issues' in item) {
+      const series = item as ScannedComicSeries
-        const series = item as ScannedComicSeries
+      const seriesKey = `${library.id}:comic_series:${series.id}`
-        const seriesKey = `${library.id}:comic_series:${series.id}`
+      allRecords.push({
-        upsertSeries.run({
+        type: 'series',
+        rec: {
           library_id: library.id,
           item_key: seriesKey,
           item_type: 'comic_series',
           title: series.title,
-          metadata: JSON.stringify({
+          metadata: JSON.stringify({ issueCount: series.issueCount, coverUrl: series.coverUrl }),
-            issueCount: series.issueCount,
-            coverUrl: series.coverUrl,
-          }),
           file_path: null,
           scanned_at: now,
-        })
+        },
-
+      })
-        for (const issue of series.issues) {
+      for (const issue of series.issues) {
-          const issueKey = `${library.id}:comic_issue:${issue.id}`
+        const issueKey = `${library.id}:comic_issue:${issue.id}`
-          upsertIssue.run({
+        allRecords.push({
+          type: 'issue',
+          rec: {
             library_id: library.id,
             item_key: issueKey,
             item_type: 'comic_issue',
@@ -609,13 +617,16 @@ async function scanComics(library: Library, libraryRoot: string): Promise<void>
             }),
             file_path: issue.filePath,
             scanned_at: now,
-          })
+          },
-          issueCount++
+        })
-        }
+        issueCount++
-      } else {
+      }
-        const issue = item as ComicIssue
+    } else {
-        const issueKey = `${library.id}:comic_issue:${issue.id}`
+      const issue = item as ComicIssue
-        upsertIssue.run({
+      const issueKey = `${library.id}:comic_issue:${issue.id}`
+      allRecords.push({
+        type: 'issue',
+        rec: {
           library_id: library.id,
           item_key: issueKey,
           item_type: 'comic_issue',
@@ -629,13 +640,27 @@ async function scanComics(library: Library, libraryRoot: string): Promise<void>
           }),
           file_path: issue.filePath,
           scanned_at: now,
-        })
+        },
-        issueCount++
+      })
-      }
+      issueCount++
     }
-  })()
+  }
 
-  // Prewarm CBZ cover thumbnails
+  // Insert in batches of 500, yielding the event loop between batches so the app
+  // remains responsive to HTTP requests during a large scan.
+  const BATCH_SIZE = 500
+  for (let i = 0; i < allRecords.length; i += BATCH_SIZE) {
+    const batch = allRecords.slice(i, i + BATCH_SIZE)
+    db.transaction(() => {
+      for (const entry of batch) {
+        if (entry.type === 'series') upsertSeries.run(entry.rec)
+        else upsertIssue.run(entry.rec)
+      }
+    })()
+    await new Promise<void>((r) => setImmediate(r))
+  }
+
+  // Prewarm CBZ cover thumbnails — fire-and-forget so they don't block scan completion.
   for (const item of items) {
     const issuesToWarm: ComicIssue[] = 'issues' in item
       ? (item as ScannedComicSeries).issues.slice(0, 1)
@@ -643,11 +668,9 @@ async function scanComics(library: Library, libraryRoot: string): Promise<void>
 
     for (const issue of issuesToWarm) {
       const absPath = path.join(libraryRoot, issue.filePath)
-      try {
+      getCbzThumbnailPath(absPath, library.id).catch((err) => {
-        await getCbzThumbnailPath(absPath, library.id)
-      } catch (err) {
         console.warn(`[scanner]   Could not generate CBZ thumbnail for ${issue.filePath}:`, err instanceof Error ? err.message : err)
-      }
+      })
     }
   }