Merge pull request 'don't block during scan' (#33) from large-library-fix into main

Reviewed-on: http://gitea.lan/gpatti/MediaLore/pulls/33
2026-04-20 12:29:51 +00:00
parent 71a026f01e a6d657d87d
commit a9461f9ae4
5 changed files with 223 additions and 112 deletions
--- a/src/app/api/comics/route.ts
+++ b/src/app/api/comics/route.ts
@@ -31,7 +31,11 @@ export async function GET(request: NextRequest) {
    return NextResponse.json(comicIssuesFromDb(libraryId, seriesId))
  }

-  return NextResponse.json(comicsFromDb(libraryId))
+  const page     = Math.max(1, parseInt(searchParams.get('page')     ?? '1',   10) || 1)
+  const pageSize = Math.min(500, Math.max(1, parseInt(searchParams.get('pageSize') ?? '200', 10) || 200))
+  const search   = (searchParams.get('search') ?? '').trim() || undefined
+
+  return NextResponse.json(comicsFromDb(libraryId, { page, pageSize, search }))
 }

 export async function DELETE(request: NextRequest) {
--- a/src/components/comics/ComicsView.tsx
+++ b/src/components/comics/ComicsView.tsx
@@ -1,6 +1,6 @@
 'use client'

-import { useCallback, useEffect, useState } from 'react'
+import { useCallback, useEffect, useRef, useState } from 'react'
 import type { ComicIssue, ComicSeries } from '@/types'
 import ComicSeriesView from './ComicSeriesView'
 import ComicIssueView from './ComicIssueView'
@@ -12,10 +12,15 @@ interface Props {
  readOnly?: boolean
 }

+const PAGE_SIZE = 200
+
 export default function ComicsView({ libraryId, readOnly }: Props) {
  const [items, setItems] = useState<(ComicIssue | ComicSeries)[]>([])
  const [loading, setLoading] = useState(true)
+  const [loadingMore, setLoadingMore] = useState(false)
  const [error, setError] = useState<string | null>(null)
+  const [page, setPage] = useState(1)
+  const [total, setTotal] = useState(0)
  const [selectedSeries, setSelectedSeries] = useState<ComicSeries | null>(null)
  const [selectedIssue, setSelectedIssue] = useState<ComicIssue | null>(null)
  const [tagPanel, setTagPanel] = useState<{ itemKey: string; title: string } | null>(null)
@@ -29,6 +34,8 @@ export default function ComicsView({ libraryId, readOnly }: Props) {
  const [showFilters, setShowFilters] = useState(
    () => typeof window !== 'undefined' && window.innerWidth >= 768
  )
+  const debounceRef = useRef<ReturnType<typeof setTimeout> | null>(null)
+  const sentinelRef = useRef<HTMLDivElement | null>(null)

  const toggleTag = (tagId: string) =>
    setSelectedTagIds((prev) => {
@@ -37,12 +44,22 @@ export default function ComicsView({ libraryId, readOnly }: Props) {
      return next
    })

-  const fetchItems = useCallback(() => {
-    fetch(`/api/comics?libraryId=${encodeURIComponent(libraryId)}`)
+  const fetchItems = useCallback((pageNum: number, searchVal: string, replace: boolean) => {
+    const params = new URLSearchParams({
+      libraryId,
+      page: String(pageNum),
+      pageSize: String(PAGE_SIZE),
+    })
+    if (searchVal) params.set('search', searchVal)
+    if (pageNum === 1) setLoading(true)
+    else setLoadingMore(true)
+    fetch(`/api/comics?${params}`)
      .then((r) => r.json())
-      .then((data: (ComicIssue | ComicSeries)[]) => {
-        setItems(data)
-        setLoading(false)
+      .then((data: { items: (ComicIssue | ComicSeries)[]; total: number }) => {
+        setItems((prev) => (replace ? data.items : [...prev, ...data.items]))
+        setTotal(data.total)
+        if (pageNum === 1) setLoading(false)
+        else setLoadingMore(false)
      })
      .catch(() => {
        setError('Failed to load comics')
@@ -50,7 +67,34 @@ export default function ComicsView({ libraryId, readOnly }: Props) {
      })
  }, [libraryId])

-  useEffect(() => { fetchItems() }, [fetchItems])
+  useEffect(() => { fetchItems(1, '', true) }, [fetchItems])
+
+  // IntersectionObserver: load next page when sentinel scrolls into view
+  useEffect(() => {
+    const sentinel = sentinelRef.current
+    if (!sentinel || items.length >= total || total === 0) return
+    const observer = new IntersectionObserver(
+      ([entry]) => {
+        if (entry.isIntersecting && !loadingMore) {
+          const next = page + 1
+          setPage(next)
+          fetchItems(next, search, false)
+        }
+      },
+      { rootMargin: '400px' }
+    )
+    observer.observe(sentinel)
+    return () => observer.disconnect()
+  }, [items.length, total, loadingMore, page, search, fetchItems])
+
+  const handleSearchChange = useCallback((val: string) => {
+    setSearch(val)
+    if (debounceRef.current) clearTimeout(debounceRef.current)
+    debounceRef.current = setTimeout(() => {
+      setPage(1)
+      fetchItems(1, val, true)
+    }, 300)
+  }, [fetchItems])

  const fetchAssignments = useCallback(() => {
    fetch(`/api/tags/library-assignments?libraryId=${encodeURIComponent(libraryId)}`)
@@ -133,7 +177,7 @@ export default function ComicsView({ libraryId, readOnly }: Props) {
              libraryId={libraryId}
              assignments={assignments}
              search={search}
-              onSearchChange={setSearch}
+              onSearchChange={handleSearchChange}
              selectedTagIds={selectedTagIds}
              onTagToggle={toggleTag}
              refreshKey={filterRefreshKey}
@@ -160,31 +204,40 @@ export default function ComicsView({ libraryId, readOnly }: Props) {
              <p className="text-sm">Add .cbz files or folders of .cbz files to this library and scan.</p>
            </div>
          ) : (
-            <div className="grid gap-4 grid-cols-2 sm:grid-cols-3 md:grid-cols-4 lg:grid-cols-5 xl:grid-cols-6">
-              {filtered.map((item) =>
-                'issueCount' in item ? (
-                  <SeriesCard
-                    key={item.id}
-                    series={item as ComicSeries}
-                    readOnly={readOnly}
-                    onClick={() => setSelectedSeries(item as ComicSeries)}
-                    onTagClick={(item as ComicSeries).item_key && !readOnly
-                      ? () => setTagPanel({ itemKey: (item as ComicSeries).item_key!, title: item.title })
-                      : undefined}
-                  />
-                ) : (
-                  <IssueCard
-                    key={item.id}
-                    issue={item as ComicIssue}
-                    readOnly={readOnly}
-                    onClick={() => setSelectedIssue(item as ComicIssue)}
-                    onTagClick={(item as ComicIssue).item_key && !readOnly
-                      ? () => setTagPanel({ itemKey: (item as ComicIssue).item_key!, title: item.title })
-                      : undefined}
-                  />
-                )
+            <>
+              {total > PAGE_SIZE && (
+                <p className="text-xs mb-3" style={{ color: 'var(--text-secondary)' }}>
+                  Showing {filtered.length.toLocaleString()} of {total.toLocaleString()}
+                </p>
              )}
-            </div>
+              <div className="grid gap-4 grid-cols-2 sm:grid-cols-3 md:grid-cols-4 lg:grid-cols-5 xl:grid-cols-6">
+                {filtered.map((item) =>
+                  'issueCount' in item ? (
+                    <SeriesCard
+                      key={item.id}
+                      series={item as ComicSeries}
+                      readOnly={readOnly}
+                      onClick={() => setSelectedSeries(item as ComicSeries)}
+                      onTagClick={(item as ComicSeries).item_key && !readOnly
+                        ? () => setTagPanel({ itemKey: (item as ComicSeries).item_key!, title: item.title })
+                        : undefined}
+                    />
+                  ) : (
+                    <IssueCard
+                      key={item.id}
+                      issue={item as ComicIssue}
+                      readOnly={readOnly}
+                      onClick={() => setSelectedIssue(item as ComicIssue)}
+                      onTagClick={(item as ComicIssue).item_key && !readOnly
+                        ? () => setTagPanel({ itemKey: (item as ComicIssue).item_key!, title: item.title })
+                        : undefined}
+                    />
+                  )
+                )}
+              </div>
+              <div ref={sentinelRef} style={{ height: 1 }} aria-hidden />
+              {loadingMore && <LoadingMore />}
+            </>
          )}
        </div>
      </div>
@@ -373,6 +426,17 @@ function IssueCard({
  )
 }

+function LoadingMore() {
+  return (
+    <div className="flex justify-center py-6">
+      <div
+        className="w-6 h-6 rounded-full border-2 animate-spin"
+        style={{ borderColor: 'var(--border)', borderTopColor: 'var(--accent)' }}
+      />
+    </div>
+  )
+}
+
 function LoadingGrid() {
  return (
    <div className="grid gap-4 grid-cols-2 sm:grid-cols-3 md:grid-cols-4 lg:grid-cols-5 xl:grid-cols-6">
--- a/src/lib/comics.ts
+++ b/src/lib/comics.ts
@@ -126,10 +126,18 @@ export function scanComicsLibrary(
  return results.sort((a, b) => naturalCompare(a.title, b.title))
 }

-// comicsFromDb returns series + standalone issues for the top-level grid.
+function escapeLike(s: string): string {
+  return `%${s.replace(/%/g, '\\%').replace(/_/g, '\\_')}%`
+}
+
+// comicsFromDb returns series + standalone issues for the top-level grid, paginated.
 // Series issues are retrieved separately via comicIssuesFromDb.
-export function comicsFromDb(libraryId: string): (ComicIssue | ComicSeries)[] {
+export function comicsFromDb(
+  libraryId: string,
+  opts: { page: number; pageSize: number; search?: string }
+): { items: (ComicIssue | ComicSeries)[]; total: number } {
  const db = getDb()
+  const offset = (opts.page - 1) * opts.pageSize

  type DbRow = {
    item_key: string
@@ -140,59 +148,63 @@ export function comicsFromDb(libraryId: string): (ComicIssue | ComicSeries)[] {
    file_path: string | null
  }

-  const allRows = db
-    .prepare(
-      `SELECT item_key, item_type, parent_key, title, metadata, file_path
-       FROM media_items
-       WHERE library_id = ? AND item_type IN ('comic_series','comic_issue')
-       ORDER BY title`
-    )
-    .all(libraryId) as DbRow[]
+  const baseWhere = `
+    WHERE library_id = ?
+      AND (item_type = 'comic_series' OR (item_type = 'comic_issue' AND parent_key IS NULL))
+  `

-  const seriesMap = new Map<string, ComicSeries>()
-  const standaloneIssues: ComicIssue[] = []
+  const total: number = opts.search
+    ? (db
+        .prepare(`SELECT COUNT(*) as cnt FROM media_items ${baseWhere} AND title LIKE ? ESCAPE '\\'`)
+        .get(libraryId, escapeLike(opts.search)) as { cnt: number }).cnt
+    : (db
+        .prepare(`SELECT COUNT(*) as cnt FROM media_items ${baseWhere}`)
+        .get(libraryId) as { cnt: number }).cnt

-  for (const row of allRows) {
-    if (row.item_type !== 'comic_series') continue
+  const rows: DbRow[] = opts.search
+    ? db
+        .prepare(
+          `SELECT item_key, item_type, parent_key, title, metadata, file_path
+           FROM media_items ${baseWhere} AND title LIKE ? ESCAPE '\\'
+           ORDER BY title LIMIT ? OFFSET ?`
+        )
+        .all(libraryId, escapeLike(opts.search), opts.pageSize, offset) as DbRow[]
+    : db
+        .prepare(
+          `SELECT item_key, item_type, parent_key, title, metadata, file_path
+           FROM media_items ${baseWhere}
+           ORDER BY title LIMIT ? OFFSET ?`
+        )
+        .all(libraryId, opts.pageSize, offset) as DbRow[]
+
+  const items: (ComicIssue | ComicSeries)[] = []
+  for (const row of rows) {
    const meta = row.metadata ? JSON.parse(row.metadata) : {}
-    const idPart = row.item_key.split(':comic_series:')[1] ?? row.item_key
-    seriesMap.set(row.item_key, {
-      id: idPart,
-      item_key: row.item_key,
-      title: row.title ?? decodeURIComponent(idPart),
-      coverUrl: meta.coverUrl ?? null,
-      issueCount: meta.issueCount ?? 0,
-    })
-  }
-
-  for (const row of allRows) {
-    if (row.item_type !== 'comic_issue') continue
-    const meta = row.metadata ? JSON.parse(row.metadata) : {}
-    const idPart = row.item_key.split(':comic_issue:')[1] ?? row.item_key
-    const issue: ComicIssue = {
-      id: idPart,
-      item_key: row.item_key,
-      title: row.title ?? decodeURIComponent(idPart.split(':').pop() ?? idPart),
-      issueNumber: meta.issueNumber ?? null,
-      pageCount: meta.pageCount ?? 0,
-      coverUrl: meta.coverUrl ?? null,
-      filePath: row.file_path ?? '',
-      isStandalone: meta.isStandalone ?? false,
-    }
-
-    if (row.parent_key && seriesMap.has(row.parent_key)) {
-      // Series issues are not included in the top-level grid — series card represents them
-      // We only include series cards + standalone issues in the grid
+    if (row.item_type === 'comic_series') {
+      const idPart = row.item_key.split(':comic_series:')[1] ?? row.item_key
+      items.push({
+        id: idPart,
+        item_key: row.item_key,
+        title: row.title ?? decodeURIComponent(idPart),
+        coverUrl: meta.coverUrl ?? null,
+        issueCount: meta.issueCount ?? 0,
+      } as ComicSeries)
    } else {
-      standaloneIssues.push(issue)
+      const idPart = row.item_key.split(':comic_issue:')[1] ?? row.item_key
+      items.push({
+        id: idPart,
+        item_key: row.item_key,
+        title: row.title ?? decodeURIComponent(idPart.split(':').pop() ?? idPart),
+        issueNumber: meta.issueNumber ?? null,
+        pageCount: meta.pageCount ?? 0,
+        coverUrl: meta.coverUrl ?? null,
+        filePath: row.file_path ?? '',
+        isStandalone: meta.isStandalone ?? true,
+      } as ComicIssue)
    }
  }

-  const results: (ComicIssue | ComicSeries)[] = [
-    ...Array.from(seriesMap.values()),
-    ...standaloneIssues,
-  ]
-  return results.sort((a, b) => naturalCompare(a.title, b.title))
+  return { items, total }
 }

 export function comicIssuesFromDb(libraryId: string, seriesId: string): ComicIssue[] {
--- a/src/lib/db.ts
+++ b/src/lib/db.ts
@@ -110,6 +110,7 @@ function initDb(db: Database.Database): void {
  migrateLibrariesAddComics(db)
  migrateComicItemTypes(db)
  migrateImportedTags(db)
+  migrateComicsIndex(db)
  seedAppSettings(db)
 }

@@ -447,3 +448,10 @@ function migrateImportedTags(db: Database.Database): void {
    );
  `)
 }
+
+function migrateComicsIndex(db: Database.Database): void {
+  db.exec(`
+    CREATE INDEX IF NOT EXISTS media_items_library_type_title
+      ON media_items(library_id, item_type, title);
+  `)
+}
--- a/src/lib/scanner.ts
+++ b/src/lib/scanner.ts
@@ -573,29 +573,37 @@ async function scanComics(library: Library, libraryRoot: string): Promise<void>
      scanned_at = excluded.scanned_at
  `)

+  type SeriesRec = Parameters<typeof upsertSeries.run>[0]
+  type IssueRec  = Parameters<typeof upsertIssue.run>[0]
+  type BatchEntry = { type: 'series'; rec: SeriesRec } | { type: 'issue'; rec: IssueRec }
+
+  // Collect all records before touching the DB so we can batch-insert with event-loop yields.
+  // Note: between clearLibraryItems and the final batch, the library will appear partially
+  // populated — acceptable for a background scan.
+  const allRecords: BatchEntry[] = []
  let issueCount = 0

-  db.transaction(() => {
-    for (const item of items) {
-      if ('issues' in item) {
-        const series = item as ScannedComicSeries
-        const seriesKey = `${library.id}:comic_series:${series.id}`
-        upsertSeries.run({
+  for (const item of items) {
+    if ('issues' in item) {
+      const series = item as ScannedComicSeries
+      const seriesKey = `${library.id}:comic_series:${series.id}`
+      allRecords.push({
+        type: 'series',
+        rec: {
          library_id: library.id,
          item_key: seriesKey,
          item_type: 'comic_series',
          title: series.title,
-          metadata: JSON.stringify({
-            issueCount: series.issueCount,
-            coverUrl: series.coverUrl,
-          }),
+          metadata: JSON.stringify({ issueCount: series.issueCount, coverUrl: series.coverUrl }),
          file_path: null,
          scanned_at: now,
-        })
-
-        for (const issue of series.issues) {
-          const issueKey = `${library.id}:comic_issue:${issue.id}`
-          upsertIssue.run({
+        },
+      })
+      for (const issue of series.issues) {
+        const issueKey = `${library.id}:comic_issue:${issue.id}`
+        allRecords.push({
+          type: 'issue',
+          rec: {
            library_id: library.id,
            item_key: issueKey,
            item_type: 'comic_issue',
@@ -609,13 +617,16 @@ async function scanComics(library: Library, libraryRoot: string): Promise<void>
            }),
            file_path: issue.filePath,
            scanned_at: now,
-          })
-          issueCount++
-        }
-      } else {
-        const issue = item as ComicIssue
-        const issueKey = `${library.id}:comic_issue:${issue.id}`
-        upsertIssue.run({
+          },
+        })
+        issueCount++
+      }
+    } else {
+      const issue = item as ComicIssue
+      const issueKey = `${library.id}:comic_issue:${issue.id}`
+      allRecords.push({
+        type: 'issue',
+        rec: {
          library_id: library.id,
          item_key: issueKey,
          item_type: 'comic_issue',
@@ -629,13 +640,27 @@ async function scanComics(library: Library, libraryRoot: string): Promise<void>
          }),
          file_path: issue.filePath,
          scanned_at: now,
-        })
-        issueCount++
-      }
+        },
+      })
+      issueCount++
    }
-  })()
+  }

-  // Prewarm CBZ cover thumbnails
+  // Insert in batches of 500, yielding the event loop between batches so the app
+  // remains responsive to HTTP requests during a large scan.
+  const BATCH_SIZE = 500
+  for (let i = 0; i < allRecords.length; i += BATCH_SIZE) {
+    const batch = allRecords.slice(i, i + BATCH_SIZE)
+    db.transaction(() => {
+      for (const entry of batch) {
+        if (entry.type === 'series') upsertSeries.run(entry.rec)
+        else upsertIssue.run(entry.rec)
+      }
+    })()
+    await new Promise<void>((r) => setImmediate(r))
+  }
+
+  // Prewarm CBZ cover thumbnails — fire-and-forget so they don't block scan completion.
  for (const item of items) {
    const issuesToWarm: ComicIssue[] = 'issues' in item
      ? (item as ScannedComicSeries).issues.slice(0, 1)
@@ -643,11 +668,9 @@ async function scanComics(library: Library, libraryRoot: string): Promise<void>

    for (const issue of issuesToWarm) {
      const absPath = path.join(libraryRoot, issue.filePath)
-      try {
-        await getCbzThumbnailPath(absPath, library.id)
-      } catch (err) {
+      getCbzThumbnailPath(absPath, library.id).catch((err) => {
        console.warn(`[scanner]   Could not generate CBZ thumbnail for ${issue.filePath}:`, err instanceof Error ? err.message : err)
-      }
+      })
    }
  }