Add AI-powered image tagging via local LLM

Adds automatic image tagging that runs as a post-scan phase, sending
thumbnails to an OpenAI-compatible vision API and applying matching
tags from the user-defined tag vocabulary.

- New ai-tagger module with batch processing, failure tolerance, and
  tag validation against existing vocabulary
- Admin settings page (Manage > AI Tagging) for endpoint, model, and
  enable toggle with connection testing
- DB migration for ai_tagged_at tracking column and AI config seeds
- Re-tag All support to queue items for reprocessing

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>

src/lib/ai-tagger.ts

@@ -0,0 +1,252 @@
+import fs from 'fs'
+import path from 'path'
+import type { Library, Tag, TagCategory } from '@/types'
+import { getDb } from './db'
+import { getAiConfig } from './app-settings'
+import { getTags, getCategories, addTagToItem } from './tags'
+import { getThumbnailPath } from './thumbnails'
+import { findFile } from './media-utils'
+
+const BATCH_LIMIT = 50
+const REQUEST_TIMEOUT_MS = 30_000
+const MAX_CONSECUTIVE_FAILURES = 3
+
+const IMAGE_EXTENSIONS = new Set(['.jpg', '.jpeg', '.png', '.gif', '.webp', '.bmp', '.tiff', '.tif'])
+
+interface MediaItemRow {
+  item_key: string
+  item_type: string
+  file_path: string | null
+  metadata: string | null
+}
+
+/**
+ * Resolve the absolute path to the best image for a media item.
+ * Returns null if no suitable image is found.
+ */
+function resolveItemImage(libraryRoot: string, item: MediaItemRow): string | null {
+  switch (item.item_type) {
+    case 'movie':
+    case 'tv_series': {
+      // metadata.posterUrl is an API URL like /api/thumbnail?libraryId=...&path=dir/poster.jpg
+      // Extract the relative path from the URL and resolve to absolute
+      const meta = item.metadata ? JSON.parse(item.metadata) : {}
+      const apiUrl = meta.posterUrl as string | undefined
+      if (!apiUrl) return null
+      try {
+        const relPath = decodeURIComponent(
+          new URL(apiUrl, 'http://localhost').searchParams.get('path') ?? ''
+        )
+        if (!relPath) return null
+        const absPath = path.join(libraryRoot, relPath)
+        if (fs.existsSync(absPath)) return absPath
+      } catch {
+        return null
+      }
+      return null
+    }
+
+    case 'game':
+    case 'game_series': {
+      const meta = item.metadata ? JSON.parse(item.metadata) : {}
+      const apiUrl = meta.coverUrl as string | undefined
+      if (!apiUrl) return null
+      try {
+        const relPath = decodeURIComponent(
+          new URL(apiUrl, 'http://localhost').searchParams.get('path') ?? ''
+        )
+        if (!relPath) return null
+        const absPath = path.join(libraryRoot, relPath)
+        if (fs.existsSync(absPath)) return absPath
+      } catch {
+        return null
+      }
+      return null
+    }
+
+    case 'tv_season': {
+      // Seasons may have a poster in their directory
+      if (!item.file_path) return null
+      const seasonDir = path.join(libraryRoot, item.file_path)
+      const posterFile = findFile(seasonDir, /^(poster|cover|folder)$/i)
+      if (posterFile) return path.join(seasonDir, posterFile)
+      return null
+    }
+
+    case 'mixed_file': {
+      // For mixed files, tag only actual images (not videos or other files)
+      if (!item.file_path) return null
+      const ext = path.extname(item.file_path).toLowerCase()
+      if (!IMAGE_EXTENSIONS.has(ext)) return null
+      return path.join(libraryRoot, item.file_path)
+    }
+
+    default:
+      return null
+  }
+}
+
+/**
+ * Build the system prompt that instructs the LLM to select matching tags.
+ */
+function buildTagPrompt(tags: Tag[], categories: TagCategory[]): string {
+  const categoryMap = new Map(categories.map((c) => [c.id, c.name]))
+
+  const grouped: Record<string, { id: string; name: string }[]> = {}
+  for (const tag of tags) {
+    const catName = categoryMap.get(tag.categoryId) ?? 'Uncategorized'
+    ;(grouped[catName] ??= []).push({ id: tag.id, name: tag.name })
+  }
+
+  const lines: string[] = []
+  for (const [catName, catTags] of Object.entries(grouped)) {
+    const tagList = catTags.map((t) => `${t.name} (id: ${t.id})`).join(', ')
+    lines.push(`[${catName}] ${tagList}`)
+  }
+
+  return [
+    'You are an image tagger. Given the image, select which of the following tags apply.',
+    'Return ONLY a JSON array of tag IDs that match the image. Do not invent new tags.',
+    'If no tags match, return an empty array: []',
+    '',
+    'Available tags:',
+    ...lines,
+  ].join('\n')
+}
+
+/**
+ * Call the OpenAI-compatible vision API to get tag suggestions for an image.
+ */
+async function callVisionApi(
+  endpoint: string,
+  model: string,
+  base64Image: string,
+  systemPrompt: string
+): Promise<string[]> {
+  const url = endpoint.replace(/\/+$/, '') + '/chat/completions'
+
+  const controller = new AbortController()
+  const timeout = setTimeout(() => controller.abort(), REQUEST_TIMEOUT_MS)
+
+  try {
+    const res = await fetch(url, {
+      method: 'POST',
+      headers: { 'Content-Type': 'application/json' },
+      signal: controller.signal,
+      body: JSON.stringify({
+        model,
+        messages: [
+          { role: 'system', content: systemPrompt },
+          {
+            role: 'user',
+            content: [
+              {
+                type: 'image_url',
+                image_url: { url: `data:image/jpeg;base64,${base64Image}` },
+              },
+            ],
+          },
+        ],
+        max_tokens: 512,
+        temperature: 0.1,
+      }),
+    })
+
+    if (!res.ok) {
+      const text = await res.text().catch(() => '')
+      throw new Error(`LLM API returned ${res.status}: ${text.slice(0, 200)}`)
+    }
+
+    const data = await res.json() as {
+      choices?: Array<{ message?: { content?: string } }>
+    }
+
+    const content = data.choices?.[0]?.message?.content?.trim() ?? ''
+
+    // Extract JSON array from the response (handle markdown code blocks)
+    const jsonMatch = content.match(/\[[\s\S]*\]/)
+    if (!jsonMatch) return []
+
+    const parsed = JSON.parse(jsonMatch[0])
+    if (!Array.isArray(parsed)) return []
+    return parsed.filter((v): v is string => typeof v === 'string')
+  } finally {
+    clearTimeout(timeout)
+  }
+}
+
+/**
+ * Run AI tagging for a single library. Called after the scanner finishes.
+ * Processes up to BATCH_LIMIT untagged items per invocation.
+ */
+export async function runAiTagging(library: Library, libraryRoot: string): Promise<void> {
+  const config = getAiConfig()
+  if (!config.enabled || !config.endpoint || !config.model) return
+
+  const tags = getTags()
+  const categories = getCategories()
+  if (tags.length === 0) return
+
+  const validTagIds = new Set(tags.map((t) => t.id))
+  const systemPrompt = buildTagPrompt(tags, categories)
+
+  const db = getDb()
+  const untaggedItems = db
+    .prepare(
+      `SELECT item_key, item_type, file_path, metadata
+       FROM media_items
+       WHERE library_id = ? AND ai_tagged_at IS NULL
+       LIMIT ?`
+    )
+    .all(library.id, BATCH_LIMIT) as MediaItemRow[]
+
+  if (untaggedItems.length === 0) return
+
+  console.log(`[ai-tagger] Processing ${untaggedItems.length} items in library "${library.name}"`)
+
+  let tagged = 0
+  let consecutiveFailures = 0
+  const markTagged = db.prepare('UPDATE media_items SET ai_tagged_at = ? WHERE item_key = ?')
+
+  for (const item of untaggedItems) {
+    if (consecutiveFailures >= MAX_CONSECUTIVE_FAILURES) {
+      console.warn(`[ai-tagger] Aborting after ${MAX_CONSECUTIVE_FAILURES} consecutive failures`)
+      break
+    }
+
+    const imagePath = resolveItemImage(libraryRoot, item)
+    if (!imagePath) {
+      // No image available — mark as tagged so we don't retry every scan
+      markTagged.run(Date.now(), item.item_key)
+      continue
+    }
+
+    try {
+      // Use the thumbnail cache for a smaller image
+      const thumbnailPath = await getThumbnailPath(imagePath, library.id, 'image')
+      const base64 = fs.readFileSync(thumbnailPath, 'base64')
+
+      const suggestedIds = await callVisionApi(config.endpoint, config.model, base64, systemPrompt)
+
+      // Filter to valid tags only
+      const validIds = suggestedIds.filter((id) => validTagIds.has(id))
+      for (const tagId of validIds) {
+        addTagToItem(item.item_key, tagId)
+      }
+
+      markTagged.run(Date.now(), item.item_key)
+      tagged++
+      consecutiveFailures = 0
+    } catch (err) {
+      consecutiveFailures++
+      console.warn(
+        `[ai-tagger] Failed to tag "${item.item_key}":`,
+        err instanceof Error ? err.message : err
+      )
+    }
+  }
+
+  if (tagged > 0) {
+    console.log(`[ai-tagger] Tagged ${tagged}/${untaggedItems.length} items in library "${library.name}"`)
+  }
+}

src/lib/app-settings.ts

@@ -36,3 +36,24 @@ export function updateScanConfig(schedule: string, enabled: boolean): void {
 export function setScanLastRan(ts: number): void {
   setSetting('scan_last_ran', String(ts))
 }
+
+// ─── AI Settings ─────────────────────────────────────────────────────────────
+
+interface AiConfig {
+  endpoint: string
+  model: string
+  enabled: boolean
+}
+
+export function getAiConfig(): AiConfig {
+  const endpoint = getSetting('ai_endpoint') ?? ''
+  const model = getSetting('ai_model') ?? ''
+  const enabled = getSetting('ai_enabled') === 'true'
+  return { endpoint, model, enabled }
+}
+
+export function updateAiConfig(endpoint: string, model: string, enabled: boolean): void {
+  setSetting('ai_endpoint', endpoint)
+  setSetting('ai_model', model)
+  setSetting('ai_enabled', enabled ? 'true' : 'false')
+}

src/lib/db.ts

@@ -102,6 +102,7 @@ function initDb(db: Database.Database): void {
   migrateMediaItemsSchema(db)
   migrateMediaItemsFingerprint(db)
   migrateMediaTagsToItemKey(db)
+  migrateMediaItemsAiTagged(db)
   seedAppSettings(db)
 }
 
@@ -110,6 +111,9 @@ function seedAppSettings(db: Database.Database): void {
     scan_schedule: '0 * * * *',
     scan_enabled: 'true',
     scan_last_ran: '',
+    ai_enabled: 'false',
+    ai_endpoint: '',
+    ai_model: '',
   }
   const insert = db.prepare(
     'INSERT OR IGNORE INTO app_settings (key, value) VALUES (?, ?)'
@@ -228,6 +232,15 @@ function migrateMediaTagsToItemKey(db: Database.Database): void {
   `)
 }
 
+function migrateMediaItemsAiTagged(db: Database.Database): void {
+  const row = db
+    .prepare("SELECT sql FROM sqlite_master WHERE type='table' AND name='media_items'")
+    .get() as { sql: string } | undefined
+  if (row && !row.sql.includes('ai_tagged_at')) {
+    db.exec('ALTER TABLE media_items ADD COLUMN ai_tagged_at INTEGER')
+  }
+}
+
 function migrateLibrariesType(db: Database.Database): void {
   const row = db
     .prepare("SELECT sql FROM sqlite_master WHERE type='table' AND name='libraries'")

src/lib/scanner.ts

@@ -10,6 +10,7 @@ import { scanGamesLibrary } from './games'
 import { getThumbnailPath } from './thumbnails'
 import { computeFingerprint } from './fingerprint'
 import { reKeyMediaItem } from './tags'
+import { runAiTagging } from './ai-tagger'
 
 let scanRunning = false
 
@@ -70,6 +71,10 @@ export async function runLibraryScan(library: Library): Promise<void> {
       await scanMixed(library, libraryRoot)
       break
   }
+
+  await runAiTagging(library, libraryRoot).catch((err) =>
+    console.error(`[ai-tagger] Error tagging library "${library.name}":`, err)
+  )
 }
 
 // ---------------------------------------------------------------------------