From 0238dbda7a9e9a849a159d19d1760653ec92bc56 Mon Sep 17 00:00:00 2001 From: Garret Patti <42485635+garretpatti@users.noreply.github.com> Date: Sun, 12 Apr 2026 15:18:03 -0400 Subject: [PATCH] Add AI-powered image tagging via local LLM Adds automatic image tagging that runs as a post-scan phase, sending thumbnails to an OpenAI-compatible vision API and applying matching tags from the user-defined tag vocabulary. - New ai-tagger module with batch processing, failure tolerance, and tag validation against existing vocabulary - Admin settings page (Manage > AI Tagging) for endpoint, model, and enable toggle with connection testing - DB migration for ai_tagged_at tracking column and AI config seeds - Re-tag All support to queue items for reprocessing Co-Authored-By: Claude Opus 4.6 --- src/app/api/ai-settings/retag/route.ts | 13 + src/app/api/ai-settings/route.ts | 38 +++ src/app/api/ai-settings/test/route.ts | 47 ++++ src/app/manage/ai-tagging/page.tsx | 318 +++++++++++++++++++++++++ src/components/ManageSubNav.tsx | 1 + src/lib/ai-tagger.ts | 252 ++++++++++++++++++++ src/lib/app-settings.ts | 21 ++ src/lib/db.ts | 13 + src/lib/scanner.ts | 5 + 9 files changed, 708 insertions(+) create mode 100644 src/app/api/ai-settings/retag/route.ts create mode 100644 src/app/api/ai-settings/route.ts create mode 100644 src/app/api/ai-settings/test/route.ts create mode 100644 src/app/manage/ai-tagging/page.tsx create mode 100644 src/lib/ai-tagger.ts diff --git a/src/app/api/ai-settings/retag/route.ts b/src/app/api/ai-settings/retag/route.ts new file mode 100644 index 0000000..93a9e9f --- /dev/null +++ b/src/app/api/ai-settings/retag/route.ts @@ -0,0 +1,13 @@ +import { NextRequest, NextResponse } from 'next/server' +import { requireAdmin } from '@/lib/auth' +import { getDb } from '@/lib/db' + +export async function POST(request: NextRequest) { + const auth = await requireAdmin(request) + if (auth instanceof NextResponse) return auth + + const db = getDb() + const result = db.prepare('UPDATE media_items SET ai_tagged_at = NULL').run() + + return NextResponse.json({ cleared: result.changes }) +} diff --git a/src/app/api/ai-settings/route.ts b/src/app/api/ai-settings/route.ts new file mode 100644 index 0000000..593f37a --- /dev/null +++ b/src/app/api/ai-settings/route.ts @@ -0,0 +1,38 @@ +import { NextRequest, NextResponse } from 'next/server' +import { requireAdmin } from '@/lib/auth' +import { getAiConfig, updateAiConfig } from '@/lib/app-settings' + +export async function GET(request: NextRequest) { + const auth = await requireAdmin(request) + if (auth instanceof NextResponse) return auth + + const { endpoint, model, enabled } = getAiConfig() + return NextResponse.json({ endpoint, model, enabled }) +} + +export async function PUT(request: NextRequest) { + const auth = await requireAdmin(request) + if (auth instanceof NextResponse) return auth + + let body: { endpoint?: string; model?: string; enabled?: boolean } + try { + body = await request.json() + } catch { + return NextResponse.json({ error: 'Invalid JSON body' }, { status: 400 }) + } + + const { endpoint, model, enabled } = body + + if (typeof endpoint !== 'string') { + return NextResponse.json({ error: 'endpoint is required' }, { status: 400 }) + } + if (typeof model !== 'string') { + return NextResponse.json({ error: 'model is required' }, { status: 400 }) + } + if (typeof enabled !== 'boolean') { + return NextResponse.json({ error: 'enabled must be a boolean' }, { status: 400 }) + } + + updateAiConfig(endpoint, model, enabled) + return NextResponse.json({ endpoint, model, enabled }) +} diff --git a/src/app/api/ai-settings/test/route.ts b/src/app/api/ai-settings/test/route.ts new file mode 100644 index 0000000..5a957a0 --- /dev/null +++ b/src/app/api/ai-settings/test/route.ts @@ -0,0 +1,47 @@ +import { NextRequest, NextResponse } from 'next/server' +import { requireAdmin } from '@/lib/auth' +import { getAiConfig } from '@/lib/app-settings' + +export async function POST(request: NextRequest) { + const auth = await requireAdmin(request) + if (auth instanceof NextResponse) return auth + + const { endpoint, model } = getAiConfig() + + if (!endpoint) { + return NextResponse.json({ error: 'No endpoint configured' }, { status: 400 }) + } + + const url = endpoint.replace(/\/+$/, '') + '/chat/completions' + + try { + const controller = new AbortController() + const timeout = setTimeout(() => controller.abort(), 10_000) + + const res = await fetch(url, { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + signal: controller.signal, + body: JSON.stringify({ + model: model || 'test', + messages: [{ role: 'user', content: 'Hi' }], + max_tokens: 1, + }), + }) + + clearTimeout(timeout) + + if (!res.ok) { + const text = await res.text().catch(() => '') + return NextResponse.json( + { error: `LLM returned ${res.status}: ${text.slice(0, 200)}` }, + { status: 502 } + ) + } + + return NextResponse.json({ ok: true }) + } catch (err) { + const message = err instanceof Error ? err.message : 'Unknown error' + return NextResponse.json({ error: `Connection failed: ${message}` }, { status: 502 }) + } +} diff --git a/src/app/manage/ai-tagging/page.tsx b/src/app/manage/ai-tagging/page.tsx new file mode 100644 index 0000000..0253d2f --- /dev/null +++ b/src/app/manage/ai-tagging/page.tsx @@ -0,0 +1,318 @@ +'use client' + +import { useEffect, useState, useCallback } from 'react' + +interface AiSettings { + endpoint: string + model: string + enabled: boolean +} + +export default function AiTaggingPage() { + const [settings, setSettings] = useState({ endpoint: '', model: '', enabled: false }) + const [loading, setLoading] = useState(true) + const [saving, setSaving] = useState(false) + const [saveError, setSaveError] = useState(null) + const [saveSuccess, setSaveSuccess] = useState(false) + const [testing, setTesting] = useState(false) + const [testResult, setTestResult] = useState<{ ok: boolean; message: string } | null>(null) + const [retagging, setRetagging] = useState(false) + const [retagResult, setRetagResult] = useState(null) + + const fetchSettings = useCallback(async () => { + try { + const res = await fetch('/api/ai-settings') + if (!res.ok) return + const data: AiSettings = await res.json() + setSettings(data) + } catch { + // ignore + } finally { + setLoading(false) + } + }, []) + + useEffect(() => { + fetchSettings() + }, [fetchSettings]) + + const handleSave = async (e: React.FormEvent) => { + e.preventDefault() + setSaveError(null) + setSaveSuccess(false) + setSaving(true) + try { + const res = await fetch('/api/ai-settings', { + method: 'PUT', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify(settings), + }) + const data = await res.json() + if (!res.ok) { + setSaveError(data.error ?? 'Failed to save settings') + } else { + setSettings(data) + setSaveSuccess(true) + setTimeout(() => setSaveSuccess(false), 3000) + } + } catch { + setSaveError('Network error. Please try again.') + } finally { + setSaving(false) + } + } + + const handleTest = async () => { + setTesting(true) + setTestResult(null) + try { + const res = await fetch('/api/ai-settings/test', { method: 'POST' }) + const data = await res.json() + if (res.ok) { + setTestResult({ ok: true, message: 'Connection successful.' }) + } else { + setTestResult({ ok: false, message: data.error ?? 'Connection failed.' }) + } + } catch { + setTestResult({ ok: false, message: 'Network error.' }) + } finally { + setTesting(false) + } + } + + const handleRetag = async () => { + if (!confirm('This will clear AI tags from all items so they get re-processed on the next scan. Continue?')) return + setRetagging(true) + setRetagResult(null) + try { + const res = await fetch('/api/ai-settings/retag', { method: 'POST' }) + const data = await res.json() + if (res.ok) { + setRetagResult(`Cleared AI tag status from ${data.cleared} items. They will be re-tagged on the next scan.`) + } else { + setRetagResult(data.error ?? 'Failed to clear tags.') + } + } catch { + setRetagResult('Network error.') + } finally { + setRetagging(false) + } + } + + return ( +
+

+ AI Tagging +

+

+ Automatically tag media using a vision-capable LLM on your network. +

+ +
+ {loading ? ( + + ) : ( +
+ + setSettings((s) => ({ ...s, endpoint: e.target.value }))} + placeholder="http://192.168.1.50:8080/v1" + className="w-full rounded-lg px-3 py-2 text-sm font-mono outline-none focus:ring-2" + style={{ + backgroundColor: 'var(--background)', + border: '1px solid var(--border)', + color: 'var(--text-primary)', + }} + onFocus={(e) => ((e.currentTarget as HTMLElement).style.borderColor = 'var(--accent)')} + onBlur={(e) => ((e.currentTarget as HTMLElement).style.borderColor = 'var(--border)')} + /> +

+ Base URL of an OpenAI-compatible API server (e.g. Ollama, vLLM, llama.cpp). +

+ + + + setSettings((s) => ({ ...s, model: e.target.value }))} + placeholder="llava" + className="w-full rounded-lg px-3 py-2 text-sm font-mono outline-none focus:ring-2" + style={{ + backgroundColor: 'var(--background)', + border: '1px solid var(--border)', + color: 'var(--text-primary)', + }} + onFocus={(e) => ((e.currentTarget as HTMLElement).style.borderColor = 'var(--accent)')} + onBlur={(e) => ((e.currentTarget as HTMLElement).style.borderColor = 'var(--border)')} + /> +

+ Model name to use for vision requests. +

+ + + + +

+ When enabled, new media will be automatically tagged during library scans. +

+ + + {saveError && ( +

+ {saveError} +

+ )} + {saveSuccess && ( +

+ Settings saved. +

+ )} + +
+ + + +
+ + {testResult && ( +

+ {testResult.message} +

+ )} +
+ )} +
+ +
+
+

+ Clear the AI tag status on all items so they get re-processed during the next scan. + Existing tag assignments are not removed. +

+
+ +
+ {retagResult && ( +

+ {retagResult} +

+ )} +
+
+
+ ) +} + +function Section({ title, children }: { title: string; children: React.ReactNode }) { + return ( +
+

+ {title} +

+
+
{children}
+
+
+ ) +} + +function Field({ label, children }: { label: string; children: React.ReactNode }) { + return ( +
+ + {children} +
+ ) +} + +function LoadingRows() { + return ( +
+ {[70, 50, 40].map((w) => ( +
+
+
+ ))} +
+ ) +} diff --git a/src/components/ManageSubNav.tsx b/src/components/ManageSubNav.tsx index f3df1f1..a57531f 100644 --- a/src/components/ManageSubNav.tsx +++ b/src/components/ManageSubNav.tsx @@ -8,6 +8,7 @@ const TABS = [ { href: '/manage/tags', label: 'Tags' }, { href: '/manage/users', label: 'Users' }, { href: '/manage/scanning', label: 'Scanning' }, + { href: '/manage/ai-tagging', label: 'AI Tagging' }, ] export default function ManageSubNav() { diff --git a/src/lib/ai-tagger.ts b/src/lib/ai-tagger.ts new file mode 100644 index 0000000..d9d6b6b --- /dev/null +++ b/src/lib/ai-tagger.ts @@ -0,0 +1,252 @@ +import fs from 'fs' +import path from 'path' +import type { Library, Tag, TagCategory } from '@/types' +import { getDb } from './db' +import { getAiConfig } from './app-settings' +import { getTags, getCategories, addTagToItem } from './tags' +import { getThumbnailPath } from './thumbnails' +import { findFile } from './media-utils' + +const BATCH_LIMIT = 50 +const REQUEST_TIMEOUT_MS = 30_000 +const MAX_CONSECUTIVE_FAILURES = 3 + +const IMAGE_EXTENSIONS = new Set(['.jpg', '.jpeg', '.png', '.gif', '.webp', '.bmp', '.tiff', '.tif']) + +interface MediaItemRow { + item_key: string + item_type: string + file_path: string | null + metadata: string | null +} + +/** + * Resolve the absolute path to the best image for a media item. + * Returns null if no suitable image is found. + */ +function resolveItemImage(libraryRoot: string, item: MediaItemRow): string | null { + switch (item.item_type) { + case 'movie': + case 'tv_series': { + // metadata.posterUrl is an API URL like /api/thumbnail?libraryId=...&path=dir/poster.jpg + // Extract the relative path from the URL and resolve to absolute + const meta = item.metadata ? JSON.parse(item.metadata) : {} + const apiUrl = meta.posterUrl as string | undefined + if (!apiUrl) return null + try { + const relPath = decodeURIComponent( + new URL(apiUrl, 'http://localhost').searchParams.get('path') ?? '' + ) + if (!relPath) return null + const absPath = path.join(libraryRoot, relPath) + if (fs.existsSync(absPath)) return absPath + } catch { + return null + } + return null + } + + case 'game': + case 'game_series': { + const meta = item.metadata ? JSON.parse(item.metadata) : {} + const apiUrl = meta.coverUrl as string | undefined + if (!apiUrl) return null + try { + const relPath = decodeURIComponent( + new URL(apiUrl, 'http://localhost').searchParams.get('path') ?? '' + ) + if (!relPath) return null + const absPath = path.join(libraryRoot, relPath) + if (fs.existsSync(absPath)) return absPath + } catch { + return null + } + return null + } + + case 'tv_season': { + // Seasons may have a poster in their directory + if (!item.file_path) return null + const seasonDir = path.join(libraryRoot, item.file_path) + const posterFile = findFile(seasonDir, /^(poster|cover|folder)$/i) + if (posterFile) return path.join(seasonDir, posterFile) + return null + } + + case 'mixed_file': { + // For mixed files, tag only actual images (not videos or other files) + if (!item.file_path) return null + const ext = path.extname(item.file_path).toLowerCase() + if (!IMAGE_EXTENSIONS.has(ext)) return null + return path.join(libraryRoot, item.file_path) + } + + default: + return null + } +} + +/** + * Build the system prompt that instructs the LLM to select matching tags. + */ +function buildTagPrompt(tags: Tag[], categories: TagCategory[]): string { + const categoryMap = new Map(categories.map((c) => [c.id, c.name])) + + const grouped: Record = {} + for (const tag of tags) { + const catName = categoryMap.get(tag.categoryId) ?? 'Uncategorized' + ;(grouped[catName] ??= []).push({ id: tag.id, name: tag.name }) + } + + const lines: string[] = [] + for (const [catName, catTags] of Object.entries(grouped)) { + const tagList = catTags.map((t) => `${t.name} (id: ${t.id})`).join(', ') + lines.push(`[${catName}] ${tagList}`) + } + + return [ + 'You are an image tagger. Given the image, select which of the following tags apply.', + 'Return ONLY a JSON array of tag IDs that match the image. Do not invent new tags.', + 'If no tags match, return an empty array: []', + '', + 'Available tags:', + ...lines, + ].join('\n') +} + +/** + * Call the OpenAI-compatible vision API to get tag suggestions for an image. + */ +async function callVisionApi( + endpoint: string, + model: string, + base64Image: string, + systemPrompt: string +): Promise { + const url = endpoint.replace(/\/+$/, '') + '/chat/completions' + + const controller = new AbortController() + const timeout = setTimeout(() => controller.abort(), REQUEST_TIMEOUT_MS) + + try { + const res = await fetch(url, { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + signal: controller.signal, + body: JSON.stringify({ + model, + messages: [ + { role: 'system', content: systemPrompt }, + { + role: 'user', + content: [ + { + type: 'image_url', + image_url: { url: `data:image/jpeg;base64,${base64Image}` }, + }, + ], + }, + ], + max_tokens: 512, + temperature: 0.1, + }), + }) + + if (!res.ok) { + const text = await res.text().catch(() => '') + throw new Error(`LLM API returned ${res.status}: ${text.slice(0, 200)}`) + } + + const data = await res.json() as { + choices?: Array<{ message?: { content?: string } }> + } + + const content = data.choices?.[0]?.message?.content?.trim() ?? '' + + // Extract JSON array from the response (handle markdown code blocks) + const jsonMatch = content.match(/\[[\s\S]*\]/) + if (!jsonMatch) return [] + + const parsed = JSON.parse(jsonMatch[0]) + if (!Array.isArray(parsed)) return [] + return parsed.filter((v): v is string => typeof v === 'string') + } finally { + clearTimeout(timeout) + } +} + +/** + * Run AI tagging for a single library. Called after the scanner finishes. + * Processes up to BATCH_LIMIT untagged items per invocation. + */ +export async function runAiTagging(library: Library, libraryRoot: string): Promise { + const config = getAiConfig() + if (!config.enabled || !config.endpoint || !config.model) return + + const tags = getTags() + const categories = getCategories() + if (tags.length === 0) return + + const validTagIds = new Set(tags.map((t) => t.id)) + const systemPrompt = buildTagPrompt(tags, categories) + + const db = getDb() + const untaggedItems = db + .prepare( + `SELECT item_key, item_type, file_path, metadata + FROM media_items + WHERE library_id = ? AND ai_tagged_at IS NULL + LIMIT ?` + ) + .all(library.id, BATCH_LIMIT) as MediaItemRow[] + + if (untaggedItems.length === 0) return + + console.log(`[ai-tagger] Processing ${untaggedItems.length} items in library "${library.name}"`) + + let tagged = 0 + let consecutiveFailures = 0 + const markTagged = db.prepare('UPDATE media_items SET ai_tagged_at = ? WHERE item_key = ?') + + for (const item of untaggedItems) { + if (consecutiveFailures >= MAX_CONSECUTIVE_FAILURES) { + console.warn(`[ai-tagger] Aborting after ${MAX_CONSECUTIVE_FAILURES} consecutive failures`) + break + } + + const imagePath = resolveItemImage(libraryRoot, item) + if (!imagePath) { + // No image available — mark as tagged so we don't retry every scan + markTagged.run(Date.now(), item.item_key) + continue + } + + try { + // Use the thumbnail cache for a smaller image + const thumbnailPath = await getThumbnailPath(imagePath, library.id, 'image') + const base64 = fs.readFileSync(thumbnailPath, 'base64') + + const suggestedIds = await callVisionApi(config.endpoint, config.model, base64, systemPrompt) + + // Filter to valid tags only + const validIds = suggestedIds.filter((id) => validTagIds.has(id)) + for (const tagId of validIds) { + addTagToItem(item.item_key, tagId) + } + + markTagged.run(Date.now(), item.item_key) + tagged++ + consecutiveFailures = 0 + } catch (err) { + consecutiveFailures++ + console.warn( + `[ai-tagger] Failed to tag "${item.item_key}":`, + err instanceof Error ? err.message : err + ) + } + } + + if (tagged > 0) { + console.log(`[ai-tagger] Tagged ${tagged}/${untaggedItems.length} items in library "${library.name}"`) + } +} diff --git a/src/lib/app-settings.ts b/src/lib/app-settings.ts index e27f2c8..f45dbf1 100644 --- a/src/lib/app-settings.ts +++ b/src/lib/app-settings.ts @@ -36,3 +36,24 @@ export function updateScanConfig(schedule: string, enabled: boolean): void { export function setScanLastRan(ts: number): void { setSetting('scan_last_ran', String(ts)) } + +// ─── AI Settings ───────────────────────────────────────────────────────────── + +interface AiConfig { + endpoint: string + model: string + enabled: boolean +} + +export function getAiConfig(): AiConfig { + const endpoint = getSetting('ai_endpoint') ?? '' + const model = getSetting('ai_model') ?? '' + const enabled = getSetting('ai_enabled') === 'true' + return { endpoint, model, enabled } +} + +export function updateAiConfig(endpoint: string, model: string, enabled: boolean): void { + setSetting('ai_endpoint', endpoint) + setSetting('ai_model', model) + setSetting('ai_enabled', enabled ? 'true' : 'false') +} diff --git a/src/lib/db.ts b/src/lib/db.ts index d1107f4..11e4905 100644 --- a/src/lib/db.ts +++ b/src/lib/db.ts @@ -102,6 +102,7 @@ function initDb(db: Database.Database): void { migrateMediaItemsSchema(db) migrateMediaItemsFingerprint(db) migrateMediaTagsToItemKey(db) + migrateMediaItemsAiTagged(db) seedAppSettings(db) } @@ -110,6 +111,9 @@ function seedAppSettings(db: Database.Database): void { scan_schedule: '0 * * * *', scan_enabled: 'true', scan_last_ran: '', + ai_enabled: 'false', + ai_endpoint: '', + ai_model: '', } const insert = db.prepare( 'INSERT OR IGNORE INTO app_settings (key, value) VALUES (?, ?)' @@ -228,6 +232,15 @@ function migrateMediaTagsToItemKey(db: Database.Database): void { `) } +function migrateMediaItemsAiTagged(db: Database.Database): void { + const row = db + .prepare("SELECT sql FROM sqlite_master WHERE type='table' AND name='media_items'") + .get() as { sql: string } | undefined + if (row && !row.sql.includes('ai_tagged_at')) { + db.exec('ALTER TABLE media_items ADD COLUMN ai_tagged_at INTEGER') + } +} + function migrateLibrariesType(db: Database.Database): void { const row = db .prepare("SELECT sql FROM sqlite_master WHERE type='table' AND name='libraries'") diff --git a/src/lib/scanner.ts b/src/lib/scanner.ts index 65d1710..33a0ccb 100644 --- a/src/lib/scanner.ts +++ b/src/lib/scanner.ts @@ -10,6 +10,7 @@ import { scanGamesLibrary } from './games' import { getThumbnailPath } from './thumbnails' import { computeFingerprint } from './fingerprint' import { reKeyMediaItem } from './tags' +import { runAiTagging } from './ai-tagger' let scanRunning = false @@ -70,6 +71,10 @@ export async function runLibraryScan(library: Library): Promise { await scanMixed(library, libraryRoot) break } + + await runAiTagging(library, libraryRoot).catch((err) => + console.error(`[ai-tagger] Error tagging library "${library.name}":`, err) + ) } // ---------------------------------------------------------------------------