From 0238dbda7a9e9a849a159d19d1760653ec92bc56 Mon Sep 17 00:00:00 2001 From: Garret Patti <42485635+garretpatti@users.noreply.github.com> Date: Sun, 12 Apr 2026 15:18:03 -0400 Subject: [PATCH 1/4] Add AI-powered image tagging via local LLM Adds automatic image tagging that runs as a post-scan phase, sending thumbnails to an OpenAI-compatible vision API and applying matching tags from the user-defined tag vocabulary. - New ai-tagger module with batch processing, failure tolerance, and tag validation against existing vocabulary - Admin settings page (Manage > AI Tagging) for endpoint, model, and enable toggle with connection testing - DB migration for ai_tagged_at tracking column and AI config seeds - Re-tag All support to queue items for reprocessing Co-Authored-By: Claude Opus 4.6 --- src/app/api/ai-settings/retag/route.ts | 13 + src/app/api/ai-settings/route.ts | 38 +++ src/app/api/ai-settings/test/route.ts | 47 ++++ src/app/manage/ai-tagging/page.tsx | 318 +++++++++++++++++++++++++ src/components/ManageSubNav.tsx | 1 + src/lib/ai-tagger.ts | 252 ++++++++++++++++++++ src/lib/app-settings.ts | 21 ++ src/lib/db.ts | 13 + src/lib/scanner.ts | 5 + 9 files changed, 708 insertions(+) create mode 100644 src/app/api/ai-settings/retag/route.ts create mode 100644 src/app/api/ai-settings/route.ts create mode 100644 src/app/api/ai-settings/test/route.ts create mode 100644 src/app/manage/ai-tagging/page.tsx create mode 100644 src/lib/ai-tagger.ts diff --git a/src/app/api/ai-settings/retag/route.ts b/src/app/api/ai-settings/retag/route.ts new file mode 100644 index 0000000..93a9e9f --- /dev/null +++ b/src/app/api/ai-settings/retag/route.ts @@ -0,0 +1,13 @@ +import { NextRequest, NextResponse } from 'next/server' +import { requireAdmin } from '@/lib/auth' +import { getDb } from '@/lib/db' + +export async function POST(request: NextRequest) { + const auth = await requireAdmin(request) + if (auth instanceof NextResponse) return auth + + const db = getDb() + const result = db.prepare('UPDATE media_items SET ai_tagged_at = NULL').run() + + return NextResponse.json({ cleared: result.changes }) +} diff --git a/src/app/api/ai-settings/route.ts b/src/app/api/ai-settings/route.ts new file mode 100644 index 0000000..593f37a --- /dev/null +++ b/src/app/api/ai-settings/route.ts @@ -0,0 +1,38 @@ +import { NextRequest, NextResponse } from 'next/server' +import { requireAdmin } from '@/lib/auth' +import { getAiConfig, updateAiConfig } from '@/lib/app-settings' + +export async function GET(request: NextRequest) { + const auth = await requireAdmin(request) + if (auth instanceof NextResponse) return auth + + const { endpoint, model, enabled } = getAiConfig() + return NextResponse.json({ endpoint, model, enabled }) +} + +export async function PUT(request: NextRequest) { + const auth = await requireAdmin(request) + if (auth instanceof NextResponse) return auth + + let body: { endpoint?: string; model?: string; enabled?: boolean } + try { + body = await request.json() + } catch { + return NextResponse.json({ error: 'Invalid JSON body' }, { status: 400 }) + } + + const { endpoint, model, enabled } = body + + if (typeof endpoint !== 'string') { + return NextResponse.json({ error: 'endpoint is required' }, { status: 400 }) + } + if (typeof model !== 'string') { + return NextResponse.json({ error: 'model is required' }, { status: 400 }) + } + if (typeof enabled !== 'boolean') { + return NextResponse.json({ error: 'enabled must be a boolean' }, { status: 400 }) + } + + updateAiConfig(endpoint, model, enabled) + return NextResponse.json({ endpoint, model, enabled }) +} diff --git a/src/app/api/ai-settings/test/route.ts b/src/app/api/ai-settings/test/route.ts new file mode 100644 index 0000000..5a957a0 --- /dev/null +++ b/src/app/api/ai-settings/test/route.ts @@ -0,0 +1,47 @@ +import { NextRequest, NextResponse } from 'next/server' +import { requireAdmin } from '@/lib/auth' +import { getAiConfig } from '@/lib/app-settings' + +export async function POST(request: NextRequest) { + const auth = await requireAdmin(request) + if (auth instanceof NextResponse) return auth + + const { endpoint, model } = getAiConfig() + + if (!endpoint) { + return NextResponse.json({ error: 'No endpoint configured' }, { status: 400 }) + } + + const url = endpoint.replace(/\/+$/, '') + '/chat/completions' + + try { + const controller = new AbortController() + const timeout = setTimeout(() => controller.abort(), 10_000) + + const res = await fetch(url, { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + signal: controller.signal, + body: JSON.stringify({ + model: model || 'test', + messages: [{ role: 'user', content: 'Hi' }], + max_tokens: 1, + }), + }) + + clearTimeout(timeout) + + if (!res.ok) { + const text = await res.text().catch(() => '') + return NextResponse.json( + { error: `LLM returned ${res.status}: ${text.slice(0, 200)}` }, + { status: 502 } + ) + } + + return NextResponse.json({ ok: true }) + } catch (err) { + const message = err instanceof Error ? err.message : 'Unknown error' + return NextResponse.json({ error: `Connection failed: ${message}` }, { status: 502 }) + } +} diff --git a/src/app/manage/ai-tagging/page.tsx b/src/app/manage/ai-tagging/page.tsx new file mode 100644 index 0000000..0253d2f --- /dev/null +++ b/src/app/manage/ai-tagging/page.tsx @@ -0,0 +1,318 @@ +'use client' + +import { useEffect, useState, useCallback } from 'react' + +interface AiSettings { + endpoint: string + model: string + enabled: boolean +} + +export default function AiTaggingPage() { + const [settings, setSettings] = useState({ endpoint: '', model: '', enabled: false }) + const [loading, setLoading] = useState(true) + const [saving, setSaving] = useState(false) + const [saveError, setSaveError] = useState(null) + const [saveSuccess, setSaveSuccess] = useState(false) + const [testing, setTesting] = useState(false) + const [testResult, setTestResult] = useState<{ ok: boolean; message: string } | null>(null) + const [retagging, setRetagging] = useState(false) + const [retagResult, setRetagResult] = useState(null) + + const fetchSettings = useCallback(async () => { + try { + const res = await fetch('/api/ai-settings') + if (!res.ok) return + const data: AiSettings = await res.json() + setSettings(data) + } catch { + // ignore + } finally { + setLoading(false) + } + }, []) + + useEffect(() => { + fetchSettings() + }, [fetchSettings]) + + const handleSave = async (e: React.FormEvent) => { + e.preventDefault() + setSaveError(null) + setSaveSuccess(false) + setSaving(true) + try { + const res = await fetch('/api/ai-settings', { + method: 'PUT', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify(settings), + }) + const data = await res.json() + if (!res.ok) { + setSaveError(data.error ?? 'Failed to save settings') + } else { + setSettings(data) + setSaveSuccess(true) + setTimeout(() => setSaveSuccess(false), 3000) + } + } catch { + setSaveError('Network error. Please try again.') + } finally { + setSaving(false) + } + } + + const handleTest = async () => { + setTesting(true) + setTestResult(null) + try { + const res = await fetch('/api/ai-settings/test', { method: 'POST' }) + const data = await res.json() + if (res.ok) { + setTestResult({ ok: true, message: 'Connection successful.' }) + } else { + setTestResult({ ok: false, message: data.error ?? 'Connection failed.' }) + } + } catch { + setTestResult({ ok: false, message: 'Network error.' }) + } finally { + setTesting(false) + } + } + + const handleRetag = async () => { + if (!confirm('This will clear AI tags from all items so they get re-processed on the next scan. Continue?')) return + setRetagging(true) + setRetagResult(null) + try { + const res = await fetch('/api/ai-settings/retag', { method: 'POST' }) + const data = await res.json() + if (res.ok) { + setRetagResult(`Cleared AI tag status from ${data.cleared} items. They will be re-tagged on the next scan.`) + } else { + setRetagResult(data.error ?? 'Failed to clear tags.') + } + } catch { + setRetagResult('Network error.') + } finally { + setRetagging(false) + } + } + + return ( +
+

+ AI Tagging +

+

+ Automatically tag media using a vision-capable LLM on your network. +

+ +
+ {loading ? ( + + ) : ( +
+ + setSettings((s) => ({ ...s, endpoint: e.target.value }))} + placeholder="http://192.168.1.50:8080/v1" + className="w-full rounded-lg px-3 py-2 text-sm font-mono outline-none focus:ring-2" + style={{ + backgroundColor: 'var(--background)', + border: '1px solid var(--border)', + color: 'var(--text-primary)', + }} + onFocus={(e) => ((e.currentTarget as HTMLElement).style.borderColor = 'var(--accent)')} + onBlur={(e) => ((e.currentTarget as HTMLElement).style.borderColor = 'var(--border)')} + /> +

+ Base URL of an OpenAI-compatible API server (e.g. Ollama, vLLM, llama.cpp). +

+ + + + setSettings((s) => ({ ...s, model: e.target.value }))} + placeholder="llava" + className="w-full rounded-lg px-3 py-2 text-sm font-mono outline-none focus:ring-2" + style={{ + backgroundColor: 'var(--background)', + border: '1px solid var(--border)', + color: 'var(--text-primary)', + }} + onFocus={(e) => ((e.currentTarget as HTMLElement).style.borderColor = 'var(--accent)')} + onBlur={(e) => ((e.currentTarget as HTMLElement).style.borderColor = 'var(--border)')} + /> +

+ Model name to use for vision requests. +

+ + + + +

+ When enabled, new media will be automatically tagged during library scans. +

+ + + {saveError && ( +

+ {saveError} +

+ )} + {saveSuccess && ( +

+ Settings saved. +

+ )} + +
+ + + +
+ + {testResult && ( +

+ {testResult.message} +

+ )} +
+ )} +
+ +
+
+

+ Clear the AI tag status on all items so they get re-processed during the next scan. + Existing tag assignments are not removed. +

+
+ +
+ {retagResult && ( +

+ {retagResult} +

+ )} +
+
+
+ ) +} + +function Section({ title, children }: { title: string; children: React.ReactNode }) { + return ( +
+

+ {title} +

+
+
{children}
+
+
+ ) +} + +function Field({ label, children }: { label: string; children: React.ReactNode }) { + return ( +
+ + {children} +
+ ) +} + +function LoadingRows() { + return ( +
+ {[70, 50, 40].map((w) => ( +
+
+
+ ))} +
+ ) +} diff --git a/src/components/ManageSubNav.tsx b/src/components/ManageSubNav.tsx index f3df1f1..a57531f 100644 --- a/src/components/ManageSubNav.tsx +++ b/src/components/ManageSubNav.tsx @@ -8,6 +8,7 @@ const TABS = [ { href: '/manage/tags', label: 'Tags' }, { href: '/manage/users', label: 'Users' }, { href: '/manage/scanning', label: 'Scanning' }, + { href: '/manage/ai-tagging', label: 'AI Tagging' }, ] export default function ManageSubNav() { diff --git a/src/lib/ai-tagger.ts b/src/lib/ai-tagger.ts new file mode 100644 index 0000000..d9d6b6b --- /dev/null +++ b/src/lib/ai-tagger.ts @@ -0,0 +1,252 @@ +import fs from 'fs' +import path from 'path' +import type { Library, Tag, TagCategory } from '@/types' +import { getDb } from './db' +import { getAiConfig } from './app-settings' +import { getTags, getCategories, addTagToItem } from './tags' +import { getThumbnailPath } from './thumbnails' +import { findFile } from './media-utils' + +const BATCH_LIMIT = 50 +const REQUEST_TIMEOUT_MS = 30_000 +const MAX_CONSECUTIVE_FAILURES = 3 + +const IMAGE_EXTENSIONS = new Set(['.jpg', '.jpeg', '.png', '.gif', '.webp', '.bmp', '.tiff', '.tif']) + +interface MediaItemRow { + item_key: string + item_type: string + file_path: string | null + metadata: string | null +} + +/** + * Resolve the absolute path to the best image for a media item. + * Returns null if no suitable image is found. + */ +function resolveItemImage(libraryRoot: string, item: MediaItemRow): string | null { + switch (item.item_type) { + case 'movie': + case 'tv_series': { + // metadata.posterUrl is an API URL like /api/thumbnail?libraryId=...&path=dir/poster.jpg + // Extract the relative path from the URL and resolve to absolute + const meta = item.metadata ? JSON.parse(item.metadata) : {} + const apiUrl = meta.posterUrl as string | undefined + if (!apiUrl) return null + try { + const relPath = decodeURIComponent( + new URL(apiUrl, 'http://localhost').searchParams.get('path') ?? '' + ) + if (!relPath) return null + const absPath = path.join(libraryRoot, relPath) + if (fs.existsSync(absPath)) return absPath + } catch { + return null + } + return null + } + + case 'game': + case 'game_series': { + const meta = item.metadata ? JSON.parse(item.metadata) : {} + const apiUrl = meta.coverUrl as string | undefined + if (!apiUrl) return null + try { + const relPath = decodeURIComponent( + new URL(apiUrl, 'http://localhost').searchParams.get('path') ?? '' + ) + if (!relPath) return null + const absPath = path.join(libraryRoot, relPath) + if (fs.existsSync(absPath)) return absPath + } catch { + return null + } + return null + } + + case 'tv_season': { + // Seasons may have a poster in their directory + if (!item.file_path) return null + const seasonDir = path.join(libraryRoot, item.file_path) + const posterFile = findFile(seasonDir, /^(poster|cover|folder)$/i) + if (posterFile) return path.join(seasonDir, posterFile) + return null + } + + case 'mixed_file': { + // For mixed files, tag only actual images (not videos or other files) + if (!item.file_path) return null + const ext = path.extname(item.file_path).toLowerCase() + if (!IMAGE_EXTENSIONS.has(ext)) return null + return path.join(libraryRoot, item.file_path) + } + + default: + return null + } +} + +/** + * Build the system prompt that instructs the LLM to select matching tags. + */ +function buildTagPrompt(tags: Tag[], categories: TagCategory[]): string { + const categoryMap = new Map(categories.map((c) => [c.id, c.name])) + + const grouped: Record = {} + for (const tag of tags) { + const catName = categoryMap.get(tag.categoryId) ?? 'Uncategorized' + ;(grouped[catName] ??= []).push({ id: tag.id, name: tag.name }) + } + + const lines: string[] = [] + for (const [catName, catTags] of Object.entries(grouped)) { + const tagList = catTags.map((t) => `${t.name} (id: ${t.id})`).join(', ') + lines.push(`[${catName}] ${tagList}`) + } + + return [ + 'You are an image tagger. Given the image, select which of the following tags apply.', + 'Return ONLY a JSON array of tag IDs that match the image. Do not invent new tags.', + 'If no tags match, return an empty array: []', + '', + 'Available tags:', + ...lines, + ].join('\n') +} + +/** + * Call the OpenAI-compatible vision API to get tag suggestions for an image. + */ +async function callVisionApi( + endpoint: string, + model: string, + base64Image: string, + systemPrompt: string +): Promise { + const url = endpoint.replace(/\/+$/, '') + '/chat/completions' + + const controller = new AbortController() + const timeout = setTimeout(() => controller.abort(), REQUEST_TIMEOUT_MS) + + try { + const res = await fetch(url, { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + signal: controller.signal, + body: JSON.stringify({ + model, + messages: [ + { role: 'system', content: systemPrompt }, + { + role: 'user', + content: [ + { + type: 'image_url', + image_url: { url: `data:image/jpeg;base64,${base64Image}` }, + }, + ], + }, + ], + max_tokens: 512, + temperature: 0.1, + }), + }) + + if (!res.ok) { + const text = await res.text().catch(() => '') + throw new Error(`LLM API returned ${res.status}: ${text.slice(0, 200)}`) + } + + const data = await res.json() as { + choices?: Array<{ message?: { content?: string } }> + } + + const content = data.choices?.[0]?.message?.content?.trim() ?? '' + + // Extract JSON array from the response (handle markdown code blocks) + const jsonMatch = content.match(/\[[\s\S]*\]/) + if (!jsonMatch) return [] + + const parsed = JSON.parse(jsonMatch[0]) + if (!Array.isArray(parsed)) return [] + return parsed.filter((v): v is string => typeof v === 'string') + } finally { + clearTimeout(timeout) + } +} + +/** + * Run AI tagging for a single library. Called after the scanner finishes. + * Processes up to BATCH_LIMIT untagged items per invocation. + */ +export async function runAiTagging(library: Library, libraryRoot: string): Promise { + const config = getAiConfig() + if (!config.enabled || !config.endpoint || !config.model) return + + const tags = getTags() + const categories = getCategories() + if (tags.length === 0) return + + const validTagIds = new Set(tags.map((t) => t.id)) + const systemPrompt = buildTagPrompt(tags, categories) + + const db = getDb() + const untaggedItems = db + .prepare( + `SELECT item_key, item_type, file_path, metadata + FROM media_items + WHERE library_id = ? AND ai_tagged_at IS NULL + LIMIT ?` + ) + .all(library.id, BATCH_LIMIT) as MediaItemRow[] + + if (untaggedItems.length === 0) return + + console.log(`[ai-tagger] Processing ${untaggedItems.length} items in library "${library.name}"`) + + let tagged = 0 + let consecutiveFailures = 0 + const markTagged = db.prepare('UPDATE media_items SET ai_tagged_at = ? WHERE item_key = ?') + + for (const item of untaggedItems) { + if (consecutiveFailures >= MAX_CONSECUTIVE_FAILURES) { + console.warn(`[ai-tagger] Aborting after ${MAX_CONSECUTIVE_FAILURES} consecutive failures`) + break + } + + const imagePath = resolveItemImage(libraryRoot, item) + if (!imagePath) { + // No image available — mark as tagged so we don't retry every scan + markTagged.run(Date.now(), item.item_key) + continue + } + + try { + // Use the thumbnail cache for a smaller image + const thumbnailPath = await getThumbnailPath(imagePath, library.id, 'image') + const base64 = fs.readFileSync(thumbnailPath, 'base64') + + const suggestedIds = await callVisionApi(config.endpoint, config.model, base64, systemPrompt) + + // Filter to valid tags only + const validIds = suggestedIds.filter((id) => validTagIds.has(id)) + for (const tagId of validIds) { + addTagToItem(item.item_key, tagId) + } + + markTagged.run(Date.now(), item.item_key) + tagged++ + consecutiveFailures = 0 + } catch (err) { + consecutiveFailures++ + console.warn( + `[ai-tagger] Failed to tag "${item.item_key}":`, + err instanceof Error ? err.message : err + ) + } + } + + if (tagged > 0) { + console.log(`[ai-tagger] Tagged ${tagged}/${untaggedItems.length} items in library "${library.name}"`) + } +} diff --git a/src/lib/app-settings.ts b/src/lib/app-settings.ts index e27f2c8..f45dbf1 100644 --- a/src/lib/app-settings.ts +++ b/src/lib/app-settings.ts @@ -36,3 +36,24 @@ export function updateScanConfig(schedule: string, enabled: boolean): void { export function setScanLastRan(ts: number): void { setSetting('scan_last_ran', String(ts)) } + +// ─── AI Settings ───────────────────────────────────────────────────────────── + +interface AiConfig { + endpoint: string + model: string + enabled: boolean +} + +export function getAiConfig(): AiConfig { + const endpoint = getSetting('ai_endpoint') ?? '' + const model = getSetting('ai_model') ?? '' + const enabled = getSetting('ai_enabled') === 'true' + return { endpoint, model, enabled } +} + +export function updateAiConfig(endpoint: string, model: string, enabled: boolean): void { + setSetting('ai_endpoint', endpoint) + setSetting('ai_model', model) + setSetting('ai_enabled', enabled ? 'true' : 'false') +} diff --git a/src/lib/db.ts b/src/lib/db.ts index d1107f4..11e4905 100644 --- a/src/lib/db.ts +++ b/src/lib/db.ts @@ -102,6 +102,7 @@ function initDb(db: Database.Database): void { migrateMediaItemsSchema(db) migrateMediaItemsFingerprint(db) migrateMediaTagsToItemKey(db) + migrateMediaItemsAiTagged(db) seedAppSettings(db) } @@ -110,6 +111,9 @@ function seedAppSettings(db: Database.Database): void { scan_schedule: '0 * * * *', scan_enabled: 'true', scan_last_ran: '', + ai_enabled: 'false', + ai_endpoint: '', + ai_model: '', } const insert = db.prepare( 'INSERT OR IGNORE INTO app_settings (key, value) VALUES (?, ?)' @@ -228,6 +232,15 @@ function migrateMediaTagsToItemKey(db: Database.Database): void { `) } +function migrateMediaItemsAiTagged(db: Database.Database): void { + const row = db + .prepare("SELECT sql FROM sqlite_master WHERE type='table' AND name='media_items'") + .get() as { sql: string } | undefined + if (row && !row.sql.includes('ai_tagged_at')) { + db.exec('ALTER TABLE media_items ADD COLUMN ai_tagged_at INTEGER') + } +} + function migrateLibrariesType(db: Database.Database): void { const row = db .prepare("SELECT sql FROM sqlite_master WHERE type='table' AND name='libraries'") diff --git a/src/lib/scanner.ts b/src/lib/scanner.ts index 65d1710..33a0ccb 100644 --- a/src/lib/scanner.ts +++ b/src/lib/scanner.ts @@ -10,6 +10,7 @@ import { scanGamesLibrary } from './games' import { getThumbnailPath } from './thumbnails' import { computeFingerprint } from './fingerprint' import { reKeyMediaItem } from './tags' +import { runAiTagging } from './ai-tagger' let scanRunning = false @@ -70,6 +71,10 @@ export async function runLibraryScan(library: Library): Promise { await scanMixed(library, libraryRoot) break } + + await runAiTagging(library, libraryRoot).catch((err) => + console.error(`[ai-tagger] Error tagging library "${library.name}":`, err) + ) } // --------------------------------------------------------------------------- -- 2.49.1 From 732e9134c368a6e223683b8fe87e32c8f150e44a Mon Sep 17 00:00:00 2001 From: Garret Patti <42485635+garretpatti@users.noreply.github.com> Date: Sun, 12 Apr 2026 15:39:48 -0400 Subject: [PATCH 2/4] ai starter implementation --- src/app/api/ai-tagging/route.ts | 39 +++++++++++++ src/app/manage/ai-tagging/page.tsx | 2 +- src/components/mixed/ImageLightbox.tsx | 42 +++++++++++++- src/components/mixed/MixedView.tsx | 77 +++++++++++++++++++++++++- src/lib/ai-tagger.ts | 57 +++++++++++++++++++ 5 files changed, 212 insertions(+), 5 deletions(-) create mode 100644 src/app/api/ai-tagging/route.ts diff --git a/src/app/api/ai-tagging/route.ts b/src/app/api/ai-tagging/route.ts new file mode 100644 index 0000000..f248a14 --- /dev/null +++ b/src/app/api/ai-tagging/route.ts @@ -0,0 +1,39 @@ +import { NextRequest, NextResponse } from 'next/server' +import { requireLibraryAccess } from '@/lib/auth' +import { tagSingleItem } from '@/lib/ai-tagger' + +export async function POST(request: NextRequest) { + let body: { itemKey?: string } + try { + body = await request.json() + } catch { + return NextResponse.json({ error: 'Invalid JSON body' }, { status: 400 }) + } + + const { itemKey } = body + if (!itemKey || typeof itemKey !== 'string') { + return NextResponse.json({ error: 'itemKey is required' }, { status: 400 }) + } + + const libraryId = itemKey.split(':')[0] + const auth = await requireLibraryAccess(request, libraryId) + if (auth instanceof NextResponse) return auth + + try { + const tagIds = await tagSingleItem(itemKey) + return NextResponse.json({ tagIds }) + } catch (err) { + const error = err as Error & { code?: string } + if (error.code === 'NOT_CONFIGURED') { + return NextResponse.json({ error: error.message }, { status: 400 }) + } + if (error.code === 'NOT_FOUND') { + return NextResponse.json({ error: error.message }, { status: 404 }) + } + if (error.code === 'NO_IMAGE') { + return NextResponse.json({ error: error.message }, { status: 404 }) + } + console.error('[ai-tagging] Error tagging item:', error) + return NextResponse.json({ error: 'AI tagging failed' }, { status: 502 }) + } +} diff --git a/src/app/manage/ai-tagging/page.tsx b/src/app/manage/ai-tagging/page.tsx index 0253d2f..b8bf1b0 100644 --- a/src/app/manage/ai-tagging/page.tsx +++ b/src/app/manage/ai-tagging/page.tsx @@ -174,7 +174,7 @@ export default function AiTaggingPage() {

- When enabled, new media will be automatically tagged during library scans. + When enabled, new media will be automatically tagged during library scans. On-demand tagging from image cards is always available when an endpoint is configured.

diff --git a/src/components/mixed/ImageLightbox.tsx b/src/components/mixed/ImageLightbox.tsx index ebbc7f6..0bd41fc 100644 --- a/src/components/mixed/ImageLightbox.tsx +++ b/src/components/mixed/ImageLightbox.tsx @@ -11,13 +11,16 @@ interface Props { onNext?: () => void itemKey?: string onTagsChanged?: () => void + onAiTag?: () => Promise } -export default function ImageLightbox({ url, name, onClose, onPrev, onNext, itemKey, onTagsChanged }: Props) { +export default function ImageLightbox({ url, name, onClose, onPrev, onNext, itemKey, onTagsChanged, onAiTag }: Props) { const overlayRef = useRef(null) const [showTags, setShowTags] = useState( () => !!itemKey && typeof window !== 'undefined' && window.innerWidth >= 1280 ) + const [aiTagging, setAiTagging] = useState(false) + const [aiTagError, setAiTagError] = useState(null) useEffect(() => { const handleKey = (e: KeyboardEvent) => { @@ -71,6 +74,43 @@ export default function ImageLightbox({ url, name, onClose, onPrev, onNext, item 🏷 )} + {onAiTag && ( + + )} {/* Kebab menu — top-right, shown on hover */} - {(onDelete || onRename) && ( + {(onDelete || onRename || (onAiTag && entry.mediaType === 'image')) && (
+ )} {onRename && (
)} + {/* AI tagging status overlay */} + {(aiTagging || aiTagError) && ( +
e.stopPropagation()} + > + + {aiTagError ?? 'AI Tagging…'} + + {aiTagError && ( + + )} +
+ )} + {/* Delete confirmation overlay */} {confirming && (
{ + const config = getAiConfig() + if (!config.endpoint || !config.model) { + throw Object.assign(new Error('AI tagging endpoint and model are not configured'), { code: 'NOT_CONFIGURED' }) + } + + const tags = getTags() + const categories = getCategories() + if (tags.length === 0) { + return [] + } + + const validTagIds = new Set(tags.map((t) => t.id)) + const systemPrompt = buildTagPrompt(tags, categories) + + const db = getDb() + const item = db + .prepare('SELECT item_key, item_type, file_path, metadata FROM media_items WHERE item_key = ?') + .get(itemKey) as MediaItemRow | undefined + + if (!item) { + throw Object.assign(new Error(`Item not found: ${itemKey}`), { code: 'NOT_FOUND' }) + } + + const libraryId = itemKey.split(':')[0] + const library = getLibrary(libraryId) + if (!library) { + throw Object.assign(new Error(`Library not found: ${libraryId}`), { code: 'NOT_FOUND' }) + } + const libraryRoot = resolveLibraryRoot(library) + + const imagePath = resolveItemImage(libraryRoot, item) + if (!imagePath) { + throw Object.assign(new Error('No image available for this item'), { code: 'NO_IMAGE' }) + } + + const thumbnailPath = await getThumbnailPath(imagePath, libraryId, 'image') + const base64 = fs.readFileSync(thumbnailPath, 'base64') + + const suggestedIds = await callVisionApi(config.endpoint, config.model, base64, systemPrompt) + const validIds = suggestedIds.filter((id) => validTagIds.has(id)) + + for (const tagId of validIds) { + addTagToItem(itemKey, tagId) + } + + db.prepare('UPDATE media_items SET ai_tagged_at = ? WHERE item_key = ?').run(Date.now(), itemKey) + + return validIds +} -- 2.49.1 From ad9920a448e30b886ed38a40349473699c19dd17 Mon Sep 17 00:00:00 2001 From: Garret Patti <42485635+garretpatti@users.noreply.github.com> Date: Sun, 12 Apr 2026 16:45:26 -0400 Subject: [PATCH 3/4] limit tags sent and send applied tags to ai --- src/lib/ai-tagger.ts | 62 ++++++++++++++++++++++++++++++-------------- src/lib/tags.ts | 19 ++++++++++++++ 2 files changed, 62 insertions(+), 19 deletions(-) diff --git a/src/lib/ai-tagger.ts b/src/lib/ai-tagger.ts index 9c7bd1d..a86587d 100644 --- a/src/lib/ai-tagger.ts +++ b/src/lib/ai-tagger.ts @@ -3,7 +3,7 @@ import path from 'path' import type { Library, Tag, TagCategory } from '@/types' import { getDb } from './db' import { getAiConfig } from './app-settings' -import { getTags, getCategories, addTagToItem } from './tags' +import { getTags, getCategories, addTagToItem, getActiveCategoryIdsForLibrary, getResolvedTagsForItem } from './tags' import { getThumbnailPath } from './thumbnails' import { findFile } from './media-utils' import { getLibrary, resolveLibraryRoot } from './libraries' @@ -89,8 +89,10 @@ function resolveItemImage(libraryRoot: string, item: MediaItemRow): string | nul /** * Build the system prompt that instructs the LLM to select matching tags. + * If currentTags are provided they are included as context to help the model + * understand the image before selecting additional tags. */ -function buildTagPrompt(tags: Tag[], categories: TagCategory[]): string { +function buildTagPrompt(tags: Tag[], categories: TagCategory[], currentTags?: Tag[]): string { const categoryMap = new Map(categories.map((c) => [c.id, c.name])) const grouped: Record = {} @@ -105,14 +107,24 @@ function buildTagPrompt(tags: Tag[], categories: TagCategory[]): string { lines.push(`[${catName}] ${tagList}`) } - return [ + const parts: string[] = [ 'You are an image tagger. Given the image, select which of the following tags apply.', - 'Return ONLY a JSON array of tag IDs that match the image. Do not invent new tags.', - 'If no tags match, return an empty array: []', - '', - 'Available tags:', - ...lines, - ].join('\n') + 'Return ONLY a JSON array of tag IDs that match the image (e.g., ["tag-apple", "tag-orange"]). Do not invent new tags. Do not return any text other than what is inside the JSON array.', + 'If no tags match, return an empty array (e.i., [])', + ] + + if (currentTags && currentTags.length > 0) { + const currentTagNames = currentTags.map((t) => t.name).join(', ') + parts.push('') + parts.push(`This image already has the following tags applied: ${currentTagNames}`) + parts.push('Use these as context to better understand the image when selecting tags.') + } + + parts.push('') + parts.push('Available tags:') + parts.push(...lines) + + return parts.join('\n') } /** @@ -148,7 +160,7 @@ async function callVisionApi( ], }, ], - max_tokens: 512, + max_tokens: 8192, temperature: 0.1, }), }) @@ -184,12 +196,15 @@ export async function runAiTagging(library: Library, libraryRoot: string): Promi const config = getAiConfig() if (!config.enabled || !config.endpoint || !config.model) return - const tags = getTags() - const categories = getCategories() + const activeCategoryIds = new Set(getActiveCategoryIdsForLibrary(library.id)) + const allTags = getTags() + const allCategories = getCategories() + + const tags = allTags.filter((t) => activeCategoryIds.has(t.categoryId)) + const categories = allCategories.filter((c) => activeCategoryIds.has(c.id)) if (tags.length === 0) return const validTagIds = new Set(tags.map((t) => t.id)) - const systemPrompt = buildTagPrompt(tags, categories) const db = getDb() const untaggedItems = db @@ -227,6 +242,9 @@ export async function runAiTagging(library: Library, libraryRoot: string): Promi const thumbnailPath = await getThumbnailPath(imagePath, library.id, 'image') const base64 = fs.readFileSync(thumbnailPath, 'base64') + const { tags: currentItemTags } = getResolvedTagsForItem(item.item_key) + const systemPrompt = buildTagPrompt(tags, categories, currentItemTags) + const suggestedIds = await callVisionApi(config.endpoint, config.model, base64, systemPrompt) // Filter to valid tags only @@ -263,14 +281,19 @@ export async function tagSingleItem(itemKey: string): Promise { throw Object.assign(new Error('AI tagging endpoint and model are not configured'), { code: 'NOT_CONFIGURED' }) } - const tags = getTags() - const categories = getCategories() + const libraryId = itemKey.split(':')[0] + + const activeCategoryIds = new Set(getActiveCategoryIdsForLibrary(libraryId)) + const allTags = getTags() + const allCategories = getCategories() + + const tags = allTags.filter((t) => activeCategoryIds.has(t.categoryId)) + const categories = allCategories.filter((c) => activeCategoryIds.has(c.id)) if (tags.length === 0) { return [] } const validTagIds = new Set(tags.map((t) => t.id)) - const systemPrompt = buildTagPrompt(tags, categories) const db = getDb() const item = db @@ -280,8 +303,6 @@ export async function tagSingleItem(itemKey: string): Promise { if (!item) { throw Object.assign(new Error(`Item not found: ${itemKey}`), { code: 'NOT_FOUND' }) } - - const libraryId = itemKey.split(':')[0] const library = getLibrary(libraryId) if (!library) { throw Object.assign(new Error(`Library not found: ${libraryId}`), { code: 'NOT_FOUND' }) @@ -296,7 +317,10 @@ export async function tagSingleItem(itemKey: string): Promise { const thumbnailPath = await getThumbnailPath(imagePath, libraryId, 'image') const base64 = fs.readFileSync(thumbnailPath, 'base64') - const suggestedIds = await callVisionApi(config.endpoint, config.model, base64, systemPrompt) + const { tags: currentItemTags } = getResolvedTagsForItem(itemKey) + const systemPromptWithContext = buildTagPrompt(tags, categories, currentItemTags) + + const suggestedIds = await callVisionApi(config.endpoint, config.model, base64, systemPromptWithContext) const validIds = suggestedIds.filter((id) => validTagIds.has(id)) for (const tagId of validIds) { diff --git a/src/lib/tags.ts b/src/lib/tags.ts index ca5e7f0..2b9dca5 100644 --- a/src/lib/tags.ts +++ b/src/lib/tags.ts @@ -18,6 +18,25 @@ export function getCategories(): TagCategory[] { return db.prepare('SELECT id, name FROM tag_categories ORDER BY name').all() as TagCategory[] } +/** + * Returns the distinct category IDs that have at least one tag assigned to any + * item in the given library. Used by the AI tagger to restrict the tag prompt + * to categories that are actually in use within the target library. + */ +export function getActiveCategoryIdsForLibrary(libraryId: string): string[] { + const db = getDb() + const rows = db + .prepare( + `SELECT DISTINCT t.category_id + FROM tags t + JOIN media_tags mt ON mt.tag_id = t.id + JOIN media_items mi ON mi.item_key = mt.item_key + WHERE mi.library_id = ?` + ) + .all(libraryId) as { category_id: string }[] + return rows.map((r) => r.category_id) +} + export function addCategory(name: string): TagCategory { const trimmed = name.trim() if (!trimmed) throw new Error('Category name is required.') -- 2.49.1 From 6c769b457f2604e9b21294ca3aa5256f8299e174 Mon Sep 17 00:00:00 2001 From: Garret Patti <42485635+garretpatti@users.noreply.github.com> Date: Sun, 12 Apr 2026 17:24:39 -0400 Subject: [PATCH 4/4] handle video tagging --- src/components/mixed/ImageLightbox.tsx | 4 +- src/components/mixed/MixedView.tsx | 13 ++++ src/components/mixed/VideoPlayerModal.tsx | 45 ++++++++++- src/components/tags/TagSelector.tsx | 9 ++- src/lib/ai-tagger.ts | 92 ++++++++++++++--------- src/lib/thumbnails.ts | 67 ++++++++++++++--- 6 files changed, 178 insertions(+), 52 deletions(-) diff --git a/src/components/mixed/ImageLightbox.tsx b/src/components/mixed/ImageLightbox.tsx index 0bd41fc..5904e04 100644 --- a/src/components/mixed/ImageLightbox.tsx +++ b/src/components/mixed/ImageLightbox.tsx @@ -21,6 +21,7 @@ export default function ImageLightbox({ url, name, onClose, onPrev, onNext, item ) const [aiTagging, setAiTagging] = useState(false) const [aiTagError, setAiTagError] = useState(null) + const [tagRefreshKey, setTagRefreshKey] = useState(0) useEffect(() => { const handleKey = (e: KeyboardEvent) => { @@ -82,6 +83,7 @@ export default function ImageLightbox({ url, name, onClose, onPrev, onNext, item setAiTagError(null) try { await onAiTag() + setTagRefreshKey((k) => k + 1) onTagsChanged?.() } catch (err) { setAiTagError(err instanceof Error ? err.message : 'AI tagging failed') @@ -165,7 +167,7 @@ export default function ImageLightbox({ url, name, onClose, onPrev, onNext, item

Tags

- +
) : ( diff --git a/src/components/mixed/MixedView.tsx b/src/components/mixed/MixedView.tsx index 1931507..42bc4db 100644 --- a/src/components/mixed/MixedView.tsx +++ b/src/components/mixed/MixedView.tsx @@ -378,6 +378,19 @@ export default function MixedView({ libraryId, initialPath }: Props) { onClose={() => setModal(null)} onPrev={modal.mediaIndex > 0 ? () => navigateModal(-1) : undefined} onNext={modal.mediaIndex < mediaEntries.length - 1 ? () => navigateModal(1) : undefined} + onAiTag={modal.itemKey ? async () => { + const res = await fetch('/api/ai-tagging', { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ itemKey: modal.itemKey }), + }) + if (!res.ok) { + const data = await res.json().catch(() => ({})) + throw new Error((data as { error?: string }).error ?? 'AI tagging failed') + } + fetchAssignments() + setFilterRefreshKey((k) => k + 1) + } : undefined} /> )} {modal?.type === 'image' && ( diff --git a/src/components/mixed/VideoPlayerModal.tsx b/src/components/mixed/VideoPlayerModal.tsx index 495aa5a..b1a1031 100644 --- a/src/components/mixed/VideoPlayerModal.tsx +++ b/src/components/mixed/VideoPlayerModal.tsx @@ -12,10 +12,11 @@ interface Props { onNext?: () => void itemKey?: string onTagsChanged?: () => void + onAiTag?: () => Promise context?: 'mixed' | 'movies' | 'tv' } -export default function VideoPlayerModal({ url, name, onClose, onPrev, onNext, itemKey, onTagsChanged, context = 'mixed' }: Props) { +export default function VideoPlayerModal({ url, name, onClose, onPrev, onNext, itemKey, onTagsChanged, onAiTag, context = 'mixed' }: Props) { const settings = useUserSettings() const autoPlay = context === 'mixed' ? settings.mixedAutoplay : context === 'movies' ? settings.moviesAutoplay : settings.tvAutoplay const loop = context === 'mixed' ? settings.mixedLoop : context === 'movies' ? settings.moviesLoop : settings.tvLoop @@ -24,6 +25,9 @@ export default function VideoPlayerModal({ url, name, onClose, onPrev, onNext, i const [showTags, setShowTags] = useState( () => !!itemKey && typeof window !== 'undefined' && window.innerWidth >= 1280 ) + const [aiTagging, setAiTagging] = useState(false) + const [aiTagError, setAiTagError] = useState(null) + const [tagRefreshKey, setTagRefreshKey] = useState(0) useEffect(() => { const handleKey = (e: KeyboardEvent) => { @@ -76,6 +80,43 @@ export default function VideoPlayerModal({ url, name, onClose, onPrev, onNext, i 🏷 )} + {onAiTag && ( + + )}