diff --git a/src/app/api/ai-settings/retag/route.ts b/src/app/api/ai-settings/retag/route.ts new file mode 100644 index 0000000..93a9e9f --- /dev/null +++ b/src/app/api/ai-settings/retag/route.ts @@ -0,0 +1,13 @@ +import { NextRequest, NextResponse } from 'next/server' +import { requireAdmin } from '@/lib/auth' +import { getDb } from '@/lib/db' + +export async function POST(request: NextRequest) { + const auth = await requireAdmin(request) + if (auth instanceof NextResponse) return auth + + const db = getDb() + const result = db.prepare('UPDATE media_items SET ai_tagged_at = NULL').run() + + return NextResponse.json({ cleared: result.changes }) +} diff --git a/src/app/api/ai-settings/route.ts b/src/app/api/ai-settings/route.ts new file mode 100644 index 0000000..593f37a --- /dev/null +++ b/src/app/api/ai-settings/route.ts @@ -0,0 +1,38 @@ +import { NextRequest, NextResponse } from 'next/server' +import { requireAdmin } from '@/lib/auth' +import { getAiConfig, updateAiConfig } from '@/lib/app-settings' + +export async function GET(request: NextRequest) { + const auth = await requireAdmin(request) + if (auth instanceof NextResponse) return auth + + const { endpoint, model, enabled } = getAiConfig() + return NextResponse.json({ endpoint, model, enabled }) +} + +export async function PUT(request: NextRequest) { + const auth = await requireAdmin(request) + if (auth instanceof NextResponse) return auth + + let body: { endpoint?: string; model?: string; enabled?: boolean } + try { + body = await request.json() + } catch { + return NextResponse.json({ error: 'Invalid JSON body' }, { status: 400 }) + } + + const { endpoint, model, enabled } = body + + if (typeof endpoint !== 'string') { + return NextResponse.json({ error: 'endpoint is required' }, { status: 400 }) + } + if (typeof model !== 'string') { + return NextResponse.json({ error: 'model is required' }, { status: 400 }) + } + if (typeof enabled !== 'boolean') { + return NextResponse.json({ error: 'enabled must be a boolean' }, { status: 400 }) + } + + updateAiConfig(endpoint, model, enabled) + return NextResponse.json({ endpoint, model, enabled }) +} diff --git a/src/app/api/ai-settings/test/route.ts b/src/app/api/ai-settings/test/route.ts new file mode 100644 index 0000000..5a957a0 --- /dev/null +++ b/src/app/api/ai-settings/test/route.ts @@ -0,0 +1,47 @@ +import { NextRequest, NextResponse } from 'next/server' +import { requireAdmin } from '@/lib/auth' +import { getAiConfig } from '@/lib/app-settings' + +export async function POST(request: NextRequest) { + const auth = await requireAdmin(request) + if (auth instanceof NextResponse) return auth + + const { endpoint, model } = getAiConfig() + + if (!endpoint) { + return NextResponse.json({ error: 'No endpoint configured' }, { status: 400 }) + } + + const url = endpoint.replace(/\/+$/, '') + '/chat/completions' + + try { + const controller = new AbortController() + const timeout = setTimeout(() => controller.abort(), 10_000) + + const res = await fetch(url, { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + signal: controller.signal, + body: JSON.stringify({ + model: model || 'test', + messages: [{ role: 'user', content: 'Hi' }], + max_tokens: 1, + }), + }) + + clearTimeout(timeout) + + if (!res.ok) { + const text = await res.text().catch(() => '') + return NextResponse.json( + { error: `LLM returned ${res.status}: ${text.slice(0, 200)}` }, + { status: 502 } + ) + } + + return NextResponse.json({ ok: true }) + } catch (err) { + const message = err instanceof Error ? err.message : 'Unknown error' + return NextResponse.json({ error: `Connection failed: ${message}` }, { status: 502 }) + } +} diff --git a/src/app/api/ai-tagging/route.ts b/src/app/api/ai-tagging/route.ts new file mode 100644 index 0000000..f248a14 --- /dev/null +++ b/src/app/api/ai-tagging/route.ts @@ -0,0 +1,39 @@ +import { NextRequest, NextResponse } from 'next/server' +import { requireLibraryAccess } from '@/lib/auth' +import { tagSingleItem } from '@/lib/ai-tagger' + +export async function POST(request: NextRequest) { + let body: { itemKey?: string } + try { + body = await request.json() + } catch { + return NextResponse.json({ error: 'Invalid JSON body' }, { status: 400 }) + } + + const { itemKey } = body + if (!itemKey || typeof itemKey !== 'string') { + return NextResponse.json({ error: 'itemKey is required' }, { status: 400 }) + } + + const libraryId = itemKey.split(':')[0] + const auth = await requireLibraryAccess(request, libraryId) + if (auth instanceof NextResponse) return auth + + try { + const tagIds = await tagSingleItem(itemKey) + return NextResponse.json({ tagIds }) + } catch (err) { + const error = err as Error & { code?: string } + if (error.code === 'NOT_CONFIGURED') { + return NextResponse.json({ error: error.message }, { status: 400 }) + } + if (error.code === 'NOT_FOUND') { + return NextResponse.json({ error: error.message }, { status: 404 }) + } + if (error.code === 'NO_IMAGE') { + return NextResponse.json({ error: error.message }, { status: 404 }) + } + console.error('[ai-tagging] Error tagging item:', error) + return NextResponse.json({ error: 'AI tagging failed' }, { status: 502 }) + } +} diff --git a/src/app/manage/ai-tagging/page.tsx b/src/app/manage/ai-tagging/page.tsx new file mode 100644 index 0000000..b8bf1b0 --- /dev/null +++ b/src/app/manage/ai-tagging/page.tsx @@ -0,0 +1,318 @@ +'use client' + +import { useEffect, useState, useCallback } from 'react' + +interface AiSettings { + endpoint: string + model: string + enabled: boolean +} + +export default function AiTaggingPage() { + const [settings, setSettings] = useState({ endpoint: '', model: '', enabled: false }) + const [loading, setLoading] = useState(true) + const [saving, setSaving] = useState(false) + const [saveError, setSaveError] = useState(null) + const [saveSuccess, setSaveSuccess] = useState(false) + const [testing, setTesting] = useState(false) + const [testResult, setTestResult] = useState<{ ok: boolean; message: string } | null>(null) + const [retagging, setRetagging] = useState(false) + const [retagResult, setRetagResult] = useState(null) + + const fetchSettings = useCallback(async () => { + try { + const res = await fetch('/api/ai-settings') + if (!res.ok) return + const data: AiSettings = await res.json() + setSettings(data) + } catch { + // ignore + } finally { + setLoading(false) + } + }, []) + + useEffect(() => { + fetchSettings() + }, [fetchSettings]) + + const handleSave = async (e: React.FormEvent) => { + e.preventDefault() + setSaveError(null) + setSaveSuccess(false) + setSaving(true) + try { + const res = await fetch('/api/ai-settings', { + method: 'PUT', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify(settings), + }) + const data = await res.json() + if (!res.ok) { + setSaveError(data.error ?? 'Failed to save settings') + } else { + setSettings(data) + setSaveSuccess(true) + setTimeout(() => setSaveSuccess(false), 3000) + } + } catch { + setSaveError('Network error. Please try again.') + } finally { + setSaving(false) + } + } + + const handleTest = async () => { + setTesting(true) + setTestResult(null) + try { + const res = await fetch('/api/ai-settings/test', { method: 'POST' }) + const data = await res.json() + if (res.ok) { + setTestResult({ ok: true, message: 'Connection successful.' }) + } else { + setTestResult({ ok: false, message: data.error ?? 'Connection failed.' }) + } + } catch { + setTestResult({ ok: false, message: 'Network error.' }) + } finally { + setTesting(false) + } + } + + const handleRetag = async () => { + if (!confirm('This will clear AI tags from all items so they get re-processed on the next scan. Continue?')) return + setRetagging(true) + setRetagResult(null) + try { + const res = await fetch('/api/ai-settings/retag', { method: 'POST' }) + const data = await res.json() + if (res.ok) { + setRetagResult(`Cleared AI tag status from ${data.cleared} items. They will be re-tagged on the next scan.`) + } else { + setRetagResult(data.error ?? 'Failed to clear tags.') + } + } catch { + setRetagResult('Network error.') + } finally { + setRetagging(false) + } + } + + return ( +
+

+ AI Tagging +

+

+ Automatically tag media using a vision-capable LLM on your network. +

+ +
+ {loading ? ( + + ) : ( +
+ + setSettings((s) => ({ ...s, endpoint: e.target.value }))} + placeholder="http://192.168.1.50:8080/v1" + className="w-full rounded-lg px-3 py-2 text-sm font-mono outline-none focus:ring-2" + style={{ + backgroundColor: 'var(--background)', + border: '1px solid var(--border)', + color: 'var(--text-primary)', + }} + onFocus={(e) => ((e.currentTarget as HTMLElement).style.borderColor = 'var(--accent)')} + onBlur={(e) => ((e.currentTarget as HTMLElement).style.borderColor = 'var(--border)')} + /> +

+ Base URL of an OpenAI-compatible API server (e.g. Ollama, vLLM, llama.cpp). +

+ + + + setSettings((s) => ({ ...s, model: e.target.value }))} + placeholder="llava" + className="w-full rounded-lg px-3 py-2 text-sm font-mono outline-none focus:ring-2" + style={{ + backgroundColor: 'var(--background)', + border: '1px solid var(--border)', + color: 'var(--text-primary)', + }} + onFocus={(e) => ((e.currentTarget as HTMLElement).style.borderColor = 'var(--accent)')} + onBlur={(e) => ((e.currentTarget as HTMLElement).style.borderColor = 'var(--border)')} + /> +

+ Model name to use for vision requests. +

+ + + + +

+ When enabled, new media will be automatically tagged during library scans. On-demand tagging from image cards is always available when an endpoint is configured. +

+ + + {saveError && ( +

+ {saveError} +

+ )} + {saveSuccess && ( +

+ Settings saved. +

+ )} + +
+ + + +
+ + {testResult && ( +

+ {testResult.message} +

+ )} +
+ )} +
+ +
+
+

+ Clear the AI tag status on all items so they get re-processed during the next scan. + Existing tag assignments are not removed. +

+
+ +
+ {retagResult && ( +

+ {retagResult} +

+ )} +
+
+
+ ) +} + +function Section({ title, children }: { title: string; children: React.ReactNode }) { + return ( +
+

+ {title} +

+
+
{children}
+
+
+ ) +} + +function Field({ label, children }: { label: string; children: React.ReactNode }) { + return ( +
+ + {children} +
+ ) +} + +function LoadingRows() { + return ( +
+ {[70, 50, 40].map((w) => ( +
+
+
+ ))} +
+ ) +} diff --git a/src/components/ManageSubNav.tsx b/src/components/ManageSubNav.tsx index f3df1f1..a57531f 100644 --- a/src/components/ManageSubNav.tsx +++ b/src/components/ManageSubNav.tsx @@ -8,6 +8,7 @@ const TABS = [ { href: '/manage/tags', label: 'Tags' }, { href: '/manage/users', label: 'Users' }, { href: '/manage/scanning', label: 'Scanning' }, + { href: '/manage/ai-tagging', label: 'AI Tagging' }, ] export default function ManageSubNav() { diff --git a/src/components/mixed/ImageLightbox.tsx b/src/components/mixed/ImageLightbox.tsx index ebbc7f6..5904e04 100644 --- a/src/components/mixed/ImageLightbox.tsx +++ b/src/components/mixed/ImageLightbox.tsx @@ -11,13 +11,17 @@ interface Props { onNext?: () => void itemKey?: string onTagsChanged?: () => void + onAiTag?: () => Promise } -export default function ImageLightbox({ url, name, onClose, onPrev, onNext, itemKey, onTagsChanged }: Props) { +export default function ImageLightbox({ url, name, onClose, onPrev, onNext, itemKey, onTagsChanged, onAiTag }: Props) { const overlayRef = useRef(null) const [showTags, setShowTags] = useState( () => !!itemKey && typeof window !== 'undefined' && window.innerWidth >= 1280 ) + const [aiTagging, setAiTagging] = useState(false) + const [aiTagError, setAiTagError] = useState(null) + const [tagRefreshKey, setTagRefreshKey] = useState(0) useEffect(() => { const handleKey = (e: KeyboardEvent) => { @@ -71,6 +75,44 @@ export default function ImageLightbox({ url, name, onClose, onPrev, onNext, item 🏷 )} + {onAiTag && ( + + )}
) : ( diff --git a/src/components/mixed/MixedView.tsx b/src/components/mixed/MixedView.tsx index f40fc94..42bc4db 100644 --- a/src/components/mixed/MixedView.tsx +++ b/src/components/mixed/MixedView.tsx @@ -321,6 +321,20 @@ export default function MixedView({ libraryId, initialPath }: Props) { entry={entry} onOpen={handleEntry} onTag={handleTagEntry} + onAiTag={async (e) => { + const itemKey = itemKeyFor(e) + const res = await fetch('/api/ai-tagging', { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ itemKey }), + }) + if (!res.ok) { + const data = await res.json().catch(() => ({})) + throw new Error((data as { error?: string }).error ?? 'AI tagging failed') + } + fetchAssignments() + setFilterRefreshKey((k) => k + 1) + }} onDelete={(e) => { const rel = filtersActive ? e.name : (currentPath ? `${currentPath}/${e.name}` : e.name) fetch(`/api/browse?libraryId=${encodeURIComponent(libraryId)}&path=${encodeURIComponent(rel)}`, { method: 'DELETE' }) @@ -364,6 +378,19 @@ export default function MixedView({ libraryId, initialPath }: Props) { onClose={() => setModal(null)} onPrev={modal.mediaIndex > 0 ? () => navigateModal(-1) : undefined} onNext={modal.mediaIndex < mediaEntries.length - 1 ? () => navigateModal(1) : undefined} + onAiTag={modal.itemKey ? async () => { + const res = await fetch('/api/ai-tagging', { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ itemKey: modal.itemKey }), + }) + if (!res.ok) { + const data = await res.json().catch(() => ({})) + throw new Error((data as { error?: string }).error ?? 'AI tagging failed') + } + fetchAssignments() + setFilterRefreshKey((k) => k + 1) + } : undefined} /> )} {modal?.type === 'image' && ( @@ -375,6 +402,19 @@ export default function MixedView({ libraryId, initialPath }: Props) { onClose={() => setModal(null)} onPrev={modal.mediaIndex > 0 ? () => navigateModal(-1) : undefined} onNext={modal.mediaIndex < mediaEntries.length - 1 ? () => navigateModal(1) : undefined} + onAiTag={async () => { + const res = await fetch('/api/ai-tagging', { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ itemKey: modal.itemKey }), + }) + if (!res.ok) { + const data = await res.json().catch(() => ({})) + throw new Error((data as { error?: string }).error ?? 'AI tagging failed') + } + fetchAssignments() + setFilterRefreshKey((k) => k + 1) + }} /> )} @@ -424,7 +464,7 @@ export default function MixedView({ libraryId, initialPath }: Props) { ) } -function EntryTile({ entry, onOpen, onTag, onDelete, onRename }: { entry: FileEntry; onOpen: (e: FileEntry) => void; onTag: (e: FileEntry) => void; onDelete?: (e: FileEntry) => void; onRename?: (e: FileEntry, newName: string) => Promise }) { +function EntryTile({ entry, onOpen, onTag, onDelete, onRename, onAiTag }: { entry: FileEntry; onOpen: (e: FileEntry) => void; onTag: (e: FileEntry) => void; onDelete?: (e: FileEntry) => void; onRename?: (e: FileEntry, newName: string) => Promise; onAiTag?: (e: FileEntry) => Promise }) { type ImgState = 'loading' | 'loaded' | 'error' const [imgState, setImgState] = useState( entry.thumbnailUrl ? 'loading' : 'error' @@ -437,6 +477,8 @@ function EntryTile({ entry, onOpen, onTag, onDelete, onRename }: { entry: FileEn const [entryRenameName, setEntryRenameName] = useState('') const [entryRenameError, setEntryRenameError] = useState(null) const [entryRenameSaving, setEntryRenameSaving] = useState(false) + const [aiTagging, setAiTagging] = useState(false) + const [aiTagError, setAiTagError] = useState(null) useEffect(() => { if (!menuOpen) return @@ -548,10 +590,10 @@ function EntryTile({ entry, onOpen, onTag, onDelete, onRename }: { entry: FileEn {/* Kebab menu β€” top-right, shown on hover */} - {(onDelete || onRename) && ( + {(onDelete || onRename || (onAiTag && entry.mediaType === 'image')) && (
+ )} {onRename && (
)} + {/* AI tagging status overlay */} + {(aiTagging || aiTagError) && ( +
e.stopPropagation()} + > + + {aiTagError ?? 'AI Tagging…'} + + {aiTagError && ( + + )} +
+ )} + {/* Delete confirmation overlay */} {confirming && (
void itemKey?: string onTagsChanged?: () => void + onAiTag?: () => Promise context?: 'mixed' | 'movies' | 'tv' } -export default function VideoPlayerModal({ url, name, onClose, onPrev, onNext, itemKey, onTagsChanged, context = 'mixed' }: Props) { +export default function VideoPlayerModal({ url, name, onClose, onPrev, onNext, itemKey, onTagsChanged, onAiTag, context = 'mixed' }: Props) { const settings = useUserSettings() const autoPlay = context === 'mixed' ? settings.mixedAutoplay : context === 'movies' ? settings.moviesAutoplay : settings.tvAutoplay const loop = context === 'mixed' ? settings.mixedLoop : context === 'movies' ? settings.moviesLoop : settings.tvLoop @@ -24,6 +25,9 @@ export default function VideoPlayerModal({ url, name, onClose, onPrev, onNext, i const [showTags, setShowTags] = useState( () => !!itemKey && typeof window !== 'undefined' && window.innerWidth >= 1280 ) + const [aiTagging, setAiTagging] = useState(false) + const [aiTagError, setAiTagError] = useState(null) + const [tagRefreshKey, setTagRefreshKey] = useState(0) useEffect(() => { const handleKey = (e: KeyboardEvent) => { @@ -76,6 +80,43 @@ export default function VideoPlayerModal({ url, name, onClose, onPrev, onNext, i 🏷 )} + {onAiTag && ( + + )}
) : ( diff --git a/src/components/tags/TagSelector.tsx b/src/components/tags/TagSelector.tsx index 527b4e4..748a7f1 100644 --- a/src/components/tags/TagSelector.tsx +++ b/src/components/tags/TagSelector.tsx @@ -7,6 +7,7 @@ import TagBadge from './TagBadge' interface Props { itemKey: string onTagsChanged?: () => void + refreshKey?: number } interface AllTags { @@ -14,7 +15,7 @@ interface AllTags { tags: Tag[] } -export default function TagSelector({ itemKey, onTagsChanged }: Props) { +export default function TagSelector({ itemKey, onTagsChanged, refreshKey }: Props) { const [assigned, setAssigned] = useState<{ tags: Tag[]; categories: TagCategory[] }>({ tags: [], categories: [], @@ -58,6 +59,12 @@ export default function TagSelector({ itemKey, onTagsChanged }: Props) { Promise.all([fetchAssigned(), fetchAll()]).finally(() => setLoading(false)) }, [fetchAssigned, fetchAll]) + useEffect(() => { + if (refreshKey !== undefined && refreshKey > 0) { + fetchAssigned() + } + }, [refreshKey, fetchAssigned]) + const isAssigned = (tagId: string) => assigned.tags.some((t) => t.id === tagId) const toggleTag = async (tag: Tag) => { diff --git a/src/lib/ai-tagger.ts b/src/lib/ai-tagger.ts new file mode 100644 index 0000000..cc704bb --- /dev/null +++ b/src/lib/ai-tagger.ts @@ -0,0 +1,353 @@ +import fs from 'fs' +import path from 'path' +import type { Library, Tag, TagCategory } from '@/types' +import { getDb } from './db' +import { getAiConfig } from './app-settings' +import { getTags, getCategories, addTagToItem, getActiveCategoryIdsForLibrary, getResolvedTagsForItem } from './tags' +import { getThumbnailPath, getVideoFramePaths } from './thumbnails' +import { findFile } from './media-utils' +import { getLibrary, resolveLibraryRoot } from './libraries' + +const BATCH_LIMIT = 50 +const REQUEST_TIMEOUT_MS = 30_000 +const MAX_CONSECUTIVE_FAILURES = 3 + +const IMAGE_EXTENSIONS = new Set(['.jpg', '.jpeg', '.png', '.gif', '.webp', '.bmp', '.tiff', '.tif']) +const VIDEO_EXTENSIONS = new Set(['.mp4', '.mkv', '.avi', '.mov', '.wmv', '.m4v', '.webm', '.flv', '.ts', '.mpg', '.mpeg']) + +const VIDEO_FRAME_PERCENTAGES = [0.10, 0.25, 0.50, 0.75, 0.90] + +interface ResolvedMedia { + path: string + mediaType: 'image' | 'video' +} + +interface MediaItemRow { + item_key: string + item_type: string + file_path: string | null + metadata: string | null +} + +/** + * Resolve the absolute path to the best image (or video) for a media item. + * Returns null if no suitable media is found. + */ +function resolveItemImage(libraryRoot: string, item: MediaItemRow): ResolvedMedia | null { + switch (item.item_type) { + case 'movie': + case 'tv_series': { + // metadata.posterUrl is an API URL like /api/thumbnail?libraryId=...&path=dir/poster.jpg + // Extract the relative path from the URL and resolve to absolute + const meta = item.metadata ? JSON.parse(item.metadata) : {} + const apiUrl = meta.posterUrl as string | undefined + if (!apiUrl) return null + try { + const relPath = decodeURIComponent( + new URL(apiUrl, 'http://localhost').searchParams.get('path') ?? '' + ) + if (!relPath) return null + const absPath = path.join(libraryRoot, relPath) + if (fs.existsSync(absPath)) return { path: absPath, mediaType: 'image' } + } catch { + return null + } + return null + } + + case 'game': + case 'game_series': { + const meta = item.metadata ? JSON.parse(item.metadata) : {} + const apiUrl = meta.coverUrl as string | undefined + if (!apiUrl) return null + try { + const relPath = decodeURIComponent( + new URL(apiUrl, 'http://localhost').searchParams.get('path') ?? '' + ) + if (!relPath) return null + const absPath = path.join(libraryRoot, relPath) + if (fs.existsSync(absPath)) return { path: absPath, mediaType: 'image' } + } catch { + return null + } + return null + } + + case 'tv_season': { + // Seasons may have a poster in their directory + if (!item.file_path) return null + const seasonDir = path.join(libraryRoot, item.file_path) + const posterFile = findFile(seasonDir, /^(poster|cover|folder)$/i) + if (posterFile) return { path: path.join(seasonDir, posterFile), mediaType: 'image' } + return null + } + + case 'mixed_file': { + if (!item.file_path) return null + const ext = path.extname(item.file_path).toLowerCase() + if (IMAGE_EXTENSIONS.has(ext)) return { path: path.join(libraryRoot, item.file_path), mediaType: 'image' } + if (VIDEO_EXTENSIONS.has(ext)) return { path: path.join(libraryRoot, item.file_path), mediaType: 'video' } + return null + } + + default: + return null + } +} + +/** + * Build the system prompt that instructs the LLM to select matching tags. + * If currentTags are provided they are included as context to help the model + * understand the content before selecting additional tags. + */ +function buildTagPrompt(tags: Tag[], categories: TagCategory[], currentTags?: Tag[], mediaContext: 'image' | 'video' = 'image'): string { + const categoryMap = new Map(categories.map((c) => [c.id, c.name])) + + const grouped: Record = {} + for (const tag of tags) { + const catName = categoryMap.get(tag.categoryId) ?? 'Uncategorized' + ;(grouped[catName] ??= []).push({ id: tag.id, name: tag.name }) + } + + const lines: string[] = [] + for (const [catName, catTags] of Object.entries(grouped)) { + const tagList = catTags.map((t) => `${t.name} (id: ${t.id})`).join(', ') + lines.push(`[${catName}] ${tagList}`) + } + + const isVideo = mediaContext === 'video' + const contentWord = isVideo ? 'video frames' : 'image' + + const parts: string[] = [ + `You are a media tagger. Given the ${contentWord}, select which of the following tags apply.`, + 'Return ONLY a JSON array of tag IDs that match (e.g., ["tag-apple", "tag-orange"]). Do not invent new tags. Do not return any text other than what is inside the JSON array.', + 'If no tags match, return an empty array (e.i., [])', + ] + + if (currentTags && currentTags.length > 0) { + const currentTagNames = currentTags.map((t) => t.name).join(', ') + parts.push('') + parts.push(`This content already has the following tags applied: ${currentTagNames}`) + parts.push('Use these as context to better understand the content when selecting tags.') + } + + parts.push('') + parts.push('Available tags:') + parts.push(...lines) + + return parts.join('\n') +} + +/** + * Call the OpenAI-compatible vision API to get tag suggestions for one or more images. + */ +async function callVisionApi( + endpoint: string, + model: string, + base64Images: string[], + systemPrompt: string +): Promise { + const url = endpoint.replace(/\/+$/, '') + '/chat/completions' + + const controller = new AbortController() + const timeout = setTimeout(() => controller.abort(), REQUEST_TIMEOUT_MS) + + try { + const res = await fetch(url, { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + signal: controller.signal, + body: JSON.stringify({ + model, + messages: [ + { role: 'system', content: systemPrompt }, + { + role: 'user', + content: base64Images.map((b64) => ({ + type: 'image_url', + image_url: { url: `data:image/jpeg;base64,${b64}` }, + })), + }, + ], + max_tokens: 8192, + temperature: 0.1, + }), + }) + + if (!res.ok) { + const text = await res.text().catch(() => '') + throw new Error(`LLM API returned ${res.status}: ${text.slice(0, 200)}`) + } + + const data = await res.json() as { + choices?: Array<{ message?: { content?: string } }> + } + + const content = data.choices?.[0]?.message?.content?.trim() ?? '' + + // Extract JSON array from the response (handle markdown code blocks) + const jsonMatch = content.match(/\[[\s\S]*\]/) + if (!jsonMatch) return [] + + const parsed = JSON.parse(jsonMatch[0]) + if (!Array.isArray(parsed)) return [] + return parsed.filter((v): v is string => typeof v === 'string') + } finally { + clearTimeout(timeout) + } +} + +/** + * Run AI tagging for a single library. Called after the scanner finishes. + * Processes up to BATCH_LIMIT untagged items per invocation. + */ +export async function runAiTagging(library: Library, libraryRoot: string): Promise { + const config = getAiConfig() + if (!config.enabled || !config.endpoint || !config.model) return + + const activeCategoryIds = new Set(getActiveCategoryIdsForLibrary(library.id)) + const allTags = getTags() + const allCategories = getCategories() + + const tags = allTags.filter((t) => activeCategoryIds.has(t.categoryId)) + const categories = allCategories.filter((c) => activeCategoryIds.has(c.id)) + if (tags.length === 0) return + + const validTagIds = new Set(tags.map((t) => t.id)) + + const db = getDb() + const untaggedItems = db + .prepare( + `SELECT item_key, item_type, file_path, metadata + FROM media_items + WHERE library_id = ? AND ai_tagged_at IS NULL + LIMIT ?` + ) + .all(library.id, BATCH_LIMIT) as MediaItemRow[] + + if (untaggedItems.length === 0) return + + console.log(`[ai-tagger] Processing ${untaggedItems.length} items in library "${library.name}"`) + + let tagged = 0 + let consecutiveFailures = 0 + const markTagged = db.prepare('UPDATE media_items SET ai_tagged_at = ? WHERE item_key = ?') + + for (const item of untaggedItems) { + if (consecutiveFailures >= MAX_CONSECUTIVE_FAILURES) { + console.warn(`[ai-tagger] Aborting after ${MAX_CONSECUTIVE_FAILURES} consecutive failures`) + break + } + + const resolvedMedia = resolveItemImage(libraryRoot, item) + if (!resolvedMedia) { + // No image or video available β€” mark as tagged so we don't retry every scan + markTagged.run(Date.now(), item.item_key) + continue + } + + try { + let base64Images: string[] + if (resolvedMedia.mediaType === 'video') { + const framePaths = await getVideoFramePaths(resolvedMedia.path, library.id, VIDEO_FRAME_PERCENTAGES) + base64Images = framePaths.map((p) => fs.readFileSync(p, 'base64')) + } else { + const thumbnailPath = await getThumbnailPath(resolvedMedia.path, library.id, 'image') + base64Images = [fs.readFileSync(thumbnailPath, 'base64')] + } + + const { tags: currentItemTags } = getResolvedTagsForItem(item.item_key) + const systemPrompt = buildTagPrompt(tags, categories, currentItemTags, resolvedMedia.mediaType) + + const suggestedIds = await callVisionApi(config.endpoint, config.model, base64Images, systemPrompt) + + // Filter to valid tags only + const validIds = suggestedIds.filter((id) => validTagIds.has(id)) + for (const tagId of validIds) { + addTagToItem(item.item_key, tagId) + } + + markTagged.run(Date.now(), item.item_key) + tagged++ + consecutiveFailures = 0 + } catch (err) { + consecutiveFailures++ + console.warn( + `[ai-tagger] Failed to tag "${item.item_key}":`, + err instanceof Error ? err.message : err + ) + } + } + + if (tagged > 0) { + console.log(`[ai-tagger] Tagged ${tagged}/${untaggedItems.length} items in library "${library.name}"`) + } +} + +/** + * Tag a single item on-demand by itemKey. + * Bypasses the ai_tagged_at check and batch limit β€” user explicitly requested this. + * Throws descriptive errors so the API route can return appropriate status codes. + */ +export async function tagSingleItem(itemKey: string): Promise { + const config = getAiConfig() + if (!config.endpoint || !config.model) { + throw Object.assign(new Error('AI tagging endpoint and model are not configured'), { code: 'NOT_CONFIGURED' }) + } + + const libraryId = itemKey.split(':')[0] + + const activeCategoryIds = new Set(getActiveCategoryIdsForLibrary(libraryId)) + const allTags = getTags() + const allCategories = getCategories() + + const tags = allTags.filter((t) => activeCategoryIds.has(t.categoryId)) + const categories = allCategories.filter((c) => activeCategoryIds.has(c.id)) + if (tags.length === 0) { + return [] + } + + const validTagIds = new Set(tags.map((t) => t.id)) + + const db = getDb() + const item = db + .prepare('SELECT item_key, item_type, file_path, metadata FROM media_items WHERE item_key = ?') + .get(itemKey) as MediaItemRow | undefined + + if (!item) { + throw Object.assign(new Error(`Item not found: ${itemKey}`), { code: 'NOT_FOUND' }) + } + const library = getLibrary(libraryId) + if (!library) { + throw Object.assign(new Error(`Library not found: ${libraryId}`), { code: 'NOT_FOUND' }) + } + const libraryRoot = resolveLibraryRoot(library) + + const imagePath = resolveItemImage(libraryRoot, item) + if (!imagePath) { + throw Object.assign(new Error('No image available for this item'), { code: 'NO_IMAGE' }) + } + + let base64Images: string[] + if (imagePath.mediaType === 'video') { + const framePaths = await getVideoFramePaths(imagePath.path, libraryId, VIDEO_FRAME_PERCENTAGES) + base64Images = framePaths.map((p) => fs.readFileSync(p, 'base64')) + } else { + const thumbnailPath = await getThumbnailPath(imagePath.path, libraryId, 'image') + base64Images = [fs.readFileSync(thumbnailPath, 'base64')] + } + + const { tags: currentItemTags } = getResolvedTagsForItem(itemKey) + const systemPromptWithContext = buildTagPrompt(tags, categories, currentItemTags, imagePath.mediaType) + + const suggestedIds = await callVisionApi(config.endpoint, config.model, base64Images, systemPromptWithContext) + const validIds = suggestedIds.filter((id) => validTagIds.has(id)) + + for (const tagId of validIds) { + addTagToItem(itemKey, tagId) + } + + db.prepare('UPDATE media_items SET ai_tagged_at = ? WHERE item_key = ?').run(Date.now(), itemKey) + + return validIds +} diff --git a/src/lib/app-settings.ts b/src/lib/app-settings.ts index e27f2c8..f45dbf1 100644 --- a/src/lib/app-settings.ts +++ b/src/lib/app-settings.ts @@ -36,3 +36,24 @@ export function updateScanConfig(schedule: string, enabled: boolean): void { export function setScanLastRan(ts: number): void { setSetting('scan_last_ran', String(ts)) } + +// ─── AI Settings ───────────────────────────────────────────────────────────── + +interface AiConfig { + endpoint: string + model: string + enabled: boolean +} + +export function getAiConfig(): AiConfig { + const endpoint = getSetting('ai_endpoint') ?? '' + const model = getSetting('ai_model') ?? '' + const enabled = getSetting('ai_enabled') === 'true' + return { endpoint, model, enabled } +} + +export function updateAiConfig(endpoint: string, model: string, enabled: boolean): void { + setSetting('ai_endpoint', endpoint) + setSetting('ai_model', model) + setSetting('ai_enabled', enabled ? 'true' : 'false') +} diff --git a/src/lib/db.ts b/src/lib/db.ts index d1107f4..11e4905 100644 --- a/src/lib/db.ts +++ b/src/lib/db.ts @@ -102,6 +102,7 @@ function initDb(db: Database.Database): void { migrateMediaItemsSchema(db) migrateMediaItemsFingerprint(db) migrateMediaTagsToItemKey(db) + migrateMediaItemsAiTagged(db) seedAppSettings(db) } @@ -110,6 +111,9 @@ function seedAppSettings(db: Database.Database): void { scan_schedule: '0 * * * *', scan_enabled: 'true', scan_last_ran: '', + ai_enabled: 'false', + ai_endpoint: '', + ai_model: '', } const insert = db.prepare( 'INSERT OR IGNORE INTO app_settings (key, value) VALUES (?, ?)' @@ -228,6 +232,15 @@ function migrateMediaTagsToItemKey(db: Database.Database): void { `) } +function migrateMediaItemsAiTagged(db: Database.Database): void { + const row = db + .prepare("SELECT sql FROM sqlite_master WHERE type='table' AND name='media_items'") + .get() as { sql: string } | undefined + if (row && !row.sql.includes('ai_tagged_at')) { + db.exec('ALTER TABLE media_items ADD COLUMN ai_tagged_at INTEGER') + } +} + function migrateLibrariesType(db: Database.Database): void { const row = db .prepare("SELECT sql FROM sqlite_master WHERE type='table' AND name='libraries'") diff --git a/src/lib/scanner.ts b/src/lib/scanner.ts index 65d1710..33a0ccb 100644 --- a/src/lib/scanner.ts +++ b/src/lib/scanner.ts @@ -10,6 +10,7 @@ import { scanGamesLibrary } from './games' import { getThumbnailPath } from './thumbnails' import { computeFingerprint } from './fingerprint' import { reKeyMediaItem } from './tags' +import { runAiTagging } from './ai-tagger' let scanRunning = false @@ -70,6 +71,10 @@ export async function runLibraryScan(library: Library): Promise { await scanMixed(library, libraryRoot) break } + + await runAiTagging(library, libraryRoot).catch((err) => + console.error(`[ai-tagger] Error tagging library "${library.name}":`, err) + ) } // --------------------------------------------------------------------------- diff --git a/src/lib/tags.ts b/src/lib/tags.ts index ca5e7f0..2b9dca5 100644 --- a/src/lib/tags.ts +++ b/src/lib/tags.ts @@ -18,6 +18,25 @@ export function getCategories(): TagCategory[] { return db.prepare('SELECT id, name FROM tag_categories ORDER BY name').all() as TagCategory[] } +/** + * Returns the distinct category IDs that have at least one tag assigned to any + * item in the given library. Used by the AI tagger to restrict the tag prompt + * to categories that are actually in use within the target library. + */ +export function getActiveCategoryIdsForLibrary(libraryId: string): string[] { + const db = getDb() + const rows = db + .prepare( + `SELECT DISTINCT t.category_id + FROM tags t + JOIN media_tags mt ON mt.tag_id = t.id + JOIN media_items mi ON mi.item_key = mt.item_key + WHERE mi.library_id = ?` + ) + .all(libraryId) as { category_id: string }[] + return rows.map((r) => r.category_id) +} + export function addCategory(name: string): TagCategory { const trimmed = name.trim() if (!trimmed) throw new Error('Category name is required.') diff --git a/src/lib/thumbnails.ts b/src/lib/thumbnails.ts index fe8c948..04f80e1 100644 --- a/src/lib/thumbnails.ts +++ b/src/lib/thumbnails.ts @@ -87,22 +87,13 @@ async function getVideoDuration(src: string): Promise { }) } -/** Generate a thumbnail from a video using ffmpeg. */ -async function generateVideoThumbnail(src: string, dest: string): Promise { +/** Extract a single frame from a video at the given offset (seconds) and write to dest. */ +async function generateVideoFrameAtOffset(src: string, dest: string, offsetSeconds: number): Promise { const tmp = dest + '.tmp' - // Seek to 10% of the video duration for a representative frame - let offset = 0 - try { - const duration = await getVideoDuration(src) - offset = Math.max(0, duration * 0.1) - } catch { - // If ffprobe fails, fall back to seeking to 0 - } - const args = [ '-y', // overwrite output - '-ss', String(offset), // seek before input (fast) + '-ss', String(offsetSeconds), // seek before input (fast) '-i', src, '-frames:v', '1', '-q:v', '5', @@ -115,6 +106,58 @@ async function generateVideoThumbnail(src: string, dest: string): Promise fs.renameSync(tmp, dest) } +/** Generate a thumbnail from a video using ffmpeg (seeks to 10% of duration). */ +async function generateVideoThumbnail(src: string, dest: string): Promise { + let offset = 0 + try { + const duration = await getVideoDuration(src) + offset = Math.max(0, duration * 0.1) + } catch { + // If ffprobe fails, fall back to seeking to 0 + } + await generateVideoFrameAtOffset(src, dest, offset) +} + +/** + * Extract frames from a video at each given percentage of its duration. + * Returns the absolute paths to the cached frame JPEGs, in the same order as `percentages`. + * Uses a per-frame cache key so each frame is cached independently. + */ +export async function getVideoFramePaths( + absoluteFilePath: string, + libraryId: string, + percentages: number[] +): Promise { + ensureCacheDir() + + let duration = 0 + try { + duration = await getVideoDuration(absoluteFilePath) + } catch { + // Fall back to 0; all frames will seek to position 0 + } + + const framePaths: string[] = [] + + for (const pct of percentages) { + const offset = Math.max(0, duration * pct) + const key = crypto + .createHash('sha1') + .update(libraryId + ':' + absoluteFilePath + ':' + pct) + .digest('hex') + const cacheFile = path.join(CACHE_DIR, key + '.jpg') + + const cached = getCachedPath(cacheFile, absoluteFilePath) + if (!cached) { + await generateVideoFrameAtOffset(absoluteFilePath, cacheFile, offset) + } + + framePaths.push(cacheFile) + } + + return framePaths +} + /** * Returns the absolute path to a cached thumbnail JPEG for the given file. * Generates it on first call (or when the source has been modified).