using System.Collections.Concurrent; using System.Net.Http.Headers; using System.Text; using System.Text.Json; using System.Text.Json.Serialization; using Microsoft.Extensions.Logging; using Microsoft.Extensions.Configuration; using TentacleBot.Models; using TentacleBot.Services.Memory; using TentacleBot.Services.Tools; namespace TentacleBot.Services; public class LlmService { private readonly HttpClient _httpClient; private readonly ILogger _logger; private readonly MemoryStore _memoryStore; private readonly ToolRegistry _toolRegistry; private readonly string _endpoint; private readonly string _apiKey; private readonly string _model; private readonly int _maxTokens; private readonly double _temperature; private readonly ConcurrentDictionary> _histories = new(); private const int MaxHistoryChars = 20000; private const int MaxToolIterations = 5; public LlmService(IConfiguration configuration, ILogger logger, MemoryStore memoryStore, ToolRegistry toolRegistry) { _logger = logger; _httpClient = new HttpClient(); _memoryStore = memoryStore; _toolRegistry = toolRegistry; _endpoint = configuration["Llm:Endpoint"] ?? throw new InvalidOperationException("Llm:Endpoint is not configured"); _apiKey = Environment.GetEnvironmentVariable("LLM_API_KEY") ?? " "; _model = configuration["Llm:Model"] ?? throw new InvalidOperationException("Llm:Model is not configured"); _maxTokens = int.Parse(configuration["Llm:MaxTokens"] ?? "1024"); _temperature = double.Parse(configuration["Llm:Temperature"] ?? "0.7"); _httpClient.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue("Bearer", _apiKey); _httpClient.DefaultRequestHeaders.Accept.Add(new MediaTypeWithQualityHeaderValue("application/json")); } public async IAsyncEnumerable SendMessageAsync(ulong channelId, ulong userId, string username, string message) { var history = _histories.GetOrAdd(channelId, _ => new List()); var systemPrompt = await BuildSystemPromptAsync(userId, username, message); if (history.Count == 0 || history[0].Role != "system") history.Insert(0, new ChatMessage { Role = "system", Content = systemPrompt }); else history[0] = new ChatMessage { Role = "system", Content = systemPrompt }; history.Add(new ChatMessage { Role = "user", Content = message, Name = username }); TrimHistory(history); var toolDefinitions = _toolRegistry.BuildToolDefinitions(); var jsonOptions = new JsonSerializerOptions { PropertyNamingPolicy = JsonNamingPolicy.CamelCase, PropertyNameCaseInsensitive = true }; for (int iteration = 0; iteration < MaxToolIterations; iteration++) { var request = new ChatRequest { Model = _model, Messages = CopyMessages(history), MaxTokens = _maxTokens, Temperature = (float)_temperature, Stream = false, Tools = toolDefinitions }; var requestJson = JsonSerializer.Serialize(request, jsonOptions); var httpContent = new StringContent(requestJson, Encoding.UTF8, "application/json"); using var response = await _httpClient.PostAsync(_endpoint, httpContent); if (!response.IsSuccessStatusCode) { var errorBody = await response.Content.ReadAsStringAsync(); _logger.LogError("LLM API error ({StatusCode}): {Error}", response.StatusCode, errorBody); throw new HttpRequestException($"LLM API returned {response.StatusCode}: {errorBody}"); } var body = await response.Content.ReadAsStringAsync(); var chatResponse = JsonSerializer.Deserialize(body, jsonOptions); var choice = chatResponse?.Choices?.FirstOrDefault(); if (choice == null) { _logger.LogError("Empty response from LLM API"); yield break; } if (choice.FinishReason == "tool_calls" && choice.Message?.ToolCalls != null && choice.Message.ToolCalls.Count > 0) { history.Add(new ChatMessage { Role = "assistant", ToolCalls = choice.Message.ToolCalls }); foreach (var tc in choice.Message.ToolCalls) { var toolResult = await _toolRegistry.ExecuteAsync( tc.Function?.Name ?? "", tc.Function?.Arguments ?? ""); history.Add(new ChatMessage { Role = "tool", Content = toolResult, ToolCallId = tc.Id }); } continue; } var responseText = choice.Message?.Content ?? ""; if (!string.IsNullOrWhiteSpace(responseText)) { history.Add(new ChatMessage { Role = "assistant", Content = responseText, Name = "TentacleBot" }); TrimHistory(history); yield return responseText; yield break; } _logger.LogWarning("LLM response had no content and no tool calls at iteration {Iteration}", iteration); break; } _logger.LogWarning("Max tool iterations reached without a final response"); } private static string? TryParseStreamToken(string data) { try { using var doc = JsonDocument.Parse(data); if (doc.RootElement.TryGetProperty("choices", out var choices) && choices.ValueKind == JsonValueKind.Array && choices.GetArrayLength() > 0) { var firstChoice = choices[0]; if (firstChoice.TryGetProperty("delta", out var delta) && delta.ValueKind == JsonValueKind.Object && delta.TryGetProperty("content", out var deltaContent) && deltaContent.ValueKind == JsonValueKind.String) { return deltaContent.GetString(); } if (firstChoice.TryGetProperty("text", out var text) && text.ValueKind == JsonValueKind.String) { return text.GetString(); } } if (doc.RootElement.TryGetProperty("delta", out var rootDelta) && rootDelta.ValueKind == JsonValueKind.Object && rootDelta.TryGetProperty("content", out var rootContent) && rootContent.ValueKind == JsonValueKind.String) { return rootContent.GetString(); } if (doc.RootElement.TryGetProperty("content", out var content) && content.ValueKind == JsonValueKind.String) { return content.GetString(); } return null; } catch { return null; } } private static string? TryParseNonStreamContent(string body) { try { using var doc = JsonDocument.Parse(body); if (doc.RootElement.TryGetProperty("choices", out var choices) && choices.ValueKind == JsonValueKind.Array && choices.GetArrayLength() > 0) { var firstChoice = choices[0]; if (firstChoice.TryGetProperty("message", out var message) && message.ValueKind == JsonValueKind.Object && message.TryGetProperty("content", out var messageContent) && messageContent.ValueKind == JsonValueKind.String) { return messageContent.GetString(); } if (firstChoice.TryGetProperty("text", out var text) && text.ValueKind == JsonValueKind.String) { return text.GetString(); } } if (doc.RootElement.TryGetProperty("content", out var content) && content.ValueKind == JsonValueKind.String) { return content.GetString(); } return null; } catch { return null; } } public void ResetHistory(ulong channelId) { _histories.TryRemove(channelId, out _); } private void TrimHistory(List history) { while (history.Sum(m => m.Content.Length) > MaxHistoryChars && history.Count > 2) { history.RemoveAt(0); } } private async Task BuildSystemPromptAsync(ulong userId, string username, string userMessage) { var sb = new StringBuilder(); sb.AppendLine("You are TentacleBot, a helpful Discord bot with memory."); sb.AppendLine($"The Discord user you are talking to is named \"{username}\". Address them by this name."); sb.AppendLine(); var rules = await _memoryStore.GetUserRulesAsync(userId.ToString()); if (rules.Count > 0) { sb.AppendLine($"## Rules for {username}:"); foreach (var rule in rules) sb.AppendLine($"- {rule.RuleText}"); sb.AppendLine(); } var recentMemories = await _memoryStore.GetRecentMemoriesAsync(userId.ToString(), 3); var searchMemories = await _memoryStore.SearchMemoriesAsync(userId.ToString(), userMessage, 3); var allMemories = recentMemories .UnionBy(searchMemories, m => m.Id) .OrderByDescending(m => m.Importance) .Take(5) .ToList(); if (allMemories.Count > 0) { sb.AppendLine($"## Memories about {username}:"); foreach (var m in allMemories) sb.AppendLine($"- [{m.CreatedAt:yyyy-MM-dd}] {m.Content} (importance: {m.Importance})"); sb.AppendLine(); } sb.AppendLine("You have access to tools for managing memory. Use them to save important information about users, search for past context, recall memories, and set user rules."); sb.AppendLine("Be concise and helpful. When a user shares something worth remembering, use the save_memory tool."); return sb.ToString(); } private static List CopyMessages(List history) { return history.Select(m => new ChatMessage { Role = m.Role, Content = m.Content, Name = m.Name, ToolCallId = m.ToolCallId, ToolCalls = m.ToolCalls }).ToList(); } private record ChatRequest { public string Model { get; init; } = ""; public List Messages { get; init; } = new(); public int MaxTokens { get; init; } public float Temperature { get; init; } public bool Stream { get; init; } [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] public List? Tools { get; init; } } private record ChatResponse { public string? Id { get; init; } public List? Choices { get; init; } } private record ChatResponseChoice { public int Index { get; init; } public ChatResponseMessage? Message { get; init; } [JsonPropertyName("finish_reason")] public string? FinishReason { get; init; } } private record ChatResponseMessage { public string? Role { get; init; } public string? Content { get; init; } [JsonPropertyName("tool_calls")] public List? ToolCalls { get; init; } } public record ChatMessage { public string Role { get; set; } = ""; public string Content { get; set; } = ""; [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] public string? Name { get; set; } [JsonPropertyName("tool_call_id")] [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] public string? ToolCallId { get; set; } [JsonPropertyName("tool_calls")] [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] public List? ToolCalls { get; set; } } }