a794e387f3
All checks were successful
Build and Publish Docker Image / build-and-push (push) Successful in 16s
341 lines
12 KiB
C#
341 lines
12 KiB
C#
using System.Collections.Concurrent;
|
|
using System.Net.Http.Headers;
|
|
using System.Text;
|
|
using System.Text.Json;
|
|
using System.Text.Json.Serialization;
|
|
using Microsoft.Extensions.Logging;
|
|
using Microsoft.Extensions.Configuration;
|
|
using TentacleBot.Models;
|
|
using TentacleBot.Services.Memory;
|
|
using TentacleBot.Services.Tools;
|
|
|
|
namespace TentacleBot.Services;
|
|
|
|
public class LlmService
|
|
{
|
|
private readonly HttpClient _httpClient;
|
|
private readonly ILogger<LlmService> _logger;
|
|
private readonly MemoryStore _memoryStore;
|
|
private readonly ToolRegistry _toolRegistry;
|
|
private readonly string _endpoint;
|
|
private readonly string _apiKey;
|
|
private readonly string _model;
|
|
private readonly int _maxTokens;
|
|
private readonly double _temperature;
|
|
private readonly ConcurrentDictionary<ulong, List<ChatMessage>> _histories = new();
|
|
private const int MaxHistoryChars = 20000;
|
|
private const int MaxToolIterations = 5;
|
|
|
|
public LlmService(IConfiguration configuration, ILogger<LlmService> logger, MemoryStore memoryStore, ToolRegistry toolRegistry)
|
|
{
|
|
_logger = logger;
|
|
_httpClient = new HttpClient();
|
|
_memoryStore = memoryStore;
|
|
_toolRegistry = toolRegistry;
|
|
|
|
_endpoint = configuration["Llm:Endpoint"] ?? throw new InvalidOperationException("Llm:Endpoint is not configured");
|
|
_apiKey = Environment.GetEnvironmentVariable("LLM_API_KEY")
|
|
?? " ";
|
|
_model = configuration["Llm:Model"] ?? throw new InvalidOperationException("Llm:Model is not configured");
|
|
_maxTokens = int.Parse(configuration["Llm:MaxTokens"] ?? "1024");
|
|
_temperature = double.Parse(configuration["Llm:Temperature"] ?? "0.7");
|
|
|
|
_httpClient.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue("Bearer", _apiKey);
|
|
_httpClient.DefaultRequestHeaders.Accept.Add(new MediaTypeWithQualityHeaderValue("application/json"));
|
|
}
|
|
|
|
public async IAsyncEnumerable<string> SendMessageAsync(ulong channelId, ulong userId, string username, string message)
|
|
{
|
|
var history = _histories.GetOrAdd(channelId, _ => new List<ChatMessage>());
|
|
|
|
var systemPrompt = await BuildSystemPromptAsync(userId, username, message);
|
|
if (history.Count == 0 || history[0].Role != "system")
|
|
history.Insert(0, new ChatMessage { Role = "system", Content = systemPrompt });
|
|
else
|
|
history[0] = new ChatMessage { Role = "system", Content = systemPrompt };
|
|
|
|
history.Add(new ChatMessage { Role = "user", Content = message, Name = username });
|
|
TrimHistory(history);
|
|
|
|
var toolDefinitions = _toolRegistry.BuildToolDefinitions();
|
|
var jsonOptions = new JsonSerializerOptions
|
|
{
|
|
PropertyNamingPolicy = JsonNamingPolicy.CamelCase,
|
|
PropertyNameCaseInsensitive = true
|
|
};
|
|
|
|
for (int iteration = 0; iteration < MaxToolIterations; iteration++)
|
|
{
|
|
var request = new ChatRequest
|
|
{
|
|
Model = _model,
|
|
Messages = CopyMessages(history),
|
|
MaxTokens = _maxTokens,
|
|
Temperature = (float)_temperature,
|
|
Stream = false,
|
|
Tools = toolDefinitions
|
|
};
|
|
|
|
var requestJson = JsonSerializer.Serialize(request, jsonOptions);
|
|
var httpContent = new StringContent(requestJson, Encoding.UTF8, "application/json");
|
|
|
|
using var response = await _httpClient.PostAsync(_endpoint, httpContent);
|
|
|
|
if (!response.IsSuccessStatusCode)
|
|
{
|
|
var errorBody = await response.Content.ReadAsStringAsync();
|
|
_logger.LogError("LLM API error ({StatusCode}): {Error}", response.StatusCode, errorBody);
|
|
throw new HttpRequestException($"LLM API returned {response.StatusCode}: {errorBody}");
|
|
}
|
|
|
|
var body = await response.Content.ReadAsStringAsync();
|
|
var chatResponse = JsonSerializer.Deserialize<ChatResponse>(body, jsonOptions);
|
|
var choice = chatResponse?.Choices?.FirstOrDefault();
|
|
|
|
if (choice == null)
|
|
{
|
|
_logger.LogError("Empty response from LLM API");
|
|
yield break;
|
|
}
|
|
|
|
if (choice.FinishReason == "tool_calls" && choice.Message?.ToolCalls != null && choice.Message.ToolCalls.Count > 0)
|
|
{
|
|
history.Add(new ChatMessage
|
|
{
|
|
Role = "assistant",
|
|
ToolCalls = choice.Message.ToolCalls
|
|
});
|
|
|
|
foreach (var tc in choice.Message.ToolCalls)
|
|
{
|
|
var toolResult = await _toolRegistry.ExecuteAsync(
|
|
tc.Function?.Name ?? "", tc.Function?.Arguments ?? "");
|
|
history.Add(new ChatMessage
|
|
{
|
|
Role = "tool",
|
|
Content = toolResult,
|
|
ToolCallId = tc.Id
|
|
});
|
|
}
|
|
|
|
continue;
|
|
}
|
|
|
|
var responseText = choice.Message?.Content ?? "";
|
|
if (!string.IsNullOrWhiteSpace(responseText))
|
|
{
|
|
history.Add(new ChatMessage { Role = "assistant", Content = responseText, Name = "TentacleBot" });
|
|
TrimHistory(history);
|
|
yield return responseText;
|
|
yield break;
|
|
}
|
|
|
|
_logger.LogWarning("LLM response had no content and no tool calls at iteration {Iteration}", iteration);
|
|
break;
|
|
}
|
|
|
|
_logger.LogWarning("Max tool iterations reached without a final response");
|
|
}
|
|
|
|
private static string? TryParseStreamToken(string data)
|
|
{
|
|
try
|
|
{
|
|
using var doc = JsonDocument.Parse(data);
|
|
|
|
if (doc.RootElement.TryGetProperty("choices", out var choices) &&
|
|
choices.ValueKind == JsonValueKind.Array &&
|
|
choices.GetArrayLength() > 0)
|
|
{
|
|
var firstChoice = choices[0];
|
|
|
|
if (firstChoice.TryGetProperty("delta", out var delta) &&
|
|
delta.ValueKind == JsonValueKind.Object &&
|
|
delta.TryGetProperty("content", out var deltaContent) &&
|
|
deltaContent.ValueKind == JsonValueKind.String)
|
|
{
|
|
return deltaContent.GetString();
|
|
}
|
|
|
|
if (firstChoice.TryGetProperty("text", out var text) &&
|
|
text.ValueKind == JsonValueKind.String)
|
|
{
|
|
return text.GetString();
|
|
}
|
|
}
|
|
|
|
if (doc.RootElement.TryGetProperty("delta", out var rootDelta) &&
|
|
rootDelta.ValueKind == JsonValueKind.Object &&
|
|
rootDelta.TryGetProperty("content", out var rootContent) &&
|
|
rootContent.ValueKind == JsonValueKind.String)
|
|
{
|
|
return rootContent.GetString();
|
|
}
|
|
|
|
if (doc.RootElement.TryGetProperty("content", out var content) &&
|
|
content.ValueKind == JsonValueKind.String)
|
|
{
|
|
return content.GetString();
|
|
}
|
|
|
|
return null;
|
|
}
|
|
catch
|
|
{
|
|
return null;
|
|
}
|
|
}
|
|
|
|
private static string? TryParseNonStreamContent(string body)
|
|
{
|
|
try
|
|
{
|
|
using var doc = JsonDocument.Parse(body);
|
|
|
|
if (doc.RootElement.TryGetProperty("choices", out var choices) &&
|
|
choices.ValueKind == JsonValueKind.Array &&
|
|
choices.GetArrayLength() > 0)
|
|
{
|
|
var firstChoice = choices[0];
|
|
|
|
if (firstChoice.TryGetProperty("message", out var message) &&
|
|
message.ValueKind == JsonValueKind.Object &&
|
|
message.TryGetProperty("content", out var messageContent) &&
|
|
messageContent.ValueKind == JsonValueKind.String)
|
|
{
|
|
return messageContent.GetString();
|
|
}
|
|
|
|
if (firstChoice.TryGetProperty("text", out var text) &&
|
|
text.ValueKind == JsonValueKind.String)
|
|
{
|
|
return text.GetString();
|
|
}
|
|
}
|
|
|
|
if (doc.RootElement.TryGetProperty("content", out var content) &&
|
|
content.ValueKind == JsonValueKind.String)
|
|
{
|
|
return content.GetString();
|
|
}
|
|
|
|
return null;
|
|
}
|
|
catch
|
|
{
|
|
return null;
|
|
}
|
|
}
|
|
|
|
public void ResetHistory(ulong channelId)
|
|
{
|
|
_histories.TryRemove(channelId, out _);
|
|
}
|
|
|
|
private void TrimHistory(List<ChatMessage> history)
|
|
{
|
|
while (history.Sum(m => m.Content.Length) > MaxHistoryChars && history.Count > 2)
|
|
{
|
|
history.RemoveAt(0);
|
|
}
|
|
}
|
|
|
|
private async Task<string> BuildSystemPromptAsync(ulong userId, string username, string userMessage)
|
|
{
|
|
var sb = new StringBuilder();
|
|
sb.AppendLine("You are TentacleBot, a helpful Discord bot with memory.");
|
|
sb.AppendLine($"The Discord user you are talking to is named \"{username}\". Address them by this name.");
|
|
sb.AppendLine();
|
|
|
|
var rules = await _memoryStore.GetUserRulesAsync(userId.ToString());
|
|
if (rules.Count > 0)
|
|
{
|
|
sb.AppendLine($"## Rules for {username}:");
|
|
foreach (var rule in rules)
|
|
sb.AppendLine($"- {rule.RuleText}");
|
|
sb.AppendLine();
|
|
}
|
|
|
|
var recentMemories = await _memoryStore.GetRecentMemoriesAsync(userId.ToString(), 3);
|
|
var searchMemories = await _memoryStore.SearchMemoriesAsync(userId.ToString(), userMessage, 3);
|
|
|
|
var allMemories = recentMemories
|
|
.UnionBy(searchMemories, m => m.Id)
|
|
.OrderByDescending(m => m.Importance)
|
|
.Take(5)
|
|
.ToList();
|
|
|
|
if (allMemories.Count > 0)
|
|
{
|
|
sb.AppendLine($"## Memories about {username}:");
|
|
foreach (var m in allMemories)
|
|
sb.AppendLine($"- [{m.CreatedAt:yyyy-MM-dd}] {m.Content} (importance: {m.Importance})");
|
|
sb.AppendLine();
|
|
}
|
|
|
|
sb.AppendLine("You have access to tools for managing memory. Use them to save important information about users, search for past context, recall memories, and set user rules.");
|
|
sb.AppendLine("Be concise and helpful. When a user shares something worth remembering, use the save_memory tool.");
|
|
|
|
return sb.ToString();
|
|
}
|
|
|
|
private static List<ChatMessage> CopyMessages(List<ChatMessage> history)
|
|
{
|
|
return history.Select(m => new ChatMessage
|
|
{
|
|
Role = m.Role,
|
|
Content = m.Content,
|
|
Name = m.Name,
|
|
ToolCallId = m.ToolCallId,
|
|
ToolCalls = m.ToolCalls
|
|
}).ToList();
|
|
}
|
|
|
|
private record ChatRequest
|
|
{
|
|
public string Model { get; init; } = "";
|
|
public List<ChatMessage> Messages { get; init; } = new();
|
|
public int MaxTokens { get; init; }
|
|
public float Temperature { get; init; }
|
|
public bool Stream { get; init; }
|
|
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
|
|
public List<object>? Tools { get; init; }
|
|
}
|
|
|
|
private record ChatResponse
|
|
{
|
|
public string? Id { get; init; }
|
|
public List<ChatResponseChoice>? Choices { get; init; }
|
|
}
|
|
|
|
private record ChatResponseChoice
|
|
{
|
|
public int Index { get; init; }
|
|
public ChatResponseMessage? Message { get; init; }
|
|
[JsonPropertyName("finish_reason")]
|
|
public string? FinishReason { get; init; }
|
|
}
|
|
|
|
private record ChatResponseMessage
|
|
{
|
|
public string? Role { get; init; }
|
|
public string? Content { get; init; }
|
|
[JsonPropertyName("tool_calls")]
|
|
public List<ToolCall>? ToolCalls { get; init; }
|
|
}
|
|
|
|
public record ChatMessage
|
|
{
|
|
public string Role { get; set; } = "";
|
|
public string Content { get; set; } = "";
|
|
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
|
|
public string? Name { get; set; }
|
|
[JsonPropertyName("tool_call_id")]
|
|
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
|
|
public string? ToolCallId { get; set; }
|
|
[JsonPropertyName("tool_calls")]
|
|
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
|
|
public List<ToolCall>? ToolCalls { get; set; }
|
|
}
|
|
}
|