// I guess it should be some kind of package with different part for different Company // I will maybe do it in the future, rn, I don't have time to learn that and I like it like that // It do need more time and try and error to do all of them but they are fully independant // And this is simple, I don't need to trick my mind to undersant that some part are share, ect // If I want to see how I go from the message to the response, I can. That what I want // If you are wondering how it work: // User send message -> Generate HTML of one user message and one bot placeholder ---- // -> Send HTML and append it to the chat container -> The placeholder do a load HTMX request to GenerateMultipleMessagesHandler ---- // -> Make multiple request in parallel to all APIs -> Send one SSE event per message receive. package main import ( "context" "encoding/json" "fmt" "strings" "sync" "time" "github.com/edgedb/edgedb-go" "github.com/flosch/pongo2" "github.com/gofiber/fiber/v2" ) type RequestMessage struct { Role string `json:"role"` Content string `json:"content"` } var lastSelectedLLMs []LLM func GeneratePlaceholderHandler(c *fiber.Ctx) error { // Step 1 I create a User message and send it as output with a placeholder // that will make a request to GenerateMultipleMessagesHandler when loading message := c.FormValue("message", "") var selectedLLMIds []string err := json.Unmarshal([]byte(c.FormValue("selectedLLMIds")), &selectedLLMIds) if err != nil { // Handle the error panic(err) } return c.SendString(GeneratePlaceholderHTML(message, selectedLLMIds, true)) } func GeneratePlaceholderHTML(message string, selectedLLMIds []string, with_user_message bool) string { var selectedLLMs []LLM var selectedLLM LLM for _, id := range selectedLLMIds { idUUID, _ := edgedb.ParseUUID(id) err := edgeClient.QuerySingle(context.Background(), ` SELECT LLM { id, name, context, temperature, custom_endpoint : { id, endpoint, key }, modelInfo : { modelID, maxToken, company : { icon, name } } } FILTER .id = $0; `, &selectedLLM, idUUID) if err != nil { panic(err) } selectedLLMs = append(selectedLLMs, selectedLLM) } lastSelectedLLMs = selectedLLMs _, position := insertArea() out := "" if with_user_message { messageID := insertUserMessage(message) messageOut, _ := userTmpl.Execute(pongo2.Context{"Content": markdownToHTML(message), "ID": messageID.String()}) out += messageOut } messageOut, _ := botTmpl.Execute(pongo2.Context{"IsPlaceholder": true, "SelectedLLMs": selectedLLMs, "ConversationAreaId": position + 1}) out += messageOut return out } func GenerateMultipleMessagesHandler(c *fiber.Ctx) error { // Step 2 generate multiple messages // And send them one by one using events insertArea() selectedLLMs := lastSelectedLLMs // Create a wait group to synchronize the goroutines var wg sync.WaitGroup // Add the length of selectedModelIds goroutines to the wait group wg.Add(len(selectedLLMs)) // Create a channel to receive the index of the first completed goroutine firstDone := make(chan int, 1) for i := range selectedLLMs { idx := i go func() { defer wg.Done() // Create a context with a 1-minute timeout ctx, cancel := context.WithTimeout(context.Background(), time.Minute) defer cancel() // Ensure the context is cancelled to free resources // Determine which message function to call based on the model var addMessageFunc func(selectedLLM LLM, selected bool) edgedb.UUID switch selectedLLMs[idx].Model.Company.Name { case "openai": addMessageFunc = addOpenaiMessage case "anthropic": addMessageFunc = addAnthropicMessage case "mistral": addMessageFunc = addMistralMessage case "groq": addMessageFunc = addGroqMessage case "gooseai": addMessageFunc = addGooseaiMessage case "huggingface": addMessageFunc = addHuggingfaceMessage } var messageID edgedb.UUID if addMessageFunc != nil { messageID = addMessageFunc(selectedLLMs[idx], idx == 0) } var message Message err := edgeClient.QuerySingle(edgeCtx, ` SELECT Message { id, content, area : { id, position }, llm : { modelInfo : { modelID, name, company : { icon, } } } } FILTER .id = $0; `, &message, messageID) if err != nil { panic(err) } templateMessage := TemplateMessage{ Icon: message.LLM.Model.Company.Icon, Content: message.Content, Hidden: false, Id: message.ID.String(), Name: message.LLM.Model.Name, ModelID: message.LLM.Model.ModelID, } // Check if the context's deadline is exceeded select { case <-ctx.Done(): // The context's deadline was exceeded fmt.Printf("Goroutine %d timed out\n", idx) default: // Send the index of the completed goroutine to the firstDone channel select { case firstDone <- idx: // Generate the HTML content outBtn, err := selectBtnTmpl.Execute(map[string]interface{}{ "message": templateMessage, "ConversationAreaId": message.Area.Position, }) if err != nil { panic(err) } outBtn = strings.ReplaceAll(outBtn, "\n", "") outIcon := `User Image` go func() { sendEvent( "swapContent-"+fmt.Sprintf("%d", message.Area.Position), ``, ) sendEvent( "swapSelectionBtn-"+selectedLLMs[idx].ID.String(), outBtn, ) sendEvent( "swapIcon-"+fmt.Sprintf("%d", message.Area.Position), outIcon, ) }() default: out, err := selectBtnTmpl.Execute(map[string]interface{}{ "message": templateMessage, "ConversationAreaId": message.Area.Position, }) if err != nil { panic(err) } // Replace newline characters to prevent premature termination outBtn := strings.ReplaceAll(out, "\n", "") // Send Content event go func() { sendEvent( "swapSelectionBtn-"+selectedLLMs[idx].ID.String(), outBtn, ) }() } } }() } // Wait for all goroutines to finish wg.Wait() return c.SendString("") } func addUsage(inputCost float32, outputCost float32, inputToken int32, outputToken int32, modelID string) { // Create a new usage err := edgeClient.Execute(edgeCtx, ` INSERT Usage { input_cost := $0, output_cost := $1, input_token := $2, output_token := $3, model_id := $4, user := global currentUser } `, inputCost, outputCost, inputToken, outputToken, modelID) if err != nil { panic(err) } }