From c3d702a09a890fa241a5ef66db96e923a475f427 Mon Sep 17 00:00:00 2001 From: MrBounty Date: Thu, 1 Aug 2024 08:31:56 +0200 Subject: [PATCH] Changed max token from ModelInfo to LLM --- EdgeDatabase.go | 2 +- LLM.go | 18 ++++++++++++++---- Request.go | 1 - RequestAnthropic.go | 16 +++++++++++----- dbschema/default.esdl | 10 +++++----- dbschema/migrations/00051-m1qx7si.edgeql | 10 ++++++++++ dbschema/migrations/00052-m1ad7ps.edgeql | 9 +++++++++ views/partials/popover-models.html | 9 ++++++--- 8 files changed, 56 insertions(+), 19 deletions(-) create mode 100644 dbschema/migrations/00051-m1qx7si.edgeql create mode 100644 dbschema/migrations/00052-m1ad7ps.edgeql diff --git a/EdgeDatabase.go b/EdgeDatabase.go index 3fe627c..1afe932 100644 --- a/EdgeDatabase.go +++ b/EdgeDatabase.go @@ -77,6 +77,7 @@ type LLM struct { ID edgedb.UUID `edgedb:"id"` Name string `edgedb:"name"` Context string `edgedb:"context"` + MaxToken int32 `edgedb:"max_token"` Temperature float32 `edgedb:"temperature"` Model ModelInfo `edgedb:"modelInfo"` Endpoint CustomEndpoint `edgedb:"custom_endpoint"` @@ -92,7 +93,6 @@ type CustomEndpoint struct { type ModelInfo struct { ID edgedb.UUID `edgedb:"id"` Name string `edgedb:"name"` - MaxToken int32 `edgedb:"maxToken"` InputPrice float32 `edgedb:"inputPrice"` OutputPrice float32 `edgedb:"outputPrice"` ModelID string `edgedb:"modelID"` diff --git a/LLM.go b/LLM.go index edfe252..af4151a 100644 --- a/LLM.go +++ b/LLM.go @@ -50,7 +50,7 @@ func deleteLLMtoDelete(c *fiber.Ctx) { } } -func createLLM(c *fiber.Ctx) error { + func createLLM(c *fiber.Ctx) error { name := c.FormValue("model-name-input") modelID := c.FormValue("selectedLLMId") temperature := c.FormValue("temperature-slider") @@ -61,7 +61,16 @@ func createLLM(c *fiber.Ctx) error { url := c.FormValue("model-url-input") token := c.FormValue("model-key-input") customID := c.FormValue("model-cid-input") + maxTokenStr := c.FormValue("max-token-input") + maxToken, err := strconv.Atoi(maxTokenStr) + if err != nil { + maxToken = 0 + } + + fmt.Println("Adding LLM with maxtoken:", maxToken) + + // TODO change the company if modelID == "custom" { err := edgeGlobalClient.WithGlobals(map[string]interface{}{"ext::auth::client_token": c.Cookies("jade-edgedb-auth-token")}).Execute(edgeCtx, ` WITH @@ -71,12 +80,12 @@ func createLLM(c *fiber.Ctx) error { context := $1, temperature := $2, position := countLLM + 1, + max_token := $6, modelInfo := (INSERT ModelInfo { name := $0, modelID := $5, inputPrice := 0.0, outputPrice := 0.0, - maxToken := 500, company := (SELECT Company FILTER .name = "huggingface" LIMIT 1), }), custom_endpoint := (INSERT CustomEndpoint { @@ -85,7 +94,7 @@ func createLLM(c *fiber.Ctx) error { }), user := global currentUser }; - `, name, systemPrompt, temperatureFloat, url, token, customID) // TODO Add real max token + `, name, systemPrompt, temperatureFloat, url, token, customID, int32(maxToken)) if err != nil { fmt.Println("Error creating LLM") panic(err) @@ -99,10 +108,11 @@ func createLLM(c *fiber.Ctx) error { context := $1, temperature := $2, position := countLLM + 1, + max_token := $4, modelInfo := (SELECT ModelInfo FILTER .modelID = $3 LIMIT 1), user := global currentUser } - `, name, systemPrompt, temperatureFloat, modelID) + `, name, systemPrompt, temperatureFloat, modelID, int32(maxToken)) if err != nil { fmt.Println("Error creating LLM") panic(err) diff --git a/Request.go b/Request.go index f98650a..bf01746 100644 --- a/Request.go +++ b/Request.go @@ -62,7 +62,6 @@ func GeneratePlaceholderHTML(c *fiber.Ctx, message string, selectedLLMIds []stri }, modelInfo : { modelID, - maxToken, company : { icon, name diff --git a/RequestAnthropic.go b/RequestAnthropic.go index 634f7d6..8cd919f 100644 --- a/RequestAnthropic.go +++ b/RequestAnthropic.go @@ -14,7 +14,7 @@ import ( type AnthropicChatCompletionRequest struct { Model string `json:"model"` Messages []RequestMessage `json:"messages"` - MaxTokens int `json:"max_tokens"` + MaxTokens int `json:"max_tokens"` Temperature float64 `json:"temperature"` Context string `json:"system"` } @@ -40,7 +40,7 @@ type AnthropicUsage struct { func addAnthropicMessage(c *fiber.Ctx, llm LLM, selected bool) edgedb.UUID { Messages := getAllSelectedMessages(c) - chatCompletion, err := RequestAnthropic(c, llm.Model.ModelID, Messages, int(llm.Model.MaxToken), float64(llm.Temperature), llm.Context) + chatCompletion, err := RequestAnthropic(c, llm.Model.ModelID, Messages, float64(llm.Temperature), llm.Context, int(llm.MaxToken)) if err != nil { fmt.Println("Error requesting Anthropic: ", err) id := insertBotMessage(c, "Error requesting Anthropic, model may not be available anymore. Better error message in development.", selected, llm.ID) @@ -110,8 +110,14 @@ func TestAnthropicKey(apiKey string) bool { return true } -func RequestAnthropic(c *fiber.Ctx, model string, messages []Message, maxTokens int, temperature float64, context string) (AnthropicChatCompletionResponse, error) { - var apiKey struct { +func RequestAnthropic(c *fiber.Ctx, model string, messages []Message, temperature float64, context string, maxTokens int) (AnthropicChatCompletionResponse, error) { + if maxTokens == 0 { + maxTokens = 4096 + } + + fmt.Println("Requesting anthropic using max token:", maxTokens) + + var apiKey struct { Key string `edgedb:"key"` } err := edgeGlobalClient.WithGlobals(map[string]interface{}{"ext::auth::client_token": c.Cookies("jade-edgedb-auth-token")}).QuerySingle(edgeCtx, ` @@ -130,7 +136,7 @@ func RequestAnthropic(c *fiber.Ctx, model string, messages []Message, maxTokens requestBody := AnthropicChatCompletionRequest{ Model: model, Messages: Message2RequestMessage(messages, ""), - MaxTokens: maxTokens, + MaxTokens: maxTokens, Temperature: temperature, Context: context, } diff --git a/dbschema/default.esdl b/dbschema/default.esdl index 7d70c52..4aac389 100644 --- a/dbschema/default.esdl +++ b/dbschema/default.esdl @@ -107,6 +107,7 @@ module default { required user: User { on target delete delete source; }; + required max_token: int32; custom_endpoint: CustomEndpoint { on source delete delete target; }; @@ -119,15 +120,14 @@ module default { type Company { required name: str; - required icon: str; + required icon: str; } type ModelInfo { required modelID: str; - required name: str; - required maxToken: int32; - required inputPrice: float32; - required outputPrice: float32; + required name: str; + required inputPrice: float32; + required outputPrice: float32; required company: Company; } } diff --git a/dbschema/migrations/00051-m1qx7si.edgeql b/dbschema/migrations/00051-m1qx7si.edgeql new file mode 100644 index 0000000..051bd3e --- /dev/null +++ b/dbschema/migrations/00051-m1qx7si.edgeql @@ -0,0 +1,10 @@ +CREATE MIGRATION m1qx7silv6ge4i7hkzlbhrpsbi5ddlat57njswhr44f2lpqcdal5sa + ONTO m1kkf4hwv2zxvzjxckc4semtent43nd4ed4bn5462ph2ffpocpo24a +{ + ALTER TYPE default::LLM { + CREATE PROPERTY max_token: std::int32; + }; + ALTER TYPE default::ModelInfo { + DROP PROPERTY maxToken; + }; +}; diff --git a/dbschema/migrations/00052-m1ad7ps.edgeql b/dbschema/migrations/00052-m1ad7ps.edgeql new file mode 100644 index 0000000..b6d657c --- /dev/null +++ b/dbschema/migrations/00052-m1ad7ps.edgeql @@ -0,0 +1,9 @@ +CREATE MIGRATION m1ad7psqkud35f6dmlabih4aqrlwrcoq2vlnmzfavf3dy6avyxsw2q + ONTO m1qx7silv6ge4i7hkzlbhrpsbi5ddlat57njswhr44f2lpqcdal5sa +{ + ALTER TYPE default::LLM { + ALTER PROPERTY max_token { + SET REQUIRED USING (1024); + }; + }; +}; diff --git a/views/partials/popover-models.html b/views/partials/popover-models.html index f0121bf..f66c23e 100644 --- a/views/partials/popover-models.html +++ b/views/partials/popover-models.html @@ -50,7 +50,7 @@ {% for modelInfo in ModelInfos %} {% endfor %} - +

System prompt:

+

Max token (optional):

+ +

System prompt (optional):

@@ -171,4 +174,4 @@ } }); -
\ No newline at end of file +