Changed max token from ModelInfo to LLM

EdgeDatabase.go

@@ -77,6 +77,7 @@ type LLM struct {
 	ID          edgedb.UUID    `edgedb:"id"`
 	Name        string         `edgedb:"name"`
 	Context     string         `edgedb:"context"`
+  MaxToken    int32          `edgedb:"max_token"`
 	Temperature float32        `edgedb:"temperature"`
 	Model       ModelInfo      `edgedb:"modelInfo"`
 	Endpoint    CustomEndpoint `edgedb:"custom_endpoint"`
@@ -92,7 +93,6 @@ type CustomEndpoint struct {
 type ModelInfo struct {
 	ID          edgedb.UUID `edgedb:"id"`
 	Name        string      `edgedb:"name"`
-	MaxToken    int32       `edgedb:"maxToken"`
 	InputPrice  float32     `edgedb:"inputPrice"`
 	OutputPrice float32     `edgedb:"outputPrice"`
 	ModelID     string      `edgedb:"modelID"`

LLM.go

@@ -61,7 +61,16 @@ func createLLM(c *fiber.Ctx) error {
 	url := c.FormValue("model-url-input")
 	token := c.FormValue("model-key-input")
 	customID := c.FormValue("model-cid-input")
+  maxTokenStr := c.FormValue("max-token-input")
 
+  maxToken, err := strconv.Atoi(maxTokenStr)
+	if err != nil {
+	  maxToken = 0
+  }
+
+  fmt.Println("Adding LLM with maxtoken:", maxToken)
+
+  // TODO change the company
 	if modelID == "custom" {
 		err := edgeGlobalClient.WithGlobals(map[string]interface{}{"ext::auth::client_token": c.Cookies("jade-edgedb-auth-token")}).Execute(edgeCtx, `
 		WITH
@@ -71,12 +80,12 @@ func createLLM(c *fiber.Ctx) error {
 			context := <str>$1,
 			temperature := <float32>$2,
 			position := countLLM + 1,
+      max_token := <int32>$6,
 			modelInfo := (INSERT ModelInfo {
 				name := <str>$0,
 				modelID := <str>$5,
 				inputPrice := 0.0,
 				outputPrice := 0.0,
-				maxToken := 500,
 				company := (SELECT Company FILTER .name = "huggingface" LIMIT 1),
 			}),
 			custom_endpoint := (INSERT CustomEndpoint {
@@ -85,7 +94,7 @@ func createLLM(c *fiber.Ctx) error {
 			}),
 			user := global currentUser
 		};
-		`, name, systemPrompt, temperatureFloat, url, token, customID) // TODO Add real max token
+		`, name, systemPrompt, temperatureFloat, url, token, customID, int32(maxToken))
 		if err != nil {
 			fmt.Println("Error creating LLM")
 			panic(err)
@@ -99,10 +108,11 @@ func createLLM(c *fiber.Ctx) error {
 			context := <str>$1,
 			temperature := <float32>$2,
 			position := countLLM + 1,
+      max_token := <int32>$4,
 			modelInfo := (SELECT ModelInfo FILTER .modelID = <str>$3 LIMIT 1),
 			user := global currentUser
 		}
-	`, name, systemPrompt, temperatureFloat, modelID)
+	`, name, systemPrompt, temperatureFloat, modelID, int32(maxToken))
 		if err != nil {
 			fmt.Println("Error creating LLM")
 			panic(err)

Request.go

@@ -62,7 +62,6 @@ func GeneratePlaceholderHTML(c *fiber.Ctx, message string, selectedLLMIds []stri
 				},
 				modelInfo : {
 					modelID,
-					maxToken,
 					company : {
 						icon,
 						name

RequestAnthropic.go

@@ -40,7 +40,7 @@ type AnthropicUsage struct {
 func addAnthropicMessage(c *fiber.Ctx, llm LLM, selected bool) edgedb.UUID {
 	Messages := getAllSelectedMessages(c)
 
-	chatCompletion, err := RequestAnthropic(c, llm.Model.ModelID, Messages, int(llm.Model.MaxToken), float64(llm.Temperature), llm.Context)
+	chatCompletion, err := RequestAnthropic(c, llm.Model.ModelID, Messages, float64(llm.Temperature), llm.Context, int(llm.MaxToken))
 	if err != nil {
     fmt.Println("Error requesting Anthropic: ", err)
     id := insertBotMessage(c, "Error requesting Anthropic, model may not be available anymore. Better error message in development.", selected, llm.ID)
@@ -110,7 +110,13 @@ func TestAnthropicKey(apiKey string) bool {
 	return true
 }
 
-func RequestAnthropic(c *fiber.Ctx, model string, messages []Message, maxTokens int, temperature float64, context string) (AnthropicChatCompletionResponse, error) {
+func RequestAnthropic(c *fiber.Ctx, model string, messages []Message, temperature float64, context string, maxTokens int) (AnthropicChatCompletionResponse, error) {
+	if maxTokens == 0 {
+    maxTokens = 4096 
+  }
+
+  fmt.Println("Requesting anthropic using max token:", maxTokens)
+
   var apiKey struct {
 		Key string `edgedb:"key"`
 	}

dbschema/default.esdl

@@ -107,6 +107,7 @@ module default {
         required user: User {
             on target delete delete source;
         };
+        required max_token: int32;
         custom_endpoint: CustomEndpoint {
             on source delete delete target;
         };
@@ -125,7 +126,6 @@ module default {
     type ModelInfo {
         required modelID:     str;
 		    required name:        str;
-		required maxToken:    int32;
 		    required inputPrice:  float32;
 		    required outputPrice: float32;
         required company: Company;

dbschema/migrations/00051-m1qx7si.edgeql

@@ -0,0 +1,10 @@
+CREATE MIGRATION m1qx7silv6ge4i7hkzlbhrpsbi5ddlat57njswhr44f2lpqcdal5sa
+    ONTO m1kkf4hwv2zxvzjxckc4semtent43nd4ed4bn5462ph2ffpocpo24a
+{
+  ALTER TYPE default::LLM {
+      CREATE PROPERTY max_token: std::int32;
+  };
+  ALTER TYPE default::ModelInfo {
+      DROP PROPERTY maxToken;
+  };
+};

dbschema/migrations/00052-m1ad7ps.edgeql

@@ -0,0 +1,9 @@
+CREATE MIGRATION m1ad7psqkud35f6dmlabih4aqrlwrcoq2vlnmzfavf3dy6avyxsw2q
+    ONTO m1qx7silv6ge4i7hkzlbhrpsbi5ddlat57njswhr44f2lpqcdal5sa
+{
+  ALTER TYPE default::LLM {
+      ALTER PROPERTY max_token {
+          SET REQUIRED USING (<std::int32>1024);
+      };
+  };
+};

views/partials/popover-models.html

@@ -50,7 +50,7 @@
                             {% for modelInfo in ModelInfos %}
                             <option value="{{ modelInfo.ModelID }}">{{ modelInfo.Company.Name }} - {{ modelInfo.Name }}</option>
                             {% endfor %}
-                            <option value="{% if IsSub %}custom{% else %}none{% endif %}">Inference Endpoints</option>
+                            <option value="{% if IsSub %}custom{% else %}none{% endif %}">Custom Endpoints</option>
                         </select>
                     </div>
                     <p class="is-hidden" style="color: red;" id="endpoint-error">
@@ -66,7 +66,10 @@
                     <input class="slider is-small mb-3" step="0.05" min="0" max="2" value="0" type="range"
                         id="temperature-slider" name="temperature-slider">
                     <output id="temperature-slider-output">0</output>
-                    <p><small>System prompt:</small></p>
+                    <p><small>Max token (optional):</small></p>
+                    <input class="input is-small mb-3" type="number" id="max-token-input" name="max-token-input"
+                        placeholder="" autocomplete="off">
+                    <p><small>System prompt (optional):</small></p>
                     <textarea class="textarea is-small mb-5 has-fixed-size" id="model-prompt-input"
                         name="model-prompt-input"></textarea>
                     <div class="is-flex is-justify-content-flex-end">