import React, { useState, useEffect } from "react"; import { Card, Title, Subtitle, Table, TableHead, TableRow, Badge, TableHeaderCell, TableCell, TableBody, Metric, Text, Grid, Button, TextInput, Select as Select2, SelectItem, Col, Accordion, AccordionBody, AccordionHeader, AccordionList, } from "@tremor/react"; import { TabPanel, TabPanels, TabGroup, TabList, Tab, Icon, } from "@tremor/react"; import { getCallbacksCall, setCallbacksCall, getGeneralSettingsCall, serviceHealthCheck, updateConfigFieldSetting, deleteConfigFieldSetting, } from "./networking"; import { Modal, Form, Input, Select, Button as Button2, message, InputNumber, } from "antd"; import { InformationCircleIcon, PencilAltIcon, PencilIcon, StatusOnlineIcon, TrashIcon, RefreshIcon, CheckCircleIcon, XCircleIcon, QuestionMarkCircleIcon, } from "@heroicons/react/outline"; import AddFallbacks from "./add_fallbacks"; import openai from "openai"; import Paragraph from "antd/es/skeleton/Paragraph"; interface GeneralSettingsPageProps { accessToken: string | null; userRole: string | null; userID: string | null; modelData: any; } async function testFallbackModelResponse( selectedModel: string, accessToken: string ) { // base url should be the current base_url const isLocal = process.env.NODE_ENV === "development"; if (isLocal != true) { console.log = function() {}; } console.log("isLocal:", isLocal); const proxyBaseUrl = isLocal ? "http://localhost:4000" : window.location.origin; const client = new openai.OpenAI({ apiKey: accessToken, // Replace with your OpenAI API key baseURL: proxyBaseUrl, // Replace with your OpenAI API base URL dangerouslyAllowBrowser: true, // using a temporary litellm proxy key }); try { const response = await client.chat.completions.create({ model: selectedModel, messages: [ { role: "user", content: "Hi, this is a test message", }, ], // @ts-ignore mock_testing_fallbacks: true, }); message.success( Test model={selectedModel}, received model= {response.model}. See{" "} window.open( "https://docs.litellm.ai/docs/proxy/reliability", "_blank" ) } style={{ textDecoration: "underline", color: "blue" }} > curl ); } catch (error) { message.error( `Error occurred while generating model response. Please try again. Error: ${error}`, 20 ); } } interface AccordionHeroProps { selectedStrategy: string | null; strategyArgs: routingStrategyArgs; paramExplanation: { [key: string]: string }; } interface routingStrategyArgs { ttl?: number; lowest_latency_buffer?: number; } interface generalSettingsItem { field_name: string; field_type: string; field_value: any; field_description: string; stored_in_db: boolean | null; } const defaultLowestLatencyArgs: routingStrategyArgs = { ttl: 3600, lowest_latency_buffer: 0, }; export const AccordionHero: React.FC = ({ selectedStrategy, strategyArgs, paramExplanation, }) => ( Routing Strategy Specific Args {selectedStrategy == "latency-based-routing" ? ( Setting Value {Object.entries(strategyArgs).map(([param, value]) => ( {param} {paramExplanation[param]} ))} ) : ( No specific settings )} ); const GeneralSettings: React.FC = ({ accessToken, userRole, userID, modelData, }) => { const [routerSettings, setRouterSettings] = useState<{ [key: string]: any }>( {} ); const [generalSettingsDict, setGeneralSettingsDict] = useState<{ [key: string]: any; }>({}); const [generalSettings, setGeneralSettings] = useState( [] ); const [isModalVisible, setIsModalVisible] = useState(false); const [form] = Form.useForm(); const [selectedCallback, setSelectedCallback] = useState(null); const [selectedStrategy, setSelectedStrategy] = useState(null); const [strategySettings, setStrategySettings] = useState(null); let paramExplanation: { [key: string]: string } = { routing_strategy_args: "(dict) Arguments to pass to the routing strategy", routing_strategy: "(string) Routing strategy to use", allowed_fails: "(int) Number of times a deployment can fail before being added to cooldown", cooldown_time: "(int) time in seconds to cooldown a deployment after failure", num_retries: "(int) Number of retries for failed requests. Defaults to 0.", timeout: "(float) Timeout for requests. Defaults to None.", retry_after: "(int) Minimum time to wait before retrying a failed request", ttl: "(int) Sliding window to look back over when calculating the average latency of a deployment. Default - 1 hour (in seconds).", lowest_latency_buffer: "(float) Shuffle between deployments within this % of the lowest latency. Default - 0 (i.e. always pick lowest latency).", }; useEffect(() => { if (!accessToken || !userRole || !userID) { return; } getCallbacksCall(accessToken, userID, userRole).then((data) => { console.log("callbacks", data); let router_settings = data.router_settings; // remove "model_group_retry_policy" from general_settings if exists if ("model_group_retry_policy" in router_settings) { delete router_settings["model_group_retry_policy"]; } setRouterSettings(router_settings); }); getGeneralSettingsCall(accessToken).then((data) => { let general_settings = data; setGeneralSettings(general_settings); }); }, [accessToken, userRole, userID]); const handleAddCallback = () => { console.log("Add callback clicked"); setIsModalVisible(true); }; const handleCancel = () => { setIsModalVisible(false); form.resetFields(); setSelectedCallback(null); }; const deleteFallbacks = async (key: string) => { /** * pop the key from the Object, if it exists */ if (!accessToken) { return; } console.log(`received key: ${key}`); console.log(`routerSettings['fallbacks']: ${routerSettings["fallbacks"]}`); routerSettings["fallbacks"].map((dict: { [key: string]: any }) => { // Check if the dictionary has the specified key and delete it if present if (key in dict) { delete dict[key]; } return dict; // Return the updated dictionary }); const payload = { router_settings: routerSettings, }; try { await setCallbacksCall(accessToken, payload); setRouterSettings({ ...routerSettings }); setSelectedStrategy(routerSettings["routing_strategy"]); message.success("Router settings updated successfully"); } catch (error) { message.error("Failed to update router settings: " + error, 20); } }; const handleInputChange = (fieldName: string, newValue: any) => { // Update the value in the state const updatedSettings = generalSettings.map((setting) => setting.field_name === fieldName ? { ...setting, field_value: newValue } : setting ); setGeneralSettings(updatedSettings); }; const handleUpdateField = (fieldName: string, idx: number) => { if (!accessToken) { return; } let fieldValue = generalSettings[idx].field_value; if (fieldValue == null || fieldValue == undefined) { return; } try { updateConfigFieldSetting(accessToken, fieldName, fieldValue); // update value in state const updatedSettings = generalSettings.map((setting) => setting.field_name === fieldName ? { ...setting, stored_in_db: true } : setting ); setGeneralSettings(updatedSettings); } catch (error) { // do something } }; const handleResetField = (fieldName: string, idx: number) => { if (!accessToken) { return; } try { deleteConfigFieldSetting(accessToken, fieldName); // update value in state const updatedSettings = generalSettings.map((setting) => setting.field_name === fieldName ? { ...setting, stored_in_db: null, field_value: null } : setting ); setGeneralSettings(updatedSettings); } catch (error) { // do something } }; const handleSaveChanges = (router_settings: any) => { if (!accessToken) { return; } console.log("router_settings", router_settings); const updatedVariables = Object.fromEntries( Object.entries(router_settings) .map(([key, value]) => { if (key !== "routing_strategy_args" && key !== "routing_strategy") { return [ key, ( document.querySelector( `input[name="${key}"]` ) as HTMLInputElement )?.value || value, ]; } else if (key == "routing_strategy") { return [key, selectedStrategy]; } else if ( key == "routing_strategy_args" && selectedStrategy == "latency-based-routing" ) { let setRoutingStrategyArgs: routingStrategyArgs = {}; const lowestLatencyBufferElement = document.querySelector( `input[name="lowest_latency_buffer"]` ) as HTMLInputElement; const ttlElement = document.querySelector( `input[name="ttl"]` ) as HTMLInputElement; if (lowestLatencyBufferElement?.value) { setRoutingStrategyArgs["lowest_latency_buffer"] = Number( lowestLatencyBufferElement.value ); } if (ttlElement?.value) { setRoutingStrategyArgs["ttl"] = Number(ttlElement.value); } console.log(`setRoutingStrategyArgs: ${setRoutingStrategyArgs}`); return ["routing_strategy_args", setRoutingStrategyArgs]; } return null; }) .filter((entry) => entry !== null && entry !== undefined) as Iterable< [string, unknown] > ); console.log("updatedVariables", updatedVariables); const payload = { router_settings: updatedVariables, }; try { setCallbacksCall(accessToken, payload); } catch (error) { message.error("Failed to update router settings: " + error, 20); } message.success("router settings updated successfully"); }; if (!accessToken) { return null; } return ( Loadbalancing Fallbacks General Router Settings Setting Value {Object.entries(routerSettings) .filter( ([param, value]) => param != "fallbacks" && param != "context_window_fallbacks" && param != "routing_strategy_args" ) .map(([param, value]) => ( {param} {paramExplanation[param]} {param == "routing_strategy" ? ( usage-based-routing latency-based-routing simple-shuffle ) : ( )} ))} 0 ? routerSettings["routing_strategy_args"] : defaultLowestLatencyArgs // default value when keys length is 0 } paramExplanation={paramExplanation} /> handleSaveChanges(routerSettings)} > Save Changes Model Name Fallbacks {routerSettings["fallbacks"] && routerSettings["fallbacks"].map( (item: Object, index: number) => Object.entries(item).map(([key, value]) => ( {key} {Array.isArray(value) ? value.join(", ") : value} testFallbackModelResponse(key, accessToken) } > Test Fallback deleteFallbacks(key)} /> )) )} data.model_name) : [] } accessToken={accessToken} routerSettings={routerSettings} setRouterSettings={setRouterSettings} /> Setting Value Status Action {generalSettings.filter((value) => value.field_type !== "TypedDictionary").map((value, index) => ( {value.field_name} {value.field_description} {value.field_type == "Integer" ? ( handleInputChange(value.field_name, newValue) } // Handle value change /> ) : null} {value.stored_in_db == true ? ( In DB ) : value.stored_in_db == false ? ( In Config ) : ( Not Set )} handleUpdateField(value.field_name, index) } > Update handleResetField(value.field_name, index) } > Reset ))} ); }; export default GeneralSettings;
{paramExplanation[param]}
{value.field_description}