Merge pull request #3869 from BerriAI/litellm_public_model_hubg

Admin UI - Public model hub

litellm/proxy/_types.py

@@ -928,6 +928,10 @@ class ConfigGeneralSettings(LiteLLMBase):
     allowed_routes: Optional[List] = Field(
         None, description="Proxy API Endpoints you want users to be able to access"
     )
+    enable_public_model_hub: bool = Field(
+        default=False,
+        description="Public model hub for users to see what models they have access to, supported openai params, etc.",
+    )
 
 
 class ConfigYAML(LiteLLMBase):
@@ -1154,3 +1158,8 @@ class WebhookEvent(CallInfo):
 class SpecialModelNames(enum.Enum):
     all_team_models = "all-team-models"
     all_proxy_models = "all-proxy-models"
+
+
+class ConfigFieldInfo(LiteLLMBase):
+    field_name: str
+    field_value: Any

litellm/proxy/proxy_server.py

@@ -11067,6 +11067,7 @@ async def update_config_general_settings(
     "/config/field/info",
     tags=["config.yaml"],
     dependencies=[Depends(user_api_key_auth)],
+    response_model=ConfigFieldInfo,
 )
 async def get_config_general_settings(
     field_name: str,
@@ -11113,10 +11114,9 @@ async def get_config_general_settings(
         general_settings = dict(db_general_settings.param_value)
 
         if field_name in general_settings:
-            return {
+            return ConfigFieldInfo(
-                "field_name": field_name,
+                field_name=field_name, field_value=general_settings[field_name]
-                "field_value": general_settings[field_name],
+            )
-            }
         else:
             raise HTTPException(
                 status_code=400,

ui/litellm-dashboard/src/app/model_hub/page.tsx

@@ -0,0 +1,25 @@
+"use client";
+import React, { Suspense, useEffect, useState } from "react";
+import { useSearchParams } from "next/navigation";
+import { modelHubCall } from "@/components/networking";
+import ModelHub from "@/components/model_hub";
+
+export default function PublicModelHub() {
+  const searchParams = useSearchParams();
+  const key = searchParams.get("key");
+  const [accessToken, setAccessToken] = useState<string | null>(null);
+
+  useEffect(() => {
+    if (!key) {
+      return;
+    }
+    setAccessToken(key);
+  }, [key]);
+  /**
+   * populate navbar
+   *
+   */
+  return (
+    <ModelHub accessToken={accessToken} publicPage={true} premiumUser={false} />
+  );
+}

ui/litellm-dashboard/src/app/page.tsx

@@ -18,6 +18,30 @@ import Usage from "../components/usage";
 import { jwtDecode } from "jwt-decode";
 import { Typography } from "antd";
 
+export function formatUserRole(userRole: string) {
+  if (!userRole) {
+    return "Undefined Role";
+  }
+  console.log(`Received user role: ${userRole.toLowerCase()}`);
+  console.log(`Received user role length: ${userRole.toLowerCase().length}`);
+  switch (userRole.toLowerCase()) {
+    case "app_owner":
+      return "App Owner";
+    case "demo_app_owner":
+      return "App Owner";
+    case "app_admin":
+      return "Admin";
+    case "proxy_admin":
+      return "Admin";
+    case "proxy_admin_viewer":
+      return "Admin Viewer";
+    case "app_user":
+      return "App User";
+    default:
+      return "Unknown Role";
+  }
+}
+
 const CreateKeyPage = () => {
   const { Title, Paragraph } = Typography;
   const [userRole, setUserRole] = useState("");
@@ -78,30 +102,6 @@ const CreateKeyPage = () => {
     }
   }, [token]);
 
-  function formatUserRole(userRole: string) {
-    if (!userRole) {
-      return "Undefined Role";
-    }
-    console.log(`Received user role: ${userRole.toLowerCase()}`);
-    console.log(`Received user role length: ${userRole.toLowerCase().length}`);
-    switch (userRole.toLowerCase()) {
-      case "app_owner":
-        return "App Owner";
-      case "demo_app_owner":
-        return "App Owner";
-      case "app_admin":
-        return "Admin";
-      case "proxy_admin":
-        return "Admin";
-      case "proxy_admin_viewer":
-        return "Admin Viewer";
-      case "app_user":
-        return "App User";
-      default:
-        return "Unknown Role";
-    }
-  }
-
   return (
     <Suspense fallback={<div>Loading...</div>}>
       <div className="flex flex-col min-h-screen">
@@ -194,17 +194,13 @@ const CreateKeyPage = () => {
               accessToken={accessToken}
               modelData={modelData}
             />
-            ) : page == "model-hub" ? (
+          ) : page == "model-hub" ? (
-              <ModelHub
+            <ModelHub
-              userID={userID}
-              userRole={userRole}
-              token={token}
               accessToken={accessToken}
-              keys={keys}
+              publicPage={false}
               premiumUser={premiumUser}
             />
-            )
+          ) : (
-            : (
             <Usage
               userID={userID}
               userRole={userRole}

ui/litellm-dashboard/src/components/model_hub.tsx

@@ -1,7 +1,8 @@
 import React, { useEffect, useState } from "react";
+import { useRouter, usePathname, useSearchParams } from "next/navigation";
 
 import { modelHubCall } from "./networking";
-
+import { getConfigFieldSetting, updateConfigFieldSetting } from "./networking";
 import {
   Card,
   Text,
@@ -15,70 +16,70 @@ import {
   TabPanel,
   TabPanels,
 } from "@tremor/react";
-
 import { RightOutlined, CopyOutlined } from "@ant-design/icons";
 
-import { Modal, Tooltip } from "antd";
+import { Modal, Tooltip, message } from "antd";
 import { Prism as SyntaxHighlighter } from "react-syntax-highlighter";
 
 interface ModelHubProps {
-  userID: string | null;
-  userRole: string | null;
-  token: string | null;
   accessToken: string | null;
-  keys: any; // Replace with the appropriate type for 'keys' prop
+  publicPage: boolean;
   premiumUser: boolean;
 }
 
 interface ModelInfo {
-    model_group: string;
+  model_group: string;
-    mode: string;
+  mode: string;
-    supports_function_calling: boolean;
+  supports_function_calling: boolean;
-    supports_vision: boolean;
+  supports_vision: boolean;
-    max_input_tokens?: number;
+  max_input_tokens?: number;
-    max_output_tokens?: number;
+  max_output_tokens?: number;
-    supported_openai_params?: string[];
+  supported_openai_params?: string[];
-
-
-  // Add other properties if needed
 }
 
 const ModelHub: React.FC<ModelHubProps> = ({
-  userID,
-
-  userRole,
-
-  token,
-
   accessToken,
-
+  publicPage,
-  keys,
-
   premiumUser,
 }) => {
+  const [publicPageAllowed, setPublicPageAllowed] = useState<boolean>(false);
   const [modelHubData, setModelHubData] = useState<ModelInfo[] | null>(null);
   const [isModalVisible, setIsModalVisible] = useState(false);
+  const [isPublicPageModalVisible, setIsPublicPageModalVisible] =
+    useState(false);
   const [selectedModel, setSelectedModel] = useState<null | ModelInfo>(null);
+  const router = useRouter();
 
   useEffect(() => {
-    if (!accessToken || !token || !userRole || !userID) {
+    if (!accessToken) {
       return;
     }
 
     const fetchData = async () => {
       try {
-        const _modelHubData = await modelHubCall(accessToken, userID, userRole);
+        const _modelHubData = await modelHubCall(accessToken);
 
         console.log("ModelHubData:", _modelHubData);
 
         setModelHubData(_modelHubData.data);
+
+        getConfigFieldSetting(accessToken, "enable_public_model_hub")
+          .then((data) => {
+            console.log(`data: ${JSON.stringify(data)}`);
+            if (data.field_value == true) {
+              setPublicPageAllowed(true);
+            }
+          })
+          .catch((error) => {
+            // do nothing
+          });
       } catch (error) {
         console.error("There was an error fetching the model data", error);
       }
     };
 
     fetchData();
-  }, [accessToken, token, userRole, userID]);
+  }, [accessToken, publicPage]);
 
   const showModal = (model: ModelInfo) => {
     setSelectedModel(model);
@@ -86,15 +87,29 @@ const ModelHub: React.FC<ModelHubProps> = ({
     setIsModalVisible(true);
   };
 
+  const goToPublicModelPage = () => {
+    router.replace(`/model_hub?key=${accessToken}`);
+  };
+  const handleMakePublicPage = async () => {
+    if (!accessToken) {
+      return;
+    }
+    updateConfigFieldSetting(accessToken, "enable_public_model_hub", true).then(
+      (data) => {
+        setIsPublicPageModalVisible(true);
+      }
+    );
+  };
+
   const handleOk = () => {
     setIsModalVisible(false);
-
+    setIsPublicPageModalVisible(false);
     setSelectedModel(null);
   };
 
   const handleCancel = () => {
     setIsModalVisible(false);
-
+    setIsPublicPageModalVisible(false);
     setSelectedModel(null);
   };
 
@@ -104,55 +119,118 @@ const ModelHub: React.FC<ModelHubProps> = ({
 
   return (
     <div>
-      <div className="w-full m-2 mt-2 p-8">
+      {(publicPage && publicPageAllowed) || publicPage == false ? (
-        <div className="relative w-full"></div>
+        <div className="w-full m-2 mt-2 p-8">
+          <div className="relative w-full"></div>
 
-        <div className="flex items-center">
+          <div
-          <Title className="ml-8 text-center ">Model Hub</Title>
+            className={`flex ${publicPage ? "justify-between" : "items-center"}`}
-          <Button className="ml-4">
+          >
-            <a href="https://forms.gle/W3U4PZpJGFHWtHyA9" target="_blank">
+            <Title className="ml-8 text-center ">Model Hub</Title>
-              ✨ Make Public
+            {publicPage == false ? (
-            </a>
+              premiumUser ? (
-          </Button>
+                <Button className="ml-4" onClick={() => handleMakePublicPage()}>
-        </div>
+                  ✨ Make Public
-
+                </Button>
-        <div className="grid grid-cols-2 gap-6 sm:grid-cols-3 lg:grid-cols-4">
+              ) : (
-          {modelHubData &&
+                <Button className="ml-4">
-            modelHubData.map((model: ModelInfo) => (
+                  <a href="https://forms.gle/W3U4PZpJGFHWtHyA9" target="_blank">
-              <Card key={model.model_group} className="mt-5 mx-8">
+                    ✨ Make Public
-                <pre className="flex justify-between">
-                  <Title>{model.model_group}</Title>
-                  <Tooltip title={model.model_group}>
-                    <CopyOutlined
-                      onClick={() => copyToClipboard(model.model_group)}
-                      style={{ cursor: "pointer", marginRight: "10px" }}
-                    />
-                  </Tooltip>
-                </pre>
-              <div className='my-5'>
-
-              <Text>Mode: {model.mode}</Text>
-              <Text>Supports Function Calling: {model?.supports_function_calling == true ? "Yes" : "No"}</Text>
-              <Text>Supports Vision: {model?.supports_vision == true ? "Yes" : "No"}</Text>
-              <Text>Max Input Tokens: {model?.max_input_tokens ? model?.max_input_tokens : "N/A"}</Text>
-              <Text>Max Output Tokens: {model?.max_output_tokens ? model?.max_output_tokens : "N/A"}</Text>
-              </div>
-                <div style={{ marginTop: "auto", textAlign: "right" }}>
-                  <a
-                    href="#"
-                    onClick={() => showModal(model)}
-                    style={{ color: "#1890ff", fontSize: "smaller" }}
-                  >
-                    View more <RightOutlined />
                   </a>
-                </div>
+                </Button>
-              </Card>
+              )
-            ))}
+            ) : (
+              <div className="flex justify-between items-center">
+                <p>Filter by key:</p>
+                <Text className="bg-gray-200 pr-2 pl-2 pt-1 pb-1 text-center">{`/ui/model_hub?key=<YOUR_KEY>`}</Text>
+              </div>
+            )}
+          </div>
+
+          <div className="grid grid-cols-2 gap-6 sm:grid-cols-3 lg:grid-cols-4">
+            {modelHubData &&
+              modelHubData.map((model: ModelInfo) => (
+                <Card key={model.model_group} className="mt-5 mx-8">
+                  <pre className="flex justify-between">
+                    <Title>{model.model_group}</Title>
+                    <Tooltip title={model.model_group}>
+                      <CopyOutlined
+                        onClick={() => copyToClipboard(model.model_group)}
+                        style={{ cursor: "pointer", marginRight: "10px" }}
+                      />
+                    </Tooltip>
+                  </pre>
+                  <div className="my-5">
+                    <Text>Mode: {model.mode}</Text>
+                    <Text>
+                      Supports Function Calling:{" "}
+                      {model?.supports_function_calling == true ? "Yes" : "No"}
+                    </Text>
+                    <Text>
+                      Supports Vision:{" "}
+                      {model?.supports_vision == true ? "Yes" : "No"}
+                    </Text>
+                    <Text>
+                      Max Input Tokens:{" "}
+                      {model?.max_input_tokens
+                        ? model?.max_input_tokens
+                        : "N/A"}
+                    </Text>
+                    <Text>
+                      Max Output Tokens:{" "}
+                      {model?.max_output_tokens
+                        ? model?.max_output_tokens
+                        : "N/A"}
+                    </Text>
+                  </div>
+                  <div style={{ marginTop: "auto", textAlign: "right" }}>
+                    <a
+                      href="#"
+                      onClick={() => showModal(model)}
+                      style={{ color: "#1890ff", fontSize: "smaller" }}
+                    >
+                      View more <RightOutlined />
+                    </a>
+                  </div>
+                </Card>
+              ))}
+          </div>
         </div>
-      </div>
+      ) : (
+        <Card className="mx-auto max-w-xl mt-10">
+          <Text className="text-xl text-center mb-2 text-black">
+            Public Model Hub not enabled.
+          </Text>
+          <p className="text-base text-center text-slate-800">
+            Ask your proxy admin to enable this on their Admin UI.
+          </p>
+        </Card>
+      )}
 
       <Modal
-        title={selectedModel && selectedModel.model_group ? selectedModel.model_group : "Unknown Model"}
+        title={"Public Model Hub"}
+        width={600}
+        visible={isPublicPageModalVisible}
+        footer={null}
+        onOk={handleOk}
+        onCancel={handleCancel}
+      >
+        <div className="pt-5 pb-5">
+          <div className="flex justify-between mb-4">
+            <Text className="text-base mr-2">Shareable Link:</Text>
+            <Text className="max-w-sm ml-2 bg-gray-200 pr-2 pl-2 pt-1 pb-1 text-center rounded">{`<proxy_base_url>/ui/model_hub?key=<YOUR_API_KEY>`}</Text>
+          </div>
+          <div className="flex justify-end">
+            <Button onClick={goToPublicModelPage}>See Page</Button>
+          </div>
+        </div>
+      </Modal>
+      <Modal
+        title={
+          selectedModel && selectedModel.model_group
+            ? selectedModel.model_group
+            : "Unknown Model"
+        }
         width={800}
         visible={isModalVisible}
         footer={null}
@@ -161,19 +239,21 @@ const ModelHub: React.FC<ModelHubProps> = ({
       >
         {selectedModel && (
           <div>
-            <p className='mb-4'><strong>Model Information & Usage</strong></p>
+            <p className="mb-4">
-           
+              <strong>Model Information & Usage</strong>
+            </p>
+
             <TabGroup>
-                  <TabList>
+              <TabList>
-                    <Tab>OpenAI Python SDK</Tab>
+                <Tab>OpenAI Python SDK</Tab>
-                    <Tab>Supported OpenAI Params</Tab>
+                <Tab>Supported OpenAI Params</Tab>
-                    <Tab>LlamaIndex</Tab>
+                <Tab>LlamaIndex</Tab>
-                    <Tab>Langchain Py</Tab>
+                <Tab>Langchain Py</Tab>
-                  </TabList>
+              </TabList>
-                  <TabPanels>
+              <TabPanels>
-                    <TabPanel>
+                <TabPanel>
-                      <SyntaxHighlighter language="python">
+                  <SyntaxHighlighter language="python">
-                        {`
+                    {`
 import openai
 client = openai.OpenAI(
     api_key="your_api_key",
@@ -192,13 +272,13 @@ response = client.chat.completions.create(
 
 print(response)
             `}
-                      </SyntaxHighlighter>
+                  </SyntaxHighlighter>
-                    </TabPanel>
+                </TabPanel>
-                    <TabPanel>
+                <TabPanel>
-                    <SyntaxHighlighter language="python">
+                  <SyntaxHighlighter language="python">
-                        {`${selectedModel.supported_openai_params?.map((param) => `${param}\n`).join('')}`}
+                    {`${selectedModel.supported_openai_params?.map((param) => `${param}\n`).join("")}`}
-                        </SyntaxHighlighter>
+                  </SyntaxHighlighter>
-                        </TabPanel>
+                </TabPanel>
                 <TabPanel>
                   <SyntaxHighlighter language="python">
                     {`

ui/litellm-dashboard/src/components/networking.tsx

@@ -579,17 +579,14 @@ export const modelInfoCall = async (
   }
 };
 
-
+export const modelHubCall = async (accessToken: String) => {
-export const modelHubCall = async (
-  accessToken: String,
-  userID: String,
-  userRole: String
-) => {
   /**
    * Get all models on proxy
    */
   try {
-    let url = proxyBaseUrl ? `${proxyBaseUrl}/model_group/info` : `/model_group/info`;
+    let url = proxyBaseUrl
+      ? `${proxyBaseUrl}/model_group/info`
+      : `/model_group/info`;
 
     //message.info("Requesting model data");
     const response = await fetch(url, {
@@ -617,8 +614,6 @@ export const modelHubCall = async (
   }
 };
 
-
-
 export const modelMetricsCall = async (
   accessToken: String,
   userID: String,
@@ -1870,6 +1865,38 @@ export const getGeneralSettingsCall = async (accessToken: String) => {
   }
 };
 
+export const getConfigFieldSetting = async (
+  accessToken: String,
+  fieldName: string
+) => {
+  try {
+    let url = proxyBaseUrl
+      ? `${proxyBaseUrl}/config/field/info?field_name=${fieldName}`
+      : `/config/field/info?field_name=${fieldName}`;
+
+    //message.info("Requesting model data");
+    const response = await fetch(url, {
+      method: "GET",
+      headers: {
+        Authorization: `Bearer ${accessToken}`,
+        "Content-Type": "application/json",
+      },
+    });
+
+    if (!response.ok) {
+      const errorData = await response.text();
+      throw new Error("Network response was not ok");
+    }
+
+    const data = await response.json();
+    return data;
+    // Handle success - you might want to update some state or UI based on the created key
+  } catch (error) {
+    console.error("Failed to set callbacks:", error);
+    throw error;
+  }
+};
+
 export const updateConfigFieldSetting = async (
   accessToken: String,
   fieldName: string,