From 6c34e481805e54da6210b93defbd68ae4db26a0f Mon Sep 17 00:00:00 2001
From: Krrish Dholakia <krrishdholakia@gmail.com>
Date: Wed, 27 Mar 2024 10:15:07 -0700
Subject: [PATCH] feat(index.ts): initial commit for proxy edge worker

testing to see if a js worker improves proxy perf (and by how much)
---
 litellm-js/proxy/README.md     |  8 +++++
 litellm-js/proxy/package.json  | 14 ++++++++
 litellm-js/proxy/src/index.ts  | 59 ++++++++++++++++++++++++++++++++++
 litellm-js/proxy/tsconfig.json | 16 +++++++++
 litellm-js/proxy/wrangler.toml | 18 +++++++++++
 5 files changed, 115 insertions(+)
 create mode 100644 litellm-js/proxy/README.md
 create mode 100644 litellm-js/proxy/package.json
 create mode 100644 litellm-js/proxy/src/index.ts
 create mode 100644 litellm-js/proxy/tsconfig.json
 create mode 100644 litellm-js/proxy/wrangler.toml

diff --git a/litellm-js/proxy/README.md b/litellm-js/proxy/README.md
new file mode 100644
index 000000000..cc58e962d
--- /dev/null
+++ b/litellm-js/proxy/README.md
@@ -0,0 +1,8 @@
+```
+npm install
+npm run dev
+```
+
+```
+npm run deploy
+```
diff --git a/litellm-js/proxy/package.json b/litellm-js/proxy/package.json
new file mode 100644
index 000000000..f63cf36d2
--- /dev/null
+++ b/litellm-js/proxy/package.json
@@ -0,0 +1,14 @@
+{
+  "scripts": {
+    "dev": "wrangler dev src/index.ts",
+    "deploy": "wrangler deploy --minify src/index.ts"
+  },
+  "dependencies": {
+    "hono": "^4.1.4",
+    "openai": "^4.29.2"
+  },
+  "devDependencies": {
+    "@cloudflare/workers-types": "^4.20240208.0",
+    "wrangler": "^3.32.0"
+  }
+}
diff --git a/litellm-js/proxy/src/index.ts b/litellm-js/proxy/src/index.ts
new file mode 100644
index 000000000..dc5dc9c68
--- /dev/null
+++ b/litellm-js/proxy/src/index.ts
@@ -0,0 +1,59 @@
+import { Hono } from 'hono'
+import { Context } from 'hono';
+import { bearerAuth } from 'hono/bearer-auth'
+import OpenAI from "openai";
+
+const openai = new OpenAI({
+  apiKey: "sk-1234",
+  baseURL: "https://openai-endpoint.ishaanjaffer0324.workers.dev"
+});
+
+async function call_proxy() {
+  const completion = await openai.chat.completions.create({
+    messages: [{ role: "system", content: "You are a helpful assistant." }],
+    model: "gpt-3.5-turbo",
+  });
+
+  return completion
+}
+
+const app = new Hono()
+
+// Middleware for API Key Authentication
+const apiKeyAuth = async (c: Context, next: Function) => {
+  const apiKey = c.req.header('Authorization');
+  if (!apiKey || apiKey !== 'Bearer sk-1234') {
+    return c.text('Unauthorized', 401);
+  }
+  await next();
+};
+
+
+app.use('/*', apiKeyAuth)
+
+
+app.get('/', (c) => {
+  return c.text('Hello Hono!')
+})
+
+
+
+
+// Handler for chat completions
+const chatCompletionHandler = async (c: Context) => {
+  // Assuming your logic for handling chat completion goes here
+  // For demonstration, just returning a simple JSON response
+  const response = await call_proxy()
+  return c.json(response);
+};
+
+// Register the above handler for different POST routes with the apiKeyAuth middleware
+app.post('/v1/chat/completions', chatCompletionHandler);
+app.post('/chat/completions', chatCompletionHandler);
+
+// Example showing how you might handle dynamic segments within the URL
+// Here, using ':model*' to capture the rest of the path as a parameter 'model'
+app.post('/openai/deployments/:model*/chat/completions', chatCompletionHandler);
+
+
+export default app
diff --git a/litellm-js/proxy/tsconfig.json b/litellm-js/proxy/tsconfig.json
new file mode 100644
index 000000000..33a96fd08
--- /dev/null
+++ b/litellm-js/proxy/tsconfig.json
@@ -0,0 +1,16 @@
+{
+  "compilerOptions": {
+    "target": "ESNext",
+    "module": "ESNext",
+    "moduleResolution": "Bundler",
+    "strict": true,
+    "lib": [
+      "ESNext"
+    ],
+    "types": [
+      "@cloudflare/workers-types"
+    ],
+    "jsx": "react-jsx",
+    "jsxImportSource": "hono/jsx"
+  },
+}
\ No newline at end of file
diff --git a/litellm-js/proxy/wrangler.toml b/litellm-js/proxy/wrangler.toml
new file mode 100644
index 000000000..e7c323dff
--- /dev/null
+++ b/litellm-js/proxy/wrangler.toml
@@ -0,0 +1,18 @@
+name = "my-app"
+compatibility_date = "2023-12-01"
+
+# [vars]
+# MY_VAR = "my-variable"
+
+# [[kv_namespaces]]
+# binding = "MY_KV_NAMESPACE"
+# id = "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx"
+
+# [[r2_buckets]]
+# binding = "MY_BUCKET"
+# bucket_name = "my-bucket"
+
+# [[d1_databases]]
+# binding = "DB"
+# database_name = "my-database"
+# database_id = ""