diff --git a/docs/my-website/docs/proxy_server.md b/docs/my-website/docs/proxy_server.md
index d2a9d521f..37e009e4c 100644
--- a/docs/my-website/docs/proxy_server.md
+++ b/docs/my-website/docs/proxy_server.md
@@ -208,6 +208,85 @@ user_proxy.initiate_chat(assistant, message="Plot a chart of META and TESLA stoc
Credits [@victordibia](https://github.com/microsoft/autogen/issues/45#issuecomment-1749921972) for this tutorial.
+
+```python
+from autogen import AssistantAgent, GroupChatManager, UserProxyAgent
+from autogen.agentchat import GroupChat
+
+config_list = [
+ {
+ "model": "ollama/mistralorca",
+ "api_base": "http://localhost:8000", # litellm compatible endpoint
+ "api_type": "open_ai",
+ "api_key": "NULL", # just a placeholder
+ }
+]
+llm_config = {"config_list": config_list, "seed": 42}
+
+code_config_list = [
+ {
+ "model": "ollama/phind-code",
+ "api_base": "http://localhost:8000", # litellm compatible endpoint
+ "api_type": "open_ai",
+ "api_key": "NULL", # just a placeholder
+ }
+]
+
+code_config = {"config_list": code_config_list, "seed": 42}
+
+admin = UserProxyAgent(
+ name="Admin",
+ system_message="A human admin. Interact with the planner to discuss the plan. Plan execution needs to be approved by this admin.",
+ llm_config=llm_config,
+ code_execution_config=False,
+)
+
+
+engineer = AssistantAgent(
+ name="Engineer",
+ llm_config=code_config,
+ system_message="""Engineer. You follow an approved plan. You write python/shell code to solve tasks. Wrap the code in a code block that specifies the script type. The user can't modify your code. So do not suggest incomplete code which requires others to modify. Don't use a code block if it's not intended to be executed by the executor.
+Don't include multiple code blocks in one response. Do not ask others to copy and paste the result. Check the execution result returned by the executor.
+If the result indicates there is an error, fix the error and output the code again. Suggest the full code instead of partial code or code changes. If the error can't be fixed or if the task is not solved even after the code is executed successfully, analyze the problem, revisit your assumption, collect additional info you need, and think of a different approach to try.
+""",
+)
+planner = AssistantAgent(
+ name="Planner",
+ system_message="""Planner. Suggest a plan. Revise the plan based on feedback from admin and critic, until admin approval.
+The plan may involve an engineer who can write code and a scientist who doesn't write code.
+Explain the plan first. Be clear which step is performed by an engineer, and which step is performed by a scientist.
+""",
+ llm_config=llm_config,
+)
+executor = UserProxyAgent(
+ name="Executor",
+ system_message="Executor. Execute the code written by the engineer and report the result.",
+ human_input_mode="NEVER",
+ llm_config=llm_config,
+ code_execution_config={"last_n_messages": 3, "work_dir": "paper"},
+)
+critic = AssistantAgent(
+ name="Critic",
+ system_message="Critic. Double check plan, claims, code from other agents and provide feedback. Check whether the plan includes adding verifiable info such as source URL.",
+ llm_config=llm_config,
+)
+groupchat = GroupChat(
+ agents=[admin, engineer, planner, executor, critic],
+ messages=[],
+ max_round=50,
+)
+manager = GroupChatManager(groupchat=groupchat, llm_config=llm_config)
+
+
+admin.initiate_chat(
+ manager,
+ message="""
+""",
+)
+```
+
+Credits [@Nathan](https://gist.github.com/CUexter) for this tutorial.
+
```python
diff --git a/litellm/__pycache__/main.cpython-311.pyc b/litellm/__pycache__/main.cpython-311.pyc
index a153c4930..f990b64c1 100644
Binary files a/litellm/__pycache__/main.cpython-311.pyc and b/litellm/__pycache__/main.cpython-311.pyc differ
diff --git a/litellm/__pycache__/utils.cpython-311.pyc b/litellm/__pycache__/utils.cpython-311.pyc
index 6fd96bed6..7a371d814 100644
Binary files a/litellm/__pycache__/utils.cpython-311.pyc and b/litellm/__pycache__/utils.cpython-311.pyc differ
diff --git a/litellm/llms/baseten.py b/litellm/llms/baseten.py
index 8f24e129e..aecacd84f 100644
--- a/litellm/llms/baseten.py
+++ b/litellm/llms/baseten.py
@@ -121,7 +121,7 @@ def completion(
sum_logprob = 0
for token in completion_response[0]["details"]["tokens"]:
sum_logprob += token["logprob"]
- model_response["choices"][0]["message"]["logprobs"] = sum_logprob
+ model_response["choices"][0]["message"]._logprobs = sum_logprob
else:
raise BasetenError(
message=f"Unable to parse response. Original response: {response.text}",
diff --git a/litellm/llms/huggingface_restapi.py b/litellm/llms/huggingface_restapi.py
index c63c31150..b3c3e5e38 100644
--- a/litellm/llms/huggingface_restapi.py
+++ b/litellm/llms/huggingface_restapi.py
@@ -141,7 +141,6 @@ def completion(
litellm_params=None,
logger_fn=None,
):
- print(f'headers inside hf rest api: {headers}')
headers = validate_environment(api_key, headers)
task = get_hf_task_for_model(model)
print_verbose(f"{model}, {task}")
@@ -254,8 +253,6 @@ def completion(
## Some servers might return streaming responses even though stream was not set to true. (e.g. Baseten)
is_streamed = False
- print(f"response keys: {response.__dict__.keys()}")
- print(f"response keys: {response.__dict__['headers']}")
if response.__dict__['headers']["Content-Type"] == "text/event-stream":
is_streamed = True
@@ -313,7 +310,7 @@ def completion(
sum_logprob = 0
for token in completion_response[0]["details"]["tokens"]:
sum_logprob += token["logprob"]
- model_response["choices"][0]["message"]["logprobs"] = sum_logprob
+ model_response["choices"][0]["message"]._logprob = sum_logprob
if "best_of" in optional_params and optional_params["best_of"] > 1:
if "details" in completion_response[0] and "best_of_sequences" in completion_response[0]["details"]:
choices_list = []
@@ -337,9 +334,14 @@ def completion(
prompt_tokens = len(
encoding.encode(input_text)
) ##[TODO] use the llama2 tokenizer here
- completion_tokens = len(
- encoding.encode(model_response["choices"][0]["message"].get("content", ""))
- ) ##[TODO] use the llama2 tokenizer here
+ print_verbose(f'output: {model_response["choices"][0]["message"]}')
+ output_text = model_response["choices"][0]["message"].get("content", "")
+ if output_text is not None and len(output_text) > 0:
+ completion_tokens = len(
+ encoding.encode(model_response["choices"][0]["message"].get("content", ""))
+ ) ##[TODO] use the llama2 tokenizer here
+ else:
+ completion_tokens = 0
model_response["created"] = time.time()
model_response["model"] = model
diff --git a/litellm/main.py b/litellm/main.py
index e24c5af73..6512ded0a 100644
--- a/litellm/main.py
+++ b/litellm/main.py
@@ -729,7 +729,6 @@ def completion(
headers
or litellm.headers
)
- print(f'headers before hf rest api: {hf_headers}')
model_response = huggingface_restapi.completion(
model=model,
messages=messages,
diff --git a/litellm/proxy/llm.py b/litellm/proxy/llm.py
index 878131697..85d34d06c 100644
--- a/litellm/proxy/llm.py
+++ b/litellm/proxy/llm.py
@@ -9,6 +9,7 @@ import backoff
import openai.error
import litellm
+from litellm.utils import trim_messages
import litellm.exceptions
cost_dict: Dict[str, Dict[str, float]] = defaultdict(dict)
@@ -113,7 +114,7 @@ def litellm_completion(data: Dict,
user_api_base: Optional[str],
user_headers: Optional[dict],
user_debug: bool) -> litellm.ModelResponse:
- try:
+ try:
global debug
debug = user_debug
if user_model:
diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py
index 90a967921..6d67ccdcc 100644
--- a/litellm/proxy/proxy_server.py
+++ b/litellm/proxy/proxy_server.py
@@ -1,6 +1,7 @@
import sys, os, platform, time, copy
import threading
import shutil, random, traceback
+messages = []
sys.path.insert(
0, os.path.abspath("../..")
) # Adds the parent directory to the system path - for litellm local dev
@@ -70,6 +71,7 @@ print()
import litellm
from fastapi import FastAPI, Request
from fastapi.routing import APIRouter
+from fastapi.encoders import jsonable_encoder
from fastapi.responses import StreamingResponse, FileResponse
from fastapi.middleware.cors import CORSMiddleware
import json
@@ -106,6 +108,12 @@ def print_verbose(print_statement):
if user_debug:
print(print_statement)
+def find_avatar_url(role):
+ role = role.replace(" ", "%20")
+ avatar_filename = f"avatars/{role}.png"
+ avatar_url = f"/static/{avatar_filename}"
+ return avatar_url
+
def usage_telemetry(feature: str): # helps us know if people are using this feature. Set `litellm --telemetry False` to your cli call to turn this off
if user_telemetry:
data = {
@@ -161,7 +169,14 @@ def save_params_to_config(data: dict):
# Write config to file
with open(user_config_path, 'wb') as f:
tomli_w.dump(config, f)
-
+
+def print_cost_logs():
+ with open('costs.json', 'r') as f:
+ # print this in green
+ print("\033[1;32m")
+ print(f.read())
+ print("\033[0m")
+ return
def load_config():
try:
@@ -469,13 +484,20 @@ async def v1_chat_completion(request: Request):
response = litellm_completion(data, type="chat_completion")
return response
-def print_cost_logs():
- with open('costs.json', 'r') as f:
- # print this in green
- print("\033[1;32m")
- print(f.read())
- print("\033[0m")
- return
+@router.post("/send_message")
+async def send_message(request: Request):
+ try:
+ data = await request.json()
+ role = data.get("role")
+ text = data.get("text")
+
+ avatarUrl = find_avatar_url(role)
+
+ message = {"role": role, "text": text, "avatarUrl": avatarUrl}
+ messages.append(message)
+ return jsonable_encoder(messages)
+ except:
+ return "An error occurred", 500
@router.get("/ollama_logs")
async def retrieve_server_log(request: Request):
diff --git a/litellm/utils.py b/litellm/utils.py
index 478eeda36..a077dbc7e 100644
--- a/litellm/utils.py
+++ b/litellm/utils.py
@@ -119,7 +119,7 @@ class Message(OpenAIObject):
super(Message, self).__init__(**params)
self.content = content
self.role = role
- self.logprobs = logprobs
+ self._logprobs = logprobs
class Delta(OpenAIObject):
def __init__(self, content=None, logprobs=None, role=None, **params):