phoenix/litellm-mirror
0
0
Fork 1
mirror of https://github.com/BerriAI/litellm.git synced 2025-04-26 11:14:04 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

use vars for batch size and flush interval seconds

Browse source
This commit is contained in:
Ishaan Jaff 2024-09-11 14:40:58 -07:00
parent 36fe499d53
commit 3e0c4448cd
2 changed files with 9 additions and 3 deletions
Download patch file Download diff file Expand all files Collapse all files

8
litellm/integrations/langsmith.py
View file

@ -24,6 +24,9 @@ from litellm.llms.custom_httpx.http_handler import (
httpxSpecialProvider, httpxSpecialProvider,
) )
DEFAULT_BATCH_SIZE = 512
DEFAULT_FLUSH_INTERVAL_SECONDS = 5
class LangsmithInputs(BaseModel): class LangsmithInputs(BaseModel):
model: Optional[str] = None model: Optional[str] = None
@ -71,11 +74,12 @@ class LangsmithLogger(CustomLogger):
) )
_batch_size = ( _batch_size = (
os.getenv("LANGSMITH_BATCH_SIZE", 100) or litellm.langsmith_batch_size os.getenv("LANGSMITH_BATCH_SIZE", DEFAULT_BATCH_SIZE)
or litellm.langsmith_batch_size
) )
self.batch_size = int(_batch_size) self.batch_size = int(_batch_size)
self.log_queue = [] self.log_queue = []
self.flush_interval = 10 # 5 seconds self.flush_interval = DEFAULT_FLUSH_INTERVAL_SECONDS # 10 seconds
self.last_flush_time = time.time() self.last_flush_time = time.time()
asyncio.create_task(self.periodic_flush()) asyncio.create_task(self.periodic_flush())
self.flush_lock = asyncio.Lock() self.flush_lock = asyncio.Lock()

4
litellm/proxy/proxy_config.yaml
View file

@ -16,4 +16,6 @@ general_settings:
master_key: sk-1234 master_key: sk-1234
litellm_settings: litellm_settings:
success_callback: ["langsmith"] success_callback: ["langsmith", "prometheus"]
service_callback: ["prometheus_system"]
callbacks: ["otel"]