(Feat) - allow including dd-trace in litellm base image (#7587)

* introduce USE_DDTRACE=true

* update dd tracer

* update

* bump dd trace

* use og slim image

* DD tracing

* fix _init_dd_tracer

Dockerfile

@@ -67,10 +67,11 @@ RUN pip install *.whl /wheels/* --no-index --find-links=/wheels/ && rm -f *.whl
 # Generate prisma client
 RUN prisma generate
 RUN chmod +x docker/entrypoint.sh
+RUN chmod +x docker/prod_entrypoint.sh
 
 EXPOSE 4000/tcp
 
-ENTRYPOINT ["litellm"]
+ENTRYPOINT ["docker/prod_entrypoint.sh"]
 
 # Append "--detailed_debug" to the end of CMD to view detailed debug logs 
 CMD ["--port", "4000"]

docker/Dockerfile.alpine

@@ -48,8 +48,11 @@ COPY --from=builder /wheels/ /wheels/
 # Install the built wheel using pip; again using a wildcard if it's the only file
 RUN pip install *.whl /wheels/* --no-index --find-links=/wheels/ && rm -f *.whl && rm -rf /wheels
 
+RUN chmod +x docker/entrypoint.sh
+RUN chmod +x docker/prod_entrypoint.sh
+
 EXPOSE 4000/tcp
 
 # Set your entrypoint and command
-ENTRYPOINT ["litellm"]
+ENTRYPOINT ["docker/prod_entrypoint.sh"]
 CMD ["--port", "4000"]

docker/Dockerfile.custom_ui

@@ -33,6 +33,7 @@ WORKDIR /app
 
 # Make sure your docker/entrypoint.sh is executable
 RUN chmod +x docker/entrypoint.sh
+RUN chmod +x docker/prod_entrypoint.sh
 
 # Expose the necessary port
 EXPOSE 4000/tcp

docker/Dockerfile.database

@@ -67,12 +67,12 @@ RUN chmod +x docker/build_admin_ui.sh && ./docker/build_admin_ui.sh
 # Generate prisma client
 RUN prisma generate
 RUN chmod +x docker/entrypoint.sh
-
+RUN chmod +x docker/prod_entrypoint.sh
 EXPOSE 4000/tcp
 
 # # Set your entrypoint and command
 
-ENTRYPOINT ["litellm"]
+ENTRYPOINT ["docker/prod_entrypoint.sh"]
 
 # Append "--detailed_debug" to the end of CMD to view detailed debug logs 
 # CMD ["--port", "4000", "--detailed_debug"]

docker/Dockerfile.non_root

@@ -71,13 +71,15 @@ RUN chmod -R 777 /.cache
 RUN pip install nodejs-bin
 RUN pip install prisma
 RUN prisma generate
+
 RUN chmod +x docker/entrypoint.sh
+RUN chmod +x docker/prod_entrypoint.sh
 
 EXPOSE 4000/tcp
 
 # # Set your entrypoint and command
 
-ENTRYPOINT ["litellm"]
+ENTRYPOINT ["docker/prod_entrypoint.sh"]
 
 # Append "--detailed_debug" to the end of CMD to view detailed debug logs 
 # CMD ["--port", "4000", "--detailed_debug"]

docker/prod_entrypoint.sh

@@ -0,0 +1,7 @@
+#!/bin/sh
+
+if [ "$USE_DDTRACE" = "true" ]; then
+    exec ddtrace-run litellm "$@"
+else
+    exec litellm "$@"
+fi

docs/my-website/docs/proxy/logging.md

@@ -1070,6 +1070,7 @@ curl --location 'http://0.0.0.0:4000/chat/completions' \
 LiteLLM Supports logging to the following Datdog Integrations:
 - `datadog` [Datadog Logs](https://docs.datadoghq.com/logs/)
 - `datadog_llm_observability` [Datadog LLM Observability](https://www.datadoghq.com/product/llm-observability/)
+- `ddtrace-run` [Datadog Tracing](#datadog-tracing)
 
 <Tabs>
 <TabItem value="datadog" label="Datadog Logs">
@@ -1142,6 +1143,21 @@ Expected output on Datadog
 
 <Image img={require('../../img/dd_small1.png')} />
 
+#### Datadog Tracing
+
+Use `ddtrace-run` to enable [Datadog Tracing](https://ddtrace.readthedocs.io/en/stable/installation_quickstart.html) on litellm proxy
+
+Pass `USE_DDTRACE=true` to the docker run command. When `USE_DDTRACE=true`, the proxy will run `ddtrace-run litellm` as the `ENTRYPOINT` instead of just `litellm`
+
+```bash
+docker run \
+    -v $(pwd)/litellm_config.yaml:/app/config.yaml \
+    -e USE_DDTRACE=true \
+    -p 4000:4000 \
+    ghcr.io/berriai/litellm:main-latest \
+    --config /app/config.yaml --detailed_debug
+```
+
 ### Set DD variables (`DD_SERVICE` etc)
 
 LiteLLM supports customizing the following Datadog environment variables

litellm/proxy/proxy_server.py

@@ -535,7 +535,13 @@ async def proxy_startup_event(app: FastAPI):
             proxy_batch_write_at=proxy_batch_write_at,
             proxy_logging_obj=proxy_logging_obj,
         )
+    ## [Optional] Initialize dd tracer
+    ProxyStartupEvent._init_dd_tracer()
+
+    # End of startup event
     yield
+
+    # Shutdown event
     await proxy_shutdown_event()
 
 
@@ -3218,6 +3224,19 @@ class ProxyStartupEvent:
             await prisma_client.health_check()
         return prisma_client
 
+    @classmethod
+    def _init_dd_tracer(cls):
+        """
+        Initialize dd tracer - if `USE_DDTRACE=true` in .env
+
+        DD tracer is used to trace Python applications.
+        Doc: https://docs.datadoghq.com/tracing/trace_collection/automatic_instrumentation/dd_libraries/python/
+        """
+        if get_secret_bool("USE_DDTRACE", False) is True:
+            import ddtrace
+
+            ddtrace.patch_all(logging=True)
+
 
 #### API ENDPOINTS ####
 @router.get(

requirements.txt

@@ -19,6 +19,7 @@ google-generativeai==0.5.0 # for vertex ai calls
 async_generator==1.10.0 # for async ollama calls
 langfuse==2.45.0 # for langfuse self-hosted logging
 prometheus_client==0.20.0 # for /metrics endpoint on proxy
+ddtrace==2.19.0rc1       # for advanced DD tracing / profiling
 orjson==3.10.12 # fast /embedding responses
 apscheduler==3.10.4 # for resetting budget in background 
 fastapi-sso==0.16.0 # admin UI, SSO