From a82d45e6ad628d6cd152e1703aba02a764873e6b Mon Sep 17 00:00:00 2001 From: Hexoplon Date: Mon, 6 Jan 2025 20:50:32 +0100 Subject: [PATCH 1/2] feat(dockerfile): pre-download tiktoken in offline non_root image --- docker/Dockerfile.non_root | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/docker/Dockerfile.non_root b/docker/Dockerfile.non_root index 4f1a2dce33..695d00d07a 100644 --- a/docker/Dockerfile.non_root +++ b/docker/Dockerfile.non_root @@ -35,6 +35,10 @@ RUN pip install dist/*.whl # install dependencies as wheels RUN pip wheel --no-cache-dir --wheel-dir=/wheels/ -r requirements.txt +# Download tiktoken for offline usage of image, see https://stackoverflow.com/a/76107077. Let tiktoken itself download the file, to ensure correct naming +ENV TIKTOKEN_CACHE_DIR="/tiktoken" +RUN mkdir /tiktoken && python -c "import tiktoken; tiktoken.get_encoding('cl100k_base'); print('tiktoken imported successfully')" + # Runtime stage FROM $LITELLM_RUNTIME_IMAGE AS runtime @@ -50,6 +54,10 @@ RUN ls -la /app COPY --from=builder /app/dist/*.whl . COPY --from=builder /wheels/ /wheels/ +# Copy tiktoken from build stage, and set env variable to stop tiktoken from downloading file +COPY --from=builder /tiktoken /tiktoken +ENV CUSTOM_TIKTOKEN_CACHE_DIR="/tiktoken" + # Install the built wheel using pip; again using a wildcard if it's the only file RUN pip install *.whl /wheels/* --no-index --find-links=/wheels/ && rm -f *.whl && rm -rf /wheels From b03fa75ca0465a89ecad32238c931ae86bea8286 Mon Sep 17 00:00:00 2001 From: Hexoplon Date: Mon, 6 Jan 2025 20:51:28 +0100 Subject: [PATCH 2/2] docs(deploy): mention tiktoken pre-download in non_root image --- docs/my-website/docs/proxy/deploy.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/my-website/docs/proxy/deploy.md b/docs/my-website/docs/proxy/deploy.md index ea8df446e0..bcfe6ae0f4 100644 --- a/docs/my-website/docs/proxy/deploy.md +++ b/docs/my-website/docs/proxy/deploy.md @@ -560,9 +560,9 @@ ghcr.io/berriai/litellm-database:main-latest --config your_config.yaml ## LiteLLM without Internet Connection -By default `prisma generate` downloads [prisma's engine binaries](https://www.prisma.io/docs/orm/reference/environment-variables-reference#custom-engine-file-locations). This might cause errors when running without internet connection. +By default `prisma generate` downloads [prisma's engine binaries](https://www.prisma.io/docs/orm/reference/environment-variables-reference#custom-engine-file-locations). LiteLLM also uses `tiktoken` for tracking the number of tokens in a given user input (for openai models). This might cause errors when running without internet connection. -Use this docker image to deploy litellm with pre-generated prisma binaries. +Use this docker image to deploy litellm with pre-generated prisma binaries, and pre-downloaded tiktoken tokenizer files. ```bash docker pull ghcr.io/berriai/litellm-non_root:main-stable