phoenix-oss/llama-stack-mirror
0
0
Fork 1
mirror of https://github.com/meta-llama/llama-stack.git synced 2025-07-30 07:39:38 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

kill llamastack-local-gpu/cpu

Browse source
This commit is contained in:
Xi Yan 2024-11-01 13:21:59 -07:00
parent 0bc087c81a
commit f1f8aa2029
2 changed files with 13 additions and 13 deletions
Download patch file Download diff file Expand all files Collapse all files

2
distributions/tgi/cpu/compose.yaml
View file

@ -17,7 +17,7 @@ services:
depends_on: depends_on:
text-generation-inference: text-generation-inference:
condition: service_healthy condition: service_healthy
image: llamastack/llamastack-local-cpu image: llamastack/llamastack-tgi
network_mode: "host" network_mode: "host"
volumes: volumes:
- ~/.llama:/root/.llama - ~/.llama:/root/.llama

24
docs/getting_started.ipynb
View file

@ -36,7 +36,7 @@
"1. Get Docker container\n", "1. Get Docker container\n",
"```\n", "```\n",
"$ docker login\n", "$ docker login\n",
"$ docker pull llamastack/llamastack-local-gpu\n", "$ docker pull llamastack/llamastack-meta-reference-gpu\n",
"```\n", "```\n",
"\n", "\n",
"2. pip install the llama stack client package \n", "2. pip install the llama stack client package \n",
@ -61,15 +61,15 @@
"```\n", "```\n",
"For GPU inference, you need to set these environment variables for specifying local directory containing your model checkpoints, and enable GPU inference to start running docker container.\n", "For GPU inference, you need to set these environment variables for specifying local directory containing your model checkpoints, and enable GPU inference to start running docker container.\n",
"$ export LLAMA_CHECKPOINT_DIR=~/.llama\n", "$ export LLAMA_CHECKPOINT_DIR=~/.llama\n",
"$ llama stack configure llamastack-local-gpu\n", "$ llama stack configure llamastack-meta-reference-gpu\n",
"```\n", "```\n",
"Follow the prompts as part of configure.\n", "Follow the prompts as part of configure.\n",
"Here is a sample output \n", "Here is a sample output \n",
"```\n", "```\n",
"$ llama stack configure llamastack-local-gpu\n", "$ llama stack configure llamastack-meta-reference-gpu\n",
"\n", "\n",
"Could not find /home/hjshah/.conda/envs/llamastack-llamastack-local-gpu/llamastack-local-gpu-build.yaml. Trying docker image name instead...\n", "Could not find ~/.conda/envs/llamastack-llamastack-meta-reference-gpu/llamastack-meta-reference-gpu-build.yaml. Trying docker image name instead...\n",
"+ podman run --network host -it -v /home/hjshah/.llama/builds/docker:/app/builds llamastack-local-gpu llama stack configure ./llamastack-build.yaml --output-dir /app/builds\n", "+ podman run --network host -it -v ~/.llama/builds/docker:/app/builds llamastack-meta-reference-gpu llama stack configure ./llamastack-build.yaml --output-dir /app/builds\n",
"\n", "\n",
"Configuring API `inference`...\n", "Configuring API `inference`...\n",
"=== Configuring provider `meta-reference` for API inference...\n", "=== Configuring provider `meta-reference` for API inference...\n",
@ -155,7 +155,7 @@
"metadata": {}, "metadata": {},
"outputs": [], "outputs": [],
"source": [ "source": [
"# For this notebook we will be working with the latest Llama3.2 vision models \n", "# For this notebook we will be working with the latest Llama3.2 vision models\n",
"model = \"Llama3.2-11B-Vision-Instruct\"" "model = \"Llama3.2-11B-Vision-Instruct\""
] ]
}, },
@ -182,7 +182,7 @@
} }
], ],
"source": [ "source": [
"# Simple text example \n", "# Simple text example\n",
"iterator = client.inference.chat_completion(\n", "iterator = client.inference.chat_completion(\n",
" model=model,\n", " model=model,\n",
" messages=[\n", " messages=[\n",
@ -224,13 +224,13 @@
], ],
"source": [ "source": [
"import base64\n", "import base64\n",
"import mimetypes \n", "import mimetypes\n",
"\n", "\n",
"from PIL import Image\n", "from PIL import Image\n",
"\n", "\n",
"# We define a simple utility function to take a local image and \n", "# We define a simple utility function to take a local image and\n",
"# convert it to as base64 encoded data url \n", "# convert it to as base64 encoded data url\n",
"# that can be passed to the server. \n", "# that can be passed to the server.\n",
"def data_url_from_image(file_path):\n", "def data_url_from_image(file_path):\n",
" mime_type, _ = mimetypes.guess_type(file_path)\n", " mime_type, _ = mimetypes.guess_type(file_path)\n",
" if mime_type is None:\n", " if mime_type is None:\n",
@ -273,7 +273,7 @@
" {\n", " {\n",
" \"role\": \"user\",\n", " \"role\": \"user\",\n",
" \"content\": [\n", " \"content\": [\n",
" { \"image\": { \"uri\": data_url } }, \n", " { \"image\": { \"uri\": data_url } },\n",
" \"Write a haiku describing the image\"\n", " \"Write a haiku describing the image\"\n",
" ]\n", " ]\n",
" }\n", " }\n",