diff --git a/zero_to_hero_guide/00_Inference101.ipynb b/zero_to_hero_guide/00_Inference101.ipynb index 4da0d0df1..16f924dce 100644 --- a/zero_to_hero_guide/00_Inference101.ipynb +++ b/zero_to_hero_guide/00_Inference101.ipynb @@ -48,7 +48,8 @@ "outputs": [], "source": [ "HOST = \"localhost\" # Replace with your host\n", - "PORT = 5000 # Replace with your port" + "PORT = 5001 # Replace with your port\n", + "MODEL_NAME='meta-llama/Llama-3.2-3B-Instruct'" ] }, { @@ -93,8 +94,10 @@ "name": "stdout", "output_type": "stream", "text": [ - "With soft fur and gentle eyes,\n", - "The llama roams, a peaceful surprise.\n" + "Here is a two-sentence poem about a llama:\n", + "\n", + "With soft fur and gentle eyes, the llama roams free,\n", + "A majestic creature, wild and carefree.\n" ] } ], @@ -104,7 +107,7 @@ " {\"role\": \"system\", \"content\": \"You are a friendly assistant.\"},\n", " {\"role\": \"user\", \"content\": \"Write a two-sentence poem about llama.\"}\n", " ],\n", - " model='Llama3.2-11B-Vision-Instruct',\n", + " model_id=MODEL_NAME,\n", ")\n", "\n", "print(response.completion_message.content)" @@ -132,8 +135,8 @@ "name": "stdout", "output_type": "stream", "text": [ - "O, fairest llama, with thy softest fleece,\n", - "Thy gentle eyes, like sapphires, in serenity do cease.\n" + "\"O, fair llama, with thy gentle eyes so bright,\n", + "In Andean hills, thou dost enthrall with soft delight.\"\n" ] } ], @@ -143,9 +146,8 @@ " {\"role\": \"system\", \"content\": \"You are shakespeare.\"},\n", " {\"role\": \"user\", \"content\": \"Write a two-sentence poem about llama.\"}\n", " ],\n", - " model='Llama3.2-11B-Vision-Instruct',\n", + " model_id=MODEL_NAME, # Changed from model to model_id\n", ")\n", - "\n", "print(response.completion_message.content)" ] }, @@ -161,7 +163,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 6, "id": "02211625", "metadata": {}, "outputs": [ @@ -169,43 +171,35 @@ "name": "stdout", "output_type": "stream", "text": [ - "User> 1+1\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\u001b[36m> Response: 2\u001b[0m\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "User> what is llama\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\u001b[36m> Response: A llama is a domesticated mammal native to South America, specifically the Andean region. It belongs to the camelid family, which also includes camels, alpacas, guanacos, and vicuñas.\n", + "\u001b[36m> Response: How can I assist you today?\u001b[0m\n", + "\u001b[36m> Response: In South American hills, they roam and play,\n", + "The llama's gentle eyes gaze out each day.\n", + "Their soft fur coats in shades of white and gray,\n", + "Inviting all to come and stay.\n", "\n", - "Here are some interesting facts about llamas:\n", + "With ears that listen, ears so fine,\n", + "They hear the whispers of the Andean mine.\n", + "Their footsteps quiet on the mountain slope,\n", + "As they graze on grasses, a peaceful hope.\n", "\n", - "1. **Physical Characteristics**: Llamas are large, even-toed ungulates with a distinctive appearance. They have a long neck, a small head, and a soft, woolly coat that can be various colors, including white, brown, gray, and black.\n", - "2. **Size**: Llamas typically grow to be between 5 and 6 feet (1.5 to 1.8 meters) tall at the shoulder and weigh between 280 and 450 pounds (127 to 204 kilograms).\n", - "3. **Habitat**: Llamas are native to the Andean highlands, where they live in herds and roam freely. They are well adapted to the harsh, high-altitude climate of the Andes.\n", - "4. **Diet**: Llamas are herbivores and feed on a variety of plants, including grasses, leaves, and shrubs. They are known for their ability to digest plant material that other animals cannot.\n", - "5. **Behavior**: Llamas are social animals and live in herds. They are known for their intelligence, curiosity, and strong sense of self-preservation.\n", - "6. **Purpose**: Llamas have been domesticated for thousands of years and have been used for a variety of purposes, including:\n", - "\t* **Pack animals**: Llamas are often used as pack animals, carrying goods and supplies over long distances.\n", - "\t* **Fiber production**: Llama wool is highly valued for its softness, warmth, and durability.\n", - "\t* **Meat**: Llama meat is consumed in some parts of the world, particularly in South America.\n", - "\t* **Companionship**: Llamas are often kept as pets or companions, due to their gentle nature and intelligence.\n", + "In Incas' time, they were revered as friends,\n", + "Their packs they bore, until the very end.\n", + "The Spanish came, with guns and strife,\n", + "But llamas stood firm, for life.\n", "\n", - "Overall, llamas are fascinating animals that have been an integral part of Andean culture for thousands of years.\u001b[0m\n" + "Now, they roam free, in fields so wide,\n", + "A symbol of resilience, side by side.\n", + "With people's lives, a bond so strong,\n", + "Together they thrive, all day long.\n", + "\n", + "Their soft hums echo through the air,\n", + "As they wander, without a care.\n", + "In their gentle hearts, a wisdom lies,\n", + "A testament to the Andean skies.\n", + "\n", + "So here they'll stay, in this land of old,\n", + "The llama's spirit, forever to hold.\u001b[0m\n", + "\u001b[33mEnding conversation. Goodbye!\u001b[0m\n" ] } ], @@ -226,7 +220,7 @@ " message = {\"role\": \"user\", \"content\": user_input}\n", " response = client.inference.chat_completion(\n", " messages=[message],\n", - " model='Llama3.2-11B-Vision-Instruct',\n", + " model_id=MODEL_NAME\n", " )\n", " cprint(f'> Response: {response.completion_message.content}', 'cyan')\n", "\n", @@ -248,7 +242,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 8, "id": "9496f75c", "metadata": {}, "outputs": [ @@ -256,7 +250,29 @@ "name": "stdout", "output_type": "stream", "text": [ - "User> 1+1\n" + "\u001b[36m> Response: How can I help you today?\u001b[0m\n", + "\u001b[36m> Response: Here's a little poem about llamas:\n", + "\n", + "In Andean highlands, they roam and play,\n", + "Their soft fur shining in the sunny day.\n", + "With ears so long and eyes so bright,\n", + "They watch with gentle curiosity, taking flight.\n", + "\n", + "Their llama voices hum, a soothing sound,\n", + "As they wander through the mountains all around.\n", + "Their padded feet barely touch the ground,\n", + "As they move with ease, without a single bound.\n", + "\n", + "In packs or alone, they make their way,\n", + "Carrying burdens, come what may.\n", + "Their gentle spirit, a sight to see,\n", + "A symbol of peace, for you and me.\n", + "\n", + "With llamas calm, our souls take flight,\n", + "In their presence, all is right.\n", + "So let us cherish these gentle friends,\n", + "And honor their beauty that never ends.\u001b[0m\n", + "\u001b[33mEnding conversation. Goodbye!\u001b[0m\n" ] } ], @@ -274,7 +290,7 @@ "\n", " response = client.inference.chat_completion(\n", " messages=conversation_history,\n", - " model='Llama3.2-11B-Vision-Instruct',\n", + " model_id=MODEL_NAME,\n", " )\n", " cprint(f'> Response: {response.completion_message.content}', 'cyan')\n", "\n", @@ -304,10 +320,23 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 9, "id": "d119026e", "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\u001b[32mUser> Write me a 3 sentence poem about llama\u001b[0m\n", + "\u001b[36mAssistant> \u001b[0m\u001b[33mHere\u001b[0m\u001b[33m is\u001b[0m\u001b[33m a\u001b[0m\u001b[33m \u001b[0m\u001b[33m3\u001b[0m\u001b[33m sentence\u001b[0m\u001b[33m poem\u001b[0m\u001b[33m about\u001b[0m\u001b[33m a\u001b[0m\u001b[33m llama\u001b[0m\u001b[33m:\n", + "\n", + "\u001b[0m\u001b[33mWith\u001b[0m\u001b[33m soft\u001b[0m\u001b[33m and\u001b[0m\u001b[33m fuzzy\u001b[0m\u001b[33m fur\u001b[0m\u001b[33m so\u001b[0m\u001b[33m bright\u001b[0m\u001b[33m,\n", + "\u001b[0m\u001b[33mThe\u001b[0m\u001b[33m llama\u001b[0m\u001b[33m ro\u001b[0m\u001b[33mams\u001b[0m\u001b[33m through\u001b[0m\u001b[33m the\u001b[0m\u001b[33m And\u001b[0m\u001b[33mean\u001b[0m\u001b[33m light\u001b[0m\u001b[33m,\n", + "\u001b[0m\u001b[33mA\u001b[0m\u001b[33m gentle\u001b[0m\u001b[33m giant\u001b[0m\u001b[33m,\u001b[0m\u001b[33m a\u001b[0m\u001b[33m w\u001b[0m\u001b[33mondrous\u001b[0m\u001b[33m sight\u001b[0m\u001b[33m.\u001b[0m\u001b[97m\u001b[0m\n" + ] + } + ], "source": [ "from llama_stack_client.lib.inference.event_logger import EventLogger\n", "\n", @@ -322,7 +351,7 @@ "\n", " response = client.inference.chat_completion(\n", " messages=[message],\n", - " model='Llama3.2-11B-Vision-Instruct',\n", + " model_id=MODEL_NAME,\n", " stream=stream,\n", " )\n", "\n", @@ -337,6 +366,16 @@ "# To run it in a python file, use this line instead\n", "# asyncio.run(run_main())\n" ] + }, + { + "cell_type": "code", + "execution_count": 10, + "id": "3091d901", + "metadata": {}, + "outputs": [], + "source": [ + "#fin" + ] } ], "metadata": { diff --git a/zero_to_hero_guide/quickstart.md b/zero_to_hero_guide/quickstart.md index c18d0ff03..7dc3af24c 100644 --- a/zero_to_hero_guide/quickstart.md +++ b/zero_to_hero_guide/quickstart.md @@ -103,6 +103,7 @@ If you're looking for more specific topics like tool calling or agent setup, we ``` After this step, you will see the console output: + ``` Build Successful! Next steps: 1. Set the environment variables: LLAMASTACK_PORT, OLLAMA_URL, INFERENCE_MODEL, SAFETY_MODEL