mirror of
https://github.com/meta-llama/llama-stack.git
synced 2025-12-28 11:51:59 +00:00
99bd39cc30
fixes #2121 this implementation splits reponsibility between litellm and openai libraries - | Inference Method | Implementation Source | |----------------------------|--------------------------| | completion | LiteLLMOpenAIMixin | | chat_completion | LiteLLMOpenAIMixin | | embedding | LiteLLMOpenAIMixin | | batch_completion | LiteLLMOpenAIMixin | | batch_chat_completion | LiteLLMOpenAIMixin | | openai_completion | AsyncOpenAI | | openai_chat_completion | AsyncOpenAI | test with - $ OPENAI_API_KEY=$LLAMA_API_KEY OPENAI_BASE_URL=https://api.llama.com/compat/v1 llama stack build --image-type conda --image-name openai --providers inference=remote::openai --run $ llama-stack-client models register Llama-4-Scout-17B-16E-Instruct-FP8 $ curl "http://localhost:8321/v1/openai/v1/chat/completions" -H "Content-Type: application/json" \ -d '{ "model": "Llama-4-Scout-17B-16E-Instruct-FP8", "messages": [ {"role": "user", "content": "Hello Llama! Can you give me a quick intro?"} ] }' {"id":"AmPwrrkc5JgVjejPdIPrpT2","choices":[{"finish_reason":"stop","index":0,"logprobs":{"content":null,"refusal":null},"message":{"content":"Hello! I'm Llama, a Meta-designed model that adapts to your conversational style. Whether you need quick answers, deep dives into ideas, or just want to vent, joke, or brainstorm—I'm here for it. What’s on your mind?","refusal":"","role":"assistant","annotations":null,"audio":null,"function_call":null,"tool_calls":null,"id":"AmPwrrkc5JgVjejPdIPrpT2"}}],"created":1747410061,"model":"Llama-4-Scout-17B-16E-Instruct-FP8","object":"chat.completions","service_tier":null,"system_fingerprint":null,"usage":{"completion_tokens":54,"prompt_tokens":22,"total_tokens":76,"completion_tokens_details":null,"prompt_tokens_details":null}} |
||
|---|---|---|
| .. | ||
| anthropic | ||
| bedrock | ||
| cerebras | ||
| cerebras_openai_compat | ||
| databricks | ||
| fireworks | ||
| fireworks_openai_compat | ||
| gemini | ||
| groq | ||
| groq_openai_compat | ||
| llama_openai_compat | ||
| nvidia | ||
| ollama | ||
| openai | ||
| passthrough | ||
| runpod | ||
| sambanova | ||
| sambanova_openai_compat | ||
| tgi | ||
| together | ||
| together_openai_compat | ||
| vllm | ||
| watsonx | ||
| __init__.py | ||