mirror of
https://github.com/meta-llama/llama-stack.git
synced 2025-12-28 08:21:59 +00:00
Merge branch 'main' into feat/litellm_sambanova_usage
This commit is contained in:
jhpiedrahitao
commit
172a918fe3
66 changed files with 9320 additions and 9446 deletions
11
docs/_static/css/my_theme.css
vendored
11
docs/_static/css/my_theme.css
vendored
|
|
@ -16,3 +16,14 @@
|
|||
.hide-title h1 {
|
||||
display: none;
|
||||
}
|
||||
|
||||
h2, h3, h4 {
|
||||
font-weight: normal;
|
||||
}
|
||||
html[data-theme="dark"] .rst-content div[class^="highlight"] {
|
||||
background-color: #0b0b0b;
|
||||
}
|
||||
pre {
|
||||
white-space: pre-wrap !important;
|
||||
word-break: break-all;
|
||||
}
|
||||
|
|
|
|||
932
docs/_static/llama-stack-spec.html
vendored
932
docs/_static/llama-stack-spec.html
vendored
|
|
@ -3092,6 +3092,125 @@
|
|||
}
|
||||
}
|
||||
},
|
||||
"/v1/openai/v1/chat/completions": {
|
||||
"post": {
|
||||
"responses": {
|
||||
"200": {
|
||||
"description": "OK",
|
||||
"content": {
|
||||
"application/json": {
|
||||
"schema": {
|
||||
"$ref": "#/components/schemas/OpenAIChatCompletion"
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
"400": {
|
||||
"$ref": "#/components/responses/BadRequest400"
|
||||
},
|
||||
"429": {
|
||||
"$ref": "#/components/responses/TooManyRequests429"
|
||||
},
|
||||
"500": {
|
||||
"$ref": "#/components/responses/InternalServerError500"
|
||||
},
|
||||
"default": {
|
||||
"$ref": "#/components/responses/DefaultError"
|
||||
}
|
||||
},
|
||||
"tags": [
|
||||
"Inference"
|
||||
],
|
||||
"description": "Generate an OpenAI-compatible chat completion for the given messages using the specified model.",
|
||||
"parameters": [],
|
||||
"requestBody": {
|
||||
"content": {
|
||||
"application/json": {
|
||||
"schema": {
|
||||
"$ref": "#/components/schemas/OpenaiChatCompletionRequest"
|
||||
}
|
||||
}
|
||||
},
|
||||
"required": true
|
||||
}
|
||||
}
|
||||
},
|
||||
"/v1/openai/v1/completions": {
|
||||
"post": {
|
||||
"responses": {
|
||||
"200": {
|
||||
"description": "OK",
|
||||
"content": {
|
||||
"application/json": {
|
||||
"schema": {
|
||||
"$ref": "#/components/schemas/OpenAICompletion"
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
"400": {
|
||||
"$ref": "#/components/responses/BadRequest400"
|
||||
},
|
||||
"429": {
|
||||
"$ref": "#/components/responses/TooManyRequests429"
|
||||
},
|
||||
"500": {
|
||||
"$ref": "#/components/responses/InternalServerError500"
|
||||
},
|
||||
"default": {
|
||||
"$ref": "#/components/responses/DefaultError"
|
||||
}
|
||||
},
|
||||
"tags": [
|
||||
"Inference"
|
||||
],
|
||||
"description": "Generate an OpenAI-compatible completion for the given prompt using the specified model.",
|
||||
"parameters": [],
|
||||
"requestBody": {
|
||||
"content": {
|
||||
"application/json": {
|
||||
"schema": {
|
||||
"$ref": "#/components/schemas/OpenaiCompletionRequest"
|
||||
}
|
||||
}
|
||||
},
|
||||
"required": true
|
||||
}
|
||||
}
|
||||
},
|
||||
"/v1/openai/v1/models": {
|
||||
"get": {
|
||||
"responses": {
|
||||
"200": {
|
||||
"description": "OK",
|
||||
"content": {
|
||||
"application/json": {
|
||||
"schema": {
|
||||
"$ref": "#/components/schemas/OpenAIListModelsResponse"
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
"400": {
|
||||
"$ref": "#/components/responses/BadRequest400"
|
||||
},
|
||||
"429": {
|
||||
"$ref": "#/components/responses/TooManyRequests429"
|
||||
},
|
||||
"500": {
|
||||
"$ref": "#/components/responses/InternalServerError500"
|
||||
},
|
||||
"default": {
|
||||
"$ref": "#/components/responses/DefaultError"
|
||||
}
|
||||
},
|
||||
"tags": [
|
||||
"Models"
|
||||
],
|
||||
"description": "",
|
||||
"parameters": []
|
||||
}
|
||||
},
|
||||
"/v1/post-training/preference-optimize": {
|
||||
"post": {
|
||||
"responses": {
|
||||
|
|
@ -8713,6 +8832,819 @@
|
|||
],
|
||||
"title": "LogEventRequest"
|
||||
},
|
||||
"OpenAIAssistantMessageParam": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"role": {
|
||||
"type": "string",
|
||||
"const": "assistant",
|
||||
"default": "assistant",
|
||||
"description": "Must be \"assistant\" to identify this as the model's response"
|
||||
},
|
||||
"content": {
|
||||
"$ref": "#/components/schemas/InterleavedContent",
|
||||
"description": "The content of the model's response"
|
||||
},
|
||||
"name": {
|
||||
"type": "string",
|
||||
"description": "(Optional) The name of the assistant message participant."
|
||||
},
|
||||
"tool_calls": {
|
||||
"type": "array",
|
||||
"items": {
|
||||
"$ref": "#/components/schemas/ToolCall"
|
||||
},
|
||||
"description": "List of tool calls. Each tool call is a ToolCall object."
|
||||
}
|
||||
},
|
||||
"additionalProperties": false,
|
||||
"required": [
|
||||
"role",
|
||||
"content"
|
||||
],
|
||||
"title": "OpenAIAssistantMessageParam",
|
||||
"description": "A message containing the model's (assistant) response in an OpenAI-compatible chat completion request."
|
||||
},
|
||||
"OpenAIDeveloperMessageParam": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"role": {
|
||||
"type": "string",
|
||||
"const": "developer",
|
||||
"default": "developer",
|
||||
"description": "Must be \"developer\" to identify this as a developer message"
|
||||
},
|
||||
"content": {
|
||||
"$ref": "#/components/schemas/InterleavedContent",
|
||||
"description": "The content of the developer message"
|
||||
},
|
||||
"name": {
|
||||
"type": "string",
|
||||
"description": "(Optional) The name of the developer message participant."
|
||||
}
|
||||
},
|
||||
"additionalProperties": false,
|
||||
"required": [
|
||||
"role",
|
||||
"content"
|
||||
],
|
||||
"title": "OpenAIDeveloperMessageParam",
|
||||
"description": "A message from the developer in an OpenAI-compatible chat completion request."
|
||||
},
|
||||
"OpenAIMessageParam": {
|
||||
"oneOf": [
|
||||
{
|
||||
"$ref": "#/components/schemas/OpenAIUserMessageParam"
|
||||
},
|
||||
{
|
||||
"$ref": "#/components/schemas/OpenAISystemMessageParam"
|
||||
},
|
||||
{
|
||||
"$ref": "#/components/schemas/OpenAIAssistantMessageParam"
|
||||
},
|
||||
{
|
||||
"$ref": "#/components/schemas/OpenAIToolMessageParam"
|
||||
},
|
||||
{
|
||||
"$ref": "#/components/schemas/OpenAIDeveloperMessageParam"
|
||||
}
|
||||
],
|
||||
"discriminator": {
|
||||
"propertyName": "role",
|
||||
"mapping": {
|
||||
"user": "#/components/schemas/OpenAIUserMessageParam",
|
||||
"system": "#/components/schemas/OpenAISystemMessageParam",
|
||||
"assistant": "#/components/schemas/OpenAIAssistantMessageParam",
|
||||
"tool": "#/components/schemas/OpenAIToolMessageParam",
|
||||
"developer": "#/components/schemas/OpenAIDeveloperMessageParam"
|
||||
}
|
||||
}
|
||||
},
|
||||
"OpenAISystemMessageParam": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"role": {
|
||||
"type": "string",
|
||||
"const": "system",
|
||||
"default": "system",
|
||||
"description": "Must be \"system\" to identify this as a system message"
|
||||
},
|
||||
"content": {
|
||||
"$ref": "#/components/schemas/InterleavedContent",
|
||||
"description": "The content of the \"system prompt\". If multiple system messages are provided, they are concatenated. The underlying Llama Stack code may also add other system messages (for example, for formatting tool definitions)."
|
||||
},
|
||||
"name": {
|
||||
"type": "string",
|
||||
"description": "(Optional) The name of the system message participant."
|
||||
}
|
||||
},
|
||||
"additionalProperties": false,
|
||||
"required": [
|
||||
"role",
|
||||
"content"
|
||||
],
|
||||
"title": "OpenAISystemMessageParam",
|
||||
"description": "A system message providing instructions or context to the model."
|
||||
},
|
||||
"OpenAIToolMessageParam": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"role": {
|
||||
"type": "string",
|
||||
"const": "tool",
|
||||
"default": "tool",
|
||||
"description": "Must be \"tool\" to identify this as a tool response"
|
||||
},
|
||||
"tool_call_id": {
|
||||
"type": "string",
|
||||
"description": "Unique identifier for the tool call this response is for"
|
||||
},
|
||||
"content": {
|
||||
"$ref": "#/components/schemas/InterleavedContent",
|
||||
"description": "The response content from the tool"
|
||||
}
|
||||
},
|
||||
"additionalProperties": false,
|
||||
"required": [
|
||||
"role",
|
||||
"tool_call_id",
|
||||
"content"
|
||||
],
|
||||
"title": "OpenAIToolMessageParam",
|
||||
"description": "A message representing the result of a tool invocation in an OpenAI-compatible chat completion request."
|
||||
},
|
||||
"OpenAIUserMessageParam": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"role": {
|
||||
"type": "string",
|
||||
"const": "user",
|
||||
"default": "user",
|
||||
"description": "Must be \"user\" to identify this as a user message"
|
||||
},
|
||||
"content": {
|
||||
"$ref": "#/components/schemas/InterleavedContent",
|
||||
"description": "The content of the message, which can include text and other media"
|
||||
},
|
||||
"name": {
|
||||
"type": "string",
|
||||
"description": "(Optional) The name of the user message participant."
|
||||
}
|
||||
},
|
||||
"additionalProperties": false,
|
||||
"required": [
|
||||
"role",
|
||||
"content"
|
||||
],
|
||||
"title": "OpenAIUserMessageParam",
|
||||
"description": "A message from the user in an OpenAI-compatible chat completion request."
|
||||
},
|
||||
"OpenaiChatCompletionRequest": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"model": {
|
||||
"type": "string",
|
||||
"description": "The identifier of the model to use. The model must be registered with Llama Stack and available via the /models endpoint."
|
||||
},
|
||||
"messages": {
|
||||
"type": "array",
|
||||
"items": {
|
||||
"$ref": "#/components/schemas/OpenAIMessageParam"
|
||||
},
|
||||
"description": "List of messages in the conversation"
|
||||
},
|
||||
"frequency_penalty": {
|
||||
"type": "number",
|
||||
"description": "(Optional) The penalty for repeated tokens"
|
||||
},
|
||||
"function_call": {
|
||||
"oneOf": [
|
||||
{
|
||||
"type": "string"
|
||||
},
|
||||
{
|
||||
"type": "object",
|
||||
"additionalProperties": {
|
||||
"oneOf": [
|
||||
{
|
||||
"type": "null"
|
||||
},
|
||||
{
|
||||
"type": "boolean"
|
||||
},
|
||||
{
|
||||
"type": "number"
|
||||
},
|
||||
{
|
||||
"type": "string"
|
||||
},
|
||||
{
|
||||
"type": "array"
|
||||
},
|
||||
{
|
||||
"type": "object"
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
],
|
||||
"description": "(Optional) The function call to use"
|
||||
},
|
||||
"functions": {
|
||||
"type": "array",
|
||||
"items": {
|
||||
"type": "object",
|
||||
"additionalProperties": {
|
||||
"oneOf": [
|
||||
{
|
||||
"type": "null"
|
||||
},
|
||||
{
|
||||
"type": "boolean"
|
||||
},
|
||||
{
|
||||
"type": "number"
|
||||
},
|
||||
{
|
||||
"type": "string"
|
||||
},
|
||||
{
|
||||
"type": "array"
|
||||
},
|
||||
{
|
||||
"type": "object"
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
"description": "(Optional) List of functions to use"
|
||||
},
|
||||
"logit_bias": {
|
||||
"type": "object",
|
||||
"additionalProperties": {
|
||||
"type": "number"
|
||||
},
|
||||
"description": "(Optional) The logit bias to use"
|
||||
},
|
||||
"logprobs": {
|
||||
"type": "boolean",
|
||||
"description": "(Optional) The log probabilities to use"
|
||||
},
|
||||
"max_completion_tokens": {
|
||||
"type": "integer",
|
||||
"description": "(Optional) The maximum number of tokens to generate"
|
||||
},
|
||||
"max_tokens": {
|
||||
"type": "integer",
|
||||
"description": "(Optional) The maximum number of tokens to generate"
|
||||
},
|
||||
"n": {
|
||||
"type": "integer",
|
||||
"description": "(Optional) The number of completions to generate"
|
||||
},
|
||||
"parallel_tool_calls": {
|
||||
"type": "boolean",
|
||||
"description": "(Optional) Whether to parallelize tool calls"
|
||||
},
|
||||
"presence_penalty": {
|
||||
"type": "number",
|
||||
"description": "(Optional) The penalty for repeated tokens"
|
||||
},
|
||||
"response_format": {
|
||||
"type": "object",
|
||||
"additionalProperties": {
|
||||
"type": "string"
|
||||
},
|
||||
"description": "(Optional) The response format to use"
|
||||
},
|
||||
"seed": {
|
||||
"type": "integer",
|
||||
"description": "(Optional) The seed to use"
|
||||
},
|
||||
"stop": {
|
||||
"oneOf": [
|
||||
{
|
||||
"type": "string"
|
||||
},
|
||||
{
|
||||
"type": "array",
|
||||
"items": {
|
||||
"type": "string"
|
||||
}
|
||||
}
|
||||
],
|
||||
"description": "(Optional) The stop tokens to use"
|
||||
},
|
||||
"stream": {
|
||||
"type": "boolean",
|
||||
"description": "(Optional) Whether to stream the response"
|
||||
},
|
||||
"stream_options": {
|
||||
"type": "object",
|
||||
"additionalProperties": {
|
||||
"oneOf": [
|
||||
{
|
||||
"type": "null"
|
||||
},
|
||||
{
|
||||
"type": "boolean"
|
||||
},
|
||||
{
|
||||
"type": "number"
|
||||
},
|
||||
{
|
||||
"type": "string"
|
||||
},
|
||||
{
|
||||
"type": "array"
|
||||
},
|
||||
{
|
||||
"type": "object"
|
||||
}
|
||||
]
|
||||
},
|
||||
"description": "(Optional) The stream options to use"
|
||||
},
|
||||
"temperature": {
|
||||
"type": "number",
|
||||
"description": "(Optional) The temperature to use"
|
||||
},
|
||||
"tool_choice": {
|
||||
"oneOf": [
|
||||
{
|
||||
"type": "string"
|
||||
},
|
||||
{
|
||||
"type": "object",
|
||||
"additionalProperties": {
|
||||
"oneOf": [
|
||||
{
|
||||
"type": "null"
|
||||
},
|
||||
{
|
||||
"type": "boolean"
|
||||
},
|
||||
{
|
||||
"type": "number"
|
||||
},
|
||||
{
|
||||
"type": "string"
|
||||
},
|
||||
{
|
||||
"type": "array"
|
||||
},
|
||||
{
|
||||
"type": "object"
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
],
|
||||
"description": "(Optional) The tool choice to use"
|
||||
},
|
||||
"tools": {
|
||||
"type": "array",
|
||||
"items": {
|
||||
"type": "object",
|
||||
"additionalProperties": {
|
||||
"oneOf": [
|
||||
{
|
||||
"type": "null"
|
||||
},
|
||||
{
|
||||
"type": "boolean"
|
||||
},
|
||||
{
|
||||
"type": "number"
|
||||
},
|
||||
{
|
||||
"type": "string"
|
||||
},
|
||||
{
|
||||
"type": "array"
|
||||
},
|
||||
{
|
||||
"type": "object"
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
"description": "(Optional) The tools to use"
|
||||
},
|
||||
"top_logprobs": {
|
||||
"type": "integer",
|
||||
"description": "(Optional) The top log probabilities to use"
|
||||
},
|
||||
"top_p": {
|
||||
"type": "number",
|
||||
"description": "(Optional) The top p to use"
|
||||
},
|
||||
"user": {
|
||||
"type": "string",
|
||||
"description": "(Optional) The user to use"
|
||||
}
|
||||
},
|
||||
"additionalProperties": false,
|
||||
"required": [
|
||||
"model",
|
||||
"messages"
|
||||
],
|
||||
"title": "OpenaiChatCompletionRequest"
|
||||
},
|
||||
"OpenAIChatCompletion": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"id": {
|
||||
"type": "string",
|
||||
"description": "The ID of the chat completion"
|
||||
},
|
||||
"choices": {
|
||||
"type": "array",
|
||||
"items": {
|
||||
"$ref": "#/components/schemas/OpenAIChoice"
|
||||
},
|
||||
"description": "List of choices"
|
||||
},
|
||||
"object": {
|
||||
"type": "string",
|
||||
"const": "chat.completion",
|
||||
"default": "chat.completion",
|
||||
"description": "The object type, which will be \"chat.completion\""
|
||||
},
|
||||
"created": {
|
||||
"type": "integer",
|
||||
"description": "The Unix timestamp in seconds when the chat completion was created"
|
||||
},
|
||||
"model": {
|
||||
"type": "string",
|
||||
"description": "The model that was used to generate the chat completion"
|
||||
}
|
||||
},
|
||||
"additionalProperties": false,
|
||||
"required": [
|
||||
"id",
|
||||
"choices",
|
||||
"object",
|
||||
"created",
|
||||
"model"
|
||||
],
|
||||
"title": "OpenAIChatCompletion",
|
||||
"description": "Response from an OpenAI-compatible chat completion request."
|
||||
},
|
||||
"OpenAIChoice": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"message": {
|
||||
"$ref": "#/components/schemas/OpenAIMessageParam",
|
||||
"description": "The message from the model"
|
||||
},
|
||||
"finish_reason": {
|
||||
"type": "string",
|
||||
"description": "The reason the model stopped generating"
|
||||
},
|
||||
"index": {
|
||||
"type": "integer"
|
||||
},
|
||||
"logprobs": {
|
||||
"$ref": "#/components/schemas/OpenAIChoiceLogprobs"
|
||||
}
|
||||
},
|
||||
"additionalProperties": false,
|
||||
"required": [
|
||||
"message",
|
||||
"finish_reason",
|
||||
"index"
|
||||
],
|
||||
"title": "OpenAIChoice",
|
||||
"description": "A choice from an OpenAI-compatible chat completion response."
|
||||
},
|
||||
"OpenAIChoiceLogprobs": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"content": {
|
||||
"type": "array",
|
||||
"items": {
|
||||
"$ref": "#/components/schemas/OpenAITokenLogProb"
|
||||
}
|
||||
},
|
||||
"refusal": {
|
||||
"type": "array",
|
||||
"items": {
|
||||
"$ref": "#/components/schemas/OpenAITokenLogProb"
|
||||
}
|
||||
}
|
||||
},
|
||||
"additionalProperties": false,
|
||||
"title": "OpenAIChoiceLogprobs",
|
||||
"description": "The log probabilities for the tokens in the message from an OpenAI-compatible chat completion response."
|
||||
},
|
||||
"OpenAITokenLogProb": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"token": {
|
||||
"type": "string"
|
||||
},
|
||||
"bytes": {
|
||||
"type": "array",
|
||||
"items": {
|
||||
"type": "integer"
|
||||
}
|
||||
},
|
||||
"logprob": {
|
||||
"type": "number"
|
||||
},
|
||||
"top_logprobs": {
|
||||
"type": "array",
|
||||
"items": {
|
||||
"$ref": "#/components/schemas/OpenAITopLogProb"
|
||||
}
|
||||
}
|
||||
},
|
||||
"additionalProperties": false,
|
||||
"required": [
|
||||
"token",
|
||||
"logprob",
|
||||
"top_logprobs"
|
||||
],
|
||||
"title": "OpenAITokenLogProb",
|
||||
"description": "The log probability for a token from an OpenAI-compatible chat completion response."
|
||||
},
|
||||
"OpenAITopLogProb": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"token": {
|
||||
"type": "string"
|
||||
},
|
||||
"bytes": {
|
||||
"type": "array",
|
||||
"items": {
|
||||
"type": "integer"
|
||||
}
|
||||
},
|
||||
"logprob": {
|
||||
"type": "number"
|
||||
}
|
||||
},
|
||||
"additionalProperties": false,
|
||||
"required": [
|
||||
"token",
|
||||
"logprob"
|
||||
],
|
||||
"title": "OpenAITopLogProb",
|
||||
"description": "The top log probability for a token from an OpenAI-compatible chat completion response."
|
||||
},
|
||||
"OpenaiCompletionRequest": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"model": {
|
||||
"type": "string",
|
||||
"description": "The identifier of the model to use. The model must be registered with Llama Stack and available via the /models endpoint."
|
||||
},
|
||||
"prompt": {
|
||||
"oneOf": [
|
||||
{
|
||||
"type": "string"
|
||||
},
|
||||
{
|
||||
"type": "array",
|
||||
"items": {
|
||||
"type": "string"
|
||||
}
|
||||
},
|
||||
{
|
||||
"type": "array",
|
||||
"items": {
|
||||
"type": "integer"
|
||||
}
|
||||
},
|
||||
{
|
||||
"type": "array",
|
||||
"items": {
|
||||
"type": "array",
|
||||
"items": {
|
||||
"type": "integer"
|
||||
}
|
||||
}
|
||||
}
|
||||
],
|
||||
"description": "The prompt to generate a completion for"
|
||||
},
|
||||
"best_of": {
|
||||
"type": "integer",
|
||||
"description": "(Optional) The number of completions to generate"
|
||||
},
|
||||
"echo": {
|
||||
"type": "boolean",
|
||||
"description": "(Optional) Whether to echo the prompt"
|
||||
},
|
||||
"frequency_penalty": {
|
||||
"type": "number",
|
||||
"description": "(Optional) The penalty for repeated tokens"
|
||||
},
|
||||
"logit_bias": {
|
||||
"type": "object",
|
||||
"additionalProperties": {
|
||||
"type": "number"
|
||||
},
|
||||
"description": "(Optional) The logit bias to use"
|
||||
},
|
||||
"logprobs": {
|
||||
"type": "boolean",
|
||||
"description": "(Optional) The log probabilities to use"
|
||||
},
|
||||
"max_tokens": {
|
||||
"type": "integer",
|
||||
"description": "(Optional) The maximum number of tokens to generate"
|
||||
},
|
||||
"n": {
|
||||
"type": "integer",
|
||||
"description": "(Optional) The number of completions to generate"
|
||||
},
|
||||
"presence_penalty": {
|
||||
"type": "number",
|
||||
"description": "(Optional) The penalty for repeated tokens"
|
||||
},
|
||||
"seed": {
|
||||
"type": "integer",
|
||||
"description": "(Optional) The seed to use"
|
||||
},
|
||||
"stop": {
|
||||
"oneOf": [
|
||||
{
|
||||
"type": "string"
|
||||
},
|
||||
{
|
||||
"type": "array",
|
||||
"items": {
|
||||
"type": "string"
|
||||
}
|
||||
}
|
||||
],
|
||||
"description": "(Optional) The stop tokens to use"
|
||||
},
|
||||
"stream": {
|
||||
"type": "boolean",
|
||||
"description": "(Optional) Whether to stream the response"
|
||||
},
|
||||
"stream_options": {
|
||||
"type": "object",
|
||||
"additionalProperties": {
|
||||
"oneOf": [
|
||||
{
|
||||
"type": "null"
|
||||
},
|
||||
{
|
||||
"type": "boolean"
|
||||
},
|
||||
{
|
||||
"type": "number"
|
||||
},
|
||||
{
|
||||
"type": "string"
|
||||
},
|
||||
{
|
||||
"type": "array"
|
||||
},
|
||||
{
|
||||
"type": "object"
|
||||
}
|
||||
]
|
||||
},
|
||||
"description": "(Optional) The stream options to use"
|
||||
},
|
||||
"temperature": {
|
||||
"type": "number",
|
||||
"description": "(Optional) The temperature to use"
|
||||
},
|
||||
"top_p": {
|
||||
"type": "number",
|
||||
"description": "(Optional) The top p to use"
|
||||
},
|
||||
"user": {
|
||||
"type": "string",
|
||||
"description": "(Optional) The user to use"
|
||||
},
|
||||
"guided_choice": {
|
||||
"type": "array",
|
||||
"items": {
|
||||
"type": "string"
|
||||
}
|
||||
},
|
||||
"prompt_logprobs": {
|
||||
"type": "integer"
|
||||
}
|
||||
},
|
||||
"additionalProperties": false,
|
||||
"required": [
|
||||
"model",
|
||||
"prompt"
|
||||
],
|
||||
"title": "OpenaiCompletionRequest"
|
||||
},
|
||||
"OpenAICompletion": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"id": {
|
||||
"type": "string"
|
||||
},
|
||||
"choices": {
|
||||
"type": "array",
|
||||
"items": {
|
||||
"$ref": "#/components/schemas/OpenAICompletionChoice"
|
||||
}
|
||||
},
|
||||
"created": {
|
||||
"type": "integer"
|
||||
},
|
||||
"model": {
|
||||
"type": "string"
|
||||
},
|
||||
"object": {
|
||||
"type": "string",
|
||||
"const": "text_completion",
|
||||
"default": "text_completion"
|
||||
}
|
||||
},
|
||||
"additionalProperties": false,
|
||||
"required": [
|
||||
"id",
|
||||
"choices",
|
||||
"created",
|
||||
"model",
|
||||
"object"
|
||||
],
|
||||
"title": "OpenAICompletion",
|
||||
"description": "Response from an OpenAI-compatible completion request."
|
||||
},
|
||||
"OpenAICompletionChoice": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"finish_reason": {
|
||||
"type": "string"
|
||||
},
|
||||
"text": {
|
||||
"type": "string"
|
||||
},
|
||||
"index": {
|
||||
"type": "integer"
|
||||
},
|
||||
"logprobs": {
|
||||
"$ref": "#/components/schemas/OpenAIChoiceLogprobs"
|
||||
}
|
||||
},
|
||||
"additionalProperties": false,
|
||||
"required": [
|
||||
"finish_reason",
|
||||
"text",
|
||||
"index"
|
||||
],
|
||||
"title": "OpenAICompletionChoice",
|
||||
"description": "A choice from an OpenAI-compatible completion response."
|
||||
},
|
||||
"OpenAIModel": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"id": {
|
||||
"type": "string"
|
||||
},
|
||||
"object": {
|
||||
"type": "string",
|
||||
"const": "model",
|
||||
"default": "model"
|
||||
},
|
||||
"created": {
|
||||
"type": "integer"
|
||||
},
|
||||
"owned_by": {
|
||||
"type": "string"
|
||||
}
|
||||
},
|
||||
"additionalProperties": false,
|
||||
"required": [
|
||||
"id",
|
||||
"object",
|
||||
"created",
|
||||
"owned_by"
|
||||
],
|
||||
"title": "OpenAIModel",
|
||||
"description": "A model from OpenAI."
|
||||
},
|
||||
"OpenAIListModelsResponse": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"data": {
|
||||
"type": "array",
|
||||
"items": {
|
||||
"$ref": "#/components/schemas/OpenAIModel"
|
||||
}
|
||||
}
|
||||
},
|
||||
"additionalProperties": false,
|
||||
"required": [
|
||||
"data"
|
||||
],
|
||||
"title": "OpenAIListModelsResponse"
|
||||
},
|
||||
"DPOAlignmentConfig": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
|
|
|
|||
665
docs/_static/llama-stack-spec.yaml
vendored
665
docs/_static/llama-stack-spec.yaml
vendored
|
|
@ -2131,6 +2131,91 @@ paths:
|
|||
schema:
|
||||
$ref: '#/components/schemas/LogEventRequest'
|
||||
required: true
|
||||
/v1/openai/v1/chat/completions:
|
||||
post:
|
||||
responses:
|
||||
'200':
|
||||
description: OK
|
||||
content:
|
||||
application/json:
|
||||
schema:
|
||||
$ref: '#/components/schemas/OpenAIChatCompletion'
|
||||
'400':
|
||||
$ref: '#/components/responses/BadRequest400'
|
||||
'429':
|
||||
$ref: >-
|
||||
#/components/responses/TooManyRequests429
|
||||
'500':
|
||||
$ref: >-
|
||||
#/components/responses/InternalServerError500
|
||||
default:
|
||||
$ref: '#/components/responses/DefaultError'
|
||||
tags:
|
||||
- Inference
|
||||
description: >-
|
||||
Generate an OpenAI-compatible chat completion for the given messages using
|
||||
the specified model.
|
||||
parameters: []
|
||||
requestBody:
|
||||
content:
|
||||
application/json:
|
||||
schema:
|
||||
$ref: '#/components/schemas/OpenaiChatCompletionRequest'
|
||||
required: true
|
||||
/v1/openai/v1/completions:
|
||||
post:
|
||||
responses:
|
||||
'200':
|
||||
description: OK
|
||||
content:
|
||||
application/json:
|
||||
schema:
|
||||
$ref: '#/components/schemas/OpenAICompletion'
|
||||
'400':
|
||||
$ref: '#/components/responses/BadRequest400'
|
||||
'429':
|
||||
$ref: >-
|
||||
#/components/responses/TooManyRequests429
|
||||
'500':
|
||||
$ref: >-
|
||||
#/components/responses/InternalServerError500
|
||||
default:
|
||||
$ref: '#/components/responses/DefaultError'
|
||||
tags:
|
||||
- Inference
|
||||
description: >-
|
||||
Generate an OpenAI-compatible completion for the given prompt using the specified
|
||||
model.
|
||||
parameters: []
|
||||
requestBody:
|
||||
content:
|
||||
application/json:
|
||||
schema:
|
||||
$ref: '#/components/schemas/OpenaiCompletionRequest'
|
||||
required: true
|
||||
/v1/openai/v1/models:
|
||||
get:
|
||||
responses:
|
||||
'200':
|
||||
description: OK
|
||||
content:
|
||||
application/json:
|
||||
schema:
|
||||
$ref: '#/components/schemas/OpenAIListModelsResponse'
|
||||
'400':
|
||||
$ref: '#/components/responses/BadRequest400'
|
||||
'429':
|
||||
$ref: >-
|
||||
#/components/responses/TooManyRequests429
|
||||
'500':
|
||||
$ref: >-
|
||||
#/components/responses/InternalServerError500
|
||||
default:
|
||||
$ref: '#/components/responses/DefaultError'
|
||||
tags:
|
||||
- Models
|
||||
description: ''
|
||||
parameters: []
|
||||
/v1/post-training/preference-optimize:
|
||||
post:
|
||||
responses:
|
||||
|
|
@ -5980,6 +6065,586 @@ components:
|
|||
- event
|
||||
- ttl_seconds
|
||||
title: LogEventRequest
|
||||
OpenAIAssistantMessageParam:
|
||||
type: object
|
||||
properties:
|
||||
role:
|
||||
type: string
|
||||
const: assistant
|
||||
default: assistant
|
||||
description: >-
|
||||
Must be "assistant" to identify this as the model's response
|
||||
content:
|
||||
$ref: '#/components/schemas/InterleavedContent'
|
||||
description: The content of the model's response
|
||||
name:
|
||||
type: string
|
||||
description: >-
|
||||
(Optional) The name of the assistant message participant.
|
||||
tool_calls:
|
||||
type: array
|
||||
items:
|
||||
$ref: '#/components/schemas/ToolCall'
|
||||
description: >-
|
||||
List of tool calls. Each tool call is a ToolCall object.
|
||||
additionalProperties: false
|
||||
required:
|
||||
- role
|
||||
- content
|
||||
title: OpenAIAssistantMessageParam
|
||||
description: >-
|
||||
A message containing the model's (assistant) response in an OpenAI-compatible
|
||||
chat completion request.
|
||||
OpenAIDeveloperMessageParam:
|
||||
type: object
|
||||
properties:
|
||||
role:
|
||||
type: string
|
||||
const: developer
|
||||
default: developer
|
||||
description: >-
|
||||
Must be "developer" to identify this as a developer message
|
||||
content:
|
||||
$ref: '#/components/schemas/InterleavedContent'
|
||||
description: The content of the developer message
|
||||
name:
|
||||
type: string
|
||||
description: >-
|
||||
(Optional) The name of the developer message participant.
|
||||
additionalProperties: false
|
||||
required:
|
||||
- role
|
||||
- content
|
||||
title: OpenAIDeveloperMessageParam
|
||||
description: >-
|
||||
A message from the developer in an OpenAI-compatible chat completion request.
|
||||
OpenAIMessageParam:
|
||||
oneOf:
|
||||
- $ref: '#/components/schemas/OpenAIUserMessageParam'
|
||||
- $ref: '#/components/schemas/OpenAISystemMessageParam'
|
||||
- $ref: '#/components/schemas/OpenAIAssistantMessageParam'
|
||||
- $ref: '#/components/schemas/OpenAIToolMessageParam'
|
||||
- $ref: '#/components/schemas/OpenAIDeveloperMessageParam'
|
||||
discriminator:
|
||||
propertyName: role
|
||||
mapping:
|
||||
user: '#/components/schemas/OpenAIUserMessageParam'
|
||||
system: '#/components/schemas/OpenAISystemMessageParam'
|
||||
assistant: '#/components/schemas/OpenAIAssistantMessageParam'
|
||||
tool: '#/components/schemas/OpenAIToolMessageParam'
|
||||
developer: '#/components/schemas/OpenAIDeveloperMessageParam'
|
||||
OpenAISystemMessageParam:
|
||||
type: object
|
||||
properties:
|
||||
role:
|
||||
type: string
|
||||
const: system
|
||||
default: system
|
||||
description: >-
|
||||
Must be "system" to identify this as a system message
|
||||
content:
|
||||
$ref: '#/components/schemas/InterleavedContent'
|
||||
description: >-
|
||||
The content of the "system prompt". If multiple system messages are provided,
|
||||
they are concatenated. The underlying Llama Stack code may also add other
|
||||
system messages (for example, for formatting tool definitions).
|
||||
name:
|
||||
type: string
|
||||
description: >-
|
||||
(Optional) The name of the system message participant.
|
||||
additionalProperties: false
|
||||
required:
|
||||
- role
|
||||
- content
|
||||
title: OpenAISystemMessageParam
|
||||
description: >-
|
||||
A system message providing instructions or context to the model.
|
||||
OpenAIToolMessageParam:
|
||||
type: object
|
||||
properties:
|
||||
role:
|
||||
type: string
|
||||
const: tool
|
||||
default: tool
|
||||
description: >-
|
||||
Must be "tool" to identify this as a tool response
|
||||
tool_call_id:
|
||||
type: string
|
||||
description: >-
|
||||
Unique identifier for the tool call this response is for
|
||||
content:
|
||||
$ref: '#/components/schemas/InterleavedContent'
|
||||
description: The response content from the tool
|
||||
additionalProperties: false
|
||||
required:
|
||||
- role
|
||||
- tool_call_id
|
||||
- content
|
||||
title: OpenAIToolMessageParam
|
||||
description: >-
|
||||
A message representing the result of a tool invocation in an OpenAI-compatible
|
||||
chat completion request.
|
||||
OpenAIUserMessageParam:
|
||||
type: object
|
||||
properties:
|
||||
role:
|
||||
type: string
|
||||
const: user
|
||||
default: user
|
||||
description: >-
|
||||
Must be "user" to identify this as a user message
|
||||
content:
|
||||
$ref: '#/components/schemas/InterleavedContent'
|
||||
description: >-
|
||||
The content of the message, which can include text and other media
|
||||
name:
|
||||
type: string
|
||||
description: >-
|
||||
(Optional) The name of the user message participant.
|
||||
additionalProperties: false
|
||||
required:
|
||||
- role
|
||||
- content
|
||||
title: OpenAIUserMessageParam
|
||||
description: >-
|
||||
A message from the user in an OpenAI-compatible chat completion request.
|
||||
OpenaiChatCompletionRequest:
|
||||
type: object
|
||||
properties:
|
||||
model:
|
||||
type: string
|
||||
description: >-
|
||||
The identifier of the model to use. The model must be registered with
|
||||
Llama Stack and available via the /models endpoint.
|
||||
messages:
|
||||
type: array
|
||||
items:
|
||||
$ref: '#/components/schemas/OpenAIMessageParam'
|
||||
description: List of messages in the conversation
|
||||
frequency_penalty:
|
||||
type: number
|
||||
description: >-
|
||||
(Optional) The penalty for repeated tokens
|
||||
function_call:
|
||||
oneOf:
|
||||
- type: string
|
||||
- type: object
|
||||
additionalProperties:
|
||||
oneOf:
|
||||
- type: 'null'
|
||||
- type: boolean
|
||||
- type: number
|
||||
- type: string
|
||||
- type: array
|
||||
- type: object
|
||||
description: (Optional) The function call to use
|
||||
functions:
|
||||
type: array
|
||||
items:
|
||||
type: object
|
||||
additionalProperties:
|
||||
oneOf:
|
||||
- type: 'null'
|
||||
- type: boolean
|
||||
- type: number
|
||||
- type: string
|
||||
- type: array
|
||||
- type: object
|
||||
description: (Optional) List of functions to use
|
||||
logit_bias:
|
||||
type: object
|
||||
additionalProperties:
|
||||
type: number
|
||||
description: (Optional) The logit bias to use
|
||||
logprobs:
|
||||
type: boolean
|
||||
description: (Optional) The log probabilities to use
|
||||
max_completion_tokens:
|
||||
type: integer
|
||||
description: >-
|
||||
(Optional) The maximum number of tokens to generate
|
||||
max_tokens:
|
||||
type: integer
|
||||
description: >-
|
||||
(Optional) The maximum number of tokens to generate
|
||||
n:
|
||||
type: integer
|
||||
description: >-
|
||||
(Optional) The number of completions to generate
|
||||
parallel_tool_calls:
|
||||
type: boolean
|
||||
description: >-
|
||||
(Optional) Whether to parallelize tool calls
|
||||
presence_penalty:
|
||||
type: number
|
||||
description: >-
|
||||
(Optional) The penalty for repeated tokens
|
||||
response_format:
|
||||
type: object
|
||||
additionalProperties:
|
||||
type: string
|
||||
description: (Optional) The response format to use
|
||||
seed:
|
||||
type: integer
|
||||
description: (Optional) The seed to use
|
||||
stop:
|
||||
oneOf:
|
||||
- type: string
|
||||
- type: array
|
||||
items:
|
||||
type: string
|
||||
description: (Optional) The stop tokens to use
|
||||
stream:
|
||||
type: boolean
|
||||
description: >-
|
||||
(Optional) Whether to stream the response
|
||||
stream_options:
|
||||
type: object
|
||||
additionalProperties:
|
||||
oneOf:
|
||||
- type: 'null'
|
||||
- type: boolean
|
||||
- type: number
|
||||
- type: string
|
||||
- type: array
|
||||
- type: object
|
||||
description: (Optional) The stream options to use
|
||||
temperature:
|
||||
type: number
|
||||
description: (Optional) The temperature to use
|
||||
tool_choice:
|
||||
oneOf:
|
||||
- type: string
|
||||
- type: object
|
||||
additionalProperties:
|
||||
oneOf:
|
||||
- type: 'null'
|
||||
- type: boolean
|
||||
- type: number
|
||||
- type: string
|
||||
- type: array
|
||||
- type: object
|
||||
description: (Optional) The tool choice to use
|
||||
tools:
|
||||
type: array
|
||||
items:
|
||||
type: object
|
||||
additionalProperties:
|
||||
oneOf:
|
||||
- type: 'null'
|
||||
- type: boolean
|
||||
- type: number
|
||||
- type: string
|
||||
- type: array
|
||||
- type: object
|
||||
description: (Optional) The tools to use
|
||||
top_logprobs:
|
||||
type: integer
|
||||
description: >-
|
||||
(Optional) The top log probabilities to use
|
||||
top_p:
|
||||
type: number
|
||||
description: (Optional) The top p to use
|
||||
user:
|
||||
type: string
|
||||
description: (Optional) The user to use
|
||||
additionalProperties: false
|
||||
required:
|
||||
- model
|
||||
- messages
|
||||
title: OpenaiChatCompletionRequest
|
||||
OpenAIChatCompletion:
|
||||
type: object
|
||||
properties:
|
||||
id:
|
||||
type: string
|
||||
description: The ID of the chat completion
|
||||
choices:
|
||||
type: array
|
||||
items:
|
||||
$ref: '#/components/schemas/OpenAIChoice'
|
||||
description: List of choices
|
||||
object:
|
||||
type: string
|
||||
const: chat.completion
|
||||
default: chat.completion
|
||||
description: >-
|
||||
The object type, which will be "chat.completion"
|
||||
created:
|
||||
type: integer
|
||||
description: >-
|
||||
The Unix timestamp in seconds when the chat completion was created
|
||||
model:
|
||||
type: string
|
||||
description: >-
|
||||
The model that was used to generate the chat completion
|
||||
additionalProperties: false
|
||||
required:
|
||||
- id
|
||||
- choices
|
||||
- object
|
||||
- created
|
||||
- model
|
||||
title: OpenAIChatCompletion
|
||||
description: >-
|
||||
Response from an OpenAI-compatible chat completion request.
|
||||
OpenAIChoice:
|
||||
type: object
|
||||
properties:
|
||||
message:
|
||||
$ref: '#/components/schemas/OpenAIMessageParam'
|
||||
description: The message from the model
|
||||
finish_reason:
|
||||
type: string
|
||||
description: The reason the model stopped generating
|
||||
index:
|
||||
type: integer
|
||||
logprobs:
|
||||
$ref: '#/components/schemas/OpenAIChoiceLogprobs'
|
||||
additionalProperties: false
|
||||
required:
|
||||
- message
|
||||
- finish_reason
|
||||
- index
|
||||
title: OpenAIChoice
|
||||
description: >-
|
||||
A choice from an OpenAI-compatible chat completion response.
|
||||
OpenAIChoiceLogprobs:
|
||||
type: object
|
||||
properties:
|
||||
content:
|
||||
type: array
|
||||
items:
|
||||
$ref: '#/components/schemas/OpenAITokenLogProb'
|
||||
refusal:
|
||||
type: array
|
||||
items:
|
||||
$ref: '#/components/schemas/OpenAITokenLogProb'
|
||||
additionalProperties: false
|
||||
title: OpenAIChoiceLogprobs
|
||||
description: >-
|
||||
The log probabilities for the tokens in the message from an OpenAI-compatible
|
||||
chat completion response.
|
||||
OpenAITokenLogProb:
|
||||
type: object
|
||||
properties:
|
||||
token:
|
||||
type: string
|
||||
bytes:
|
||||
type: array
|
||||
items:
|
||||
type: integer
|
||||
logprob:
|
||||
type: number
|
||||
top_logprobs:
|
||||
type: array
|
||||
items:
|
||||
$ref: '#/components/schemas/OpenAITopLogProb'
|
||||
additionalProperties: false
|
||||
required:
|
||||
- token
|
||||
- logprob
|
||||
- top_logprobs
|
||||
title: OpenAITokenLogProb
|
||||
description: >-
|
||||
The log probability for a token from an OpenAI-compatible chat completion
|
||||
response.
|
||||
OpenAITopLogProb:
|
||||
type: object
|
||||
properties:
|
||||
token:
|
||||
type: string
|
||||
bytes:
|
||||
type: array
|
||||
items:
|
||||
type: integer
|
||||
logprob:
|
||||
type: number
|
||||
additionalProperties: false
|
||||
required:
|
||||
- token
|
||||
- logprob
|
||||
title: OpenAITopLogProb
|
||||
description: >-
|
||||
The top log probability for a token from an OpenAI-compatible chat completion
|
||||
response.
|
||||
OpenaiCompletionRequest:
|
||||
type: object
|
||||
properties:
|
||||
model:
|
||||
type: string
|
||||
description: >-
|
||||
The identifier of the model to use. The model must be registered with
|
||||
Llama Stack and available via the /models endpoint.
|
||||
prompt:
|
||||
oneOf:
|
||||
- type: string
|
||||
- type: array
|
||||
items:
|
||||
type: string
|
||||
- type: array
|
||||
items:
|
||||
type: integer
|
||||
- type: array
|
||||
items:
|
||||
type: array
|
||||
items:
|
||||
type: integer
|
||||
description: The prompt to generate a completion for
|
||||
best_of:
|
||||
type: integer
|
||||
description: >-
|
||||
(Optional) The number of completions to generate
|
||||
echo:
|
||||
type: boolean
|
||||
description: (Optional) Whether to echo the prompt
|
||||
frequency_penalty:
|
||||
type: number
|
||||
description: >-
|
||||
(Optional) The penalty for repeated tokens
|
||||
logit_bias:
|
||||
type: object
|
||||
additionalProperties:
|
||||
type: number
|
||||
description: (Optional) The logit bias to use
|
||||
logprobs:
|
||||
type: boolean
|
||||
description: (Optional) The log probabilities to use
|
||||
max_tokens:
|
||||
type: integer
|
||||
description: >-
|
||||
(Optional) The maximum number of tokens to generate
|
||||
n:
|
||||
type: integer
|
||||
description: >-
|
||||
(Optional) The number of completions to generate
|
||||
presence_penalty:
|
||||
type: number
|
||||
description: >-
|
||||
(Optional) The penalty for repeated tokens
|
||||
seed:
|
||||
type: integer
|
||||
description: (Optional) The seed to use
|
||||
stop:
|
||||
oneOf:
|
||||
- type: string
|
||||
- type: array
|
||||
items:
|
||||
type: string
|
||||
description: (Optional) The stop tokens to use
|
||||
stream:
|
||||
type: boolean
|
||||
description: >-
|
||||
(Optional) Whether to stream the response
|
||||
stream_options:
|
||||
type: object
|
||||
additionalProperties:
|
||||
oneOf:
|
||||
- type: 'null'
|
||||
- type: boolean
|
||||
- type: number
|
||||
- type: string
|
||||
- type: array
|
||||
- type: object
|
||||
description: (Optional) The stream options to use
|
||||
temperature:
|
||||
type: number
|
||||
description: (Optional) The temperature to use
|
||||
top_p:
|
||||
type: number
|
||||
description: (Optional) The top p to use
|
||||
user:
|
||||
type: string
|
||||
description: (Optional) The user to use
|
||||
guided_choice:
|
||||
type: array
|
||||
items:
|
||||
type: string
|
||||
prompt_logprobs:
|
||||
type: integer
|
||||
additionalProperties: false
|
||||
required:
|
||||
- model
|
||||
- prompt
|
||||
title: OpenaiCompletionRequest
|
||||
OpenAICompletion:
|
||||
type: object
|
||||
properties:
|
||||
id:
|
||||
type: string
|
||||
choices:
|
||||
type: array
|
||||
items:
|
||||
$ref: '#/components/schemas/OpenAICompletionChoice'
|
||||
created:
|
||||
type: integer
|
||||
model:
|
||||
type: string
|
||||
object:
|
||||
type: string
|
||||
const: text_completion
|
||||
default: text_completion
|
||||
additionalProperties: false
|
||||
required:
|
||||
- id
|
||||
- choices
|
||||
- created
|
||||
- model
|
||||
- object
|
||||
title: OpenAICompletion
|
||||
description: >-
|
||||
Response from an OpenAI-compatible completion request.
|
||||
OpenAICompletionChoice:
|
||||
type: object
|
||||
properties:
|
||||
finish_reason:
|
||||
type: string
|
||||
text:
|
||||
type: string
|
||||
index:
|
||||
type: integer
|
||||
logprobs:
|
||||
$ref: '#/components/schemas/OpenAIChoiceLogprobs'
|
||||
additionalProperties: false
|
||||
required:
|
||||
- finish_reason
|
||||
- text
|
||||
- index
|
||||
title: OpenAICompletionChoice
|
||||
description: >-
|
||||
A choice from an OpenAI-compatible completion response.
|
||||
OpenAIModel:
|
||||
type: object
|
||||
properties:
|
||||
id:
|
||||
type: string
|
||||
object:
|
||||
type: string
|
||||
const: model
|
||||
default: model
|
||||
created:
|
||||
type: integer
|
||||
owned_by:
|
||||
type: string
|
||||
additionalProperties: false
|
||||
required:
|
||||
- id
|
||||
- object
|
||||
- created
|
||||
- owned_by
|
||||
title: OpenAIModel
|
||||
description: A model from OpenAI.
|
||||
OpenAIListModelsResponse:
|
||||
type: object
|
||||
properties:
|
||||
data:
|
||||
type: array
|
||||
items:
|
||||
$ref: '#/components/schemas/OpenAIModel'
|
||||
additionalProperties: false
|
||||
required:
|
||||
- data
|
||||
title: OpenAIListModelsResponse
|
||||
DPOAlignmentConfig:
|
||||
type: object
|
||||
properties:
|
||||
|
|
|
|||
|
|
@ -2,7 +2,7 @@
|
|||
|
||||
The Llama Stack runtime configuration is specified as a YAML file. Here is a simplified version of an example configuration file for the Ollama distribution:
|
||||
|
||||
```{dropdown} Sample Configuration File
|
||||
```{dropdown} 👋 Click here for a Sample Configuration File
|
||||
|
||||
```yaml
|
||||
version: 2
|
||||
|
|
|
|||
|
|
@ -11,7 +11,12 @@ First, create a local Kubernetes cluster via Kind:
|
|||
kind create cluster --image kindest/node:v1.32.0 --name llama-stack-test
|
||||
```
|
||||
|
||||
First, create a Kubernetes PVC and Secret for downloading and storing Hugging Face model:
|
||||
First set your hugging face token as an environment variable.
|
||||
```
|
||||
export HF_TOKEN=$(echo -n "your-hf-token" | base64)
|
||||
```
|
||||
|
||||
Now create a Kubernetes PVC and Secret for downloading and storing Hugging Face model:
|
||||
|
||||
```
|
||||
cat <<EOF |kubectl apply -f -
|
||||
|
|
@ -33,7 +38,8 @@ metadata:
|
|||
name: hf-token-secret
|
||||
type: Opaque
|
||||
data:
|
||||
token: $(HF_TOKEN)
|
||||
token: $HF_TOKEN
|
||||
EOF
|
||||
```
|
||||
|
||||
|
||||
|
|
@ -120,7 +126,7 @@ providers:
|
|||
Once we have defined the run configuration for Llama Stack, we can build an image with that configuration and the server source code:
|
||||
|
||||
```
|
||||
cat >/tmp/test-vllm-llama-stack/Containerfile.llama-stack-run-k8s <<EOF
|
||||
tmp_dir=$(mktemp -d) && cat >$tmp_dir/Containerfile.llama-stack-run-k8s <<EOF
|
||||
FROM distribution-myenv:dev
|
||||
|
||||
RUN apt-get update && apt-get install -y git
|
||||
|
|
@ -128,7 +134,7 @@ RUN git clone https://github.com/meta-llama/llama-stack.git /app/llama-stack-sou
|
|||
|
||||
ADD ./vllm-llama-stack-run-k8s.yaml /app/config.yaml
|
||||
EOF
|
||||
podman build -f /tmp/test-vllm-llama-stack/Containerfile.llama-stack-run-k8s -t llama-stack-run-k8s /tmp/test-vllm-llama-stack
|
||||
podman build -f $tmp_dir/Containerfile.llama-stack-run-k8s -t llama-stack-run-k8s $tmp_dir
|
||||
```
|
||||
|
||||
### Deploying Llama Stack Server in Kubernetes
|
||||
|
|
|
|||
|
|
@ -1,88 +0,0 @@
|
|||
<!-- This file was auto-generated by distro_codegen.py, please edit source -->
|
||||
# NVIDIA Distribution
|
||||
|
||||
The `llamastack/distribution-nvidia` distribution consists of the following provider configurations.
|
||||
|
||||
| API | Provider(s) |
|
||||
|-----|-------------|
|
||||
| agents | `inline::meta-reference` |
|
||||
| datasetio | `inline::localfs` |
|
||||
| eval | `inline::meta-reference` |
|
||||
| inference | `remote::nvidia` |
|
||||
| post_training | `remote::nvidia` |
|
||||
| safety | `remote::nvidia` |
|
||||
| scoring | `inline::basic` |
|
||||
| telemetry | `inline::meta-reference` |
|
||||
| tool_runtime | `inline::rag-runtime` |
|
||||
| vector_io | `inline::faiss` |
|
||||
|
||||
|
||||
### Environment Variables
|
||||
|
||||
The following environment variables can be configured:
|
||||
|
||||
- `NVIDIA_API_KEY`: NVIDIA API Key (default: ``)
|
||||
- `NVIDIA_USER_ID`: NVIDIA User ID (default: `llama-stack-user`)
|
||||
- `NVIDIA_DATASET_NAMESPACE`: NVIDIA Dataset Namespace (default: `default`)
|
||||
- `NVIDIA_ACCESS_POLICIES`: NVIDIA Access Policies (default: `{}`)
|
||||
- `NVIDIA_PROJECT_ID`: NVIDIA Project ID (default: `test-project`)
|
||||
- `NVIDIA_CUSTOMIZER_URL`: NVIDIA Customizer URL (default: `https://customizer.api.nvidia.com`)
|
||||
- `NVIDIA_OUTPUT_MODEL_DIR`: NVIDIA Output Model Directory (default: `test-example-model@v1`)
|
||||
- `GUARDRAILS_SERVICE_URL`: URL for the NeMo Guardrails Service (default: `http://0.0.0.0:7331`)
|
||||
- `INFERENCE_MODEL`: Inference model (default: `Llama3.1-8B-Instruct`)
|
||||
- `SAFETY_MODEL`: Name of the model to use for safety (default: `meta/llama-3.1-8b-instruct`)
|
||||
|
||||
### Models
|
||||
|
||||
The following models are available by default:
|
||||
|
||||
- `meta/llama3-8b-instruct (aliases: meta-llama/Llama-3-8B-Instruct)`
|
||||
- `meta/llama3-70b-instruct (aliases: meta-llama/Llama-3-70B-Instruct)`
|
||||
- `meta/llama-3.1-8b-instruct (aliases: meta-llama/Llama-3.1-8B-Instruct)`
|
||||
- `meta/llama-3.1-70b-instruct (aliases: meta-llama/Llama-3.1-70B-Instruct)`
|
||||
- `meta/llama-3.1-405b-instruct (aliases: meta-llama/Llama-3.1-405B-Instruct-FP8)`
|
||||
- `meta/llama-3.2-1b-instruct (aliases: meta-llama/Llama-3.2-1B-Instruct)`
|
||||
- `meta/llama-3.2-3b-instruct (aliases: meta-llama/Llama-3.2-3B-Instruct)`
|
||||
- `meta/llama-3.2-11b-vision-instruct (aliases: meta-llama/Llama-3.2-11B-Vision-Instruct)`
|
||||
- `meta/llama-3.2-90b-vision-instruct (aliases: meta-llama/Llama-3.2-90B-Vision-Instruct)`
|
||||
- `nvidia/llama-3.2-nv-embedqa-1b-v2 `
|
||||
- `nvidia/nv-embedqa-e5-v5 `
|
||||
- `nvidia/nv-embedqa-mistral-7b-v2 `
|
||||
- `snowflake/arctic-embed-l `
|
||||
|
||||
|
||||
### Prerequisite: API Keys
|
||||
|
||||
Make sure you have access to a NVIDIA API Key. You can get one by visiting [https://build.nvidia.com/](https://build.nvidia.com/).
|
||||
|
||||
|
||||
## Running Llama Stack with NVIDIA
|
||||
|
||||
You can do this via Conda (build code) or Docker which has a pre-built image.
|
||||
|
||||
### Via Docker
|
||||
|
||||
This method allows you to get started quickly without having to build the distribution code.
|
||||
|
||||
```bash
|
||||
LLAMA_STACK_PORT=8321
|
||||
docker run \
|
||||
-it \
|
||||
--pull always \
|
||||
-p $LLAMA_STACK_PORT:$LLAMA_STACK_PORT \
|
||||
-v ./run.yaml:/root/my-run.yaml \
|
||||
llamastack/distribution-nvidia \
|
||||
--yaml-config /root/my-run.yaml \
|
||||
--port $LLAMA_STACK_PORT \
|
||||
--env NVIDIA_API_KEY=$NVIDIA_API_KEY
|
||||
```
|
||||
|
||||
### Via Conda
|
||||
|
||||
```bash
|
||||
llama stack build --template nvidia --image-type conda
|
||||
llama stack run ./run.yaml \
|
||||
--port 8321 \
|
||||
--env NVIDIA_API_KEY=$NVIDIA_API_KEY
|
||||
--env INFERENCE_MODEL=$INFERENCE_MODEL
|
||||
```
|
||||
|
|
@ -1,3 +1,4 @@
|
|||
<!-- This file was auto-generated by distro_codegen.py, please edit source -->
|
||||
# NVIDIA Distribution
|
||||
|
||||
The `llamastack/distribution-nvidia` distribution consists of the following provider configurations.
|
||||
|
|
@ -5,24 +6,49 @@ The `llamastack/distribution-nvidia` distribution consists of the following prov
|
|||
| API | Provider(s) |
|
||||
|-----|-------------|
|
||||
| agents | `inline::meta-reference` |
|
||||
| datasetio | `inline::localfs` |
|
||||
| eval | `inline::meta-reference` |
|
||||
| inference | `remote::nvidia` |
|
||||
| memory | `inline::faiss`, `remote::chromadb`, `remote::pgvector` |
|
||||
| safety | `inline::llama-guard` |
|
||||
| post_training | `remote::nvidia` |
|
||||
| safety | `remote::nvidia` |
|
||||
| scoring | `inline::basic` |
|
||||
| telemetry | `inline::meta-reference` |
|
||||
| tool_runtime | `inline::rag-runtime` |
|
||||
| vector_io | `inline::faiss` |
|
||||
|
||||
|
||||
### Environment Variables
|
||||
|
||||
The following environment variables can be configured:
|
||||
|
||||
- `LLAMASTACK_PORT`: Port for the Llama Stack distribution server (default: `8321`)
|
||||
- `NVIDIA_API_KEY`: NVIDIA API Key (default: ``)
|
||||
- `NVIDIA_USER_ID`: NVIDIA User ID (default: `llama-stack-user`)
|
||||
- `NVIDIA_DATASET_NAMESPACE`: NVIDIA Dataset Namespace (default: `default`)
|
||||
- `NVIDIA_ACCESS_POLICIES`: NVIDIA Access Policies (default: `{}`)
|
||||
- `NVIDIA_PROJECT_ID`: NVIDIA Project ID (default: `test-project`)
|
||||
- `NVIDIA_CUSTOMIZER_URL`: NVIDIA Customizer URL (default: `https://customizer.api.nvidia.com`)
|
||||
- `NVIDIA_OUTPUT_MODEL_DIR`: NVIDIA Output Model Directory (default: `test-example-model@v1`)
|
||||
- `GUARDRAILS_SERVICE_URL`: URL for the NeMo Guardrails Service (default: `http://0.0.0.0:7331`)
|
||||
- `INFERENCE_MODEL`: Inference model (default: `Llama3.1-8B-Instruct`)
|
||||
- `SAFETY_MODEL`: Name of the model to use for safety (default: `meta/llama-3.1-8b-instruct`)
|
||||
|
||||
### Models
|
||||
|
||||
The following models are available by default:
|
||||
|
||||
- `${env.INFERENCE_MODEL} (None)`
|
||||
- `meta/llama3-8b-instruct (aliases: meta-llama/Llama-3-8B-Instruct)`
|
||||
- `meta/llama3-70b-instruct (aliases: meta-llama/Llama-3-70B-Instruct)`
|
||||
- `meta/llama-3.1-8b-instruct (aliases: meta-llama/Llama-3.1-8B-Instruct)`
|
||||
- `meta/llama-3.1-70b-instruct (aliases: meta-llama/Llama-3.1-70B-Instruct)`
|
||||
- `meta/llama-3.1-405b-instruct (aliases: meta-llama/Llama-3.1-405B-Instruct-FP8)`
|
||||
- `meta/llama-3.2-1b-instruct (aliases: meta-llama/Llama-3.2-1B-Instruct)`
|
||||
- `meta/llama-3.2-3b-instruct (aliases: meta-llama/Llama-3.2-3B-Instruct)`
|
||||
- `meta/llama-3.2-11b-vision-instruct (aliases: meta-llama/Llama-3.2-11B-Vision-Instruct)`
|
||||
- `meta/llama-3.2-90b-vision-instruct (aliases: meta-llama/Llama-3.2-90B-Vision-Instruct)`
|
||||
- `nvidia/llama-3.2-nv-embedqa-1b-v2 `
|
||||
- `nvidia/nv-embedqa-e5-v5 `
|
||||
- `nvidia/nv-embedqa-mistral-7b-v2 `
|
||||
- `snowflake/arctic-embed-l `
|
||||
|
||||
|
||||
### Prerequisite: API Keys
|
||||
|
|
@ -58,4 +84,5 @@ llama stack build --template nvidia --image-type conda
|
|||
llama stack run ./run.yaml \
|
||||
--port 8321 \
|
||||
--env NVIDIA_API_KEY=$NVIDIA_API_KEY
|
||||
--env INFERENCE_MODEL=$INFERENCE_MODEL
|
||||
```
|
||||
|
|
|
|||
|
|
@ -2,22 +2,22 @@
|
|||
|
||||
You can run a Llama Stack server in one of the following ways:
|
||||
|
||||
**As a Library**:
|
||||
## As a Library:
|
||||
|
||||
This is the simplest way to get started. Using Llama Stack as a library means you do not need to start a server. This is especially useful when you are not running inference locally and relying on an external inference service (eg. fireworks, together, groq, etc.) See [Using Llama Stack as a Library](importing_as_library)
|
||||
|
||||
|
||||
**Container**:
|
||||
## Container:
|
||||
|
||||
Another simple way to start interacting with Llama Stack is to just spin up a container (via Docker or Podman) which is pre-built with all the providers you need. We provide a number of pre-built images so you can start a Llama Stack server instantly. You can also build your own custom container. Which distribution to choose depends on the hardware you have. See [Selection of a Distribution](selection) for more details.
|
||||
|
||||
|
||||
**Conda**:
|
||||
## Conda:
|
||||
|
||||
If you have a custom or an advanced setup or you are developing on Llama Stack you can also build a custom Llama Stack server. Using `llama stack build` and `llama stack run` you can build/run a custom Llama Stack server containing the exact combination of providers you wish. We have also provided various templates to make getting started easier. See [Building a Custom Distribution](building_distro) for more details.
|
||||
|
||||
|
||||
**Kubernetes**:
|
||||
## Kubernetes:
|
||||
|
||||
If you have built a container image and want to deploy it in a Kubernetes cluster instead of starting the Llama Stack server locally. See [Kubernetes Deployment Guide](kubernetes_deployment) for more details.
|
||||
|
||||
|
|
|
|||
541
docs/source/getting_started/detailed_tutorial.md
Normal file
541
docs/source/getting_started/detailed_tutorial.md
Normal file
|
|
@ -0,0 +1,541 @@
|
|||
# Detailed Tutorial
|
||||
|
||||
In this guide, we'll walk through how you can use the Llama Stack (server and client SDK) to test a simple agent.
|
||||
A Llama Stack agent is a simple integrated system that can perform tasks by combining a Llama model for reasoning with
|
||||
tools (e.g., RAG, web search, code execution, etc.) for taking actions.
|
||||
In Llama Stack, we provide a server exposing multiple APIs. These APIs are backed by implementations from different providers.
|
||||
|
||||
Llama Stack is a stateful service with REST APIs to support seamless transition of AI applications across different environments. The server can be run in a variety of ways, including as a standalone binary, Docker container, or hosted service. You can build and test using a local server first and deploy to a hosted endpoint for production.
|
||||
|
||||
In this guide, we'll walk through how to build a RAG agent locally using Llama Stack with [Ollama](https://ollama.com/)
|
||||
as the inference [provider](../providers/index.md#inference) for a Llama Model.
|
||||
|
||||
## Step 1: Installation and Setup
|
||||
|
||||
Install Ollama by following the instructions on the [Ollama website](https://ollama.com/download), then
|
||||
download Llama 3.2 3B model, and then start the Ollama service.
|
||||
```bash
|
||||
ollama pull llama3.2:3b
|
||||
ollama run llama3.2:3b --keepalive 60m
|
||||
```
|
||||
|
||||
Install [uv](https://docs.astral.sh/uv/) to setup your virtual environment
|
||||
|
||||
::::{tab-set}
|
||||
|
||||
:::{tab-item} macOS and Linux
|
||||
Use `curl` to download the script and execute it with `sh`:
|
||||
```console
|
||||
curl -LsSf https://astral.sh/uv/install.sh | sh
|
||||
```
|
||||
:::
|
||||
|
||||
:::{tab-item} Windows
|
||||
Use `irm` to download the script and execute it with `iex`:
|
||||
|
||||
```console
|
||||
powershell -ExecutionPolicy ByPass -c "irm https://astral.sh/uv/install.ps1 | iex"
|
||||
```
|
||||
:::
|
||||
::::
|
||||
|
||||
Setup your virtual environment.
|
||||
|
||||
```bash
|
||||
uv venv --python 3.10
|
||||
source .venv/bin/activate
|
||||
```
|
||||
## Step 2: Run Llama Stack
|
||||
Llama Stack is a server that exposes multiple APIs, you connect with it using the Llama Stack client SDK.
|
||||
|
||||
::::{tab-set}
|
||||
|
||||
:::{tab-item} Using `venv`
|
||||
You can use Python to build and run the Llama Stack server, which is useful for testing and development.
|
||||
|
||||
Llama Stack uses a [YAML configuration file](../distributions/configuration.md) to specify the stack setup,
|
||||
which defines the providers and their settings.
|
||||
Now let's build and run the Llama Stack config for Ollama.
|
||||
|
||||
```bash
|
||||
INFERENCE_MODEL=llama3.2:3b llama stack build --template ollama --image-type venv --run
|
||||
```
|
||||
:::
|
||||
:::{tab-item} Using `conda`
|
||||
You can use Python to build and run the Llama Stack server, which is useful for testing and development.
|
||||
|
||||
Llama Stack uses a [YAML configuration file](../distributions/configuration.md) to specify the stack setup,
|
||||
which defines the providers and their settings.
|
||||
Now let's build and run the Llama Stack config for Ollama.
|
||||
|
||||
```bash
|
||||
INFERENCE_MODEL=llama3.2:3b llama stack build --template ollama --image-type conda --image-name llama3-3b-conda --run
|
||||
```
|
||||
:::
|
||||
:::{tab-item} Using a Container
|
||||
You can use a container image to run the Llama Stack server. We provide several container images for the server
|
||||
component that works with different inference providers out of the box. For this guide, we will use
|
||||
`llamastack/distribution-ollama` as the container image. If you'd like to build your own image or customize the
|
||||
configurations, please check out [this guide](../references/index.md).
|
||||
First lets setup some environment variables and create a local directory to mount into the container’s file system.
|
||||
```bash
|
||||
export INFERENCE_MODEL="llama3.2:3b"
|
||||
export LLAMA_STACK_PORT=8321
|
||||
mkdir -p ~/.llama
|
||||
```
|
||||
Then start the server using the container tool of your choice. For example, if you are running Docker you can use the
|
||||
following command:
|
||||
```bash
|
||||
docker run -it \
|
||||
--pull always \
|
||||
-p $LLAMA_STACK_PORT:$LLAMA_STACK_PORT \
|
||||
-v ~/.llama:/root/.llama \
|
||||
llamastack/distribution-ollama \
|
||||
--port $LLAMA_STACK_PORT \
|
||||
--env INFERENCE_MODEL=$INFERENCE_MODEL \
|
||||
--env OLLAMA_URL=http://host.docker.internal:11434
|
||||
```
|
||||
Note to start the container with Podman, you can do the same but replace `docker` at the start of the command with
|
||||
`podman`. If you are using `podman` older than `4.7.0`, please also replace `host.docker.internal` in the `OLLAMA_URL`
|
||||
with `host.containers.internal`.
|
||||
|
||||
The configuration YAML for the Ollama distribution is available at `distributions/ollama/run.yaml`.
|
||||
|
||||
```{tip}
|
||||
|
||||
Docker containers run in their own isolated network namespaces on Linux. To allow the container to communicate with services running on the host via `localhost`, you need `--network=host`. This makes the container use the host’s network directly so it can connect to Ollama running on `localhost:11434`.
|
||||
|
||||
Linux users having issues running the above command should instead try the following:
|
||||
```bash
|
||||
docker run -it \
|
||||
--pull always \
|
||||
-p $LLAMA_STACK_PORT:$LLAMA_STACK_PORT \
|
||||
-v ~/.llama:/root/.llama \
|
||||
--network=host \
|
||||
llamastack/distribution-ollama \
|
||||
--port $LLAMA_STACK_PORT \
|
||||
--env INFERENCE_MODEL=$INFERENCE_MODEL \
|
||||
--env OLLAMA_URL=http://localhost:11434
|
||||
```
|
||||
:::
|
||||
::::
|
||||
You will see output like below:
|
||||
```
|
||||
INFO: Application startup complete.
|
||||
INFO: Uvicorn running on http://['::', '0.0.0.0']:8321 (Press CTRL+C to quit)
|
||||
```
|
||||
|
||||
Now you can use the Llama Stack client to run inference and build agents!
|
||||
|
||||
You can reuse the server setup or use the [Llama Stack Client](https://github.com/meta-llama/llama-stack-client-python/).
|
||||
Note that the client package is already included in the `llama-stack` package.
|
||||
|
||||
## Step 3: Run Client CLI
|
||||
|
||||
Open a new terminal and navigate to the same directory you started the server from. Then set up a new or activate your
|
||||
existing server virtual environment.
|
||||
|
||||
::::{tab-set}
|
||||
|
||||
:::{tab-item} Reuse Server `venv`
|
||||
```bash
|
||||
# The client is included in the llama-stack package so we just activate the server venv
|
||||
source .venv/bin/activate
|
||||
```
|
||||
:::
|
||||
|
||||
:::{tab-item} Install with `venv`
|
||||
```bash
|
||||
uv venv client --python 3.10
|
||||
source client/bin/activate
|
||||
pip install llama-stack-client
|
||||
```
|
||||
:::
|
||||
|
||||
:::{tab-item} Install with `conda`
|
||||
```bash
|
||||
yes | conda create -n stack-client python=3.10
|
||||
conda activate stack-client
|
||||
pip install llama-stack-client
|
||||
```
|
||||
:::
|
||||
::::
|
||||
|
||||
Now let's use the `llama-stack-client` [CLI](../references/llama_stack_client_cli_reference.md) to check the
|
||||
connectivity to the server.
|
||||
|
||||
```bash
|
||||
llama-stack-client configure --endpoint http://localhost:8321 --api-key none
|
||||
```
|
||||
You will see the below:
|
||||
```
|
||||
Done! You can now use the Llama Stack Client CLI with endpoint http://localhost:8321
|
||||
```
|
||||
|
||||
List the models
|
||||
```bash
|
||||
llama-stack-client models list
|
||||
Available Models
|
||||
|
||||
┏━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━┓
|
||||
┃ model_type ┃ identifier ┃ provider_resource_id ┃ metadata ┃ provider_id ┃
|
||||
┡━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━┩
|
||||
│ embedding │ all-MiniLM-L6-v2 │ all-minilm:latest │ {'embedding_dimension': 384.0} │ ollama │
|
||||
├─────────────────┼─────────────────────────────────────┼─────────────────────────────────────┼───────────────────────────────────────────┼─────────────────┤
|
||||
│ llm │ llama3.2:3b │ llama3.2:3b │ │ ollama │
|
||||
└─────────────────┴─────────────────────────────────────┴─────────────────────────────────────┴───────────────────────────────────────────┴─────────────────┘
|
||||
|
||||
Total models: 2
|
||||
|
||||
```
|
||||
You can test basic Llama inference completion using the CLI.
|
||||
|
||||
```bash
|
||||
llama-stack-client inference chat-completion --message "tell me a joke"
|
||||
```
|
||||
Sample output:
|
||||
```python
|
||||
ChatCompletionResponse(
|
||||
completion_message=CompletionMessage(
|
||||
content="Here's one:\n\nWhat do you call a fake noodle?\n\nAn impasta!",
|
||||
role="assistant",
|
||||
stop_reason="end_of_turn",
|
||||
tool_calls=[],
|
||||
),
|
||||
logprobs=None,
|
||||
metrics=[
|
||||
Metric(metric="prompt_tokens", value=14.0, unit=None),
|
||||
Metric(metric="completion_tokens", value=27.0, unit=None),
|
||||
Metric(metric="total_tokens", value=41.0, unit=None),
|
||||
],
|
||||
)
|
||||
```
|
||||
|
||||
## Step 4: Run the Demos
|
||||
|
||||
Note that these demos show the [Python Client SDK](../references/python_sdk_reference/index.md).
|
||||
Other SDKs are also available, please refer to the [Client SDK](../index.md#client-sdks) list for the complete options.
|
||||
|
||||
::::{tab-set}
|
||||
|
||||
:::{tab-item} Basic Inference
|
||||
Now you can run inference using the Llama Stack client SDK.
|
||||
|
||||
### i. Create the Script
|
||||
|
||||
Create a file `inference.py` and add the following code:
|
||||
```python
|
||||
from llama_stack_client import LlamaStackClient
|
||||
|
||||
client = LlamaStackClient(base_url="http://localhost:8321")
|
||||
|
||||
# List available models
|
||||
models = client.models.list()
|
||||
|
||||
# Select the first LLM
|
||||
llm = next(m for m in models if m.model_type == "llm")
|
||||
model_id = llm.identifier
|
||||
|
||||
print("Model:", model_id)
|
||||
|
||||
response = client.inference.chat_completion(
|
||||
model_id=model_id,
|
||||
messages=[
|
||||
{"role": "system", "content": "You are a helpful assistant."},
|
||||
{"role": "user", "content": "Write a haiku about coding"},
|
||||
],
|
||||
)
|
||||
print(response.completion_message.content)
|
||||
```
|
||||
|
||||
### ii. Run the Script
|
||||
Let's run the script using `uv`
|
||||
```bash
|
||||
uv run python inference.py
|
||||
```
|
||||
Which will output:
|
||||
```
|
||||
Model: llama3.2:3b
|
||||
Here is a haiku about coding:
|
||||
|
||||
Lines of code unfold
|
||||
Logic flows through digital night
|
||||
Beauty in the bits
|
||||
```
|
||||
:::
|
||||
|
||||
:::{tab-item} Build a Simple Agent
|
||||
Next we can move beyond simple inference and build an agent that can perform tasks using the Llama Stack server.
|
||||
### i. Create the Script
|
||||
Create a file `agent.py` and add the following code:
|
||||
|
||||
```python
|
||||
from llama_stack_client import LlamaStackClient
|
||||
from llama_stack_client import Agent, AgentEventLogger
|
||||
from rich.pretty import pprint
|
||||
import uuid
|
||||
|
||||
client = LlamaStackClient(base_url=f"http://localhost:8321")
|
||||
|
||||
models = client.models.list()
|
||||
llm = next(m for m in models if m.model_type == "llm")
|
||||
model_id = llm.identifier
|
||||
|
||||
agent = Agent(client, model=model_id, instructions="You are a helpful assistant.")
|
||||
|
||||
s_id = agent.create_session(session_name=f"s{uuid.uuid4().hex}")
|
||||
|
||||
print("Non-streaming ...")
|
||||
response = agent.create_turn(
|
||||
messages=[{"role": "user", "content": "Who are you?"}],
|
||||
session_id=s_id,
|
||||
stream=False,
|
||||
)
|
||||
print("agent>", response.output_message.content)
|
||||
|
||||
print("Streaming ...")
|
||||
stream = agent.create_turn(
|
||||
messages=[{"role": "user", "content": "Who are you?"}], session_id=s_id, stream=True
|
||||
)
|
||||
for event in stream:
|
||||
pprint(event)
|
||||
|
||||
print("Streaming with print helper...")
|
||||
stream = agent.create_turn(
|
||||
messages=[{"role": "user", "content": "Who are you?"}], session_id=s_id, stream=True
|
||||
)
|
||||
for event in AgentEventLogger().log(stream):
|
||||
event.print()
|
||||
```
|
||||
### ii. Run the Script
|
||||
Let's run the script using `uv`
|
||||
```bash
|
||||
uv run python agent.py
|
||||
```
|
||||
|
||||
```{dropdown} 👋 Click here to see the sample output
|
||||
Non-streaming ...
|
||||
agent> I'm an artificial intelligence designed to assist and communicate with users like you. I don't have a personal identity, but I'm here to provide information, answer questions, and help with tasks to the best of my abilities.
|
||||
|
||||
I can be used for a wide range of purposes, such as:
|
||||
|
||||
* Providing definitions and explanations
|
||||
* Offering suggestions and ideas
|
||||
* Helping with language translation
|
||||
* Assisting with writing and proofreading
|
||||
* Generating text or responses to questions
|
||||
* Playing simple games or chatting about topics of interest
|
||||
|
||||
I'm constantly learning and improving my abilities, so feel free to ask me anything, and I'll do my best to help!
|
||||
|
||||
Streaming ...
|
||||
AgentTurnResponseStreamChunk(
|
||||
│ event=TurnResponseEvent(
|
||||
│ │ payload=AgentTurnResponseStepStartPayload(
|
||||
│ │ │ event_type='step_start',
|
||||
│ │ │ step_id='69831607-fa75-424a-949b-e2049e3129d1',
|
||||
│ │ │ step_type='inference',
|
||||
│ │ │ metadata={}
|
||||
│ │ )
|
||||
│ )
|
||||
)
|
||||
AgentTurnResponseStreamChunk(
|
||||
│ event=TurnResponseEvent(
|
||||
│ │ payload=AgentTurnResponseStepProgressPayload(
|
||||
│ │ │ delta=TextDelta(text='As', type='text'),
|
||||
│ │ │ event_type='step_progress',
|
||||
│ │ │ step_id='69831607-fa75-424a-949b-e2049e3129d1',
|
||||
│ │ │ step_type='inference'
|
||||
│ │ )
|
||||
│ )
|
||||
)
|
||||
AgentTurnResponseStreamChunk(
|
||||
│ event=TurnResponseEvent(
|
||||
│ │ payload=AgentTurnResponseStepProgressPayload(
|
||||
│ │ │ delta=TextDelta(text=' a', type='text'),
|
||||
│ │ │ event_type='step_progress',
|
||||
│ │ │ step_id='69831607-fa75-424a-949b-e2049e3129d1',
|
||||
│ │ │ step_type='inference'
|
||||
│ │ )
|
||||
│ )
|
||||
)
|
||||
...
|
||||
AgentTurnResponseStreamChunk(
|
||||
│ event=TurnResponseEvent(
|
||||
│ │ payload=AgentTurnResponseStepCompletePayload(
|
||||
│ │ │ event_type='step_complete',
|
||||
│ │ │ step_details=InferenceStep(
|
||||
│ │ │ │ api_model_response=CompletionMessage(
|
||||
│ │ │ │ │ content='As a conversational AI, I don\'t have a personal identity in the classical sense. I exist as a program running on computer servers, designed to process and respond to text-based inputs.\n\nI\'m an instance of a type of artificial intelligence called a "language model," which is trained on vast amounts of text data to generate human-like responses. My primary function is to understand and respond to natural language inputs, like our conversation right now.\n\nThink of me as a virtual assistant, a chatbot, or a conversational interface – I\'m here to provide information, answer questions, and engage in conversation to the best of my abilities. I don\'t have feelings, emotions, or consciousness like humans do, but I\'m designed to simulate human-like interactions to make our conversations feel more natural and helpful.\n\nSo, that\'s me in a nutshell! What can I help you with today?',
|
||||
│ │ │ │ │ role='assistant',
|
||||
│ │ │ │ │ stop_reason='end_of_turn',
|
||||
│ │ │ │ │ tool_calls=[]
|
||||
│ │ │ │ ),
|
||||
│ │ │ │ step_id='69831607-fa75-424a-949b-e2049e3129d1',
|
||||
│ │ │ │ step_type='inference',
|
||||
│ │ │ │ turn_id='8b360202-f7cb-4786-baa9-166a1b46e2ca',
|
||||
│ │ │ │ completed_at=datetime.datetime(2025, 4, 3, 1, 15, 21, 716174, tzinfo=TzInfo(UTC)),
|
||||
│ │ │ │ started_at=datetime.datetime(2025, 4, 3, 1, 15, 14, 28823, tzinfo=TzInfo(UTC))
|
||||
│ │ │ ),
|
||||
│ │ │ step_id='69831607-fa75-424a-949b-e2049e3129d1',
|
||||
│ │ │ step_type='inference'
|
||||
│ │ )
|
||||
│ )
|
||||
)
|
||||
AgentTurnResponseStreamChunk(
|
||||
│ event=TurnResponseEvent(
|
||||
│ │ payload=AgentTurnResponseTurnCompletePayload(
|
||||
│ │ │ event_type='turn_complete',
|
||||
│ │ │ turn=Turn(
|
||||
│ │ │ │ input_messages=[UserMessage(content='Who are you?', role='user', context=None)],
|
||||
│ │ │ │ output_message=CompletionMessage(
|
||||
│ │ │ │ │ content='As a conversational AI, I don\'t have a personal identity in the classical sense. I exist as a program running on computer servers, designed to process and respond to text-based inputs.\n\nI\'m an instance of a type of artificial intelligence called a "language model," which is trained on vast amounts of text data to generate human-like responses. My primary function is to understand and respond to natural language inputs, like our conversation right now.\n\nThink of me as a virtual assistant, a chatbot, or a conversational interface – I\'m here to provide information, answer questions, and engage in conversation to the best of my abilities. I don\'t have feelings, emotions, or consciousness like humans do, but I\'m designed to simulate human-like interactions to make our conversations feel more natural and helpful.\n\nSo, that\'s me in a nutshell! What can I help you with today?',
|
||||
│ │ │ │ │ role='assistant',
|
||||
│ │ │ │ │ stop_reason='end_of_turn',
|
||||
│ │ │ │ │ tool_calls=[]
|
||||
│ │ │ │ ),
|
||||
│ │ │ │ session_id='abd4afea-4324-43f4-9513-cfe3970d92e8',
|
||||
│ │ │ │ started_at=datetime.datetime(2025, 4, 3, 1, 15, 14, 28722, tzinfo=TzInfo(UTC)),
|
||||
│ │ │ │ steps=[
|
||||
│ │ │ │ │ InferenceStep(
|
||||
│ │ │ │ │ │ api_model_response=CompletionMessage(
|
||||
│ │ │ │ │ │ │ content='As a conversational AI, I don\'t have a personal identity in the classical sense. I exist as a program running on computer servers, designed to process and respond to text-based inputs.\n\nI\'m an instance of a type of artificial intelligence called a "language model," which is trained on vast amounts of text data to generate human-like responses. My primary function is to understand and respond to natural language inputs, like our conversation right now.\n\nThink of me as a virtual assistant, a chatbot, or a conversational interface – I\'m here to provide information, answer questions, and engage in conversation to the best of my abilities. I don\'t have feelings, emotions, or consciousness like humans do, but I\'m designed to simulate human-like interactions to make our conversations feel more natural and helpful.\n\nSo, that\'s me in a nutshell! What can I help you with today?',
|
||||
│ │ │ │ │ │ │ role='assistant',
|
||||
│ │ │ │ │ │ │ stop_reason='end_of_turn',
|
||||
│ │ │ │ │ │ │ tool_calls=[]
|
||||
│ │ │ │ │ │ ),
|
||||
│ │ │ │ │ │ step_id='69831607-fa75-424a-949b-e2049e3129d1',
|
||||
│ │ │ │ │ │ step_type='inference',
|
||||
│ │ │ │ │ │ turn_id='8b360202-f7cb-4786-baa9-166a1b46e2ca',
|
||||
│ │ │ │ │ │ completed_at=datetime.datetime(2025, 4, 3, 1, 15, 21, 716174, tzinfo=TzInfo(UTC)),
|
||||
│ │ │ │ │ │ started_at=datetime.datetime(2025, 4, 3, 1, 15, 14, 28823, tzinfo=TzInfo(UTC))
|
||||
│ │ │ │ │ )
|
||||
│ │ │ │ ],
|
||||
│ │ │ │ turn_id='8b360202-f7cb-4786-baa9-166a1b46e2ca',
|
||||
│ │ │ │ completed_at=datetime.datetime(2025, 4, 3, 1, 15, 21, 727364, tzinfo=TzInfo(UTC)),
|
||||
│ │ │ │ output_attachments=[]
|
||||
│ │ │ )
|
||||
│ │ )
|
||||
│ )
|
||||
)
|
||||
|
||||
|
||||
Streaming with print helper...
|
||||
inference> Déjà vu!
|
||||
|
||||
As I mentioned earlier, I'm an artificial intelligence language model. I don't have a personal identity or consciousness like humans do. I exist solely to process and respond to text-based inputs, providing information and assistance on a wide range of topics.
|
||||
|
||||
I'm a computer program designed to simulate human-like conversations, using natural language processing (NLP) and machine learning algorithms to understand and generate responses. My purpose is to help users like you with their questions, provide information, and engage in conversation.
|
||||
|
||||
Think of me as a virtual companion, a helpful tool designed to make your interactions more efficient and enjoyable. I don't have personal opinions, emotions, or biases, but I'm here to provide accurate and informative responses to the best of my abilities.
|
||||
|
||||
So, who am I? I'm just a computer program designed to help you!
|
||||
```
|
||||
:::
|
||||
|
||||
:::{tab-item} Build a RAG Agent
|
||||
|
||||
For our last demo, we can build a RAG agent that can answer questions about the Torchtune project using the documents
|
||||
in a vector database.
|
||||
### i. Create the Script
|
||||
Create a file `rag_agent.py` and add the following code:
|
||||
|
||||
```python
|
||||
from llama_stack_client import LlamaStackClient
|
||||
from llama_stack_client import Agent, AgentEventLogger
|
||||
from llama_stack_client.types import Document
|
||||
import uuid
|
||||
from termcolor import cprint
|
||||
|
||||
client = LlamaStackClient(base_url="http://localhost:8321")
|
||||
|
||||
# Create a vector database instance
|
||||
embed_lm = next(m for m in client.models.list() if m.model_type == "embedding")
|
||||
embedding_model = embed_lm.identifier
|
||||
vector_db_id = f"v{uuid.uuid4().hex}"
|
||||
client.vector_dbs.register(
|
||||
vector_db_id=vector_db_id,
|
||||
embedding_model=embedding_model,
|
||||
)
|
||||
|
||||
# Create Documents
|
||||
urls = [
|
||||
"memory_optimizations.rst",
|
||||
"chat.rst",
|
||||
"llama3.rst",
|
||||
"datasets.rst",
|
||||
"qat_finetune.rst",
|
||||
"lora_finetune.rst",
|
||||
]
|
||||
documents = [
|
||||
Document(
|
||||
document_id=f"num-{i}",
|
||||
content=f"https://raw.githubusercontent.com/pytorch/torchtune/main/docs/source/tutorials/{url}",
|
||||
mime_type="text/plain",
|
||||
metadata={},
|
||||
)
|
||||
for i, url in enumerate(urls)
|
||||
]
|
||||
|
||||
# Insert documents
|
||||
client.tool_runtime.rag_tool.insert(
|
||||
documents=documents,
|
||||
vector_db_id=vector_db_id,
|
||||
chunk_size_in_tokens=512,
|
||||
)
|
||||
|
||||
# Get the model being served
|
||||
llm = next(m for m in client.models.list() if m.model_type == "llm")
|
||||
model = llm.identifier
|
||||
|
||||
# Create the RAG agent
|
||||
rag_agent = Agent(
|
||||
client,
|
||||
model=model,
|
||||
instructions="You are a helpful assistant. Use the RAG tool to answer questions as needed.",
|
||||
tools=[
|
||||
{
|
||||
"name": "builtin::rag/knowledge_search",
|
||||
"args": {"vector_db_ids": [vector_db_id]},
|
||||
}
|
||||
],
|
||||
)
|
||||
|
||||
session_id = rag_agent.create_session(session_name=f"s{uuid.uuid4().hex}")
|
||||
|
||||
turns = ["what is torchtune", "tell me about dora"]
|
||||
|
||||
for t in turns:
|
||||
print("user>", t)
|
||||
stream = rag_agent.create_turn(
|
||||
messages=[{"role": "user", "content": t}], session_id=session_id, stream=True
|
||||
)
|
||||
for event in AgentEventLogger().log(stream):
|
||||
event.print()
|
||||
```
|
||||
### ii. Run the Script
|
||||
Let's run the script using `uv`
|
||||
```bash
|
||||
uv run python rag_agent.py
|
||||
```
|
||||
|
||||
```{dropdown} 👋 Click here to see the sample output
|
||||
user> what is torchtune
|
||||
inference> [knowledge_search(query='TorchTune')]
|
||||
tool_execution> Tool:knowledge_search Args:{'query': 'TorchTune'}
|
||||
tool_execution> Tool:knowledge_search Response:[TextContentItem(text='knowledge_search tool found 5 chunks:\nBEGIN of knowledge_search tool results.\n', type='text'), TextContentItem(text='Result 1:\nDocument_id:num-1\nContent: conversational data, :func:`~torchtune.datasets.chat_dataset` seems to be a good fit. ..., type='text'), TextContentItem(text='END of knowledge_search tool results.\n', type='text')]
|
||||
inference> Here is a high-level overview of the text:
|
||||
|
||||
**LoRA Finetuning with PyTorch Tune**
|
||||
|
||||
PyTorch Tune provides a recipe for LoRA (Low-Rank Adaptation) finetuning, which is a technique to adapt pre-trained models to new tasks. The recipe uses the `lora_finetune_distributed` command.
|
||||
...
|
||||
Overall, DORA is a powerful reinforcement learning algorithm that can learn complex tasks from human demonstrations. However, it requires careful consideration of the challenges and limitations to achieve optimal results.
|
||||
```
|
||||
:::
|
||||
|
||||
::::
|
||||
|
||||
**You're Ready to Build Your Own Apps!**
|
||||
|
||||
Congrats! 🥳 Now you're ready to [build your own Llama Stack applications](../building_applications/index)! 🚀
|
||||
|
|
@ -1,414 +1,65 @@
|
|||
# Quick Start
|
||||
# Quickstart
|
||||
|
||||
Get started with Llama Stack in minutes!
|
||||
|
||||
Llama Stack is a stateful service with REST APIs to support seamless transition of AI applications across different environments. The server can be run in a variety of ways, including as a standalone binary, Docker container, or hosted service. You can build and test using a local server first and deploy to a hosted endpoint for production.
|
||||
Llama Stack is a stateful service with REST APIs to support the seamless transition of AI applications across different
|
||||
environments. You can build and test using a local server first and deploy to a hosted endpoint for production.
|
||||
|
||||
In this guide, we'll walk through how to build a RAG agent locally using Llama Stack with [Ollama](https://ollama.com/) to run inference on a Llama Model.
|
||||
|
||||
|
||||
### 1. Download a Llama model with Ollama
|
||||
In this guide, we'll walk through how to build a RAG application locally using Llama Stack with [Ollama](https://ollama.com/)
|
||||
as the inference [provider](../providers/index.md#inference) for a Llama Model.
|
||||
|
||||
#### Step 1: Install and setup
|
||||
1. Install [uv](https://docs.astral.sh/uv/)
|
||||
2. Run inference on a Llama model with [Ollama](https://ollama.com/download)
|
||||
```bash
|
||||
ollama pull llama3.2:3b-instruct-fp16
|
||||
ollama run llama3.2:3b --keepalive 60m
|
||||
```
|
||||
|
||||
This will instruct the Ollama service to download the Llama 3.2 3B Instruct model, which we'll use in the rest of this guide.
|
||||
|
||||
```{admonition} Note
|
||||
:class: tip
|
||||
|
||||
If you do not have ollama, you can install it from [here](https://ollama.com/download).
|
||||
```
|
||||
|
||||
### 2. Run Llama Stack locally
|
||||
|
||||
We use `uv` to setup a virtual environment and install the Llama Stack package.
|
||||
|
||||
:::{dropdown} [Click to Open] Instructions to setup uv
|
||||
|
||||
Install [uv](https://docs.astral.sh/uv/) to setup your virtual environment.
|
||||
|
||||
|
||||
#### For macOS and Linux:
|
||||
#### Step 2: Run the Llama Stack server
|
||||
We will use `uv` to run the Llama Stack server.
|
||||
```bash
|
||||
curl -LsSf https://astral.sh/uv/install.sh | sh
|
||||
```
|
||||
#### For Windows:
|
||||
Use `irm` to download the script and execute it with `iex`:
|
||||
```powershell
|
||||
powershell -ExecutionPolicy ByPass -c "irm https://astral.sh/uv/install.ps1 | iex"
|
||||
INFERENCE_MODEL=llama3.2:3b uv run --with llama-stack llama stack build --template ollama --image-type venv --run
|
||||
```
|
||||
#### Step 3: Run the demo
|
||||
Now open up a new terminal and copy the following script into a file named `demo_script.py`.
|
||||
|
||||
Setup venv
|
||||
```bash
|
||||
uv venv --python 3.10
|
||||
source .venv/bin/activate
|
||||
```
|
||||
:::
|
||||
|
||||
**Install the Llama Stack package**
|
||||
```bash
|
||||
uv pip install -U llama-stack
|
||||
```
|
||||
|
||||
**Build and Run the Llama Stack server for Ollama.**
|
||||
```bash
|
||||
INFERENCE_MODEL=llama3.2:3b llama stack build --template ollama --image-type venv --run
|
||||
```
|
||||
|
||||
You will see the output end like below:
|
||||
```
|
||||
...
|
||||
INFO: Application startup complete.
|
||||
INFO: Uvicorn running on http://['::', '0.0.0.0']:8321 (Press CTRL+C to quit)
|
||||
```
|
||||
|
||||
Now you can use the llama stack client to run inference and build agents!
|
||||
|
||||
### 3. Client CLI
|
||||
|
||||
Install the client package
|
||||
```bash
|
||||
pip install llama-stack-client
|
||||
```
|
||||
|
||||
:::{dropdown} OR reuse server setup
|
||||
Open a new terminal and navigate to the same directory you started the server from.
|
||||
|
||||
Setup venv (llama-stack already includes the llama-stack-client package)
|
||||
```bash
|
||||
source .venv/bin/activate
|
||||
```
|
||||
:::
|
||||
|
||||
#### 3.1 Configure the client to point to the local server
|
||||
```bash
|
||||
llama-stack-client configure --endpoint http://localhost:8321 --api-key none
|
||||
```
|
||||
You will see the below:
|
||||
```
|
||||
Done! You can now use the Llama Stack Client CLI with endpoint http://localhost:8321
|
||||
```
|
||||
|
||||
#### 3.2 List available models
|
||||
```
|
||||
llama-stack-client models list
|
||||
```
|
||||
|
||||
```
|
||||
Available Models
|
||||
|
||||
┏━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━┓
|
||||
┃ model_type ┃ identifier ┃ provider_resource_id ┃ metadata ┃ provider_id ┃
|
||||
┡━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━┩
|
||||
│ embedding │ all-MiniLM-L6-v2 │ all-minilm:latest │ {'embedding_dimension': 384.0} │ ollama │
|
||||
├─────────────────┼─────────────────────────────────────┼─────────────────────────────────────┼───────────────────────────────────────────┼─────────────────┤
|
||||
│ llm │ llama3.2:3b │ llama3.2:3b │ │ ollama │
|
||||
└─────────────────┴─────────────────────────────────────┴─────────────────────────────────────┴───────────────────────────────────────────┴─────────────────┘
|
||||
|
||||
Total models: 2
|
||||
|
||||
```
|
||||
|
||||
#### 3.3 Test basic inference
|
||||
```bash
|
||||
llama-stack-client inference chat-completion --message "tell me a joke"
|
||||
```
|
||||
Sample output:
|
||||
```python
|
||||
ChatCompletionResponse(
|
||||
completion_message=CompletionMessage(
|
||||
content="Here's one:\n\nWhat do you call a fake noodle?\n\nAn impasta!",
|
||||
role="assistant",
|
||||
stop_reason="end_of_turn",
|
||||
tool_calls=[],
|
||||
),
|
||||
logprobs=None,
|
||||
metrics=[
|
||||
Metric(metric="prompt_tokens", value=14.0, unit=None),
|
||||
Metric(metric="completion_tokens", value=27.0, unit=None),
|
||||
Metric(metric="total_tokens", value=41.0, unit=None),
|
||||
],
|
||||
)
|
||||
```
|
||||
from llama_stack_client import Agent, AgentEventLogger, RAGDocument, LlamaStackClient
|
||||
|
||||
### 4. Python SDK
|
||||
Install the python client
|
||||
```bash
|
||||
pip install llama-stack-client
|
||||
```
|
||||
:::{dropdown} OR reuse server setup
|
||||
Open a new terminal and navigate to the same directory you started the server from.
|
||||
vector_db_id = "my_demo_vector_db"
|
||||
client = LlamaStackClient(base_url="http://localhost:8321")
|
||||
|
||||
Setup venv (llama-stack already includes the llama-stack-client package)
|
||||
```bash
|
||||
source .venv/bin/activate
|
||||
```
|
||||
:::
|
||||
#### 4.1 Basic Inference
|
||||
Create a file `inference.py` and add the following code:
|
||||
```python
|
||||
from llama_stack_client import LlamaStackClient
|
||||
|
||||
client = LlamaStackClient(base_url=f"http://localhost:8321")
|
||||
|
||||
# List available models
|
||||
models = client.models.list()
|
||||
|
||||
# Select the first LLM
|
||||
llm = next(m for m in models if m.model_type == "llm")
|
||||
model_id = llm.identifier
|
||||
# Select the first LLM and first embedding models
|
||||
model_id = next(m for m in models if m.model_type == "llm").identifier
|
||||
embedding_model_id = (
|
||||
em := next(m for m in models if m.model_type == "embedding")
|
||||
).identifier
|
||||
embedding_dimension = em.metadata["embedding_dimension"]
|
||||
|
||||
print("Model:", model_id)
|
||||
|
||||
response = client.inference.chat_completion(
|
||||
model_id=model_id,
|
||||
messages=[
|
||||
{"role": "system", "content": "You are a helpful assistant."},
|
||||
{"role": "user", "content": "Write a haiku about coding"},
|
||||
],
|
||||
)
|
||||
print(response.completion_message.content)
|
||||
```
|
||||
Run the script
|
||||
```bash
|
||||
python inference.py
|
||||
```
|
||||
Sample output:
|
||||
```
|
||||
Model: llama3.2:3b-instruct-fp16
|
||||
Here is a haiku about coding:
|
||||
|
||||
Lines of code unfold
|
||||
Logic flows through digital night
|
||||
Beauty in the bits
|
||||
```
|
||||
|
||||
#### 4.2. Basic Agent
|
||||
|
||||
Create a file `agent.py` and add the following code:
|
||||
```python
|
||||
from llama_stack_client import LlamaStackClient
|
||||
from llama_stack_client import Agent, AgentEventLogger
|
||||
from rich.pretty import pprint
|
||||
import uuid
|
||||
|
||||
client = LlamaStackClient(base_url=f"http://localhost:8321")
|
||||
|
||||
models = client.models.list()
|
||||
llm = next(m for m in models if m.model_type == "llm")
|
||||
model_id = llm.identifier
|
||||
|
||||
agent = Agent(client, model=model_id, instructions="You are a helpful assistant.")
|
||||
|
||||
s_id = agent.create_session(session_name=f"s{uuid.uuid4().hex}")
|
||||
|
||||
print("Non-streaming ...")
|
||||
response = agent.create_turn(
|
||||
messages=[{"role": "user", "content": "Who are you?"}],
|
||||
session_id=s_id,
|
||||
stream=False,
|
||||
)
|
||||
print("agent>", response.output_message.content)
|
||||
|
||||
print("Streaming ...")
|
||||
stream = agent.create_turn(
|
||||
messages=[{"role": "user", "content": "Who are you?"}], session_id=s_id, stream=True
|
||||
)
|
||||
for event in stream:
|
||||
pprint(event)
|
||||
|
||||
print("Streaming with print helper...")
|
||||
stream = agent.create_turn(
|
||||
messages=[{"role": "user", "content": "Who are you?"}], session_id=s_id, stream=True
|
||||
)
|
||||
for event in AgentEventLogger().log(stream):
|
||||
event.print()
|
||||
```
|
||||
|
||||
Run the script:
|
||||
```bash
|
||||
python agent.py
|
||||
```
|
||||
|
||||
:::{dropdown} `Sample output`
|
||||
```
|
||||
Non-streaming ...
|
||||
agent> I'm an artificial intelligence designed to assist and communicate with users like you. I don't have a personal identity, but I'm here to provide information, answer questions, and help with tasks to the best of my abilities.
|
||||
|
||||
I can be used for a wide range of purposes, such as:
|
||||
|
||||
* Providing definitions and explanations
|
||||
* Offering suggestions and ideas
|
||||
* Helping with language translation
|
||||
* Assisting with writing and proofreading
|
||||
* Generating text or responses to questions
|
||||
* Playing simple games or chatting about topics of interest
|
||||
|
||||
I'm constantly learning and improving my abilities, so feel free to ask me anything, and I'll do my best to help!
|
||||
|
||||
Streaming ...
|
||||
AgentTurnResponseStreamChunk(
|
||||
│ event=TurnResponseEvent(
|
||||
│ │ payload=AgentTurnResponseStepStartPayload(
|
||||
│ │ │ event_type='step_start',
|
||||
│ │ │ step_id='69831607-fa75-424a-949b-e2049e3129d1',
|
||||
│ │ │ step_type='inference',
|
||||
│ │ │ metadata={}
|
||||
│ │ )
|
||||
│ )
|
||||
)
|
||||
AgentTurnResponseStreamChunk(
|
||||
│ event=TurnResponseEvent(
|
||||
│ │ payload=AgentTurnResponseStepProgressPayload(
|
||||
│ │ │ delta=TextDelta(text='As', type='text'),
|
||||
│ │ │ event_type='step_progress',
|
||||
│ │ │ step_id='69831607-fa75-424a-949b-e2049e3129d1',
|
||||
│ │ │ step_type='inference'
|
||||
│ │ )
|
||||
│ )
|
||||
)
|
||||
AgentTurnResponseStreamChunk(
|
||||
│ event=TurnResponseEvent(
|
||||
│ │ payload=AgentTurnResponseStepProgressPayload(
|
||||
│ │ │ delta=TextDelta(text=' a', type='text'),
|
||||
│ │ │ event_type='step_progress',
|
||||
│ │ │ step_id='69831607-fa75-424a-949b-e2049e3129d1',
|
||||
│ │ │ step_type='inference'
|
||||
│ │ )
|
||||
│ )
|
||||
)
|
||||
...
|
||||
AgentTurnResponseStreamChunk(
|
||||
│ event=TurnResponseEvent(
|
||||
│ │ payload=AgentTurnResponseStepCompletePayload(
|
||||
│ │ │ event_type='step_complete',
|
||||
│ │ │ step_details=InferenceStep(
|
||||
│ │ │ │ api_model_response=CompletionMessage(
|
||||
│ │ │ │ │ content='As a conversational AI, I don\'t have a personal identity in the classical sense. I exist as a program running on computer servers, designed to process and respond to text-based inputs.\n\nI\'m an instance of a type of artificial intelligence called a "language model," which is trained on vast amounts of text data to generate human-like responses. My primary function is to understand and respond to natural language inputs, like our conversation right now.\n\nThink of me as a virtual assistant, a chatbot, or a conversational interface – I\'m here to provide information, answer questions, and engage in conversation to the best of my abilities. I don\'t have feelings, emotions, or consciousness like humans do, but I\'m designed to simulate human-like interactions to make our conversations feel more natural and helpful.\n\nSo, that\'s me in a nutshell! What can I help you with today?',
|
||||
│ │ │ │ │ role='assistant',
|
||||
│ │ │ │ │ stop_reason='end_of_turn',
|
||||
│ │ │ │ │ tool_calls=[]
|
||||
│ │ │ │ ),
|
||||
│ │ │ │ step_id='69831607-fa75-424a-949b-e2049e3129d1',
|
||||
│ │ │ │ step_type='inference',
|
||||
│ │ │ │ turn_id='8b360202-f7cb-4786-baa9-166a1b46e2ca',
|
||||
│ │ │ │ completed_at=datetime.datetime(2025, 4, 3, 1, 15, 21, 716174, tzinfo=TzInfo(UTC)),
|
||||
│ │ │ │ started_at=datetime.datetime(2025, 4, 3, 1, 15, 14, 28823, tzinfo=TzInfo(UTC))
|
||||
│ │ │ ),
|
||||
│ │ │ step_id='69831607-fa75-424a-949b-e2049e3129d1',
|
||||
│ │ │ step_type='inference'
|
||||
│ │ )
|
||||
│ )
|
||||
)
|
||||
AgentTurnResponseStreamChunk(
|
||||
│ event=TurnResponseEvent(
|
||||
│ │ payload=AgentTurnResponseTurnCompletePayload(
|
||||
│ │ │ event_type='turn_complete',
|
||||
│ │ │ turn=Turn(
|
||||
│ │ │ │ input_messages=[UserMessage(content='Who are you?', role='user', context=None)],
|
||||
│ │ │ │ output_message=CompletionMessage(
|
||||
│ │ │ │ │ content='As a conversational AI, I don\'t have a personal identity in the classical sense. I exist as a program running on computer servers, designed to process and respond to text-based inputs.\n\nI\'m an instance of a type of artificial intelligence called a "language model," which is trained on vast amounts of text data to generate human-like responses. My primary function is to understand and respond to natural language inputs, like our conversation right now.\n\nThink of me as a virtual assistant, a chatbot, or a conversational interface – I\'m here to provide information, answer questions, and engage in conversation to the best of my abilities. I don\'t have feelings, emotions, or consciousness like humans do, but I\'m designed to simulate human-like interactions to make our conversations feel more natural and helpful.\n\nSo, that\'s me in a nutshell! What can I help you with today?',
|
||||
│ │ │ │ │ role='assistant',
|
||||
│ │ │ │ │ stop_reason='end_of_turn',
|
||||
│ │ │ │ │ tool_calls=[]
|
||||
│ │ │ │ ),
|
||||
│ │ │ │ session_id='abd4afea-4324-43f4-9513-cfe3970d92e8',
|
||||
│ │ │ │ started_at=datetime.datetime(2025, 4, 3, 1, 15, 14, 28722, tzinfo=TzInfo(UTC)),
|
||||
│ │ │ │ steps=[
|
||||
│ │ │ │ │ InferenceStep(
|
||||
│ │ │ │ │ │ api_model_response=CompletionMessage(
|
||||
│ │ │ │ │ │ │ content='As a conversational AI, I don\'t have a personal identity in the classical sense. I exist as a program running on computer servers, designed to process and respond to text-based inputs.\n\nI\'m an instance of a type of artificial intelligence called a "language model," which is trained on vast amounts of text data to generate human-like responses. My primary function is to understand and respond to natural language inputs, like our conversation right now.\n\nThink of me as a virtual assistant, a chatbot, or a conversational interface – I\'m here to provide information, answer questions, and engage in conversation to the best of my abilities. I don\'t have feelings, emotions, or consciousness like humans do, but I\'m designed to simulate human-like interactions to make our conversations feel more natural and helpful.\n\nSo, that\'s me in a nutshell! What can I help you with today?',
|
||||
│ │ │ │ │ │ │ role='assistant',
|
||||
│ │ │ │ │ │ │ stop_reason='end_of_turn',
|
||||
│ │ │ │ │ │ │ tool_calls=[]
|
||||
│ │ │ │ │ │ ),
|
||||
│ │ │ │ │ │ step_id='69831607-fa75-424a-949b-e2049e3129d1',
|
||||
│ │ │ │ │ │ step_type='inference',
|
||||
│ │ │ │ │ │ turn_id='8b360202-f7cb-4786-baa9-166a1b46e2ca',
|
||||
│ │ │ │ │ │ completed_at=datetime.datetime(2025, 4, 3, 1, 15, 21, 716174, tzinfo=TzInfo(UTC)),
|
||||
│ │ │ │ │ │ started_at=datetime.datetime(2025, 4, 3, 1, 15, 14, 28823, tzinfo=TzInfo(UTC))
|
||||
│ │ │ │ │ )
|
||||
│ │ │ │ ],
|
||||
│ │ │ │ turn_id='8b360202-f7cb-4786-baa9-166a1b46e2ca',
|
||||
│ │ │ │ completed_at=datetime.datetime(2025, 4, 3, 1, 15, 21, 727364, tzinfo=TzInfo(UTC)),
|
||||
│ │ │ │ output_attachments=[]
|
||||
│ │ │ )
|
||||
│ │ )
|
||||
│ )
|
||||
)
|
||||
|
||||
|
||||
Streaming with print helper...
|
||||
inference> Déjà vu!
|
||||
|
||||
As I mentioned earlier, I'm an artificial intelligence language model. I don't have a personal identity or consciousness like humans do. I exist solely to process and respond to text-based inputs, providing information and assistance on a wide range of topics.
|
||||
|
||||
I'm a computer program designed to simulate human-like conversations, using natural language processing (NLP) and machine learning algorithms to understand and generate responses. My purpose is to help users like you with their questions, provide information, and engage in conversation.
|
||||
|
||||
Think of me as a virtual companion, a helpful tool designed to make your interactions more efficient and enjoyable. I don't have personal opinions, emotions, or biases, but I'm here to provide accurate and informative responses to the best of my abilities.
|
||||
|
||||
So, who am I? I'm just a computer program designed to help you!
|
||||
|
||||
```
|
||||
:::
|
||||
|
||||
#### 4.3. RAG agent
|
||||
|
||||
Create a file `rag_agent.py` and add the following code:
|
||||
|
||||
```python
|
||||
from llama_stack_client import LlamaStackClient
|
||||
from llama_stack_client import Agent, AgentEventLogger
|
||||
from llama_stack_client.types import Document
|
||||
import uuid
|
||||
|
||||
client = LlamaStackClient(base_url=f"http://localhost:8321")
|
||||
|
||||
# Create a vector database instance
|
||||
embedlm = next(m for m in client.models.list() if m.model_type == "embedding")
|
||||
embedding_model = embedlm.identifier
|
||||
vector_db_id = f"v{uuid.uuid4().hex}"
|
||||
client.vector_dbs.register(
|
||||
_ = client.vector_dbs.register(
|
||||
vector_db_id=vector_db_id,
|
||||
embedding_model=embedding_model,
|
||||
embedding_model=embedding_model_id,
|
||||
embedding_dimension=embedding_dimension,
|
||||
provider_id="faiss",
|
||||
)
|
||||
source = "https://www.paulgraham.com/greatwork.html"
|
||||
print("rag_tool> Ingesting document:", source)
|
||||
document = RAGDocument(
|
||||
document_id="document_1",
|
||||
content=source,
|
||||
mime_type="text/html",
|
||||
metadata={},
|
||||
)
|
||||
|
||||
# Create Documents
|
||||
urls = [
|
||||
"memory_optimizations.rst",
|
||||
"chat.rst",
|
||||
"llama3.rst",
|
||||
"datasets.rst",
|
||||
"qat_finetune.rst",
|
||||
"lora_finetune.rst",
|
||||
]
|
||||
documents = [
|
||||
Document(
|
||||
document_id=f"num-{i}",
|
||||
content=f"https://raw.githubusercontent.com/pytorch/torchtune/main/docs/source/tutorials/{url}",
|
||||
mime_type="text/plain",
|
||||
metadata={},
|
||||
)
|
||||
for i, url in enumerate(urls)
|
||||
]
|
||||
|
||||
# Insert documents
|
||||
client.tool_runtime.rag_tool.insert(
|
||||
documents=documents,
|
||||
documents=[document],
|
||||
vector_db_id=vector_db_id,
|
||||
chunk_size_in_tokens=512,
|
||||
chunk_size_in_tokens=50,
|
||||
)
|
||||
|
||||
# Get the model being served
|
||||
llm = next(m for m in client.models.list() if m.model_type == "llm")
|
||||
model = llm.identifier
|
||||
|
||||
# Create RAG agent
|
||||
ragagent = Agent(
|
||||
agent = Agent(
|
||||
client,
|
||||
model=model,
|
||||
instructions="You are a helpful assistant. Use the RAG tool to answer questions as needed.",
|
||||
model=model_id,
|
||||
instructions="You are a helpful assistant",
|
||||
tools=[
|
||||
{
|
||||
"name": "builtin::rag/knowledge_search",
|
||||
|
|
@ -417,39 +68,54 @@ ragagent = Agent(
|
|||
],
|
||||
)
|
||||
|
||||
s_id = ragagent.create_session(session_name=f"s{uuid.uuid4().hex}")
|
||||
prompt = "How do you do great work?"
|
||||
print("prompt>", prompt)
|
||||
|
||||
turns = ["what is torchtune", "tell me about dora"]
|
||||
response = agent.create_turn(
|
||||
messages=[{"role": "user", "content": prompt}],
|
||||
session_id=agent.create_session("rag_session"),
|
||||
stream=True,
|
||||
)
|
||||
|
||||
for t in turns:
|
||||
print("user>", t)
|
||||
stream = ragagent.create_turn(
|
||||
messages=[{"role": "user", "content": t}], session_id=s_id, stream=True
|
||||
)
|
||||
for event in AgentEventLogger().log(stream):
|
||||
event.print()
|
||||
for log in AgentEventLogger().log(response):
|
||||
log.print()
|
||||
```
|
||||
Run the script:
|
||||
We will use `uv` to run the script
|
||||
```
|
||||
python rag_agent.py
|
||||
uv run --with llama-stack-client demo_script.py
|
||||
```
|
||||
:::{dropdown} `Sample output`
|
||||
And you should see output like below.
|
||||
```
|
||||
user> what is torchtune
|
||||
inference> [knowledge_search(query='TorchTune')]
|
||||
tool_execution> Tool:knowledge_search Args:{'query': 'TorchTune'}
|
||||
tool_execution> Tool:knowledge_search Response:[TextContentItem(text='knowledge_search tool found 5 chunks:\nBEGIN of knowledge_search tool results.\n', type='text'), TextContentItem(text='Result 1:\nDocument_id:num-1\nContent: conversational data, :func:`~torchtune.datasets.chat_dataset` seems to be a good fit. ..., type='text'), TextContentItem(text='END of knowledge_search tool results.\n', type='text')]
|
||||
inference> Here is a high-level overview of the text:
|
||||
rag_tool> Ingesting document: https://www.paulgraham.com/greatwork.html
|
||||
|
||||
**LoRA Finetuning with PyTorch Tune**
|
||||
prompt> How do you do great work?
|
||||
|
||||
PyTorch Tune provides a recipe for LoRA (Low-Rank Adaptation) finetuning, which is a technique to adapt pre-trained models to new tasks. The recipe uses the `lora_finetune_distributed` command.
|
||||
...
|
||||
Overall, DORA is a powerful reinforcement learning algorithm that can learn complex tasks from human demonstrations. However, it requires careful consideration of the challenges and limitations to achieve optimal results.
|
||||
inference> [knowledge_search(query="What is the key to doing great work")]
|
||||
|
||||
tool_execution> Tool:knowledge_search Args:{'query': 'What is the key to doing great work'}
|
||||
|
||||
tool_execution> Tool:knowledge_search Response:[TextContentItem(text='knowledge_search tool found 5 chunks:\nBEGIN of knowledge_search tool results.\n', type='text'), TextContentItem(text="Result 1:\nDocument_id:docum\nContent: work. Doing great work means doing something important\nso well that you expand people's ideas of what's possible. But\nthere's no threshold for importance. It's a matter of degree, and\noften hard to judge at the time anyway.\n", type='text'), TextContentItem(text="Result 2:\nDocument_id:docum\nContent: work. Doing great work means doing something important\nso well that you expand people's ideas of what's possible. But\nthere's no threshold for importance. It's a matter of degree, and\noften hard to judge at the time anyway.\n", type='text'), TextContentItem(text="Result 3:\nDocument_id:docum\nContent: work. Doing great work means doing something important\nso well that you expand people's ideas of what's possible. But\nthere's no threshold for importance. It's a matter of degree, and\noften hard to judge at the time anyway.\n", type='text'), TextContentItem(text="Result 4:\nDocument_id:docum\nContent: work. Doing great work means doing something important\nso well that you expand people's ideas of what's possible. But\nthere's no threshold for importance. It's a matter of degree, and\noften hard to judge at the time anyway.\n", type='text'), TextContentItem(text="Result 5:\nDocument_id:docum\nContent: work. Doing great work means doing something important\nso well that you expand people's ideas of what's possible. But\nthere's no threshold for importance. It's a matter of degree, and\noften hard to judge at the time anyway.\n", type='text'), TextContentItem(text='END of knowledge_search tool results.\n', type='text')]
|
||||
|
||||
inference> Based on the search results, it seems that doing great work means doing something important so well that you expand people's ideas of what's possible. However, there is no clear threshold for importance, and it can be difficult to judge at the time.
|
||||
|
||||
To further clarify, I would suggest that doing great work involves:
|
||||
|
||||
* Completing tasks with high quality and attention to detail
|
||||
* Expanding on existing knowledge or ideas
|
||||
* Making a positive impact on others through your work
|
||||
* Striving for excellence and continuous improvement
|
||||
|
||||
Ultimately, great work is about making a meaningful contribution and leaving a lasting impression.
|
||||
```
|
||||
:::
|
||||
Congratulations! You've successfully built your first RAG application using Llama Stack! 🎉🥳
|
||||
|
||||
## Next Steps
|
||||
- Go through the [Getting Started Notebook](https://github.com/meta-llama/llama-stack/blob/main/docs/getting_started.ipynb)
|
||||
- Checkout more [Notebooks on GitHub](https://github.com/meta-llama/llama-stack/tree/main/docs/notebooks)
|
||||
- See [References](../references/index.md) for more details about the llama CLI and Python SDK
|
||||
- For example applications and more detailed tutorials, visit our [llama-stack-apps](https://github.com/meta-llama/llama-stack-apps/tree/main/examples) repository.
|
||||
|
||||
Now you're ready to dive deeper into Llama Stack!
|
||||
- Explore the [Detailed Tutorial](./detailed_tutorial.md).
|
||||
- Try the [Getting Started Notebook](https://github.com/meta-llama/llama-stack/blob/main/docs/getting_started.ipynb).
|
||||
- Browse more [Notebooks on GitHub](https://github.com/meta-llama/llama-stack/tree/main/docs/notebooks).
|
||||
- Learn about Llama Stack [Concepts](../concepts/index.md).
|
||||
- Discover how to [Build Llama Stacks](../distributions/index.md).
|
||||
- Refer to our [References](../references/index.md) for details on the Llama CLI and Python SDK.
|
||||
- Check out the [llama-stack-apps](https://github.com/meta-llama/llama-stack-apps/tree/main/examples) repository for example applications and tutorials.
|
||||
|
|
|
|||
|
|
@ -1,3 +1,5 @@
|
|||
# Llama Stack
|
||||
Welcome to Llama Stack, the open-source framework for building generative AI applications.
|
||||
```{admonition} Llama 4 is here!
|
||||
:class: tip
|
||||
|
||||
|
|
@ -9,7 +11,6 @@ Check out [Getting Started with Llama 4](https://colab.research.google.com/githu
|
|||
Llama Stack {{ llama_stack_version }} is now available! See the {{ llama_stack_version_link }} for more details.
|
||||
```
|
||||
|
||||
# Llama Stack
|
||||
|
||||
## What is Llama Stack?
|
||||
|
||||
|
|
@ -98,8 +99,9 @@ A number of "adapters" are available for some popular Inference and Vector Store
|
|||
:maxdepth: 3
|
||||
|
||||
self
|
||||
introduction/index
|
||||
getting_started/index
|
||||
getting_started/detailed_tutorial
|
||||
introduction/index
|
||||
concepts/index
|
||||
providers/index
|
||||
distributions/index
|
||||
|
|
|
|||
|
|
@ -1,8 +1,8 @@
|
|||
# Providers Overview
|
||||
|
||||
The goal of Llama Stack is to build an ecosystem where users can easily swap out different implementations for the same API. Examples for these include:
|
||||
- LLM inference providers (e.g., Fireworks, Together, AWS Bedrock, Groq, Cerebras, SambaNova, vLLM, etc.),
|
||||
- Vector databases (e.g., ChromaDB, Weaviate, Qdrant, Milvus, FAISS, PGVector, etc.),
|
||||
- LLM inference providers (e.g., Ollama, Fireworks, Together, AWS Bedrock, Groq, Cerebras, SambaNova, vLLM, etc.),
|
||||
- Vector databases (e.g., ChromaDB, Weaviate, Qdrant, Milvus, FAISS, PGVector, SQLite-Vec, etc.),
|
||||
- Safety providers (e.g., Meta's Llama Guard, AWS Bedrock Guardrails, etc.)
|
||||
|
||||
Providers come in two flavors:
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue