diff --git a/llama_toolchain/stack.py b/llama_toolchain/stack.py index 6ec05896d..875bc5802 100644 --- a/llama_toolchain/stack.py +++ b/llama_toolchain/stack.py @@ -11,7 +11,7 @@ from llama_toolchain.evaluations.api import * # noqa: F403 from llama_toolchain.inference.api import * # noqa: F403 from llama_toolchain.batch_inference.api import * # noqa: F403 from llama_toolchain.memory.api import * # noqa: F403 -from llama_toolchain.observability.api import * # noqa: F403 +from llama_toolchain.telemetry.api import * # noqa: F403 from llama_toolchain.post_training.api import * # noqa: F403 from llama_toolchain.reward_scoring.api import * # noqa: F403 from llama_toolchain.synthetic_data_generation.api import * # noqa: F403 @@ -24,7 +24,7 @@ class LlamaStack( RewardScoring, SyntheticDataGeneration, Datasets, - Observability, + Telemetry, PostTraining, Memory, Evaluations, diff --git a/llama_toolchain/observability/__init__.py b/llama_toolchain/telemetry/__init__.py similarity index 100% rename from llama_toolchain/observability/__init__.py rename to llama_toolchain/telemetry/__init__.py diff --git a/llama_toolchain/observability/api/__init__.py b/llama_toolchain/telemetry/api/__init__.py similarity index 100% rename from llama_toolchain/observability/api/__init__.py rename to llama_toolchain/telemetry/api/__init__.py diff --git a/llama_toolchain/observability/api/api.py b/llama_toolchain/telemetry/api/api.py similarity index 99% rename from llama_toolchain/observability/api/api.py rename to llama_toolchain/telemetry/api/api.py index 86a5cc703..ae784428b 100644 --- a/llama_toolchain/observability/api/api.py +++ b/llama_toolchain/telemetry/api/api.py @@ -134,7 +134,7 @@ class LogSearchRequest(BaseModel): filters: Optional[Dict[str, Any]] = None -class Observability(Protocol): +class Telemetry(Protocol): @webmethod(route="/experiments/create") def create_experiment(self, request: CreateExperimentRequest) -> Experiment: ... diff --git a/rfcs/RFC-0001-llama-stack-assets/llama-stack-spec.html b/rfcs/RFC-0001-llama-stack-assets/llama-stack-spec.html index d417f02f3..38efd4ca3 100644 --- a/rfcs/RFC-0001-llama-stack-assets/llama-stack-spec.html +++ b/rfcs/RFC-0001-llama-stack-assets/llama-stack-spec.html @@ -21,7 +21,7 @@ "info": { "title": "[DRAFT] Llama Stack Specification", "version": "0.0.1", - "description": "This is the specification of the llama stack that provides\n a set of endpoints and their corresponding interfaces that are tailored to\n best leverage Llama Models. The specification is still in draft and subject to change.\n Generated at 2024-09-04 10:28:38.779789" + "description": "This is the specification of the llama stack that provides\n a set of endpoints and their corresponding interfaces that are tailored to\n best leverage Llama Models. The specification is still in draft and subject to change.\n Generated at 2024-09-07 15:23:29.488676" }, "servers": [ { @@ -323,7 +323,7 @@ } }, "tags": [ - "Observability" + "Telemetry" ], "parameters": [], "requestBody": { @@ -383,7 +383,7 @@ } }, "tags": [ - "Observability" + "Telemetry" ], "parameters": [], "requestBody": { @@ -784,7 +784,7 @@ } }, "tags": [ - "Observability" + "Telemetry" ], "parameters": [ { @@ -988,7 +988,7 @@ } }, "tags": [ - "Observability" + "Telemetry" ], "parameters": [ { @@ -1017,7 +1017,7 @@ } }, "tags": [ - "Observability" + "Telemetry" ], "parameters": [], "requestBody": { @@ -1083,7 +1083,7 @@ } }, "tags": [ - "Observability" + "Telemetry" ], "parameters": [ { @@ -1242,7 +1242,7 @@ } }, "tags": [ - "Observability" + "Telemetry" ], "parameters": [], "requestBody": { @@ -1272,7 +1272,7 @@ } }, "tags": [ - "Observability" + "Telemetry" ], "parameters": [] } @@ -1305,7 +1305,7 @@ } }, "tags": [ - "Observability" + "Telemetry" ], "parameters": [], "requestBody": { @@ -1328,7 +1328,7 @@ } }, "tags": [ - "Observability" + "Telemetry" ], "parameters": [], "requestBody": { @@ -1531,7 +1531,7 @@ } }, "tags": [ - "Observability" + "Telemetry" ], "parameters": [], "requestBody": { @@ -1561,7 +1561,7 @@ } }, "tags": [ - "Observability" + "Telemetry" ], "parameters": [], "requestBody": { @@ -1591,7 +1591,7 @@ } }, "tags": [ - "Observability" + "Telemetry" ], "parameters": [], "requestBody": { @@ -2376,7 +2376,183 @@ "$ref": "#/components/schemas/FunctionCallToolDefinition" }, { - "$ref": "#/components/schemas/MemoryToolDefinition" + "type": "object", + "properties": { + "input_shields": { + "type": "array", + "items": { + "$ref": "#/components/schemas/ShieldDefinition" + } + }, + "output_shields": { + "type": "array", + "items": { + "$ref": "#/components/schemas/ShieldDefinition" + } + }, + "type": { + "type": "string", + "const": "memory" + }, + "memory_bank_configs": { + "type": "array", + "items": { + "oneOf": [ + { + "type": "object", + "properties": { + "bank_id": { + "type": "string" + }, + "type": { + "type": "string", + "const": "vector" + } + }, + "additionalProperties": false, + "required": [ + "bank_id", + "type" + ] + }, + { + "type": "object", + "properties": { + "bank_id": { + "type": "string" + }, + "type": { + "type": "string", + "const": "keyvalue" + }, + "keys": { + "type": "array", + "items": { + "type": "string" + } + } + }, + "additionalProperties": false, + "required": [ + "bank_id", + "type", + "keys" + ] + }, + { + "type": "object", + "properties": { + "bank_id": { + "type": "string" + }, + "type": { + "type": "string", + "const": "keyword" + } + }, + "additionalProperties": false, + "required": [ + "bank_id", + "type" + ] + }, + { + "type": "object", + "properties": { + "bank_id": { + "type": "string" + }, + "type": { + "type": "string", + "const": "graph" + }, + "entities": { + "type": "array", + "items": { + "type": "string" + } + } + }, + "additionalProperties": false, + "required": [ + "bank_id", + "type", + "entities" + ] + } + ] + } + }, + "query_generator_config": { + "oneOf": [ + { + "type": "object", + "properties": { + "type": { + "type": "string", + "const": "default" + }, + "sep": { + "type": "string" + } + }, + "additionalProperties": false, + "required": [ + "type", + "sep" + ] + }, + { + "type": "object", + "properties": { + "type": { + "type": "string", + "const": "llm" + }, + "model": { + "type": "string" + }, + "template": { + "type": "string" + } + }, + "additionalProperties": false, + "required": [ + "type", + "model", + "template" + ] + }, + { + "type": "object", + "properties": { + "type": { + "type": "string", + "const": "custom" + } + }, + "additionalProperties": false, + "required": [ + "type" + ] + } + ] + }, + "max_tokens_in_context": { + "type": "integer" + }, + "max_chunks": { + "type": "integer" + } + }, + "additionalProperties": false, + "required": [ + "type", + "memory_bank_configs", + "query_generator_config", + "max_tokens_in_context", + "max_chunks" + ] } ] } @@ -2513,129 +2689,6 @@ "parameters" ] }, - "MemoryToolDefinition": { - "type": "object", - "properties": { - "input_shields": { - "type": "array", - "items": { - "$ref": "#/components/schemas/ShieldDefinition" - } - }, - "output_shields": { - "type": "array", - "items": { - "$ref": "#/components/schemas/ShieldDefinition" - } - }, - "type": { - "type": "string", - "const": "memory" - }, - "memory_bank_configs": { - "type": "array", - "items": { - "oneOf": [ - { - "type": "object", - "properties": { - "bank_id": { - "type": "string" - }, - "type": { - "type": "string", - "const": "vector" - } - }, - "additionalProperties": false, - "required": [ - "bank_id", - "type" - ] - }, - { - "type": "object", - "properties": { - "bank_id": { - "type": "string" - }, - "type": { - "type": "string", - "const": "keyvalue" - }, - "keys": { - "type": "array", - "items": { - "type": "string" - } - } - }, - "additionalProperties": false, - "required": [ - "bank_id", - "type", - "keys" - ] - }, - { - "type": "object", - "properties": { - "bank_id": { - "type": "string" - }, - "type": { - "type": "string", - "const": "keyword" - } - }, - "additionalProperties": false, - "required": [ - "bank_id", - "type" - ] - }, - { - "type": "object", - "properties": { - "bank_id": { - "type": "string" - }, - "type": { - "type": "string", - "const": "graph" - }, - "entities": { - "type": "array", - "items": { - "type": "string" - } - } - }, - "additionalProperties": false, - "required": [ - "bank_id", - "type", - "entities" - ] - } - ] - } - }, - "max_tokens_in_context": { - "type": "integer" - }, - "max_chunks": { - "type": "integer" - } - }, - "additionalProperties": false, - "required": [ - "type", - "memory_bank_configs", - "max_tokens_in_context", - "max_chunks" - ] - }, "OnViolationAction": { "type": "integer", "enum": [ @@ -2873,7 +2926,183 @@ "$ref": "#/components/schemas/FunctionCallToolDefinition" }, { - "$ref": "#/components/schemas/MemoryToolDefinition" + "type": "object", + "properties": { + "input_shields": { + "type": "array", + "items": { + "$ref": "#/components/schemas/ShieldDefinition" + } + }, + "output_shields": { + "type": "array", + "items": { + "$ref": "#/components/schemas/ShieldDefinition" + } + }, + "type": { + "type": "string", + "const": "memory" + }, + "memory_bank_configs": { + "type": "array", + "items": { + "oneOf": [ + { + "type": "object", + "properties": { + "bank_id": { + "type": "string" + }, + "type": { + "type": "string", + "const": "vector" + } + }, + "additionalProperties": false, + "required": [ + "bank_id", + "type" + ] + }, + { + "type": "object", + "properties": { + "bank_id": { + "type": "string" + }, + "type": { + "type": "string", + "const": "keyvalue" + }, + "keys": { + "type": "array", + "items": { + "type": "string" + } + } + }, + "additionalProperties": false, + "required": [ + "bank_id", + "type", + "keys" + ] + }, + { + "type": "object", + "properties": { + "bank_id": { + "type": "string" + }, + "type": { + "type": "string", + "const": "keyword" + } + }, + "additionalProperties": false, + "required": [ + "bank_id", + "type" + ] + }, + { + "type": "object", + "properties": { + "bank_id": { + "type": "string" + }, + "type": { + "type": "string", + "const": "graph" + }, + "entities": { + "type": "array", + "items": { + "type": "string" + } + } + }, + "additionalProperties": false, + "required": [ + "bank_id", + "type", + "entities" + ] + } + ] + } + }, + "query_generator_config": { + "oneOf": [ + { + "type": "object", + "properties": { + "type": { + "type": "string", + "const": "default" + }, + "sep": { + "type": "string" + } + }, + "additionalProperties": false, + "required": [ + "type", + "sep" + ] + }, + { + "type": "object", + "properties": { + "type": { + "type": "string", + "const": "llm" + }, + "model": { + "type": "string" + }, + "template": { + "type": "string" + } + }, + "additionalProperties": false, + "required": [ + "type", + "model", + "template" + ] + }, + { + "type": "object", + "properties": { + "type": { + "type": "string", + "const": "custom" + } + }, + "additionalProperties": false, + "required": [ + "type" + ] + } + ] + }, + "max_tokens_in_context": { + "type": "integer" + }, + "max_chunks": { + "type": "integer" + } + }, + "additionalProperties": false, + "required": [ + "type", + "memory_bank_configs", + "query_generator_config", + "max_tokens_in_context", + "max_chunks" + ] } ] } @@ -5582,35 +5811,35 @@ ], "tags": [ { - "name": "Observability" + "name": "Telemetry" + }, + { + "name": "Evaluations" + }, + { + "name": "AgenticSystem" }, { "name": "Inference" }, + { + "name": "BatchInference" + }, + { + "name": "PostTraining" + }, { "name": "Datasets" }, { "name": "Memory" }, - { - "name": "AgenticSystem" - }, - { - "name": "Evaluations" - }, { "name": "SyntheticDataGeneration" }, { "name": "RewardScoring" }, - { - "name": "PostTraining" - }, - { - "name": "BatchInference" - }, { "name": "BatchChatCompletionRequest", "description": "" @@ -5743,10 +5972,6 @@ "name": "FunctionCallToolDefinition", "description": "" }, - { - "name": "MemoryToolDefinition", - "description": "" - }, { "name": "OnViolationAction", "description": "" @@ -6134,10 +6359,10 @@ "Evaluations", "Inference", "Memory", - "Observability", "PostTraining", "RewardScoring", - "SyntheticDataGeneration" + "SyntheticDataGeneration", + "Telemetry" ] }, { @@ -6216,7 +6441,6 @@ "MemoryBank", "MemoryBankDocument", "MemoryRetrievalStep", - "MemoryToolDefinition", "Metric", "OnViolationAction", "OptimizerConfig", diff --git a/rfcs/RFC-0001-llama-stack-assets/llama-stack-spec.yaml b/rfcs/RFC-0001-llama-stack-assets/llama-stack-spec.yaml index 7560f3c88..11f092549 100644 --- a/rfcs/RFC-0001-llama-stack-assets/llama-stack-spec.yaml +++ b/rfcs/RFC-0001-llama-stack-assets/llama-stack-spec.yaml @@ -30,7 +30,123 @@ components: - $ref: '#/components/schemas/PhotogenToolDefinition' - $ref: '#/components/schemas/CodeInterpreterToolDefinition' - $ref: '#/components/schemas/FunctionCallToolDefinition' - - $ref: '#/components/schemas/MemoryToolDefinition' + - additionalProperties: false + properties: + input_shields: + items: + $ref: '#/components/schemas/ShieldDefinition' + type: array + max_chunks: + type: integer + max_tokens_in_context: + type: integer + memory_bank_configs: + items: + oneOf: + - additionalProperties: false + properties: + bank_id: + type: string + type: + const: vector + type: string + required: + - bank_id + - type + type: object + - additionalProperties: false + properties: + bank_id: + type: string + keys: + items: + type: string + type: array + type: + const: keyvalue + type: string + required: + - bank_id + - type + - keys + type: object + - additionalProperties: false + properties: + bank_id: + type: string + type: + const: keyword + type: string + required: + - bank_id + - type + type: object + - additionalProperties: false + properties: + bank_id: + type: string + entities: + items: + type: string + type: array + type: + const: graph + type: string + required: + - bank_id + - type + - entities + type: object + type: array + output_shields: + items: + $ref: '#/components/schemas/ShieldDefinition' + type: array + query_generator_config: + oneOf: + - additionalProperties: false + properties: + sep: + type: string + type: + const: default + type: string + required: + - type + - sep + type: object + - additionalProperties: false + properties: + model: + type: string + template: + type: string + type: + const: llm + type: string + required: + - type + - model + - template + type: object + - additionalProperties: false + properties: + type: + const: custom + type: string + required: + - type + type: object + type: + const: memory + type: string + required: + - type + - memory_bank_configs + - query_generator_config + - max_tokens_in_context + - max_chunks + type: object type: array required: - model @@ -107,7 +223,123 @@ components: - $ref: '#/components/schemas/PhotogenToolDefinition' - $ref: '#/components/schemas/CodeInterpreterToolDefinition' - $ref: '#/components/schemas/FunctionCallToolDefinition' - - $ref: '#/components/schemas/MemoryToolDefinition' + - additionalProperties: false + properties: + input_shields: + items: + $ref: '#/components/schemas/ShieldDefinition' + type: array + max_chunks: + type: integer + max_tokens_in_context: + type: integer + memory_bank_configs: + items: + oneOf: + - additionalProperties: false + properties: + bank_id: + type: string + type: + const: vector + type: string + required: + - bank_id + - type + type: object + - additionalProperties: false + properties: + bank_id: + type: string + keys: + items: + type: string + type: array + type: + const: keyvalue + type: string + required: + - bank_id + - type + - keys + type: object + - additionalProperties: false + properties: + bank_id: + type: string + type: + const: keyword + type: string + required: + - bank_id + - type + type: object + - additionalProperties: false + properties: + bank_id: + type: string + entities: + items: + type: string + type: array + type: + const: graph + type: string + required: + - bank_id + - type + - entities + type: object + type: array + output_shields: + items: + $ref: '#/components/schemas/ShieldDefinition' + type: array + query_generator_config: + oneOf: + - additionalProperties: false + properties: + sep: + type: string + type: + const: default + type: string + required: + - type + - sep + type: object + - additionalProperties: false + properties: + model: + type: string + template: + type: string + type: + const: llm + type: string + required: + - type + - model + - template + type: object + - additionalProperties: false + properties: + type: + const: custom + type: string + required: + - type + type: object + type: + const: memory + type: string + required: + - type + - memory_bank_configs + - query_generator_config + - max_tokens_in_context + - max_chunks + type: object type: array required: - agent_id @@ -1310,88 +1542,6 @@ components: - memory_bank_ids - inserted_context type: object - MemoryToolDefinition: - additionalProperties: false - properties: - input_shields: - items: - $ref: '#/components/schemas/ShieldDefinition' - type: array - max_chunks: - type: integer - max_tokens_in_context: - type: integer - memory_bank_configs: - items: - oneOf: - - additionalProperties: false - properties: - bank_id: - type: string - type: - const: vector - type: string - required: - - bank_id - - type - type: object - - additionalProperties: false - properties: - bank_id: - type: string - keys: - items: - type: string - type: array - type: - const: keyvalue - type: string - required: - - bank_id - - type - - keys - type: object - - additionalProperties: false - properties: - bank_id: - type: string - type: - const: keyword - type: string - required: - - bank_id - - type - type: object - - additionalProperties: false - properties: - bank_id: - type: string - entities: - items: - type: string - type: array - type: - const: graph - type: string - required: - - bank_id - - type - - entities - type: object - type: array - output_shields: - items: - $ref: '#/components/schemas/ShieldDefinition' - type: array - type: - const: memory - type: string - required: - - type - - memory_bank_configs - - max_tokens_in_context - - max_chunks - type: object Metric: additionalProperties: false properties: @@ -2459,7 +2609,7 @@ info: description: "This is the specification of the llama stack that provides\n \ \ a set of endpoints and their corresponding interfaces that are tailored\ \ to\n best leverage Llama Models. The specification is still in\ - \ draft and subject to change.\n Generated at 2024-09-04 10:28:38.779789" + \ draft and subject to change.\n Generated at 2024-09-07 15:23:29.488676" title: '[DRAFT] Llama Stack Specification' version: 0.0.1 jsonSchemaDialect: https://json-schema.org/draft/2020-12/schema @@ -2640,7 +2790,7 @@ paths: $ref: '#/components/schemas/Artifact' description: OK tags: - - Observability + - Telemetry /batch_inference/chat_completion: post: parameters: [] @@ -2870,7 +3020,7 @@ paths: $ref: '#/components/schemas/Artifact' description: OK tags: - - Observability + - Telemetry /experiments/artifacts/upload: post: parameters: [] @@ -2888,7 +3038,7 @@ paths: $ref: '#/components/schemas/Artifact' description: OK tags: - - Observability + - Telemetry /experiments/create: post: parameters: [] @@ -2906,7 +3056,7 @@ paths: $ref: '#/components/schemas/Experiment' description: OK tags: - - Observability + - Telemetry /experiments/create_run: post: parameters: [] @@ -2924,7 +3074,7 @@ paths: $ref: '#/components/schemas/Run' description: OK tags: - - Observability + - Telemetry /experiments/get: get: parameters: @@ -2941,7 +3091,7 @@ paths: $ref: '#/components/schemas/Experiment' description: OK tags: - - Observability + - Telemetry /experiments/list: get: parameters: [] @@ -2953,7 +3103,7 @@ paths: $ref: '#/components/schemas/Experiment' description: OK tags: - - Observability + - Telemetry /experiments/update: post: parameters: [] @@ -2971,7 +3121,7 @@ paths: $ref: '#/components/schemas/Experiment' description: OK tags: - - Observability + - Telemetry /inference/chat_completion: post: parameters: [] @@ -3043,7 +3193,7 @@ paths: $ref: '#/components/schemas/Log' description: OK tags: - - Observability + - Telemetry /logging/log_messages: post: parameters: [] @@ -3057,7 +3207,7 @@ paths: '200': description: OK tags: - - Observability + - Telemetry /memory_bank/documents/delete: post: parameters: [] @@ -3352,7 +3502,7 @@ paths: '200': description: OK tags: - - Observability + - Telemetry /runs/metrics: get: parameters: @@ -3369,7 +3519,7 @@ paths: $ref: '#/components/schemas/Metric' description: OK tags: - - Observability + - Telemetry /runs/update: post: parameters: [] @@ -3387,7 +3537,7 @@ paths: $ref: '#/components/schemas/Run' description: OK tags: - - Observability + - Telemetry /synthetic_data_generation/generate: post: parameters: [] @@ -3411,16 +3561,16 @@ security: servers: - url: http://any-hosted-llama-stack.com tags: -- name: Observability +- name: Telemetry +- name: Evaluations +- name: AgenticSystem - name: Inference +- name: BatchInference +- name: PostTraining - name: Datasets - name: Memory -- name: AgenticSystem -- name: Evaluations - name: SyntheticDataGeneration - name: RewardScoring -- name: PostTraining -- name: BatchInference - description: name: BatchChatCompletionRequest @@ -3525,9 +3675,6 @@ tags: - description: name: FunctionCallToolDefinition -- description: - name: MemoryToolDefinition - description: name: OnViolationAction @@ -3852,10 +3999,10 @@ x-tagGroups: - Evaluations - Inference - Memory - - Observability - PostTraining - RewardScoring - SyntheticDataGeneration + - Telemetry - name: Types tags: - AgentConfig @@ -3931,7 +4078,6 @@ x-tagGroups: - MemoryBank - MemoryBankDocument - MemoryRetrievalStep - - MemoryToolDefinition - Metric - OnViolationAction - OptimizerConfig diff --git a/rfcs/openapi_generator/run_openapi_generator.sh b/rfcs/openapi_generator/run_openapi_generator.sh index cf4265ae5..1b2f979cc 100755 --- a/rfcs/openapi_generator/run_openapi_generator.sh +++ b/rfcs/openapi_generator/run_openapi_generator.sh @@ -28,4 +28,4 @@ if [ ${#missing_packages[@]} -ne 0 ]; then exit 1 fi -PYTHONPATH=$PYTHONPATH:../.. python3 -m rfcs.openapi_generator.generate $* +PYTHONPATH=$PYTHONPATH:../.. python -m rfcs.openapi_generator.generate $*