Merge branch 'main' into add-nvidia-inference-adapter

2025-12-17 07:32:36 +00:00 · 2024-11-15 14:09:12 -05:00 · 2024-11-15 14:09:12 -05:00 · 43262df033
commit 43262df033
parent dbe665ed19 e8112b31ab
399 changed files with 17826 additions and 10490 deletions
--- a/llama_stack/providers/registry/agents.py
+++ b/llama_stack/providers/registry/agents.py
@ -14,7 +14,7 @@ def available_providers() -> List[ProviderSpec]:
    return [
        InlineProviderSpec(
            api=Api.agents,
-            provider_type="meta-reference",
+            provider_type="inline::meta-reference",
            pip_packages=[
                "matplotlib",
                "pillow",
@ -22,8 +22,8 @@ def available_providers() -> List[ProviderSpec]:
                "scikit-learn",
            ]
            + kvstore_dependencies(),
-            module="llama_stack.providers.impls.meta_reference.agents",
-            config_class="llama_stack.providers.impls.meta_reference.agents.MetaReferenceAgentsImplConfig",
+            module="llama_stack.providers.inline.agents.meta_reference",
+            config_class="llama_stack.providers.inline.agents.meta_reference.MetaReferenceAgentsImplConfig",
            api_dependencies=[
                Api.inference,
                Api.safety,
@ -36,8 +36,8 @@ def available_providers() -> List[ProviderSpec]:
            adapter=AdapterSpec(
                adapter_type="sample",
                pip_packages=[],
-                module="llama_stack.providers.adapters.agents.sample",
-                config_class="llama_stack.providers.adapters.agents.sample.SampleConfig",
+                module="llama_stack.providers.remote.agents.sample",
+                config_class="llama_stack.providers.remote.agents.sample.SampleConfig",
            ),
        ),
    ]
--- a/llama_stack/providers/registry/datasetio.py
+++ b/llama_stack/providers/registry/datasetio.py
@ -13,10 +13,21 @@ def available_providers() -> List[ProviderSpec]:
    return [
        InlineProviderSpec(
            api=Api.datasetio,
-            provider_type="meta-reference",
+            provider_type="inline::localfs",
            pip_packages=["pandas"],
-            module="llama_stack.providers.impls.meta_reference.datasetio",
-            config_class="llama_stack.providers.impls.meta_reference.datasetio.MetaReferenceDatasetIOConfig",
+            module="llama_stack.providers.inline.datasetio.localfs",
+            config_class="llama_stack.providers.inline.datasetio.localfs.LocalFSDatasetIOConfig",
            api_dependencies=[],
        ),
+        remote_provider_spec(
+            api=Api.datasetio,
+            adapter=AdapterSpec(
+                adapter_type="huggingface",
+                pip_packages=[
+                    "datasets",
+                ],
+                module="llama_stack.providers.remote.datasetio.huggingface",
+                config_class="llama_stack.providers.remote.datasetio.huggingface.HuggingfaceDatasetIOConfig",
+            ),
+        ),
    ]
--- a/llama_stack/providers/registry/eval.py
+++ b/llama_stack/providers/registry/eval.py
@ -13,10 +13,10 @@ def available_providers() -> List[ProviderSpec]:
    return [
        InlineProviderSpec(
            api=Api.eval,
-            provider_type="meta-reference",
+            provider_type="inline::meta-reference",
            pip_packages=[],
-            module="llama_stack.providers.impls.meta_reference.eval",
-            config_class="llama_stack.providers.impls.meta_reference.eval.MetaReferenceEvalConfig",
+            module="llama_stack.providers.inline.eval.meta_reference",
+            config_class="llama_stack.providers.inline.eval.meta_reference.MetaReferenceEvalConfig",
            api_dependencies=[
                Api.datasetio,
                Api.datasets,
--- a/llama_stack/providers/registry/inference.py
+++ b/llama_stack/providers/registry/inference.py
@ -25,14 +25,14 @@ def available_providers() -> List[ProviderSpec]:
    return [
        InlineProviderSpec(
            api=Api.inference,
-            provider_type="meta-reference",
+            provider_type="inline::meta-reference",
            pip_packages=META_REFERENCE_DEPS,
-            module="llama_stack.providers.impls.meta_reference.inference",
-            config_class="llama_stack.providers.impls.meta_reference.inference.MetaReferenceInferenceConfig",
+            module="llama_stack.providers.inline.inference.meta_reference",
+            config_class="llama_stack.providers.inline.inference.meta_reference.MetaReferenceInferenceConfig",
        ),
        InlineProviderSpec(
            api=Api.inference,
-            provider_type="meta-reference-quantized",
+            provider_type="inline::meta-reference-quantized",
            pip_packages=(
                META_REFERENCE_DEPS
                + [
@ -40,16 +40,25 @@ def available_providers() -> List[ProviderSpec]:
                    "torchao==0.5.0",
                ]
            ),
-            module="llama_stack.providers.impls.meta_reference.inference",
-            config_class="llama_stack.providers.impls.meta_reference.inference.MetaReferenceQuantizedInferenceConfig",
+            module="llama_stack.providers.inline.inference.meta_reference",
+            config_class="llama_stack.providers.inline.inference.meta_reference.MetaReferenceQuantizedInferenceConfig",
+        ),
+        InlineProviderSpec(
+            api=Api.inference,
+            provider_type="inline::vllm",
+            pip_packages=[
+                "vllm",
+            ],
+            module="llama_stack.providers.inline.inference.vllm",
+            config_class="llama_stack.providers.inline.inference.vllm.VLLMConfig",
        ),
        remote_provider_spec(
            api=Api.inference,
            adapter=AdapterSpec(
                adapter_type="sample",
                pip_packages=[],
-                module="llama_stack.providers.adapters.inference.sample",
-                config_class="llama_stack.providers.adapters.inference.sample.SampleConfig",
+                module="llama_stack.providers.remote.inference.sample",
+                config_class="llama_stack.providers.remote.inference.sample.SampleConfig",
            ),
        ),
        remote_provider_spec(
@ -57,26 +66,26 @@ def available_providers() -> List[ProviderSpec]:
            adapter=AdapterSpec(
                adapter_type="ollama",
                pip_packages=["ollama", "aiohttp"],
-                config_class="llama_stack.providers.adapters.inference.ollama.OllamaImplConfig",
-                module="llama_stack.providers.adapters.inference.ollama",
+                config_class="llama_stack.providers.remote.inference.ollama.OllamaImplConfig",
+                module="llama_stack.providers.remote.inference.ollama",
+            ),
+        ),
+        remote_provider_spec(
+            api=Api.inference,
+            adapter=AdapterSpec(
+                adapter_type="vllm",
+                pip_packages=["openai"],
+                module="llama_stack.providers.remote.inference.vllm",
+                config_class="llama_stack.providers.remote.inference.vllm.VLLMInferenceAdapterConfig",
            ),
        ),
-        # remote_provider_spec(
-        #     api=Api.inference,
-        #     adapter=AdapterSpec(
-        #         adapter_type="vllm",
-        #         pip_packages=["openai"],
-        #         module="llama_stack.providers.adapters.inference.vllm",
-        #         config_class="llama_stack.providers.adapters.inference.vllm.VLLMImplConfig",
-        #     ),
-        # ),
        remote_provider_spec(
            api=Api.inference,
            adapter=AdapterSpec(
                adapter_type="tgi",
                pip_packages=["huggingface_hub", "aiohttp"],
-                module="llama_stack.providers.adapters.inference.tgi",
-                config_class="llama_stack.providers.adapters.inference.tgi.TGIImplConfig",
+                module="llama_stack.providers.remote.inference.tgi",
+                config_class="llama_stack.providers.remote.inference.tgi.TGIImplConfig",
            ),
        ),
        remote_provider_spec(
@ -84,8 +93,8 @@ def available_providers() -> List[ProviderSpec]:
            adapter=AdapterSpec(
                adapter_type="hf::serverless",
                pip_packages=["huggingface_hub", "aiohttp"],
-                module="llama_stack.providers.adapters.inference.tgi",
-                config_class="llama_stack.providers.adapters.inference.tgi.InferenceAPIImplConfig",
+                module="llama_stack.providers.remote.inference.tgi",
+                config_class="llama_stack.providers.remote.inference.tgi.InferenceAPIImplConfig",
            ),
        ),
        remote_provider_spec(
@ -93,8 +102,8 @@ def available_providers() -> List[ProviderSpec]:
            adapter=AdapterSpec(
                adapter_type="hf::endpoint",
                pip_packages=["huggingface_hub", "aiohttp"],
-                module="llama_stack.providers.adapters.inference.tgi",
-                config_class="llama_stack.providers.adapters.inference.tgi.InferenceEndpointImplConfig",
+                module="llama_stack.providers.remote.inference.tgi",
+                config_class="llama_stack.providers.remote.inference.tgi.InferenceEndpointImplConfig",
            ),
        ),
        remote_provider_spec(
@ -104,8 +113,9 @@ def available_providers() -> List[ProviderSpec]:
                pip_packages=[
                    "fireworks-ai",
                ],
-                module="llama_stack.providers.adapters.inference.fireworks",
-                config_class="llama_stack.providers.adapters.inference.fireworks.FireworksImplConfig",
+                module="llama_stack.providers.remote.inference.fireworks",
+                config_class="llama_stack.providers.remote.inference.fireworks.FireworksImplConfig",
+                provider_data_validator="llama_stack.providers.remote.inference.fireworks.FireworksProviderDataValidator",
            ),
        ),
        remote_provider_spec(
@ -115,9 +125,9 @@ def available_providers() -> List[ProviderSpec]:
                pip_packages=[
                    "together",
                ],
-                module="llama_stack.providers.adapters.inference.together",
-                config_class="llama_stack.providers.adapters.inference.together.TogetherImplConfig",
-                provider_data_validator="llama_stack.providers.adapters.safety.together.TogetherProviderDataValidator",
+                module="llama_stack.providers.remote.inference.together",
+                config_class="llama_stack.providers.remote.inference.together.TogetherImplConfig",
+                provider_data_validator="llama_stack.providers.remote.inference.together.TogetherProviderDataValidator",
            ),
        ),
        remote_provider_spec(
@ -125,8 +135,8 @@ def available_providers() -> List[ProviderSpec]:
            adapter=AdapterSpec(
                adapter_type="bedrock",
                pip_packages=["boto3"],
-                module="llama_stack.providers.adapters.inference.bedrock",
-                config_class="llama_stack.providers.adapters.inference.bedrock.BedrockConfig",
+                module="llama_stack.providers.remote.inference.bedrock",
+                config_class="llama_stack.providers.remote.inference.bedrock.BedrockConfig",
            ),
        ),
        remote_provider_spec(
@ -136,8 +146,8 @@ def available_providers() -> List[ProviderSpec]:
                pip_packages=[
                    "openai",
                ],
-                module="llama_stack.providers.adapters.inference.databricks",
-                config_class="llama_stack.providers.adapters.inference.databricks.DatabricksImplConfig",
+                module="llama_stack.providers.remote.inference.databricks",
+                config_class="llama_stack.providers.remote.inference.databricks.DatabricksImplConfig",
            ),
        ),
        remote_provider_spec(
@ -148,16 +158,7 @@ def available_providers() -> List[ProviderSpec]:
                    "openai",
                ],
                module="llama_stack.providers.adapters.inference.nvidia",
-                config_class="llama_stack.providers.adapters.inference.nvidia.NVIDIAConfig",
+                config_class="llama_stack.providers.remote.inference.nvidia.NVIDIAConfig",
            ),
        ),
-        InlineProviderSpec(
-            api=Api.inference,
-            provider_type="vllm",
-            pip_packages=[
-                "vllm",
-            ],
-            module="llama_stack.providers.impls.vllm",
-            config_class="llama_stack.providers.impls.vllm.VLLMConfig",
-        ),
    ]
--- a/llama_stack/providers/registry/memory.py
+++ b/llama_stack/providers/registry/memory.py
@ -34,17 +34,26 @@ def available_providers() -> List[ProviderSpec]:
    return [
        InlineProviderSpec(
            api=Api.memory,
-            provider_type="meta-reference",
+            provider_type="inline::meta-reference",
            pip_packages=EMBEDDING_DEPS + ["faiss-cpu"],
-            module="llama_stack.providers.impls.meta_reference.memory",
-            config_class="llama_stack.providers.impls.meta_reference.memory.FaissImplConfig",
+            module="llama_stack.providers.inline.memory.faiss",
+            config_class="llama_stack.providers.inline.memory.faiss.FaissImplConfig",
+            deprecation_warning="Please use the `inline::faiss` provider instead.",
+        ),
+        InlineProviderSpec(
+            api=Api.memory,
+            provider_type="inline::faiss",
+            pip_packages=EMBEDDING_DEPS + ["faiss-cpu"],
+            module="llama_stack.providers.inline.memory.faiss",
+            config_class="llama_stack.providers.inline.memory.faiss.FaissImplConfig",
        ),
        remote_provider_spec(
            Api.memory,
            AdapterSpec(
                adapter_type="chromadb",
                pip_packages=EMBEDDING_DEPS + ["chromadb-client"],
-                module="llama_stack.providers.adapters.memory.chroma",
+                module="llama_stack.providers.remote.memory.chroma",
+                config_class="llama_stack.distribution.datatypes.RemoteProviderConfig",
            ),
        ),
        remote_provider_spec(
@ -52,8 +61,8 @@ def available_providers() -> List[ProviderSpec]:
            AdapterSpec(
                adapter_type="pgvector",
                pip_packages=EMBEDDING_DEPS + ["psycopg2-binary"],
-                module="llama_stack.providers.adapters.memory.pgvector",
-                config_class="llama_stack.providers.adapters.memory.pgvector.PGVectorConfig",
+                module="llama_stack.providers.remote.memory.pgvector",
+                config_class="llama_stack.providers.remote.memory.pgvector.PGVectorConfig",
            ),
        ),
        remote_provider_spec(
@ -61,9 +70,9 @@ def available_providers() -> List[ProviderSpec]:
            AdapterSpec(
                adapter_type="weaviate",
                pip_packages=EMBEDDING_DEPS + ["weaviate-client"],
-                module="llama_stack.providers.adapters.memory.weaviate",
-                config_class="llama_stack.providers.adapters.memory.weaviate.WeaviateConfig",
-                provider_data_validator="llama_stack.providers.adapters.memory.weaviate.WeaviateRequestProviderData",
+                module="llama_stack.providers.remote.memory.weaviate",
+                config_class="llama_stack.providers.remote.memory.weaviate.WeaviateConfig",
+                provider_data_validator="llama_stack.providers.remote.memory.weaviate.WeaviateRequestProviderData",
            ),
        ),
        remote_provider_spec(
@ -71,8 +80,8 @@ def available_providers() -> List[ProviderSpec]:
            adapter=AdapterSpec(
                adapter_type="sample",
                pip_packages=[],
-                module="llama_stack.providers.adapters.memory.sample",
-                config_class="llama_stack.providers.adapters.memory.sample.SampleConfig",
+                module="llama_stack.providers.remote.memory.sample",
+                config_class="llama_stack.providers.remote.memory.sample.SampleConfig",
            ),
        ),
        remote_provider_spec(
@ -80,8 +89,8 @@ def available_providers() -> List[ProviderSpec]:
            AdapterSpec(
                adapter_type="qdrant",
                pip_packages=EMBEDDING_DEPS + ["qdrant-client"],
-                module="llama_stack.providers.adapters.memory.qdrant",
-                config_class="llama_stack.providers.adapters.memory.qdrant.QdrantConfig",
+                module="llama_stack.providers.remote.memory.qdrant",
+                config_class="llama_stack.providers.remote.memory.qdrant.QdrantConfig",
            ),
        ),
    ]
--- a/llama_stack/providers/registry/safety.py
+++ b/llama_stack/providers/registry/safety.py
@ -19,24 +19,61 @@ def available_providers() -> List[ProviderSpec]:
    return [
        InlineProviderSpec(
            api=Api.safety,
-            provider_type="meta-reference",
+            provider_type="inline::meta-reference",
            pip_packages=[
                "transformers",
                "torch --index-url https://download.pytorch.org/whl/cpu",
            ],
-            module="llama_stack.providers.impls.meta_reference.safety",
-            config_class="llama_stack.providers.impls.meta_reference.safety.SafetyConfig",
+            module="llama_stack.providers.inline.safety.meta_reference",
+            config_class="llama_stack.providers.inline.safety.meta_reference.SafetyConfig",
            api_dependencies=[
                Api.inference,
            ],
+            deprecation_error="""
+Provider `inline::meta-reference` for API `safety` does not work with the latest Llama Stack.
+
+- if you are using Llama Guard v3, please use the `inline::llama-guard` provider instead.
+- if you are using Prompt Guard, please use the `inline::prompt-guard` provider instead.
+- if you are using Code Scanner, please use the `inline::code-scanner` provider instead.
+
+            """,
+        ),
+        InlineProviderSpec(
+            api=Api.safety,
+            provider_type="inline::llama-guard",
+            pip_packages=[],
+            module="llama_stack.providers.inline.safety.llama_guard",
+            config_class="llama_stack.providers.inline.safety.llama_guard.LlamaGuardConfig",
+            api_dependencies=[
+                Api.inference,
+            ],
+        ),
+        InlineProviderSpec(
+            api=Api.safety,
+            provider_type="inline::prompt-guard",
+            pip_packages=[
+                "transformers",
+                "torch --index-url https://download.pytorch.org/whl/cpu",
+            ],
+            module="llama_stack.providers.inline.safety.prompt_guard",
+            config_class="llama_stack.providers.inline.safety.prompt_guard.PromptGuardConfig",
+        ),
+        InlineProviderSpec(
+            api=Api.safety,
+            provider_type="inline::code-scanner",
+            pip_packages=[
+                "codeshield",
+            ],
+            module="llama_stack.providers.inline.safety.code_scanner",
+            config_class="llama_stack.providers.inline.safety.code_scanner.CodeScannerConfig",
        ),
        remote_provider_spec(
            api=Api.safety,
            adapter=AdapterSpec(
                adapter_type="sample",
                pip_packages=[],
-                module="llama_stack.providers.adapters.safety.sample",
-                config_class="llama_stack.providers.adapters.safety.sample.SampleConfig",
+                module="llama_stack.providers.remote.safety.sample",
+                config_class="llama_stack.providers.remote.safety.sample.SampleConfig",
            ),
        ),
        remote_provider_spec(
@ -44,30 +81,8 @@ def available_providers() -> List[ProviderSpec]:
            adapter=AdapterSpec(
                adapter_type="bedrock",
                pip_packages=["boto3"],
-                module="llama_stack.providers.adapters.safety.bedrock",
-                config_class="llama_stack.providers.adapters.safety.bedrock.BedrockSafetyConfig",
+                module="llama_stack.providers.remote.safety.bedrock",
+                config_class="llama_stack.providers.remote.safety.bedrock.BedrockSafetyConfig",
            ),
        ),
-        remote_provider_spec(
-            api=Api.safety,
-            adapter=AdapterSpec(
-                adapter_type="together",
-                pip_packages=[
-                    "together",
-                ],
-                module="llama_stack.providers.adapters.safety.together",
-                config_class="llama_stack.providers.adapters.safety.together.TogetherSafetyConfig",
-                provider_data_validator="llama_stack.providers.adapters.safety.together.TogetherProviderDataValidator",
-            ),
-        ),
-        InlineProviderSpec(
-            api=Api.safety,
-            provider_type="meta-reference/codeshield",
-            pip_packages=[
-                "codeshield",
-            ],
-            module="llama_stack.providers.impls.meta_reference.codeshield",
-            config_class="llama_stack.providers.impls.meta_reference.codeshield.CodeShieldConfig",
-            api_dependencies=[],
-        ),
    ]
--- a/llama_stack/providers/registry/scoring.py
+++ b/llama_stack/providers/registry/scoring.py
@ -13,10 +13,21 @@ def available_providers() -> List[ProviderSpec]:
    return [
        InlineProviderSpec(
            api=Api.scoring,
-            provider_type="meta-reference",
+            provider_type="inline::basic",
            pip_packages=[],
-            module="llama_stack.providers.impls.meta_reference.scoring",
-            config_class="llama_stack.providers.impls.meta_reference.scoring.MetaReferenceScoringConfig",
+            module="llama_stack.providers.inline.scoring.basic",
+            config_class="llama_stack.providers.inline.scoring.basic.BasicScoringConfig",
+            api_dependencies=[
+                Api.datasetio,
+                Api.datasets,
+            ],
+        ),
+        InlineProviderSpec(
+            api=Api.scoring,
+            provider_type="inline::llm-as-judge",
+            pip_packages=[],
+            module="llama_stack.providers.inline.scoring.llm_as_judge",
+            config_class="llama_stack.providers.inline.scoring.llm_as_judge.LlmAsJudgeScoringConfig",
            api_dependencies=[
                Api.datasetio,
                Api.datasets,
@ -25,10 +36,10 @@ def available_providers() -> List[ProviderSpec]:
        ),
        InlineProviderSpec(
            api=Api.scoring,
-            provider_type="braintrust",
+            provider_type="inline::braintrust",
            pip_packages=["autoevals", "openai"],
-            module="llama_stack.providers.impls.braintrust.scoring",
-            config_class="llama_stack.providers.impls.braintrust.scoring.BraintrustScoringConfig",
+            module="llama_stack.providers.inline.scoring.braintrust",
+            config_class="llama_stack.providers.inline.scoring.braintrust.BraintrustScoringConfig",
            api_dependencies=[
                Api.datasetio,
                Api.datasets,
--- a/llama_stack/providers/registry/telemetry.py
+++ b/llama_stack/providers/registry/telemetry.py
@ -13,18 +13,18 @@ def available_providers() -> List[ProviderSpec]:
    return [
        InlineProviderSpec(
            api=Api.telemetry,
-            provider_type="meta-reference",
+            provider_type="inline::meta-reference",
            pip_packages=[],
-            module="llama_stack.providers.impls.meta_reference.telemetry",
-            config_class="llama_stack.providers.impls.meta_reference.telemetry.ConsoleConfig",
+            module="llama_stack.providers.inline.meta_reference.telemetry",
+            config_class="llama_stack.providers.inline.meta_reference.telemetry.ConsoleConfig",
        ),
        remote_provider_spec(
            api=Api.telemetry,
            adapter=AdapterSpec(
                adapter_type="sample",
                pip_packages=[],
-                module="llama_stack.providers.adapters.telemetry.sample",
-                config_class="llama_stack.providers.adapters.telemetry.sample.SampleConfig",
+                module="llama_stack.providers.remote.telemetry.sample",
+                config_class="llama_stack.providers.remote.telemetry.sample.SampleConfig",
            ),
        ),
        remote_provider_spec(
@ -37,8 +37,8 @@ def available_providers() -> List[ProviderSpec]:
                    "opentelemetry-exporter-jaeger",
                    "opentelemetry-semantic-conventions",
                ],
-                module="llama_stack.providers.adapters.telemetry.opentelemetry",
-                config_class="llama_stack.providers.adapters.telemetry.opentelemetry.OpenTelemetryConfig",
+                module="llama_stack.providers.remote.telemetry.opentelemetry",
+                config_class="llama_stack.providers.remote.telemetry.opentelemetry.OpenTelemetryConfig",
            ),
        ),
    ]