nvdia fix

llama_stack/providers/remote/inference/nvidia/openai_utils.py

@@ -279,7 +279,6 @@ def convert_chat_completion_request(
             nvext.update(top_k=strategy.top_k)
         elif isinstance(strategy, GreedySamplingStrategy):
             nvext.update(top_k=-1)
-            payload.update(temperature=strategy.temperature)
         else:
             raise ValueError(f"Unsupported sampling strategy: {strategy}")
 

llama_stack/templates/nvidia/nvidia.py

@@ -6,8 +6,11 @@
 
 from pathlib import Path
 
+from llama_models.sku_list import all_registered_models
+
 from llama_stack.distribution.datatypes import ModelInput, Provider
 from llama_stack.providers.remote.inference.nvidia import NVIDIAConfig
+from llama_stack.providers.remote.inference.nvidia.nvidia import _MODEL_ALIASES
 
 from llama_stack.templates.template import DistributionTemplate, RunConfigSettings
 
@@ -36,10 +39,21 @@ def get_distribution_template() -> DistributionTemplate:
         config=NVIDIAConfig.sample_run_config(),
     )
 
-    inference_model = ModelInput(
-        model_id="${env.INFERENCE_MODEL}",
-        provider_id="nvidia",
-    )
+    # inference_model = ModelInput(
+    #     model_id="${env.INFERENCE_MODEL}",
+    #     provider_id="nvidia",
+    # )
+    core_model_to_hf_repo = {
+        m.descriptor(): m.huggingface_repo for m in all_registered_models()
+    }
+    default_models = [
+        ModelInput(
+            model_id=core_model_to_hf_repo[m.llama_model],
+            provider_model_id=m.provider_model_id,
+            provider_id="nvidia",
+        )
+        for m in _MODEL_ALIASES
+    ]
 
     return DistributionTemplate(
         name="nvidia",
@@ -48,13 +62,13 @@ def get_distribution_template() -> DistributionTemplate:
         docker_image=None,
         template_path=Path(__file__).parent / "doc_template.md",
         providers=providers,
-        default_models=[inference_model],
+        default_models=default_models,
         run_configs={
             "run.yaml": RunConfigSettings(
                 provider_overrides={
                     "inference": [inference_provider],
                 },
-                default_models=[inference_model],
+                default_models=default_models,
             ),
         },
         run_config_env_vars={

llama_stack/templates/nvidia/run.yaml

@@ -89,8 +89,49 @@ metadata_store:
   db_path: ${env.SQLITE_STORE_DIR:~/.llama/distributions/nvidia}/registry.db
 models:
 - metadata: {}
-  model_id: ${env.INFERENCE_MODEL}
+  model_id: meta-llama/Llama-3-8B-Instruct
   provider_id: nvidia
+  provider_model_id: meta/llama3-8b-instruct
+  model_type: llm
+- metadata: {}
+  model_id: meta-llama/Llama-3-70B-Instruct
+  provider_id: nvidia
+  provider_model_id: meta/llama3-70b-instruct
+  model_type: llm
+- metadata: {}
+  model_id: meta-llama/Llama-3.1-8B-Instruct
+  provider_id: nvidia
+  provider_model_id: meta/llama-3.1-8b-instruct
+  model_type: llm
+- metadata: {}
+  model_id: meta-llama/Llama-3.1-70B-Instruct
+  provider_id: nvidia
+  provider_model_id: meta/llama-3.1-70b-instruct
+  model_type: llm
+- metadata: {}
+  model_id: meta-llama/Llama-3.1-405B-Instruct-FP8
+  provider_id: nvidia
+  provider_model_id: meta/llama-3.1-405b-instruct
+  model_type: llm
+- metadata: {}
+  model_id: meta-llama/Llama-3.2-1B-Instruct
+  provider_id: nvidia
+  provider_model_id: meta/llama-3.2-1b-instruct
+  model_type: llm
+- metadata: {}
+  model_id: meta-llama/Llama-3.2-3B-Instruct
+  provider_id: nvidia
+  provider_model_id: meta/llama-3.2-3b-instruct
+  model_type: llm
+- metadata: {}
+  model_id: meta-llama/Llama-3.2-11B-Vision-Instruct
+  provider_id: nvidia
+  provider_model_id: meta/llama-3.2-11b-vision-instruct
+  model_type: llm
+- metadata: {}
+  model_id: meta-llama/Llama-3.2-90B-Vision-Instruct
+  provider_id: nvidia
+  provider_model_id: meta/llama-3.2-90b-vision-instruct
   model_type: llm
 shields: []
 memory_banks: []