Docs for meta-reference-gpu

llama_stack/providers/inline/inference/meta_reference/config.py

@@ -49,6 +49,18 @@ class MetaReferenceInferenceConfig(BaseModel):
         resolved = resolve_model(self.model)
         return resolved.pth_file_count
 
+    @classmethod
+    def sample_run_config(
+        cls,
+        model: str = "Llama3.2-3B-Instruct",
+        checkpoint_dir: str = "${env.CHECKPOINT_DIR:null}",
+    ) -> Dict[str, Any]:
+        return {
+            "model": model,
+            "max_seq_len": 4096,
+            "checkpoint_dir": checkpoint_dir,
+        }
+
 
 class MetaReferenceQuantizedInferenceConfig(MetaReferenceInferenceConfig):
     quantization: QuantizationConfig

llama_stack/providers/inline/inference/meta_reference/generation.py

@@ -107,7 +107,7 @@ class Llama:
             sys.stdout = open(os.devnull, "w")
 
         start_time = time.time()
-        if config.checkpoint_dir:
+        if config.checkpoint_dir and config.checkpoint_dir != "null":
             ckpt_dir = config.checkpoint_dir
         else:
             ckpt_dir = model_checkpoint_dir(model)
@@ -137,7 +137,6 @@ class Llama:
         ), f"model_args vocab = {model_args.vocab_size} but tokenizer vocab = {tokenizer.n_words}"
 
         if isinstance(config, MetaReferenceQuantizedInferenceConfig):
-
             if isinstance(config.quantization, Fp8QuantizationConfig):
                 from .quantization.loader import convert_to_fp8_quantized_model