wip

llama_stack/distribution/templates/build_configs/local-gpu-docker-build.yaml

@@ -1,4 +1,4 @@
-name: local
+name: local-gpu
 distribution_spec:
   description: Use code from `llama_stack` itself to serve all llama stack APIs
   providers:
@@ -7,4 +7,4 @@ distribution_spec:
     safety: meta-reference
     agents: meta-reference
     telemetry: meta-reference
-image_type: conda
+image_type: docker

llama_stack/distribution/templates/build_configs/local-tgi-chroma-docker.yaml

@@ -1,11 +1,11 @@
-name: local-gpu
+name: local-tgi-chroma
 distribution_spec:
-  description: local meta reference
+  description: remote tgi inference + chromadb memory
   docker_image: null
   providers:
-    inference: meta-reference
+    inference: remote::tgi
     safety: meta-reference
     agents: meta-reference
-    memory: meta-reference
+    memory: remote::chromadb
     telemetry: meta-reference
 image_type: docker

llama_stack/distribution/templates/docker/llamastack-local-cpu/run.yaml

@@ -1,42 +0,0 @@
-version: '2'
-built_at: '2024-10-08T17:42:07.505267'
-image_name: local-cpu
-docker_image: local-cpu
-conda_env: null
-apis:
-- agents
-- inference
-- models
-- memory
-- safety
-- shields
-- memory_banks
-providers:
-  inference:
-  - provider_id: remote::ollama
-    provider_type: remote::ollama
-    config:
-      host: localhost
-      port: 6000
-  safety:
-  - provider_id: meta-reference
-    provider_type: meta-reference
-    config:
-      llama_guard_shield: null
-      prompt_guard_shield: null
-  memory:
-  - provider_id: meta-reference
-    provider_type: meta-reference
-    config: {}
-  agents:
-  - provider_id: meta-reference
-    provider_type: meta-reference
-    config:
-      persistence_store:
-        namespace: null
-        type: sqlite
-        db_path: ~/.llama/runtime/kvstore.db
-  telemetry:
-  - provider_id: meta-reference
-    provider_type: meta-reference
-    config: {}

llama_stack/distribution/templates/run_configs/local-run.yaml

@@ -1,16 +1,16 @@
 version: '2'
-built_at: '2024-10-08T17:42:33.690666'
-image_name: local-gpu
-docker_image: local-gpu
-conda_env: null
+built_at: '2024-10-08T17:40:45.325529'
+image_name: local
+docker_image: null
+conda_env: local
 apis:
-- memory
-- inference
-- agents
 - shields
-- safety
+- agents
 - models
+- memory
 - memory_banks
+- inference
+- safety
 providers:
   inference:
   - provider_id: meta-reference
@@ -25,8 +25,13 @@ providers:
   - provider_id: meta-reference
     provider_type: meta-reference
     config:
-      llama_guard_shield: null
-      prompt_guard_shield: null
+      llama_guard_shield:
+        model: Llama-Guard-3-1B
+        excluded_categories: []
+        disable_input_check: false
+        disable_output_check: false
+      prompt_guard_shield:
+        model: Prompt-Guard-86M
   memory:
   - provider_id: meta-reference
     provider_type: meta-reference
@@ -38,7 +43,7 @@ providers:
       persistence_store:
         namespace: null
         type: sqlite
-        db_path: ~/.llama/runtime/kvstore.db
+        db_path: /home/xiyan/.llama/runtime/kvstore.db
   telemetry:
   - provider_id: meta-reference
     provider_type: meta-reference

llama_stack/distribution/templates/run_configs/local-tgi-run.yaml

@@ -0,0 +1,50 @@
+version: '2'
+built_at: '2024-10-08T17:40:45.325529'
+image_name: local
+docker_image: null
+conda_env: local
+apis:
+- shields
+- agents
+- models
+- memory
+- memory_banks
+- inference
+- safety
+providers:
+  inference:
+  - provider_id: meta-reference
+    provider_type: meta-reference
+    config:
+      model: Llama3.1-8B-Instruct
+      quantization: null
+      torch_seed: null
+      max_seq_len: 4096
+      max_batch_size: 1
+  safety:
+  - provider_id: meta-reference
+    provider_type: meta-reference
+    config:
+      llama_guard_shield:
+        model: Llama-Guard-3-1B
+        excluded_categories: []
+        disable_input_check: false
+        disable_output_check: false
+      prompt_guard_shield:
+        model: Prompt-Guard-86M
+  memory:
+  - provider_id: meta-reference
+    provider_type: meta-reference
+    config: {}
+  agents:
+  - provider_id: meta-reference
+    provider_type: meta-reference
+    config:
+      persistence_store:
+        namespace: null
+        type: sqlite
+        db_path: /home/xiyan/.llama/runtime/kvstore.db
+  telemetry:
+  - provider_id: meta-reference
+    provider_type: meta-reference
+    config: {}