diff --git a/llama_stack/core/build.py b/llama_stack/core/build.py
index fa1fe632b..2ceb9e9be 100644
--- a/llama_stack/core/build.py
+++ b/llama_stack/core/build.py
@@ -80,7 +80,7 @@ def get_provider_dependencies(
     normal_deps = []
     special_deps = []
     for package in deps:
-        if "--no-deps" in package or "--index-url" in package:
+        if any(f in package for f in ["--no-deps", "--index-url", "--extra-index-url"]):
             special_deps.append(package)
         else:
             normal_deps.append(package)
diff --git a/llama_stack/providers/registry/inference.py b/llama_stack/providers/registry/inference.py
index 4af7b8e31..82b771a28 100644
--- a/llama_stack/providers/registry/inference.py
+++ b/llama_stack/providers/registry/inference.py
@@ -42,7 +42,7 @@ def available_providers() -> list[ProviderSpec]:
             provider_type="inline::sentence-transformers",
             # CrossEncoder depends on torchao.quantization
             pip_packages=[
-                "torch torchvision torchao>=0.12.0 --index-url https://download.pytorch.org/whl/cpu",
+                "torch torchvision torchao>=0.12.0 --extra-index-url https://download.pytorch.org/whl/cpu",
                 "sentence-transformers --no-deps",
             ],
             module="llama_stack.providers.inline.inference.sentence_transformers",