feat: Add Kubernetes authentication (#1778)

# What does this PR do?

This commit adds a new authentication system to the Llama Stack server
with support for Kubernetes and custom authentication providers. Key
changes include:

- Implemented KubernetesAuthProvider for validating Kubernetes service
account tokens
- Implemented CustomAuthProvider for validating tokens against external
endpoints - this is the same code that was already present.
- Added test for Kubernetes
- Updated server configuration to support authentication settings
- Added documentation for authentication configuration and usage

The authentication system supports:
- Bearer token validation
- Kubernetes service account token validation
- Custom authentication endpoints

## Test Plan

Setup a Kube cluster using Kind or Minikube.

Run a server with:

```
server:
  port: 8321
  auth:
    provider_type: kubernetes
    config:
      api_server_url: http://url
      ca_cert_path: path/to/cert (optional)
```

Run:

```
curl -s -L -H "Authorization: Bearer $(kubectl create token my-user)" http://127.0.0.1:8321/v1/providers
```

Or replace "my-user" with your service account.

Signed-off-by: Sébastien Han <seb@redhat.com>

tests/unit/registry/test_registry_acl.py

@@ -12,7 +12,7 @@ import pytest
 
 from llama_stack.apis.models import ModelType
 from llama_stack.distribution.datatypes import ModelWithACL
-from llama_stack.distribution.server.auth import AccessAttributes
+from llama_stack.distribution.server.auth_providers import AccessAttributes
 from llama_stack.distribution.store.registry import CachedDiskDistributionRegistry
 from llama_stack.providers.utils.kvstore.config import SqliteKVStoreConfig
 from llama_stack.providers.utils.kvstore.sqlite import SqliteKVStoreImpl

tests/unit/server/test_auth.py

@@ -10,7 +10,9 @@ import pytest
 from fastapi import FastAPI
 from fastapi.testclient import TestClient
 
+from llama_stack.distribution.datatypes import AccessAttributes
 from llama_stack.distribution.server.auth import AuthenticationMiddleware
+from llama_stack.distribution.server.auth_providers import AuthProviderConfig, AuthProviderType
 
 
 class MockResponse:
@@ -38,9 +40,23 @@ def invalid_api_key():
 
 
 @pytest.fixture
-def app(mock_auth_endpoint):
+def valid_token():
+    return "valid.jwt.token"
+
+
+@pytest.fixture
+def invalid_token():
+    return "invalid.jwt.token"
+
+
+@pytest.fixture
+def http_app(mock_auth_endpoint):
     app = FastAPI()
-    app.add_middleware(AuthenticationMiddleware, auth_endpoint=mock_auth_endpoint)
+    auth_config = AuthProviderConfig(
+        provider_type=AuthProviderType.CUSTOM,
+        config={"endpoint": mock_auth_endpoint},
+    )
+    app.add_middleware(AuthenticationMiddleware, auth_config=auth_config)
 
     @app.get("/test")
     def test_endpoint():
@@ -50,8 +66,29 @@ def app(mock_auth_endpoint):
 
 
 @pytest.fixture
-def client(app):
-    return TestClient(app)
+def k8s_app():
+    app = FastAPI()
+    auth_config = AuthProviderConfig(
+        provider_type=AuthProviderType.KUBERNETES,
+        config={"api_server_url": "https://kubernetes.default.svc"},
+    )
+    app.add_middleware(AuthenticationMiddleware, auth_config=auth_config)
+
+    @app.get("/test")
+    def test_endpoint():
+        return {"message": "Authentication successful"}
+
+    return app
+
+
+@pytest.fixture
+def http_client(http_app):
+    return TestClient(http_app)
+
+
+@pytest.fixture
+def k8s_client(k8s_app):
+    return TestClient(k8s_app)
 
 
 @pytest.fixture
@@ -61,7 +98,7 @@ def mock_scope():
         "path": "/models/list",
         "headers": [
             (b"content-type", b"application/json"),
-            (b"authorization", b"Bearer test-api-key"),
+            (b"authorization", b"Bearer test.jwt.token"),
             (b"user-agent", b"test-user-agent"),
         ],
         "query_string": b"limit=100&offset=0",
@@ -69,13 +106,38 @@ def mock_scope():
 
 
 @pytest.fixture
-def mock_middleware(mock_auth_endpoint):
+def mock_http_middleware(mock_auth_endpoint):
     mock_app = AsyncMock()
-    return AuthenticationMiddleware(mock_app, mock_auth_endpoint), mock_app
+    auth_config = AuthProviderConfig(
+        provider_type=AuthProviderType.CUSTOM,
+        config={"endpoint": mock_auth_endpoint},
+    )
+    return AuthenticationMiddleware(mock_app, auth_config), mock_app
+
+
+@pytest.fixture
+def mock_k8s_middleware():
+    mock_app = AsyncMock()
+    auth_config = AuthProviderConfig(
+        provider_type=AuthProviderType.KUBERNETES,
+        config={"api_server_url": "https://kubernetes.default.svc"},
+    )
+    return AuthenticationMiddleware(mock_app, auth_config), mock_app
 
 
 async def mock_post_success(*args, **kwargs):
-    return MockResponse(200, {"message": "Authentication successful"})
+    return MockResponse(
+        200,
+        {
+            "message": "Authentication successful",
+            "access_attributes": {
+                "roles": ["admin", "user"],
+                "teams": ["ml-team", "nlp-team"],
+                "projects": ["llama-3", "project-x"],
+                "namespaces": ["research", "production"],
+            },
+        },
+    )
 
 
 async def mock_post_failure(*args, **kwargs):
@@ -86,45 +148,46 @@ async def mock_post_exception(*args, **kwargs):
     raise Exception("Connection error")
 
 
-def test_missing_auth_header(client):
-    response = client.get("/test")
+# HTTP Endpoint Tests
+def test_missing_auth_header(http_client):
+    response = http_client.get("/test")
     assert response.status_code == 401
     assert "Missing or invalid Authorization header" in response.json()["error"]["message"]
 
 
-def test_invalid_auth_header_format(client):
-    response = client.get("/test", headers={"Authorization": "InvalidFormat token123"})
+def test_invalid_auth_header_format(http_client):
+    response = http_client.get("/test", headers={"Authorization": "InvalidFormat token123"})
     assert response.status_code == 401
     assert "Missing or invalid Authorization header" in response.json()["error"]["message"]
 
 
 @patch("httpx.AsyncClient.post", new=mock_post_success)
-def test_valid_authentication(client, valid_api_key):
-    response = client.get("/test", headers={"Authorization": f"Bearer {valid_api_key}"})
+def test_valid_http_authentication(http_client, valid_api_key):
+    response = http_client.get("/test", headers={"Authorization": f"Bearer {valid_api_key}"})
     assert response.status_code == 200
     assert response.json() == {"message": "Authentication successful"}
 
 
 @patch("httpx.AsyncClient.post", new=mock_post_failure)
-def test_invalid_authentication(client, invalid_api_key):
-    response = client.get("/test", headers={"Authorization": f"Bearer {invalid_api_key}"})
+def test_invalid_http_authentication(http_client, invalid_api_key):
+    response = http_client.get("/test", headers={"Authorization": f"Bearer {invalid_api_key}"})
     assert response.status_code == 401
     assert "Authentication failed" in response.json()["error"]["message"]
 
 
 @patch("httpx.AsyncClient.post", new=mock_post_exception)
-def test_auth_service_error(client, valid_api_key):
-    response = client.get("/test", headers={"Authorization": f"Bearer {valid_api_key}"})
+def test_http_auth_service_error(http_client, valid_api_key):
+    response = http_client.get("/test", headers={"Authorization": f"Bearer {valid_api_key}"})
     assert response.status_code == 401
     assert "Authentication service error" in response.json()["error"]["message"]
 
 
-def test_auth_request_payload(client, valid_api_key, mock_auth_endpoint):
+def test_http_auth_request_payload(http_client, valid_api_key, mock_auth_endpoint):
     with patch("httpx.AsyncClient.post") as mock_post:
         mock_response = MockResponse(200, {"message": "Authentication successful"})
         mock_post.return_value = mock_response
 
-        client.get(
+        http_client.get(
             "/test?param1=value1&param2=value2",
             headers={
                 "Authorization": f"Bearer {valid_api_key}",
@@ -149,40 +212,43 @@ def test_auth_request_payload(client, valid_api_key, mock_auth_endpoint):
 
 
 @pytest.mark.asyncio
-async def test_auth_middleware_with_access_attributes(mock_middleware, mock_scope):
-    middleware, mock_app = mock_middleware
+async def test_http_middleware_with_access_attributes(mock_http_middleware, mock_scope):
+    """Test HTTP middleware behavior with access attributes"""
+    middleware, mock_app = mock_http_middleware
     mock_receive = AsyncMock()
     mock_send = AsyncMock()
 
-    with patch("httpx.AsyncClient") as mock_client:
-        mock_client_instance = AsyncMock()
-        mock_client.return_value.__aenter__.return_value = mock_client_instance
-
-        mock_client_instance.post.return_value = MockResponse(
+    with patch("httpx.AsyncClient.post") as mock_post:
+        mock_response = MockResponse(
             200,
             {
+                "message": "Authentication successful",
                 "access_attributes": {
                     "roles": ["admin", "user"],
-                    "teams": ["ml-team"],
-                    "projects": ["project-x", "project-y"],
-                }
+                    "teams": ["ml-team", "nlp-team"],
+                    "projects": ["llama-3", "project-x"],
+                    "namespaces": ["research", "production"],
+                },
             },
         )
+        mock_post.return_value = mock_response
 
         await middleware(mock_scope, mock_receive, mock_send)
 
         assert "user_attributes" in mock_scope
-        assert mock_scope["user_attributes"]["roles"] == ["admin", "user"]
-        assert mock_scope["user_attributes"]["teams"] == ["ml-team"]
-        assert mock_scope["user_attributes"]["projects"] == ["project-x", "project-y"]
+        attributes = mock_scope["user_attributes"]
+        assert attributes["roles"] == ["admin", "user"]
+        assert attributes["teams"] == ["ml-team", "nlp-team"]
+        assert attributes["projects"] == ["llama-3", "project-x"]
+        assert attributes["namespaces"] == ["research", "production"]
 
         mock_app.assert_called_once_with(mock_scope, mock_receive, mock_send)
 
 
 @pytest.mark.asyncio
-async def test_auth_middleware_no_attributes(mock_middleware, mock_scope):
+async def test_http_middleware_no_attributes(mock_http_middleware, mock_scope):
     """Test middleware behavior with no access attributes"""
-    middleware, mock_app = mock_middleware
+    middleware, mock_app = mock_http_middleware
     mock_receive = AsyncMock()
     mock_send = AsyncMock()
 
@@ -203,4 +269,104 @@ async def test_auth_middleware_no_attributes(mock_middleware, mock_scope):
         assert "user_attributes" in mock_scope
         attributes = mock_scope["user_attributes"]
         assert "namespaces" in attributes
-        assert attributes["namespaces"] == ["test-api-key"]
+        assert attributes["namespaces"] == ["test.jwt.token"]
+
+
+# Kubernetes Tests
+def test_missing_auth_header_k8s(k8s_client):
+    response = k8s_client.get("/test")
+    assert response.status_code == 401
+    assert "Missing or invalid Authorization header" in response.json()["error"]["message"]
+
+
+def test_invalid_auth_header_format_k8s(k8s_client):
+    response = k8s_client.get("/test", headers={"Authorization": "InvalidFormat token123"})
+    assert response.status_code == 401
+    assert "Missing or invalid Authorization header" in response.json()["error"]["message"]
+
+
+@patch("kubernetes.client.ApiClient")
+def test_valid_k8s_authentication(mock_api_client, k8s_client, valid_token):
+    # Mock the Kubernetes client
+    mock_client = AsyncMock()
+    mock_api_client.return_value = mock_client
+
+    # Mock successful token validation
+    mock_client.set_default_header = AsyncMock()
+
+    # Mock the token validation to return valid access attributes
+    with patch("llama_stack.distribution.server.auth_providers.KubernetesAuthProvider.validate_token") as mock_validate:
+        mock_validate.return_value = AccessAttributes(
+            roles=["admin"], teams=["ml-team"], projects=["llama-3"], namespaces=["research"]
+        )
+        response = k8s_client.get("/test", headers={"Authorization": f"Bearer {valid_token}"})
+        assert response.status_code == 200
+        assert response.json() == {"message": "Authentication successful"}
+
+
+@patch("kubernetes.client.ApiClient")
+def test_invalid_k8s_authentication(mock_api_client, k8s_client, invalid_token):
+    # Mock the Kubernetes client
+    mock_client = AsyncMock()
+    mock_api_client.return_value = mock_client
+
+    # Mock failed token validation by raising an exception
+    with patch("llama_stack.distribution.server.auth_providers.KubernetesAuthProvider.validate_token") as mock_validate:
+        mock_validate.side_effect = ValueError("Invalid or expired token")
+        response = k8s_client.get("/test", headers={"Authorization": f"Bearer {invalid_token}"})
+        assert response.status_code == 401
+        assert "Invalid or expired token" in response.json()["error"]["message"]
+
+
+@pytest.mark.asyncio
+async def test_k8s_middleware_with_access_attributes(mock_k8s_middleware, mock_scope):
+    middleware, mock_app = mock_k8s_middleware
+    mock_receive = AsyncMock()
+    mock_send = AsyncMock()
+
+    with patch("kubernetes.client.ApiClient") as mock_api_client:
+        mock_client = AsyncMock()
+        mock_api_client.return_value = mock_client
+
+        # Mock successful token validation
+        mock_client.set_default_header = AsyncMock()
+
+        # Mock token payload with access attributes
+        mock_token_parts = ["header", "eyJzdWIiOiJhZG1pbiIsImdyb3VwcyI6WyJtbC10ZWFtIl19", "signature"]
+        mock_scope["headers"][1] = (b"authorization", f"Bearer {'.'.join(mock_token_parts)}".encode())
+
+        await middleware(mock_scope, mock_receive, mock_send)
+
+        assert "user_attributes" in mock_scope
+        assert mock_scope["user_attributes"]["roles"] == ["admin"]
+        assert mock_scope["user_attributes"]["teams"] == ["ml-team"]
+
+        mock_app.assert_called_once_with(mock_scope, mock_receive, mock_send)
+
+
+@pytest.mark.asyncio
+async def test_k8s_middleware_no_attributes(mock_k8s_middleware, mock_scope):
+    """Test middleware behavior with no access attributes"""
+    middleware, mock_app = mock_k8s_middleware
+    mock_receive = AsyncMock()
+    mock_send = AsyncMock()
+
+    with patch("kubernetes.client.ApiClient") as mock_api_client:
+        mock_client = AsyncMock()
+        mock_api_client.return_value = mock_client
+
+        # Mock successful token validation
+        mock_client.set_default_header = AsyncMock()
+
+        # Mock token payload without access attributes
+        mock_token_parts = ["header", "eyJzdWIiOiJhZG1pbiJ9", "signature"]
+        mock_scope["headers"][1] = (b"authorization", f"Bearer {'.'.join(mock_token_parts)}".encode())
+
+        await middleware(mock_scope, mock_receive, mock_send)
+
+        assert "user_attributes" in mock_scope
+        attributes = mock_scope["user_attributes"]
+        assert "roles" in attributes
+        assert attributes["roles"] == ["admin"]
+
+        mock_app.assert_called_once_with(mock_scope, mock_receive, mock_send)