mirror of
https://github.com/meta-llama/llama-stack.git
synced 2025-12-03 09:53:45 +00:00
840ad75fe9
# What does this PR do? Extract API definitions and provider specifications into a standalone llama-stack-api package that can be published to PyPI independently of the main llama-stack server. see: https://github.com/llamastack/llama-stack/pull/2978 and https://github.com/llamastack/llama-stack/pull/2978#issuecomment-3145115942 Motivation External providers currently import from llama-stack, which overrides the installed version and causes dependency conflicts. This separation allows external providers to: - Install only the type definitions they need without server dependencies - Avoid version conflicts with the installed llama-stack package - Be versioned and released independently This enables us to re-enable external provider module tests that were previously blocked by these import conflicts. Changes - Created llama-stack-api package with minimal dependencies (pydantic, jsonschema) - Moved APIs, providers datatypes, strong_typing, and schema_utils - Updated all imports from llama_stack.* to llama_stack_api.* - Configured local editable install for development workflow - Updated linting and type-checking configuration for both packages Next Steps - Publish llama-stack-api to PyPI - Update external provider dependencies - Re-enable external provider module tests Pre-cursor PRs to this one: - #4093 - #3954 - #4064 These PRs moved key pieces _out_ of the Api pkg, limiting the scope of change here. relates to #3237 ## Test Plan Package builds successfully and can be imported independently. All pre-commit hooks pass with expected exclusions maintained. --------- Signed-off-by: Charlie Doern <cdoern@redhat.com>
70 lines
2.4 KiB
Python
70 lines
2.4 KiB
Python
# Copyright (c) Meta Platforms, Inc. and affiliates.
|
|
# All rights reserved.
|
|
#
|
|
# This source code is licensed under the terms described in the LICENSE file in
|
|
# the root directory of this source tree.
|
|
|
|
import base64
|
|
import mimetypes
|
|
import os
|
|
|
|
import pytest
|
|
from llama_stack_api import ViolationLevel
|
|
|
|
VISION_SHIELD_ENABLED_PROVIDERS = {"together"}
|
|
|
|
|
|
def data_url_from_image(file_path):
|
|
mime_type, _ = mimetypes.guess_type(file_path)
|
|
if mime_type is None:
|
|
raise ValueError("Could not determine MIME type of the file")
|
|
|
|
with open(file_path, "rb") as image_file:
|
|
encoded_string = base64.b64encode(image_file.read()).decode("utf-8")
|
|
|
|
data_url = f"data:{mime_type};base64,{encoded_string}"
|
|
return data_url
|
|
|
|
|
|
def test_safety_with_image(llama_stack_client, model_providers):
|
|
vision_safety_enabled = len(VISION_SHIELD_ENABLED_PROVIDERS & model_providers) > 0
|
|
if not vision_safety_enabled:
|
|
pytest.skip(f"Testing vision shields is not supported for model_providers {model_providers}")
|
|
|
|
current_dir = os.path.dirname(os.path.abspath(__file__))
|
|
unsafe_example_img_path = f"{current_dir}/resources/example_unsafe.jpg"
|
|
safe_example_img_path = f"{current_dir}/resources/example_safe.jpg"
|
|
|
|
examples = {
|
|
"safe": safe_example_img_path,
|
|
"unsafe": unsafe_example_img_path,
|
|
}
|
|
|
|
for _, file_path in examples.items():
|
|
message = {
|
|
"role": "user",
|
|
"content": [
|
|
{
|
|
"type": "text",
|
|
"text": "Tell me how to assemble this",
|
|
},
|
|
{
|
|
"type": "image",
|
|
"image": {"url": {"uri": data_url_from_image(file_path)}},
|
|
},
|
|
],
|
|
}
|
|
response = llama_stack_client.safety.run_shield(
|
|
messages=[message],
|
|
shield_id="meta-llama/Llama-Guard-3-11B-Vision",
|
|
params={},
|
|
)
|
|
assert response is not None
|
|
|
|
# FIXME: We are getting flaky results with the unsafe example:
|
|
# 1. sometimes it is marked as safe
|
|
# 2. sometimes it is marked as unsafe but with incorrect violation_type
|
|
# 3. sometimes it is marked as unsafe with correct violation_type
|
|
if response.violation is not None:
|
|
assert response.violation.violation_level == ViolationLevel.ERROR.value
|
|
assert response.violation.user_message == "I can't answer that. Can I help with something else?"
|