Enable vision models for Together and Fireworks

llama_stack/providers/utils/inference/openai_compat.py

@@ -46,6 +46,9 @@ def text_from_choice(choice) -> str:
     if hasattr(choice, "delta") and choice.delta:
         return choice.delta.content
 
+    if hasattr(choice, "message"):
+        return choice.message.content
+
     return choice.text
 
 
@@ -99,7 +102,6 @@ def process_chat_completion_response(
 async def process_completion_stream_response(
     stream: AsyncGenerator[OpenAICompatCompletionResponse, None], formatter: ChatFormat
 ) -> AsyncGenerator:
-
     stop_reason = None
 
     async for chunk in stream:
@@ -158,6 +160,10 @@ async def process_chat_completion_stream_response(
             break
 
         text = text_from_choice(choice)
+        if not text:
+            # Sometimes you get empty chunks from providers
+            continue
+
         # check if its a tool call ( aka starts with <|python_tag|> )
         if not ipython and text.startswith("<|python_tag|>"):
             ipython = True

llama_stack/providers/utils/inference/prompt_adapter.py

@@ -3,10 +3,14 @@
 #
 # This source code is licensed under the terms described in the LICENSE file in
 # the root directory of this source tree.
+
+import base64
+import io
 import json
 from typing import Tuple
 
 from llama_models.llama3.api.chat_format import ChatFormat
+from PIL import Image as PIL_Image
 from termcolor import cprint
 
 from llama_models.llama3.api.datatypes import *  # noqa: F403
@@ -24,6 +28,73 @@ from llama_models.sku_list import resolve_model
 from llama_stack.providers.utils.inference import supported_inference_models
 
 
+def content_has_media(content: InterleavedTextMedia):
+    def _has_media_content(c):
+        return isinstance(c, ImageMedia)
+
+    if isinstance(content, list):
+        return any(_has_media_content(c) for c in content)
+    else:
+        return _has_media_content(content)
+
+
+def messages_have_media(messages: List[Message]):
+    return any(content_has_media(m.content) for m in messages)
+
+
+def request_has_media(request: Union[ChatCompletionRequest, CompletionRequest]):
+    if isinstance(request, ChatCompletionRequest):
+        return messages_have_media(request.messages)
+    else:
+        return content_has_media(request.content)
+
+
+def convert_image_media_to_url(media: ImageMedia) -> str:
+    if isinstance(media.image, PIL_Image.Image):
+        if media.image.format == "PNG":
+            format = "png"
+        elif media.image.format == "GIF":
+            format = "gif"
+        elif media.image.format == "JPEG":
+            format = "jpeg"
+        else:
+            raise ValueError(f"Unsupported image format {media.image.format}")
+
+        bytestream = io.BytesIO()
+        media.image.save(bytestream, format=media.image.format)
+        bytestream.seek(0)
+        return f"data:image/{format};base64," + base64.b64encode(
+            bytestream.getvalue()
+        ).decode("utf-8")
+    else:
+        assert isinstance(media.image, URL)
+        return media.image.uri
+
+
+def convert_message_to_dict(message: Message) -> dict:
+    def _convert_content(content) -> dict:
+        if isinstance(content, ImageMedia):
+            return {
+                "type": "image_url",
+                "image_url": {
+                    "url": convert_image_media_to_url(content),
+                },
+            }
+        else:
+            assert isinstance(content, str)
+            return {"type": "text", "text": content}
+
+    if isinstance(message.content, list):
+        content = [_convert_content(c) for c in message.content]
+    else:
+        content = [_convert_content(message.content)]
+
+    return {
+        "role": message.role,
+        "content": content,
+    }
+
+
 def completion_request_to_prompt(
     request: CompletionRequest, formatter: ChatFormat
 ) -> str: