mirror of
https://github.com/meta-llama/llama-stack.git
synced 2025-06-27 18:50:41 +00:00
96e158eaac
This PR does the following: 1) adds the ability to generate embeddings in all supported inference providers. 2) Moves all the memory providers to use the inference API and improved the memory tests to setup the inference stack correctly and use the embedding models This is a merge from #589 and #598
47 lines
1.4 KiB
Python
47 lines
1.4 KiB
Python
# Copyright (c) Meta Platforms, Inc. and affiliates.
|
|
# All rights reserved.
|
|
#
|
|
# This source code is licensed under the terms described in the LICENSE file in
|
|
# the root directory of this source tree.
|
|
|
|
import logging
|
|
from typing import List
|
|
|
|
from llama_models.llama3.api.datatypes import InterleavedTextMedia
|
|
|
|
from llama_stack.apis.inference.inference import EmbeddingsResponse, ModelStore
|
|
|
|
EMBEDDING_MODELS = {}
|
|
|
|
|
|
log = logging.getLogger(__name__)
|
|
|
|
|
|
class SentenceTransformerEmbeddingMixin:
|
|
model_store: ModelStore
|
|
|
|
async def embeddings(
|
|
self,
|
|
model_id: str,
|
|
contents: List[InterleavedTextMedia],
|
|
) -> EmbeddingsResponse:
|
|
model = await self.model_store.get_model(model_id)
|
|
embedding_model = self._load_sentence_transformer_model(
|
|
model.provider_resource_id
|
|
)
|
|
embeddings = embedding_model.encode(contents)
|
|
return EmbeddingsResponse(embeddings=embeddings)
|
|
|
|
def _load_sentence_transformer_model(self, model: str) -> "SentenceTransformer":
|
|
global EMBEDDING_MODELS
|
|
|
|
loaded_model = EMBEDDING_MODELS.get(model)
|
|
if loaded_model is not None:
|
|
return loaded_model
|
|
|
|
log.info(f"Loading sentence transformer for {model}...")
|
|
from sentence_transformers import SentenceTransformer
|
|
|
|
loaded_model = SentenceTransformer(model)
|
|
EMBEDDING_MODELS[model] = loaded_model
|
|
return loaded_model
|