phoenix-oss/llama-stack-mirror
0
0
Fork 1
mirror of https://github.com/meta-llama/llama-stack.git synced 2025-12-16 19:22:37 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

send/recv pydantic json over socket

Browse source
This commit is contained in:
dltn 2024-10-09 09:20:13 -07:00
parent 2d4f7d8acf
commit 0deeb4abec
3 changed files with 145 additions and 52 deletions
Download patch file Download diff file Expand all files Collapse all files

7
llama_stack/providers/impls/meta_reference/inference/generation.py
View file

@ -35,12 +35,14 @@ from llama_models.llama3.reference_impl.multimodal.model import (
CrossAttentionTransformer,
)
from llama_models.sku_list import resolve_model
from termcolor import cprint
from llama_stack.apis.inference import QuantizationType
from llama_stack.distribution.utils.model_utils import model_local_dir
from pydantic import BaseModel
from termcolor import cprint
from .config import MetaReferenceImplConfig
@ -58,8 +60,7 @@ def model_checkpoint_dir(model) -> str:
return str(checkpoint_dir)
@dataclass
class TokenResult:
class TokenResult(BaseModel):
token: int
text: str
logprobs: Optional[List[float]] = None