phoenix-oss/llama-stack-mirror
0
0
Fork 1
mirror of https://github.com/meta-llama/llama-stack.git synced 2025-12-25 13:52:00 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

chore: Updating chunk id generation to ensure uniqueness

Browse source 
Signed-off-by: Francisco Javier Arceo <farceo@redhat.com>
This commit is contained in:
Francisco Javier Arceo 2025-07-03 23:34:38 -04:00
parent 4afd619c56
commit 0bd7a07454
3 changed files with 14 additions and 3 deletions
Download patch file Download diff file Expand all files Collapse all files

4
llama_stack/providers/utils/vector_io/chunk_utils.py
View file

@ -8,7 +8,7 @@ import hashlib
import uuid
def generate_chunk_id(document_id: str, chunk_text: str) -> str:
def generate_chunk_id(document_id: str, chunk_text: str, chunk_window: str | None = None) -> str:
"""
Generate a unique chunk ID using a hash of the document ID and chunk text.
@ -16,4 +16,6 @@ def generate_chunk_id(document_id: str, chunk_text: str) -> str:
Adding usedforsecurity=False for compatibility with FIPS environments.
"""
hash_input = f"{document_id}:{chunk_text}".encode()
if chunk_window:
hash_input += f":{chunk_window}".encode()
return str(uuid.UUID(hashlib.md5(hash_input, usedforsecurity=False).hexdigest()))