phoenix-oss/llama-stack-mirror
0
0
Fork 1
mirror of https://github.com/meta-llama/llama-stack.git synced 2025-07-06 22:10:41 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

chore: Updating chunk id generation to ensure uniqueness (#2618)

Browse source 
# What does this PR do?
This handles an edge case for `generate_chunk_id` if the concatenation
of the `document_id` and `chunk_text` combination are not unique. Adding
the window location ensures uniqueness.

## Test Plan
Added unit test

Signed-off-by: Francisco Javier Arceo <farceo@redhat.com>
This commit is contained in:
Francisco Arceo 2025-07-04 00:56:35 -04:00 • committed by GitHub
parent 4afd619c56
commit ea80ea63ac
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
3 changed files with 14 additions and 3 deletions
Download patch file Download diff file Expand all files Collapse all files

5
llama_stack/providers/utils/memory/vector_store.py
View file

@ -164,7 +164,8 @@ def make_overlapped_chunks(
for i in range(0, len(tokens), window_len - overlap_len):
toks = tokens[i : i + window_len]
chunk = tokenizer.decode(toks)
chunk_id = generate_chunk_id(chunk, text)
chunk_window = f"{i}-{i + len(toks)}"
chunk_id = generate_chunk_id(chunk, text, chunk_window)
chunk_metadata = metadata.copy()
chunk_metadata["chunk_id"] = chunk_id
chunk_metadata["document_id"] = document_id
@ -177,7 +178,7 @@ def make_overlapped_chunks(
source=metadata.get("source", None),
created_timestamp=metadata.get("created_timestamp", int(time.time())),
updated_timestamp=int(time.time()),
chunk_window=f"{i}-{i + len(toks)}",
chunk_window=chunk_window,
chunk_tokenizer=default_tokenizer,
chunk_embedding_model=None, # This will be set in `VectorDBWithIndex.insert_chunks`
content_token_count=len(toks),

4
llama_stack/providers/utils/vector_io/chunk_utils.py
View file

@ -8,7 +8,7 @@ import hashlib
import uuid
def generate_chunk_id(document_id: str, chunk_text: str) -> str:
def generate_chunk_id(document_id: str, chunk_text: str, chunk_window: str | None = None) -> str:
"""
Generate a unique chunk ID using a hash of the document ID and chunk text.
@ -16,4 +16,6 @@ def generate_chunk_id(document_id: str, chunk_text: str) -> str:
Adding usedforsecurity=False for compatibility with FIPS environments.
"""
hash_input = f"{document_id}:{chunk_text}".encode()
if chunk_window:
hash_input += f":{chunk_window}".encode()
return str(uuid.UUID(hashlib.md5(hash_input, usedforsecurity=False).hexdigest()))