phoenix-oss/llama-stack-mirror
0
0
Fork 1
mirror of https://github.com/meta-llama/llama-stack.git synced 2025-12-16 08:59:26 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Changed from config to model_args

Browse source
This commit is contained in:
Sachin Mehta 2024-10-25 12:24:50 -07:00
parent 93472042f8
commit db4f18099f
2 changed files with 9 additions and 18 deletions
Download patch file Download diff file Expand all files Collapse all files

6
llama_stack/providers/impls/meta_reference/inference/generation.py
View file

@ -151,15 +151,11 @@ class Llama:
elif isinstance(config.quantization, Int4QuantizationConfig):
from .quantization.loader import convert_to_int4_quantized_model
assert (
config.quantization.scheme is not None
), "Please specify a quantization scheme."
model = Transformer(model_args)
model = convert_to_int4_quantized_model(model, model_args, config)
model.load_state_dict(state_dict, strict=True)
if config.quantization.spinquant:
if model_args.quantization_args.spinquant:
# Add a wrapper for adding hadamard transform for spinquant.
# This needs to be done after loading the state dict otherwise an error will be raised while
# loading the state dict.