Merge pull request #3835 from unslothai/quant-config-respect
Respect user quantization_config
This commit is contained in:
Daniel Han
GitHub
commit
eef05330ca
2 changed files with 134 additions and 37 deletions
|
|
@ -151,8 +151,41 @@ class FastLanguageModel(FastLlamaModel):
|
|||
*args,
|
||||
**kwargs,
|
||||
):
|
||||
# Respect user-provided quantization_config (e.g. BitsAndBytesConfig)
|
||||
quantization_config = kwargs.get("quantization_config", None)
|
||||
if quantization_config is not None:
|
||||
if isinstance(quantization_config, dict):
|
||||
q_load_in_4bit = quantization_config.get("load_in_4bit", False)
|
||||
q_load_in_8bit = quantization_config.get("load_in_8bit", False)
|
||||
else:
|
||||
q_load_in_4bit = getattr(quantization_config, "load_in_4bit", False)
|
||||
q_load_in_8bit = getattr(quantization_config, "load_in_8bit", False)
|
||||
if q_load_in_4bit:
|
||||
load_in_4bit = True
|
||||
load_in_8bit = False
|
||||
if q_load_in_8bit:
|
||||
load_in_8bit = True
|
||||
load_in_4bit = False
|
||||
|
||||
# Login to allow private models
|
||||
token = hf_login(token)
|
||||
# Align dtype with bnb_4bit_compute_dtype if provided and dtype is unset.
|
||||
if dtype is None and quantization_config is not None:
|
||||
bnb_compute_dtype = None
|
||||
if isinstance(quantization_config, dict):
|
||||
if quantization_config.get("load_in_4bit", False):
|
||||
bnb_compute_dtype = quantization_config.get(
|
||||
"bnb_4bit_compute_dtype", None
|
||||
)
|
||||
else:
|
||||
if getattr(quantization_config, "load_in_4bit", False):
|
||||
bnb_compute_dtype = getattr(
|
||||
quantization_config, "bnb_4bit_compute_dtype", None
|
||||
)
|
||||
if isinstance(bnb_compute_dtype, str):
|
||||
bnb_compute_dtype = getattr(torch, bnb_compute_dtype, None)
|
||||
if isinstance(bnb_compute_dtype, torch.dtype):
|
||||
dtype = bnb_compute_dtype
|
||||
if load_in_8bit or full_finetuning or qat_scheme is not None:
|
||||
return FastModel.from_pretrained(
|
||||
model_name = model_name,
|
||||
|
|
@ -542,11 +575,17 @@ class FastLanguageModel(FastLlamaModel):
|
|||
if fast_inference:
|
||||
fast_inference, model_name = fast_inference_setup(model_name, model_config)
|
||||
|
||||
load_in_4bit_kwargs = load_in_4bit
|
||||
load_in_8bit_kwargs = load_in_8bit
|
||||
if quantization_config is not None and not fast_inference:
|
||||
load_in_4bit_kwargs = False
|
||||
load_in_8bit_kwargs = False
|
||||
|
||||
model, tokenizer = dispatch_model.from_pretrained(
|
||||
model_name = model_name,
|
||||
max_seq_length = max_seq_length,
|
||||
dtype = _get_dtype(dtype),
|
||||
load_in_4bit = load_in_4bit,
|
||||
load_in_4bit = load_in_4bit_kwargs,
|
||||
token = token,
|
||||
device_map = device_map,
|
||||
rope_scaling = rope_scaling,
|
||||
|
|
@ -583,22 +622,30 @@ class FastLanguageModel(FastLlamaModel):
|
|||
)
|
||||
|
||||
if load_in_4bit:
|
||||
# Fix up bitsandbytes config
|
||||
compute_dtype = dtype_from_config(model.config)
|
||||
quantization_config = {
|
||||
# Sometimes compute_dtype is not a string!!
|
||||
"bnb_4bit_compute_dtype": compute_dtype,
|
||||
"bnb_4bit_quant_type": "nf4",
|
||||
"bnb_4bit_use_double_quant": True,
|
||||
"llm_int8_enable_fp32_cpu_offload": False,
|
||||
"llm_int8_has_fp16_weight": False,
|
||||
"llm_int8_skip_modules": None,
|
||||
"llm_int8_threshold": 6.0,
|
||||
"load_in_4bit": True,
|
||||
"load_in_8bit": False,
|
||||
"quant_method": "bitsandbytes",
|
||||
}
|
||||
model.config.update({"quantization_config": quantization_config})
|
||||
# Fix up bitsandbytes config, but respect user-provided quantization_config
|
||||
if quantization_config is None:
|
||||
compute_dtype = dtype_from_config(model.config)
|
||||
quantization_config = {
|
||||
# Sometimes compute_dtype is not a string!!
|
||||
"bnb_4bit_compute_dtype": compute_dtype,
|
||||
"bnb_4bit_quant_type": "nf4",
|
||||
"bnb_4bit_use_double_quant": True,
|
||||
"llm_int8_enable_fp32_cpu_offload": False,
|
||||
"llm_int8_has_fp16_weight": False,
|
||||
"llm_int8_skip_modules": None,
|
||||
"llm_int8_threshold": 6.0,
|
||||
"load_in_4bit": True,
|
||||
"load_in_8bit": False,
|
||||
"quant_method": "bitsandbytes",
|
||||
}
|
||||
model.config.update({"quantization_config": quantization_config})
|
||||
else:
|
||||
if hasattr(quantization_config, "to_dict"):
|
||||
model.config.update(
|
||||
{"quantization_config": quantization_config.to_dict()}
|
||||
)
|
||||
elif isinstance(quantization_config, dict):
|
||||
model.config.update({"quantization_config": quantization_config})
|
||||
|
||||
if load_in_fp8 != False:
|
||||
_tag_model_with_fp8_torchao_config(model, fp8_mode)
|
||||
|
|
@ -690,12 +737,45 @@ class FastModel(FastBaseModel):
|
|||
*args,
|
||||
**kwargs,
|
||||
):
|
||||
# Respect user-provided quantization_config (e.g. BitsAndBytesConfig)
|
||||
quantization_config = kwargs.get("quantization_config", None)
|
||||
if quantization_config is not None:
|
||||
if isinstance(quantization_config, dict):
|
||||
q_load_in_4bit = quantization_config.get("load_in_4bit", False)
|
||||
q_load_in_8bit = quantization_config.get("load_in_8bit", False)
|
||||
else:
|
||||
q_load_in_4bit = getattr(quantization_config, "load_in_4bit", False)
|
||||
q_load_in_8bit = getattr(quantization_config, "load_in_8bit", False)
|
||||
if q_load_in_4bit:
|
||||
load_in_4bit = True
|
||||
load_in_8bit = False
|
||||
if q_load_in_8bit:
|
||||
load_in_8bit = True
|
||||
load_in_4bit = False
|
||||
|
||||
# Login to allow private models
|
||||
token = hf_login(token)
|
||||
if whisper_language is not None:
|
||||
assert type(whisper_language) is str
|
||||
if whisper_task is not None:
|
||||
assert type(whisper_task) is str
|
||||
# Align dtype with bnb_4bit_compute_dtype if provided and dtype is unset.
|
||||
if dtype is None and quantization_config is not None:
|
||||
bnb_compute_dtype = None
|
||||
if isinstance(quantization_config, dict):
|
||||
if quantization_config.get("load_in_4bit", False):
|
||||
bnb_compute_dtype = quantization_config.get(
|
||||
"bnb_4bit_compute_dtype", None
|
||||
)
|
||||
else:
|
||||
if getattr(quantization_config, "load_in_4bit", False):
|
||||
bnb_compute_dtype = getattr(
|
||||
quantization_config, "bnb_4bit_compute_dtype", None
|
||||
)
|
||||
if isinstance(bnb_compute_dtype, str):
|
||||
bnb_compute_dtype = getattr(torch, bnb_compute_dtype, None)
|
||||
if isinstance(bnb_compute_dtype, torch.dtype):
|
||||
dtype = bnb_compute_dtype
|
||||
SUPPORTS_BFLOAT16 = is_bfloat16_supported()
|
||||
if dtype is None:
|
||||
dtype = torch.float16 if not SUPPORTS_BFLOAT16 else torch.bfloat16
|
||||
|
|
@ -1169,12 +1249,18 @@ class FastModel(FastBaseModel):
|
|||
if auto_model is None:
|
||||
auto_model = AutoModelForVision2Seq if is_vlm else AutoModelForCausalLM
|
||||
|
||||
load_in_4bit_kwargs = load_in_4bit
|
||||
load_in_8bit_kwargs = load_in_8bit
|
||||
if quantization_config is not None and not fast_inference:
|
||||
load_in_4bit_kwargs = False
|
||||
load_in_8bit_kwargs = False
|
||||
|
||||
model, tokenizer = FastBaseModel.from_pretrained(
|
||||
model_name = model_name,
|
||||
max_seq_length = max_seq_length,
|
||||
dtype = _get_dtype(dtype),
|
||||
load_in_4bit = load_in_4bit,
|
||||
load_in_8bit = load_in_8bit,
|
||||
load_in_4bit = load_in_4bit_kwargs,
|
||||
load_in_8bit = load_in_8bit_kwargs,
|
||||
load_in_16bit = load_in_16bit,
|
||||
full_finetuning = full_finetuning,
|
||||
token = token,
|
||||
|
|
@ -1220,22 +1306,30 @@ class FastModel(FastBaseModel):
|
|||
)
|
||||
|
||||
if load_in_4bit:
|
||||
# Fix up bitsandbytes config
|
||||
compute_dtype = dtype_from_config(model.config)
|
||||
quantization_config = {
|
||||
# Sometimes compute_dtype is not a string!!
|
||||
"bnb_4bit_compute_dtype": compute_dtype,
|
||||
"bnb_4bit_quant_type": "nf4",
|
||||
"bnb_4bit_use_double_quant": True,
|
||||
"llm_int8_enable_fp32_cpu_offload": False,
|
||||
"llm_int8_has_fp16_weight": False,
|
||||
"llm_int8_skip_modules": None,
|
||||
"llm_int8_threshold": 6.0,
|
||||
"load_in_4bit": True,
|
||||
"load_in_8bit": False,
|
||||
"quant_method": "bitsandbytes",
|
||||
}
|
||||
model.config.update({"quantization_config": quantization_config})
|
||||
# Fix up bitsandbytes config, but respect user-provided quantization_config
|
||||
if quantization_config is None:
|
||||
compute_dtype = dtype_from_config(model.config)
|
||||
quantization_config = {
|
||||
# Sometimes compute_dtype is not a string!!
|
||||
"bnb_4bit_compute_dtype": compute_dtype,
|
||||
"bnb_4bit_quant_type": "nf4",
|
||||
"bnb_4bit_use_double_quant": True,
|
||||
"llm_int8_enable_fp32_cpu_offload": False,
|
||||
"llm_int8_has_fp16_weight": False,
|
||||
"llm_int8_skip_modules": None,
|
||||
"llm_int8_threshold": 6.0,
|
||||
"load_in_4bit": True,
|
||||
"load_in_8bit": False,
|
||||
"quant_method": "bitsandbytes",
|
||||
}
|
||||
model.config.update({"quantization_config": quantization_config})
|
||||
else:
|
||||
if hasattr(quantization_config, "to_dict"):
|
||||
model.config.update(
|
||||
{"quantization_config": quantization_config.to_dict()}
|
||||
)
|
||||
elif isinstance(quantization_config, dict):
|
||||
model.config.update({"quantization_config": quantization_config})
|
||||
|
||||
if load_in_fp8 != False:
|
||||
_tag_model_with_fp8_torchao_config(model, fp8_mode)
|
||||
|
|
|
|||
|
|
@ -529,6 +529,7 @@ class FastBaseModel:
|
|||
del kwargs["attn_implementation"]
|
||||
|
||||
bnb_config = None
|
||||
user_quantization_config = kwargs.get("quantization_config", None)
|
||||
if full_finetuning and (load_in_4bit or load_in_8bit):
|
||||
print(
|
||||
"Unsloth: You selected full finetuning support, but 4bit / 8bit is enabled - disabling LoRA / QLoRA."
|
||||
|
|
@ -596,7 +597,8 @@ class FastBaseModel:
|
|||
):
|
||||
pass
|
||||
else:
|
||||
kwargs["quantization_config"] = bnb_config
|
||||
if user_quantization_config is None:
|
||||
kwargs["quantization_config"] = bnb_config
|
||||
else:
|
||||
if auto_config is None:
|
||||
auto_config = AutoConfig.from_pretrained(
|
||||
|
|
@ -641,7 +643,8 @@ class FastBaseModel:
|
|||
)
|
||||
except:
|
||||
pass
|
||||
kwargs["quantization_config"] = quantization_config
|
||||
if user_quantization_config is None:
|
||||
kwargs["quantization_config"] = quantization_config
|
||||
|
||||
# Check if using forced float32 - we load it in bfloat16, then cast to float16!
|
||||
torch_dtype = dtype
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue