diff --git a/unsloth/models/loader.py b/unsloth/models/loader.py index 247c72f43f..eb3b21e206 100644 --- a/unsloth/models/loader.py +++ b/unsloth/models/loader.py @@ -151,8 +151,41 @@ class FastLanguageModel(FastLlamaModel): *args, **kwargs, ): + # Respect user-provided quantization_config (e.g. BitsAndBytesConfig) + quantization_config = kwargs.get("quantization_config", None) + if quantization_config is not None: + if isinstance(quantization_config, dict): + q_load_in_4bit = quantization_config.get("load_in_4bit", False) + q_load_in_8bit = quantization_config.get("load_in_8bit", False) + else: + q_load_in_4bit = getattr(quantization_config, "load_in_4bit", False) + q_load_in_8bit = getattr(quantization_config, "load_in_8bit", False) + if q_load_in_4bit: + load_in_4bit = True + load_in_8bit = False + if q_load_in_8bit: + load_in_8bit = True + load_in_4bit = False + # Login to allow private models token = hf_login(token) + # Align dtype with bnb_4bit_compute_dtype if provided and dtype is unset. + if dtype is None and quantization_config is not None: + bnb_compute_dtype = None + if isinstance(quantization_config, dict): + if quantization_config.get("load_in_4bit", False): + bnb_compute_dtype = quantization_config.get( + "bnb_4bit_compute_dtype", None + ) + else: + if getattr(quantization_config, "load_in_4bit", False): + bnb_compute_dtype = getattr( + quantization_config, "bnb_4bit_compute_dtype", None + ) + if isinstance(bnb_compute_dtype, str): + bnb_compute_dtype = getattr(torch, bnb_compute_dtype, None) + if isinstance(bnb_compute_dtype, torch.dtype): + dtype = bnb_compute_dtype if load_in_8bit or full_finetuning or qat_scheme is not None: return FastModel.from_pretrained( model_name = model_name, @@ -542,11 +575,17 @@ class FastLanguageModel(FastLlamaModel): if fast_inference: fast_inference, model_name = fast_inference_setup(model_name, model_config) + load_in_4bit_kwargs = load_in_4bit + load_in_8bit_kwargs = load_in_8bit + if quantization_config is not None and not fast_inference: + load_in_4bit_kwargs = False + load_in_8bit_kwargs = False + model, tokenizer = dispatch_model.from_pretrained( model_name = model_name, max_seq_length = max_seq_length, dtype = _get_dtype(dtype), - load_in_4bit = load_in_4bit, + load_in_4bit = load_in_4bit_kwargs, token = token, device_map = device_map, rope_scaling = rope_scaling, @@ -583,22 +622,30 @@ class FastLanguageModel(FastLlamaModel): ) if load_in_4bit: - # Fix up bitsandbytes config - compute_dtype = dtype_from_config(model.config) - quantization_config = { - # Sometimes compute_dtype is not a string!! - "bnb_4bit_compute_dtype": compute_dtype, - "bnb_4bit_quant_type": "nf4", - "bnb_4bit_use_double_quant": True, - "llm_int8_enable_fp32_cpu_offload": False, - "llm_int8_has_fp16_weight": False, - "llm_int8_skip_modules": None, - "llm_int8_threshold": 6.0, - "load_in_4bit": True, - "load_in_8bit": False, - "quant_method": "bitsandbytes", - } - model.config.update({"quantization_config": quantization_config}) + # Fix up bitsandbytes config, but respect user-provided quantization_config + if quantization_config is None: + compute_dtype = dtype_from_config(model.config) + quantization_config = { + # Sometimes compute_dtype is not a string!! + "bnb_4bit_compute_dtype": compute_dtype, + "bnb_4bit_quant_type": "nf4", + "bnb_4bit_use_double_quant": True, + "llm_int8_enable_fp32_cpu_offload": False, + "llm_int8_has_fp16_weight": False, + "llm_int8_skip_modules": None, + "llm_int8_threshold": 6.0, + "load_in_4bit": True, + "load_in_8bit": False, + "quant_method": "bitsandbytes", + } + model.config.update({"quantization_config": quantization_config}) + else: + if hasattr(quantization_config, "to_dict"): + model.config.update( + {"quantization_config": quantization_config.to_dict()} + ) + elif isinstance(quantization_config, dict): + model.config.update({"quantization_config": quantization_config}) if load_in_fp8 != False: _tag_model_with_fp8_torchao_config(model, fp8_mode) @@ -690,12 +737,45 @@ class FastModel(FastBaseModel): *args, **kwargs, ): + # Respect user-provided quantization_config (e.g. BitsAndBytesConfig) + quantization_config = kwargs.get("quantization_config", None) + if quantization_config is not None: + if isinstance(quantization_config, dict): + q_load_in_4bit = quantization_config.get("load_in_4bit", False) + q_load_in_8bit = quantization_config.get("load_in_8bit", False) + else: + q_load_in_4bit = getattr(quantization_config, "load_in_4bit", False) + q_load_in_8bit = getattr(quantization_config, "load_in_8bit", False) + if q_load_in_4bit: + load_in_4bit = True + load_in_8bit = False + if q_load_in_8bit: + load_in_8bit = True + load_in_4bit = False + # Login to allow private models token = hf_login(token) if whisper_language is not None: assert type(whisper_language) is str if whisper_task is not None: assert type(whisper_task) is str + # Align dtype with bnb_4bit_compute_dtype if provided and dtype is unset. + if dtype is None and quantization_config is not None: + bnb_compute_dtype = None + if isinstance(quantization_config, dict): + if quantization_config.get("load_in_4bit", False): + bnb_compute_dtype = quantization_config.get( + "bnb_4bit_compute_dtype", None + ) + else: + if getattr(quantization_config, "load_in_4bit", False): + bnb_compute_dtype = getattr( + quantization_config, "bnb_4bit_compute_dtype", None + ) + if isinstance(bnb_compute_dtype, str): + bnb_compute_dtype = getattr(torch, bnb_compute_dtype, None) + if isinstance(bnb_compute_dtype, torch.dtype): + dtype = bnb_compute_dtype SUPPORTS_BFLOAT16 = is_bfloat16_supported() if dtype is None: dtype = torch.float16 if not SUPPORTS_BFLOAT16 else torch.bfloat16 @@ -1169,12 +1249,18 @@ class FastModel(FastBaseModel): if auto_model is None: auto_model = AutoModelForVision2Seq if is_vlm else AutoModelForCausalLM + load_in_4bit_kwargs = load_in_4bit + load_in_8bit_kwargs = load_in_8bit + if quantization_config is not None and not fast_inference: + load_in_4bit_kwargs = False + load_in_8bit_kwargs = False + model, tokenizer = FastBaseModel.from_pretrained( model_name = model_name, max_seq_length = max_seq_length, dtype = _get_dtype(dtype), - load_in_4bit = load_in_4bit, - load_in_8bit = load_in_8bit, + load_in_4bit = load_in_4bit_kwargs, + load_in_8bit = load_in_8bit_kwargs, load_in_16bit = load_in_16bit, full_finetuning = full_finetuning, token = token, @@ -1220,22 +1306,30 @@ class FastModel(FastBaseModel): ) if load_in_4bit: - # Fix up bitsandbytes config - compute_dtype = dtype_from_config(model.config) - quantization_config = { - # Sometimes compute_dtype is not a string!! - "bnb_4bit_compute_dtype": compute_dtype, - "bnb_4bit_quant_type": "nf4", - "bnb_4bit_use_double_quant": True, - "llm_int8_enable_fp32_cpu_offload": False, - "llm_int8_has_fp16_weight": False, - "llm_int8_skip_modules": None, - "llm_int8_threshold": 6.0, - "load_in_4bit": True, - "load_in_8bit": False, - "quant_method": "bitsandbytes", - } - model.config.update({"quantization_config": quantization_config}) + # Fix up bitsandbytes config, but respect user-provided quantization_config + if quantization_config is None: + compute_dtype = dtype_from_config(model.config) + quantization_config = { + # Sometimes compute_dtype is not a string!! + "bnb_4bit_compute_dtype": compute_dtype, + "bnb_4bit_quant_type": "nf4", + "bnb_4bit_use_double_quant": True, + "llm_int8_enable_fp32_cpu_offload": False, + "llm_int8_has_fp16_weight": False, + "llm_int8_skip_modules": None, + "llm_int8_threshold": 6.0, + "load_in_4bit": True, + "load_in_8bit": False, + "quant_method": "bitsandbytes", + } + model.config.update({"quantization_config": quantization_config}) + else: + if hasattr(quantization_config, "to_dict"): + model.config.update( + {"quantization_config": quantization_config.to_dict()} + ) + elif isinstance(quantization_config, dict): + model.config.update({"quantization_config": quantization_config}) if load_in_fp8 != False: _tag_model_with_fp8_torchao_config(model, fp8_mode) diff --git a/unsloth/models/vision.py b/unsloth/models/vision.py index 6c5356e0b9..6de942d7d2 100644 --- a/unsloth/models/vision.py +++ b/unsloth/models/vision.py @@ -529,6 +529,7 @@ class FastBaseModel: del kwargs["attn_implementation"] bnb_config = None + user_quantization_config = kwargs.get("quantization_config", None) if full_finetuning and (load_in_4bit or load_in_8bit): print( "Unsloth: You selected full finetuning support, but 4bit / 8bit is enabled - disabling LoRA / QLoRA." @@ -596,7 +597,8 @@ class FastBaseModel: ): pass else: - kwargs["quantization_config"] = bnb_config + if user_quantization_config is None: + kwargs["quantization_config"] = bnb_config else: if auto_config is None: auto_config = AutoConfig.from_pretrained( @@ -641,7 +643,8 @@ class FastBaseModel: ) except: pass - kwargs["quantization_config"] = quantization_config + if user_quantization_config is None: + kwargs["quantization_config"] = quantization_config # Check if using forced float32 - we load it in bfloat16, then cast to float16! torch_dtype = dtype