diff --git a/unsloth/models/cohere.py b/unsloth/models/cohere.py index a091a0173f..e9f56763d6 100644 --- a/unsloth/models/cohere.py +++ b/unsloth/models/cohere.py @@ -344,8 +344,8 @@ def CohereAttention_fast_forward_inference( Kn = Kn.view(bsz, 1, n_kv_heads, head_dim).transpose(1, 2) Vn = Vn.view(bsz, 1, n_kv_heads, head_dim).transpose(1, 2) if self.use_qk_norm: - Q = fast_layernorm_inference(self.q_norm, Q, self.q_norm_out_weight) - K = fast_layernorm_inference(self.k_norm, K, self.k_norm_out_weight) + Qn = fast_layernorm_inference(self.q_norm, Qn, self.q_norm_out_weight) + Kn = fast_layernorm_inference(self.k_norm, Kn, self.k_norm_out_weight) # cos, sin = self.rotary_emb(Vn, seq_len = kv_seq_len) # Qn, Kn = inplace_rope_embedding(Qn, Kn, cos, sin, position_ids) @@ -479,7 +479,7 @@ def CohereModel_fast_forward_inference( ) ) - hidden_states_mlp = fast_swiglu_inference(self.mlp, hidden_states) + hidden_states_mlp = fast_swiglu_inference(decoder_layer.mlp, hidden_states) residual += hidden_states_attention residual += hidden_states_mlp hidden_states = residual diff --git a/unsloth/models/falcon_h1.py b/unsloth/models/falcon_h1.py index fc5ea458a6..428f49d727 100644 --- a/unsloth/models/falcon_h1.py +++ b/unsloth/models/falcon_h1.py @@ -456,9 +456,9 @@ def FalconH1DecoderLayer_fast_forward( # Fully Connected residual = hidden_states hidden_states = fast_rms_layernorm_inference( - self.post_attention_layernorm, hidden_states + self.pre_ff_layernorm, hidden_states ) - hidden_states = fast_swiglu_inference(self.mlp, hidden_states) + hidden_states = fast_swiglu_inference(self.feed_forward, hidden_states) hidden_states += residual else: residual = hidden_states diff --git a/unsloth/models/granite.py b/unsloth/models/granite.py index 2632ab6914..f85f1b641f 100644 --- a/unsloth/models/granite.py +++ b/unsloth/models/granite.py @@ -46,9 +46,9 @@ except: transformers_version = Version(transformers_version) if not transformers_version >= Version("4.45.0"): raise ImportError( - f"Unsloth: Your transformers version of {transformers_version} does not support Gemma2.\n" - f"The minimum required version is 4.42.3.\n" - f'Try `pip install --upgrade "transformers>=4.42.3"`\n' + f"Unsloth: Your transformers version of {transformers_version} does not support Granite.\n" + f"The minimum required version is 4.45.0.\n" + f'Try `pip install --upgrade "transformers>=4.45.0"`\n' f"to obtain the latest transformers build, then restart this session." ) diff --git a/unsloth/models/qwen3_moe.py b/unsloth/models/qwen3_moe.py index bec3fa7b0d..e1f8c71b6b 100644 --- a/unsloth/models/qwen3_moe.py +++ b/unsloth/models/qwen3_moe.py @@ -207,7 +207,7 @@ class FastQwen3MoeModel(FastQwen3Model): # https://github.com/huggingface/transformers/blob/v4.37.2/src/transformers/models/llama/modeling_llama.py\ import transformers.models.qwen3_moe.modeling_qwen3_moe - transformers.models.Qwen3Moe.modeling_qwen3_moe.Qwen3MoeRotaryEmbedding = ( + transformers.models.qwen3_moe.modeling_qwen3_moe.Qwen3MoeRotaryEmbedding = ( LlamaRotaryEmbedding ) return @@ -236,7 +236,7 @@ class FastQwen3MoeModel(FastQwen3Model): device_map = device_map, rope_scaling = rope_scaling, fix_tokenizer = fix_tokenizer, - model_patcher = FastQwen3Model, + model_patcher = FastQwen3MoeModel, tokenizer_name = tokenizer_name, trust_remote_code = trust_remote_code, **kwargs, diff --git a/unsloth/models/rl.py b/unsloth/models/rl.py index e1c43b8b85..22189f459c 100644 --- a/unsloth/models/rl.py +++ b/unsloth/models/rl.py @@ -559,8 +559,12 @@ def _patch_trl_rl_trainers(trainer_file = "grpo_trainer"): " if args_max_seq_length is None and model_max_seq_length is not None:\n" " max_seq_length = model.max_seq_length\n" " if hasattr(args, 'max_seq_length'): args.max_seq_length = max_seq_length\n" + " elif args_max_seq_length is not None and model_max_seq_length is not None:\n" + " if args_max_seq_length > model_max_seq_length:\n" + " print('Unsloth: You set `max_seq_length` as ' + str(args_max_seq_length) + ' but '\n" + " 'the maximum the model supports is ' + str(model_max_seq_length) + '. We shall reduce it.')\n" + " args.max_seq_length = model_max_seq_length\n" ) - " elif args_max_seq_length is not None and model_max_seq_length is not None:\n" " if args_max_seq_length > model_max_seq_length:\n" " print('Unsloth: You set `max_seq_length` as ' + str(args_max_seq_length) + ' but \n" " the maximum the model supports is ' + str(model_max_seq_length) + '. We shall reduce it.')\n" " args.max_seq_length = model_max_seq_length\n" extra_args += length_check # At this point max_seq_length might be set, but trl is moving to max_length diff --git a/unsloth/save.py b/unsloth/save.py index 3a275cf0c3..ceb36854d2 100644 --- a/unsloth/save.py +++ b/unsloth/save.py @@ -547,7 +547,7 @@ def unsloth_save_model( elif mb_found: sharded_ram_usage = int(mb_found.group(1)) * 1024 * 1024 elif type(max_shard_size) is int: - sharded_ram_usage = sharded_ram_usage + sharded_ram_usage = max_shard_size # Switch to our fast saving modules if it's a slow PC! n_cpus = psutil.cpu_count(logical = False) diff --git a/unsloth/utils/hf_hub.py b/unsloth/utils/hf_hub.py index 75df00fbf0..e3960ba0ce 100644 --- a/unsloth/utils/hf_hub.py +++ b/unsloth/utils/hf_hub.py @@ -19,7 +19,9 @@ def formatted_int(value: int) -> str: elif value < MILLION: return f"{float(value) / 1000:,.1f}K" elif value < BILLION: - return f"{float(value) // 1000000:,.1f}M" + return f"{float(value) / 1000000:,.1f}M" + else: + return f"{float(value) / 1000000000:,.1f}B" def get_model_info(