From 5d9e68181eeef30d0c0758a68ff0ff3f5bfb4cdf Mon Sep 17 00:00:00 2001 From: Daniel Han-Chen Date: Tue, 23 Jan 2024 19:11:17 +1100 Subject: [PATCH] hidden_states --- unsloth/models/llama.py | 3 ++- unsloth/models/mistral.py | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/unsloth/models/llama.py b/unsloth/models/llama.py index 84707eaee2..e9ac9bfaf5 100644 --- a/unsloth/models/llama.py +++ b/unsloth/models/llama.py @@ -232,6 +232,7 @@ def LlamaAttention_fast_forward( ) -> Tuple[torch.Tensor, Optional[torch.Tensor], Optional[Tuple[torch.Tensor]]]: bsz, q_len, _ = hidden_states.size() + print(hidden_states.size()) # Check for inference if past_key_value is not None and q_len == 1 and bsz == 1: @@ -722,7 +723,7 @@ class FastLlamaModel: statistics = \ f"==((====))== Unsloth: Fast Llama patching release {__version__}\n"\ f" \\\ /| GPU: {gpu_stats.name}. Max memory: {max_memory} GB. Platform = {platform_system}.\n"\ - f"O^O/ \_/ \\ Pytorch: {torch.__version__}. CUDA = {gpu_stats.major}.{gpu_stats.minor}. CUDA Toolkit = {torch.version.cuda}.\n"\ + f"O^O/ \_/ \\ Pytorch: {torch.__version__}. CUDA = {gpu_stats.major}.{gpu_stats.minor}. CUDA Toolkit = {torch.version.cuda}.\n"\ f"\ / Bfloat16 = {str(SUPPORTS_BFLOAT16).upper()}. Xformers = {xformers_version}. FA = {HAS_FLASH_ATTENTION}.\n"\ f' "-____-" Apache 2 free license: http://github.com/unslothai/unsloth' logger.warning_once(statistics) diff --git a/unsloth/models/mistral.py b/unsloth/models/mistral.py index bde68a0a51..baf3695b22 100644 --- a/unsloth/models/mistral.py +++ b/unsloth/models/mistral.py @@ -278,7 +278,7 @@ class FastMistralModel(FastLlamaModel): statistics = \ f"==((====))== Unsloth: Fast Mistral patching release {__version__}\n"\ f" \\\ /| GPU: {gpu_stats.name}. Max memory: {max_memory} GB. Platform = {platform_system}.\n"\ - f"O^O/ \_/ \\ Pytorch: {torch.__version__}. CUDA = {gpu_stats.major}.{gpu_stats.minor}. CUDA Toolkit = {torch.version.cuda}.\n"\ + f"O^O/ \_/ \\ Pytorch: {torch.__version__}. CUDA = {gpu_stats.major}.{gpu_stats.minor}. CUDA Toolkit = {torch.version.cuda}.\n"\ f"\ / Bfloat16 = {str(SUPPORTS_BFLOAT16).upper()}. Xformers = {xformers_version}. FA = {HAS_FLASH_ATTENTION}.\n"\ f' "-____-" Apache 2 free license: http://github.com/unslothai/unsloth' logger.warning_once(statistics)