ceos/unsloth
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

hidden_states

Browse source
This commit is contained in:
Daniel Han-Chen 2024-01-23 19:11:17 +11:00
commit 5d9e68181e
2 changed files with 3 additions and 2 deletions
Download patch file Download diff file Expand all files Collapse all files

3
unsloth/models/llama.py
View file

@ -232,6 +232,7 @@ def LlamaAttention_fast_forward(
) -> Tuple[torch.Tensor, Optional[torch.Tensor], Optional[Tuple[torch.Tensor]]]:
bsz, q_len, _ = hidden_states.size()
print(hidden_states.size())
# Check for inference
if past_key_value is not None and q_len == 1 and bsz == 1:
@ -722,7 +723,7 @@ class FastLlamaModel:
statistics = \
f"==((====))== Unsloth: Fast Llama patching release {__version__}\n"\
f" \\\ /| GPU: {gpu_stats.name}. Max memory: {max_memory} GB. Platform = {platform_system}.\n"\
f"O^O/ \_/ \\ Pytorch: {torch.__version__}. CUDA = {gpu_stats.major}.{gpu_stats.minor}. CUDA Toolkit = {torch.version.cuda}.\n"\
f"O^O/ \_/ \\ Pytorch: {torch.__version__}. CUDA = {gpu_stats.major}.{gpu_stats.minor}. CUDA Toolkit = {torch.version.cuda}.\n"\
f"\ / Bfloat16 = {str(SUPPORTS_BFLOAT16).upper()}. Xformers = {xformers_version}. FA = {HAS_FLASH_ATTENTION}.\n"\
f' "-____-" Apache 2 free license: http://github.com/unslothai/unsloth'
logger.warning_once(statistics)

2
unsloth/models/mistral.py
View file

@ -278,7 +278,7 @@ class FastMistralModel(FastLlamaModel):
statistics = \
f"==((====))== Unsloth: Fast Mistral patching release {__version__}\n"\
f" \\\ /| GPU: {gpu_stats.name}. Max memory: {max_memory} GB. Platform = {platform_system}.\n"\
f"O^O/ \_/ \\ Pytorch: {torch.__version__}. CUDA = {gpu_stats.major}.{gpu_stats.minor}. CUDA Toolkit = {torch.version.cuda}.\n"\
f"O^O/ \_/ \\ Pytorch: {torch.__version__}. CUDA = {gpu_stats.major}.{gpu_stats.minor}. CUDA Toolkit = {torch.version.cuda}.\n"\
f"\ / Bfloat16 = {str(SUPPORTS_BFLOAT16).upper()}. Xformers = {xformers_version}. FA = {HAS_FLASH_ATTENTION}.\n"\
f' "-____-" Apache 2 free license: http://github.com/unslothai/unsloth'
logger.warning_once(statistics)