Merge branch 'nightly' of https://github.com/unslothai/unsloth into nightly
This commit is contained in:
commit
65d24536fd
4 changed files with 3 additions and 10 deletions
|
|
@ -115,7 +115,7 @@ See [here](https://github.com/unslothai/unsloth/edit/main/README.md#advanced-pip
|
|||
7. **Install Unsloth:**
|
||||
|
||||
```python
|
||||
pip install "unsloth[windows] @ git+https://github.com/unslothai/unsloth.git"
|
||||
pip install unsloth
|
||||
```
|
||||
|
||||
#### Notes
|
||||
|
|
|
|||
|
|
@ -33,10 +33,7 @@ exclude = ["images*"]
|
|||
|
||||
[project.optional-dependencies]
|
||||
triton = [
|
||||
"triton @ https://github.com/woct0rdho/triton-windows/releases/download/v3.2.0-windows.post10/triton-3.2.0-cp39-cp39-win_amd64.whl ; python_version=='3.9' and platform_system == 'Windows'",
|
||||
"triton @ https://github.com/woct0rdho/triton-windows/releases/download/v3.2.0-windows.post10/triton-3.2.0-cp310-cp310-win_amd64.whl ; python_version=='3.10' and platform_system == 'Windows'",
|
||||
"triton @ https://github.com/woct0rdho/triton-windows/releases/download/v3.2.0-windows.post10/triton-3.2.0-cp311-cp311-win_amd64.whl ; python_version=='3.11' and platform_system == 'Windows'",
|
||||
"triton @ https://github.com/woct0rdho/triton-windows/releases/download/v3.2.0-windows.post10/triton-3.2.0-cp312-cp312-win_amd64.whl ; python_version=='3.12' and platform_system == 'Windows'"
|
||||
"triton-windows ; platform_system == 'Windows'",
|
||||
]
|
||||
|
||||
huggingface = [
|
||||
|
|
|
|||
|
|
@ -1512,10 +1512,7 @@ def get_ollama_eos_tokens(tokenizer, extra_eos_tokens = []):
|
|||
|
||||
# Remove duplicates
|
||||
splitted = joined_text.split("\x01\x00")
|
||||
final_eos_tokens = []
|
||||
for old, new in zip(added_tokens_decoder, splitted):
|
||||
if old == new: final_eos_tokens.append(old)
|
||||
pass
|
||||
final_eos_tokens = [old for old, new in zip(added_tokens_decoder, splitted) if old == new]
|
||||
final_eos_tokens += extra_eos_tokens
|
||||
final_eos_tokens += repeatted_tokens
|
||||
|
||||
|
|
|
|||
|
|
@ -256,7 +256,6 @@ def unpatch_rms_layernorm():
|
|||
except:
|
||||
pass
|
||||
return
|
||||
return
|
||||
pass
|
||||
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue