Compare commits
1 commit
main
...
fix/venv-t
| Author | SHA1 | Date | |
|---|---|---|---|
|
fc3a9a2b40 |
7 changed files with 111 additions and 102 deletions
|
|
@ -56,39 +56,32 @@ def _activate_transformers_version(model_name: str) -> None:
|
|||
import subprocess as sp
|
||||
|
||||
os.makedirs(venv_t5, exist_ok = True)
|
||||
r1 = sp.run(
|
||||
[
|
||||
sys.executable,
|
||||
"-m",
|
||||
"pip",
|
||||
"install",
|
||||
"--target",
|
||||
venv_t5,
|
||||
"--no-deps",
|
||||
"transformers==5.3.0",
|
||||
],
|
||||
stdout = sp.PIPE,
|
||||
stderr = sp.STDOUT,
|
||||
)
|
||||
r2 = sp.run(
|
||||
[
|
||||
sys.executable,
|
||||
"-m",
|
||||
"pip",
|
||||
"install",
|
||||
"--target",
|
||||
venv_t5,
|
||||
"--no-deps",
|
||||
"huggingface_hub==1.3.0",
|
||||
],
|
||||
stdout = sp.PIPE,
|
||||
stderr = sp.STDOUT,
|
||||
)
|
||||
if r1.returncode != 0 or r2.returncode != 0:
|
||||
raise RuntimeError(
|
||||
f"Failed to install transformers 5.x into {venv_t5}. "
|
||||
f"pip returncode: transformers={r1.returncode}, huggingface_hub={r2.returncode}"
|
||||
_t5_packages = [
|
||||
"transformers==5.3.0",
|
||||
"huggingface_hub==1.3.0",
|
||||
"sentencepiece>=0.2.0",
|
||||
"tiktoken",
|
||||
]
|
||||
for _pkg in _t5_packages:
|
||||
_r = sp.run(
|
||||
[
|
||||
sys.executable,
|
||||
"-m",
|
||||
"pip",
|
||||
"install",
|
||||
"--target",
|
||||
venv_t5,
|
||||
"--no-deps",
|
||||
_pkg,
|
||||
],
|
||||
stdout = sp.PIPE,
|
||||
stderr = sp.STDOUT,
|
||||
)
|
||||
if _r.returncode != 0:
|
||||
raise RuntimeError(
|
||||
f"Failed to install {_pkg} into {venv_t5}. "
|
||||
f"pip returncode: {_r.returncode}"
|
||||
)
|
||||
sys.path.insert(0, venv_t5)
|
||||
# Propagate to child subprocesses (e.g. GGUF converter)
|
||||
_pp = os.environ.get("PYTHONPATH", "")
|
||||
|
|
|
|||
|
|
@ -58,39 +58,32 @@ def _activate_transformers_version(model_name: str) -> None:
|
|||
import subprocess as sp
|
||||
|
||||
os.makedirs(venv_t5, exist_ok = True)
|
||||
r1 = sp.run(
|
||||
[
|
||||
sys.executable,
|
||||
"-m",
|
||||
"pip",
|
||||
"install",
|
||||
"--target",
|
||||
venv_t5,
|
||||
"--no-deps",
|
||||
"transformers==5.3.0",
|
||||
],
|
||||
stdout = sp.PIPE,
|
||||
stderr = sp.STDOUT,
|
||||
)
|
||||
r2 = sp.run(
|
||||
[
|
||||
sys.executable,
|
||||
"-m",
|
||||
"pip",
|
||||
"install",
|
||||
"--target",
|
||||
venv_t5,
|
||||
"--no-deps",
|
||||
"huggingface_hub==1.3.0",
|
||||
],
|
||||
stdout = sp.PIPE,
|
||||
stderr = sp.STDOUT,
|
||||
)
|
||||
if r1.returncode != 0 or r2.returncode != 0:
|
||||
raise RuntimeError(
|
||||
f"Failed to install transformers 5.x into {venv_t5}. "
|
||||
f"pip returncode: transformers={r1.returncode}, huggingface_hub={r2.returncode}"
|
||||
_t5_packages = [
|
||||
"transformers==5.3.0",
|
||||
"huggingface_hub==1.3.0",
|
||||
"sentencepiece>=0.2.0",
|
||||
"tiktoken",
|
||||
]
|
||||
for _pkg in _t5_packages:
|
||||
_r = sp.run(
|
||||
[
|
||||
sys.executable,
|
||||
"-m",
|
||||
"pip",
|
||||
"install",
|
||||
"--target",
|
||||
venv_t5,
|
||||
"--no-deps",
|
||||
_pkg,
|
||||
],
|
||||
stdout = sp.PIPE,
|
||||
stderr = sp.STDOUT,
|
||||
)
|
||||
if _r.returncode != 0:
|
||||
raise RuntimeError(
|
||||
f"Failed to install {_pkg} into {venv_t5}. "
|
||||
f"pip returncode: {_r.returncode}"
|
||||
)
|
||||
sys.path.insert(0, venv_t5)
|
||||
# Propagate to child subprocesses (e.g. GGUF converter)
|
||||
_pp = os.environ.get("PYTHONPATH", "")
|
||||
|
|
|
|||
|
|
@ -188,11 +188,20 @@ class UnslothTrainer:
|
|||
else:
|
||||
from transformers import AutoTokenizer
|
||||
|
||||
self.tokenizer = AutoTokenizer.from_pretrained(
|
||||
model_name,
|
||||
trust_remote_code = trust_remote_code,
|
||||
token = hf_token,
|
||||
)
|
||||
try:
|
||||
self.tokenizer = AutoTokenizer.from_pretrained(
|
||||
model_name,
|
||||
trust_remote_code = trust_remote_code,
|
||||
token = hf_token,
|
||||
)
|
||||
except ValueError as exc:
|
||||
if "sentencepiece" in str(exc) or "tiktoken" in str(exc):
|
||||
raise ValueError(
|
||||
f"Tokenizer for '{model_name}' requires sentencepiece or tiktoken. "
|
||||
f"Please run: pip install sentencepiece tiktoken\n"
|
||||
f"Original error: {exc}"
|
||||
) from exc
|
||||
raise
|
||||
|
||||
logger.info("Pre-loaded tokenizer for %s", model_name)
|
||||
|
||||
|
|
|
|||
|
|
@ -52,39 +52,32 @@ def _activate_transformers_version(model_name: str) -> None:
|
|||
import subprocess as sp
|
||||
|
||||
os.makedirs(venv_t5, exist_ok = True)
|
||||
r1 = sp.run(
|
||||
[
|
||||
sys.executable,
|
||||
"-m",
|
||||
"pip",
|
||||
"install",
|
||||
"--target",
|
||||
venv_t5,
|
||||
"--no-deps",
|
||||
"transformers==5.3.0",
|
||||
],
|
||||
stdout = sp.PIPE,
|
||||
stderr = sp.STDOUT,
|
||||
)
|
||||
r2 = sp.run(
|
||||
[
|
||||
sys.executable,
|
||||
"-m",
|
||||
"pip",
|
||||
"install",
|
||||
"--target",
|
||||
venv_t5,
|
||||
"--no-deps",
|
||||
"huggingface_hub==1.3.0",
|
||||
],
|
||||
stdout = sp.PIPE,
|
||||
stderr = sp.STDOUT,
|
||||
)
|
||||
if r1.returncode != 0 or r2.returncode != 0:
|
||||
raise RuntimeError(
|
||||
f"Failed to install transformers 5.x into {venv_t5}. "
|
||||
f"pip returncode: transformers={r1.returncode}, huggingface_hub={r2.returncode}"
|
||||
_t5_packages = [
|
||||
"transformers==5.3.0",
|
||||
"huggingface_hub==1.3.0",
|
||||
"sentencepiece>=0.2.0",
|
||||
"tiktoken",
|
||||
]
|
||||
for _pkg in _t5_packages:
|
||||
_r = sp.run(
|
||||
[
|
||||
sys.executable,
|
||||
"-m",
|
||||
"pip",
|
||||
"install",
|
||||
"--target",
|
||||
venv_t5,
|
||||
"--no-deps",
|
||||
_pkg,
|
||||
],
|
||||
stdout = sp.PIPE,
|
||||
stderr = sp.STDOUT,
|
||||
)
|
||||
if _r.returncode != 0:
|
||||
raise RuntimeError(
|
||||
f"Failed to install {_pkg} into {venv_t5}. "
|
||||
f"pip returncode: {_r.returncode}"
|
||||
)
|
||||
sys.path.insert(0, venv_t5)
|
||||
# Propagate to child subprocesses (e.g. GGUF converter)
|
||||
_pp = os.environ.get("PYTHONPATH", "")
|
||||
|
|
|
|||
|
|
@ -223,7 +223,12 @@ def _ensure_venv_t5_exists() -> bool:
|
|||
|
||||
logger.warning(".venv_t5 not found at %s — installing at runtime", _VENV_T5_DIR)
|
||||
os.makedirs(_VENV_T5_DIR, exist_ok = True)
|
||||
for pkg in (f"transformers=={TRANSFORMERS_5_VERSION}", "huggingface_hub==1.3.0"):
|
||||
for pkg in (
|
||||
f"transformers=={TRANSFORMERS_5_VERSION}",
|
||||
"huggingface_hub==1.3.0",
|
||||
"sentencepiece>=0.2.0",
|
||||
"tiktoken",
|
||||
):
|
||||
cmd = [
|
||||
sys.executable,
|
||||
"-m",
|
||||
|
|
|
|||
|
|
@ -918,6 +918,19 @@ if ($LASTEXITCODE -ne 0) {
|
|||
$ErrorActionPreference = $prevEAP_t5
|
||||
exit 1
|
||||
}
|
||||
# Tokenizer backends — required by AutoTokenizer for many model families
|
||||
pip install --target $VenvT5Dir --no-deps "sentencepiece>=0.2.0" 2>&1 | Out-Null
|
||||
if ($LASTEXITCODE -ne 0) {
|
||||
Write-Host "[FAIL] Could not install sentencepiece into .venv_t5/" -ForegroundColor Red
|
||||
$ErrorActionPreference = $prevEAP_t5
|
||||
exit 1
|
||||
}
|
||||
pip install --target $VenvT5Dir --no-deps "tiktoken" 2>&1 | Out-Null
|
||||
if ($LASTEXITCODE -ne 0) {
|
||||
Write-Host "[FAIL] Could not install tiktoken into .venv_t5/" -ForegroundColor Red
|
||||
$ErrorActionPreference = $prevEAP_t5
|
||||
exit 1
|
||||
}
|
||||
$ErrorActionPreference = $prevEAP_t5
|
||||
Write-Host "[OK] Transformers 5.x pre-installed to .venv_t5/" -ForegroundColor Green
|
||||
|
||||
|
|
|
|||
|
|
@ -254,6 +254,9 @@ else
|
|||
mkdir -p "$VENV_T5_DIR"
|
||||
run_quiet "pip install transformers 5.x" pip install --target "$VENV_T5_DIR" --no-deps "transformers==5.3.0"
|
||||
run_quiet "pip install huggingface_hub for t5" pip install --target "$VENV_T5_DIR" --no-deps "huggingface_hub==1.3.0"
|
||||
# Tokenizer backends — required by AutoTokenizer for many model families
|
||||
run_quiet "pip install sentencepiece for t5" pip install --target "$VENV_T5_DIR" --no-deps "sentencepiece>=0.2.0"
|
||||
run_quiet "pip install tiktoken for t5" pip install --target "$VENV_T5_DIR" --no-deps "tiktoken"
|
||||
echo "✅ Transformers 5.x pre-installed to $VENV_T5_DIR/"
|
||||
|
||||
# ── 7. WSL: pre-install GGUF build dependencies ──
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue