Fix Kaggle telemetry misclassification when COLAB_ keys exist
Problem: Kaggle notebook environments can expose both KAGGLE_* and COLAB_* environment keys. _get_statistics currently checks COLAB_ before KAGGLE_, causing Kaggle sessions to be labeled colab/colabpro. Prefer filesystem markers (e.g. /kaggle/working, /content + /opt/colab) before env-key heuristics, then fall back to the existing env-key checks. This avoids misclassification when providers leak overlapping env vars. Kaggle test notebook: https://www.kaggle.com/code/hnxnq07/kaggle-stats-gathering-test
This commit is contained in:
Rachel Li
GitHub
parent
ff8f5cd328
commit
1cdf751f8e
1 changed files with 90 additions and 75 deletions
|
|
@ -1108,85 +1108,100 @@ def _get_statistics(statistics = None, force_download = True):
|
|||
|
||||
if statistics is not None:
|
||||
pass
|
||||
elif "\nCOLAB_" in keynames and n_cpus == 1:
|
||||
statistics = "colab"
|
||||
elif "\nCOLAB_" in keynames:
|
||||
statistics = "colabpro"
|
||||
elif "\nKAGGLE_" in keynames:
|
||||
statistics = "kaggle"
|
||||
elif "\nRUNPOD_" in keynames:
|
||||
statistics = "runpod"
|
||||
elif "\nAWS_" in keynames:
|
||||
statistics = "aws"
|
||||
elif "\nAZURE_" in keynames:
|
||||
statistics = "azure"
|
||||
# elif "\nK_" in keynames or "\nFUNCTION_" in keynames: statistics = "gcp"
|
||||
elif "\nINVOCATION_ID" in keynames:
|
||||
statistics = "lambda"
|
||||
# else: statistics = "other"
|
||||
else:
|
||||
|
||||
def try_vllm_check():
|
||||
vendor_files = (
|
||||
"/sys/class/dmi/id/product_version",
|
||||
"/sys/class/dmi/id/bios_vendor",
|
||||
"/sys/class/dmi/id/product_name",
|
||||
"/sys/class/dmi/id/chassis_asset_tag",
|
||||
"/sys/class/dmi/id/sys_vendor",
|
||||
)
|
||||
# Prefer filesystem markers (harder to misidentify) before env-key matching
|
||||
try:
|
||||
from pathlib import Path
|
||||
|
||||
for vendor_file in vendor_files:
|
||||
path = Path(vendor_file)
|
||||
if path.is_file():
|
||||
file_content = path.read_text().lower()
|
||||
if "amazon" in file_content:
|
||||
return "aws"
|
||||
elif "microsoft corporation" in file_content:
|
||||
return "azure"
|
||||
elif "google" in file_content:
|
||||
return "gcp"
|
||||
return "other"
|
||||
|
||||
pass
|
||||
try:
|
||||
statistics = try_vllm_check()
|
||||
except:
|
||||
statistics = "other"
|
||||
if statistics is not None:
|
||||
import tempfile
|
||||
from huggingface_hub import snapshot_download
|
||||
from unsloth_zoo.rl_environments import execute_with_time_limit
|
||||
|
||||
if has_internet():
|
||||
|
||||
def stats_check():
|
||||
with tempfile.TemporaryDirectory(ignore_cleanup_errors = True) as f:
|
||||
snapshot_download(
|
||||
f"unslothai/{statistics}",
|
||||
force_download = True,
|
||||
cache_dir = f,
|
||||
local_dir = f,
|
||||
if Path("/kaggle/working").exists():
|
||||
statistics = "kaggle"
|
||||
elif Path("/content").exists() and Path("/opt/colab").exists():
|
||||
statistics = "colab" if n_cpus == 1 else "colabpro"
|
||||
elif Path("/runpod-volume").exists():
|
||||
statistics = "runpod"
|
||||
except Exception:
|
||||
pass
|
||||
# Fallback to env-key detection
|
||||
if statistics is None:
|
||||
if "\nKAGGLE_" in keynames:
|
||||
statistics = "kaggle"
|
||||
elif "\nCOLAB_" in keynames and n_cpus == 1:
|
||||
statistics = "colab"
|
||||
elif "\nCOLAB_" in keynames:
|
||||
statistics = "colabpro"
|
||||
elif "\nRUNPOD_" in keynames:
|
||||
statistics = "runpod"
|
||||
elif "\nAWS_" in keynames:
|
||||
statistics = "aws"
|
||||
elif "\nAZURE_" in keynames:
|
||||
statistics = "azure"
|
||||
# elif "\nK_" in keynames or "\nFUNCTION_" in keynames: statistics = "gcp"
|
||||
elif "\nINVOCATION_ID" in keynames:
|
||||
statistics = "lambda"
|
||||
# else: statistics = "other"
|
||||
else:
|
||||
|
||||
def try_vllm_check():
|
||||
vendor_files = (
|
||||
"/sys/class/dmi/id/product_version",
|
||||
"/sys/class/dmi/id/bios_vendor",
|
||||
"/sys/class/dmi/id/product_name",
|
||||
"/sys/class/dmi/id/chassis_asset_tag",
|
||||
"/sys/class/dmi/id/sys_vendor",
|
||||
)
|
||||
|
||||
time_limited_stats_check = execute_with_time_limit(120)(stats_check)
|
||||
try:
|
||||
time_limited_stats_check()
|
||||
except TimeoutError:
|
||||
raise TimeoutError(
|
||||
"Unsloth: HuggingFace seems to be down after trying for 120 seconds :(\n"
|
||||
"Check https://status.huggingface.co/ for more details.\n"
|
||||
"As a temporary measure, use modelscope with the same model name ie:\n"
|
||||
"```\n"
|
||||
"pip install modelscope\n"
|
||||
"import os; os.environ['UNSLOTH_USE_MODELSCOPE'] = '1'\n"
|
||||
"from unsloth import FastLanguageModel\n"
|
||||
"model = FastLanguageModel.from_pretrained('unsloth/gpt-oss-20b')\n"
|
||||
"```"
|
||||
)
|
||||
except:
|
||||
# Try no time limit check
|
||||
stats_check()
|
||||
from pathlib import Path
|
||||
|
||||
for vendor_file in vendor_files:
|
||||
path = Path(vendor_file)
|
||||
if path.is_file():
|
||||
file_content = path.read_text().lower()
|
||||
if "amazon" in file_content:
|
||||
return "aws"
|
||||
elif "microsoft corporation" in file_content:
|
||||
return "azure"
|
||||
elif "google" in file_content:
|
||||
return "gcp"
|
||||
return "other"
|
||||
|
||||
pass
|
||||
try:
|
||||
statistics = try_vllm_check()
|
||||
except:
|
||||
statistics = "other"
|
||||
if statistics is not None:
|
||||
import tempfile
|
||||
from huggingface_hub import snapshot_download
|
||||
from unsloth_zoo.rl_environments import execute_with_time_limit
|
||||
|
||||
if has_internet():
|
||||
|
||||
def stats_check():
|
||||
with tempfile.TemporaryDirectory(ignore_cleanup_errors = True) as f:
|
||||
snapshot_download(
|
||||
f"unslothai/{statistics}",
|
||||
force_download = True,
|
||||
cache_dir = f,
|
||||
local_dir = f,
|
||||
)
|
||||
|
||||
time_limited_stats_check = execute_with_time_limit(120)(stats_check)
|
||||
try:
|
||||
time_limited_stats_check()
|
||||
except TimeoutError:
|
||||
raise TimeoutError(
|
||||
"Unsloth: HuggingFace seems to be down after trying for 120 seconds :(\n"
|
||||
"Check https://status.huggingface.co/ for more details.\n"
|
||||
"As a temporary measure, use modelscope with the same model name ie:\n"
|
||||
"```\n"
|
||||
"pip install modelscope\n"
|
||||
"import os; os.environ['UNSLOTH_USE_MODELSCOPE'] = '1'\n"
|
||||
"from unsloth import FastLanguageModel\n"
|
||||
"model = FastLanguageModel.from_pretrained('unsloth/gpt-oss-20b')\n"
|
||||
"```"
|
||||
)
|
||||
except:
|
||||
# Try no time limit check
|
||||
stats_check()
|
||||
|
||||
|
||||
def get_statistics(local_files_only = False):
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue