feat: add embedding model training support
Add end-to-end embedding/sentence-transformer training pipeline using FastSentenceTransformer, SentenceTransformerTrainer, and MultipleNegativesRankingLoss with BatchSamplers.NO_DUPLICATES. Backend: - Add is_embedding_model() detection via HF tags + pipeline_tag - Add /check-embedding/ API route and EmbeddingCheckResponse - Extend derive_model_type() to return "embeddings" - Add _run_embedding_training() in worker.py with progress callbacks, stop handling, LoRA (task_type=FEATURE_EXTRACTION), and model saving - Add is_embedding field to TrainingStartRequest and ModelDetails - Add YAML configs for 5 models: all-MiniLM-L6-v2, bge-m3, embeddinggemma-300m, gte-modernbert-base, Qwen3-Embedding-0.6B Frontend: - Wire isEmbeddingModel flag through store, API types, and mappers - Force packing=false, train_on_completions=false, warmup_ratio=0.03 - Hide packing and train_on_completions checkboxes for embedding models - Auto-set modelType to "embeddings" from backend model_type response
This commit is contained in:
parent
cb389fb756
commit
5a086353ab
20 changed files with 758 additions and 26 deletions
|
|
@ -8,6 +8,7 @@ from .model_config import (
|
|||
ModelConfig,
|
||||
GgufVariantInfo,
|
||||
is_vision_model,
|
||||
is_embedding_model,
|
||||
detect_audio_type,
|
||||
is_audio_input_type,
|
||||
VALID_AUDIO_TYPES,
|
||||
|
|
@ -26,6 +27,7 @@ __all__ = [
|
|||
'ModelConfig',
|
||||
'GgufVariantInfo',
|
||||
'is_vision_model',
|
||||
'is_embedding_model',
|
||||
'detect_audio_type',
|
||||
'is_audio_input_type',
|
||||
'VALID_AUDIO_TYPES',
|
||||
|
|
|
|||
|
|
@ -25,6 +25,30 @@ logger = logging.getLogger(__name__)
|
|||
# Format: "canonical_model_name.yaml": [list of all equivalent model names]
|
||||
# Based on the model mapper provided - canonical filename is based on the first model name in the mapper
|
||||
MODEL_NAME_MAPPING = {
|
||||
# ── Embedding models ──
|
||||
"unsloth_all-MiniLM-L6-v2.yaml": [
|
||||
"unsloth/all-MiniLM-L6-v2",
|
||||
"sentence-transformers/all-MiniLM-L6-v2",
|
||||
],
|
||||
"unsloth_bge-m3.yaml": [
|
||||
"unsloth/bge-m3",
|
||||
"BAAI/bge-m3",
|
||||
],
|
||||
"unsloth_embeddinggemma-300m.yaml": [
|
||||
"unsloth/embeddinggemma-300m",
|
||||
"google/embeddinggemma-300m",
|
||||
],
|
||||
"unsloth_gte-modernbert-base.yaml": [
|
||||
"unsloth/gte-modernbert-base",
|
||||
"Alibaba-NLP/gte-modernbert-base",
|
||||
],
|
||||
"unsloth_Qwen3-Embedding-0.6B.yaml": [
|
||||
"unsloth/Qwen3-Embedding-0.6B",
|
||||
"Qwen/Qwen3-Embedding-0.6B",
|
||||
"unsloth/Qwen3-Embedding-4B",
|
||||
"Qwen/Qwen3-Embedding-4B",
|
||||
],
|
||||
# ── Other models ──
|
||||
"unsloth_answerdotai_ModernBERT-large.yaml": [
|
||||
"answerdotai/ModernBERT-large",
|
||||
],
|
||||
|
|
@ -894,6 +918,67 @@ def download_gguf_file(
|
|||
return local_path
|
||||
|
||||
|
||||
# Cache embedding detection results per session to avoid repeated HF API calls
|
||||
_embedding_detection_cache: Dict[str, bool] = {}
|
||||
|
||||
|
||||
def is_embedding_model(model_name: str, hf_token: Optional[str] = None) -> bool:
|
||||
"""
|
||||
Detect embedding/sentence-transformer models using HuggingFace model metadata.
|
||||
|
||||
Uses a belt-and-suspenders approach combining three signals:
|
||||
1. "sentence-transformers" in model tags
|
||||
2. "feature-extraction" in model tags
|
||||
3. pipeline_tag is "sentence-similarity" or "feature-extraction"
|
||||
|
||||
This catches all known embedding models including those like gte-modernbert
|
||||
whose library_name is "transformers" rather than "sentence-transformers".
|
||||
|
||||
Args:
|
||||
model_name: Model identifier (HF repo or local path)
|
||||
hf_token: Optional HF token for accessing gated/private models
|
||||
|
||||
Returns:
|
||||
True if the model is an embedding model, False otherwise.
|
||||
Defaults to False for local paths or on errors.
|
||||
"""
|
||||
if model_name in _embedding_detection_cache:
|
||||
return _embedding_detection_cache[model_name]
|
||||
|
||||
# Local paths have no HF metadata to query
|
||||
if is_local_path(model_name):
|
||||
_embedding_detection_cache[model_name] = False
|
||||
return False
|
||||
|
||||
try:
|
||||
from huggingface_hub import model_info as hf_model_info
|
||||
|
||||
info = hf_model_info(model_name, token=hf_token)
|
||||
tags = set(info.tags or [])
|
||||
pipeline_tag = info.pipeline_tag or ""
|
||||
|
||||
is_emb = (
|
||||
"sentence-transformers" in tags
|
||||
or "feature-extraction" in tags
|
||||
or pipeline_tag in ("sentence-similarity", "feature-extraction")
|
||||
)
|
||||
|
||||
_embedding_detection_cache[model_name] = is_emb
|
||||
if is_emb:
|
||||
logger.info(
|
||||
f"Model {model_name} detected as embedding model: "
|
||||
f"pipeline_tag={pipeline_tag}, "
|
||||
f"sentence-transformers in tags={('sentence-transformers' in tags)}, "
|
||||
f"feature-extraction in tags={('feature-extraction' in tags)}"
|
||||
)
|
||||
return is_emb
|
||||
|
||||
except Exception as e:
|
||||
logger.warning(f"Could not determine if {model_name} is embedding model: {e}")
|
||||
_embedding_detection_cache[model_name] = False
|
||||
return False
|
||||
|
||||
|
||||
def scan_trained_loras(outputs_dir: str = "./outputs") -> List[Tuple[str, str]]:
|
||||
"""
|
||||
Scan outputs folder for trained LoRA adapters.
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue