Fix step count mismatch when sequence packing is enabled (#5967)

* Fix step count mismatch when sequence packing is enabled

* [pre-commit.ci] auto fixes from pre-commit.com hooks

for more information, see https://pre-commit.ci

* Emit a single step-0 progress event and guard applyStatus totalSteps

Merge the two consecutive _update_progress calls before train() so the
step-0 gate in _on_progress fires once instead of twice, avoiding a
duplicate startup event and a null-metric step-0 row in training_metrics.

Apply the same positive-number guard to applyStatus that applyProgress
uses, so a stale or startup status poll can no longer overwrite the
packed step count with 0 or replace it with a stale total.

* Log debug message when train_dataset length is unavailable

The TypeError fallback for length-less datasets (e.g. streaming
IterableDataset) was silent, leaving no trace that the step estimate
came from the raw dataset rather than the packed one.

* [pre-commit.ci] auto fixes from pre-commit.com hooks

for more information, see https://pre-commit.ci

---------

Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com>
Co-authored-by: Etherll <61019402+Etherll@users.noreply.github.com>
This commit is contained in:
Irakli 2026-06-12 13:30:53 +04:00 committed by GitHub
commit 95a2627bf6
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
3 changed files with 21 additions and 8 deletions

View file

@ -3367,7 +3367,19 @@ class UnslothTrainer:
# ========== PROGRESS TRACKING ==========
self.trainer.add_callback(self._create_progress_callback())
num_samples = len(dataset["dataset"] if isinstance(dataset, dict) else dataset)
num_samples = None
if hasattr(self.trainer, "train_dataset") and self.trainer.train_dataset is not None:
try:
num_samples = len(self.trainer.train_dataset)
except TypeError:
logger.debug(
"train_dataset does not support len(); falling back to "
"raw dataset size for step estimation."
)
if num_samples is None:
num_samples = len(dataset["dataset"] if isinstance(dataset, dict) else dataset)
batch_size = training_args.get("batch_size", 2)
total_steps = self._calculate_total_steps(
num_samples,
@ -3376,10 +3388,8 @@ class UnslothTrainer:
training_args.get("num_epochs", 3),
training_args.get("max_steps", 0),
)
self._update_progress(total_steps = total_steps)
# ========== START TRAINING ==========
self._update_progress(status_message = "Starting training...")
self._update_progress(total_steps = total_steps, status_message = "Starting training...")
logger.info("Starting training...\n")
self.trainer.train(resume_from_checkpoint = training_args.get("resume_from_checkpoint"))

View file

@ -2462,7 +2462,7 @@ def run_training_process(*, event_queue: Any, stop_queue: Any, config: dict) ->
def _on_progress(progress: TrainingProgress):
has_train_loss = progress.step > 0 and progress.loss is not None
has_eval_loss = progress.eval_loss is not None
if has_train_loss or has_eval_loss:
if (progress.step == 0 and progress.total_steps > 0) or has_train_loss or has_eval_loss:
event_queue.put(
{
"type": "progress",

View file

@ -209,8 +209,8 @@ export const useTrainingRuntimeStore = create<TrainingRuntimeStore>()((set) => (
currentStep:
typeof detailStep === "number" ? Math.max(detailStep, 0) : state.currentStep,
totalSteps:
typeof detailTotal === "number"
? Math.max(detailTotal, 0)
typeof detailTotal === "number" && detailTotal > 0
? detailTotal
: state.totalSteps,
currentLoss:
typeof detailLoss === "number" ? detailLoss : state.currentLoss,
@ -273,7 +273,10 @@ export const useTrainingRuntimeStore = create<TrainingRuntimeStore>()((set) => (
...state,
jobId: payload.job_id || state.jobId,
currentStep: step,
totalSteps: Math.max(payload.total_steps, state.totalSteps),
totalSteps:
typeof payload.total_steps === "number" && payload.total_steps > 0
? payload.total_steps
: state.totalSteps,
// A null loss at a new step means the backend reported a non-finite
// loss; clear the display instead of keeping the stale value.
currentLoss: