formatting and typo fix
This commit is contained in:
parent
1a214716bf
commit
b60c138131
2 changed files with 4 additions and 1 deletions
|
|
@ -50,6 +50,7 @@ def _layer_norm_fwd_fused(
|
|||
a = tl.load(X + cols, mask=cols < N, other=0.).to(tl.float32)
|
||||
_mean += a
|
||||
mean = tl.sum(_mean, axis=0) / N
|
||||
|
||||
# Compute variance
|
||||
_var = tl.zeros([BLOCK_SIZE], dtype=tl.float32)
|
||||
for off in range(0, N, BLOCK_SIZE):
|
||||
|
|
@ -59,9 +60,11 @@ def _layer_norm_fwd_fused(
|
|||
_var += x * x
|
||||
var = tl.sum(_var, axis=0) / N
|
||||
rstd = 1 / tl.sqrt(var + eps)
|
||||
|
||||
# Write mean / rstd
|
||||
tl.store(Mean + row, mean)
|
||||
tl.store(Rstd + row, rstd)
|
||||
|
||||
# Normalize and apply linear transformation
|
||||
for off in range(0, N, BLOCK_SIZE):
|
||||
cols = off + tl.arange(0, BLOCK_SIZE)
|
||||
|
|
|
|||
|
|
@ -785,7 +785,7 @@ class FastLlamaModel:
|
|||
layers = model.model.layers
|
||||
|
||||
# Torch.compile fails on embedding matrix??
|
||||
# Workaround randomnly fixes it for torch versions < 2.2
|
||||
# Workaround randomly fixes it for torch versions < 2.2
|
||||
model.model.embed_tokens = torch.nn.Embedding.from_pretrained(model.model.embed_tokens.weight)
|
||||
model.config.update({"unsloth_version" : __version__})
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue