Fix GPT-OSS BlockMask error during inference (#3982)
GPT-OSS models use eager attention during inference because flex attention returns incorrect results (likely due to left padding). However, when _attn_implementation is set to "flex_attention", transformers creates BlockMask objects which cause a TypeError when passed to the eager attention path: TypeError: unsupported operand type(s) for +=: 'Tensor' and 'BlockMask' This fix excludes GPT-OSS from using flex_attention, keeping it on the eager path to avoid the BlockMask/Tensor type mismatch.
This commit is contained in:
parent
620d4648ff
commit
0166c6266d
1 changed files with 6 additions and 0 deletions
|
|
@ -197,6 +197,12 @@ def prefer_flex_attn_if_supported(model_class, config):
|
|||
model_class, "_supports_flex_attn", False
|
||||
):
|
||||
return None
|
||||
# GPT-OSS uses eager attention during inference since flex attention
|
||||
# returns incorrect results (likely due to left padding issues).
|
||||
# Skip setting flex_attention to avoid BlockMask type errors.
|
||||
model_type = getattr(config, "model_type", "") if config else ""
|
||||
if model_type == "gpt_oss":
|
||||
return None
|
||||
if config is not None:
|
||||
setattr(config, "_attn_implementation", "flex_attention")
|
||||
if hasattr(config, "attn_implementation"):
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue