15 changed files with
42 additions and
0 deletions
|
|
|
|
@ -1,6 +1,7 @@
|
|
|
|
|
## MoE Grouped GEMM
|
|
|
|
|
|
|
|
|
|
Optimized implementation of `MoE MLP Block`.
|
|
|
|
|
Licensed under AGPLv3.
|
|
|
|
|
|
|
|
|
|
### Background
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
@ -1,3 +1,6 @@
|
|
|
|
|
# SPDX-License-Identifier: GNU Affero General Public License v3.0
|
|
|
|
|
# Copyright 2023-present the Unsloth team. All rights reserved.
|
|
|
|
|
|
|
|
|
|
import logging
|
|
|
|
|
import warnings
|
|
|
|
|
from dataclasses import asdict
|
|
|
|
|
|
|
|
|
|
@ -1,3 +1,6 @@
|
|
|
|
|
# SPDX-License-Identifier: GNU Affero General Public License v3.0
|
|
|
|
|
# Copyright 2023-present the Unsloth team. All rights reserved.
|
|
|
|
|
|
|
|
|
|
"""
|
|
|
|
|
Autotuning utils
|
|
|
|
|
"""
|
|
|
|
|
|
|
|
|
|
@ -1,3 +1,6 @@
|
|
|
|
|
# SPDX-License-Identifier: GNU Affero General Public License v3.0
|
|
|
|
|
# Copyright 2023-present the Unsloth team. All rights reserved.
|
|
|
|
|
|
|
|
|
|
import torch
|
|
|
|
|
import triton
|
|
|
|
|
import triton.language as tl
|
|
|
|
|
|
|
|
|
|
@ -1,3 +1,6 @@
|
|
|
|
|
# SPDX-License-Identifier: GNU Affero General Public License v3.0
|
|
|
|
|
# Copyright 2023-present the Unsloth team. All rights reserved.
|
|
|
|
|
|
|
|
|
|
import torch
|
|
|
|
|
import triton
|
|
|
|
|
import triton.language as tl
|
|
|
|
|
|
|
|
|
|
@ -1,3 +1,6 @@
|
|
|
|
|
# SPDX-License-Identifier: GNU Affero General Public License v3.0
|
|
|
|
|
# Copyright 2023-present the Unsloth team. All rights reserved.
|
|
|
|
|
|
|
|
|
|
"""
|
|
|
|
|
Manual tuning utils
|
|
|
|
|
"""
|
|
|
|
|
|
|
|
|
|
@ -1,3 +1,6 @@
|
|
|
|
|
# SPDX-License-Identifier: GNU Affero General Public License v3.0
|
|
|
|
|
# Copyright 2023-present the Unsloth team. All rights reserved.
|
|
|
|
|
|
|
|
|
|
from dataclasses import dataclass
|
|
|
|
|
from typing import Tuple
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
@ -1,3 +1,6 @@
|
|
|
|
|
# SPDX-License-Identifier: GNU Affero General Public License v3.0
|
|
|
|
|
# Copyright 2023-present the Unsloth team. All rights reserved.
|
|
|
|
|
|
|
|
|
|
from dataclasses import dataclass
|
|
|
|
|
from typing import Tuple
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
@ -1,3 +1,6 @@
|
|
|
|
|
# SPDX-License-Identifier: GNU Affero General Public License v3.0
|
|
|
|
|
# Copyright 2023-present the Unsloth team. All rights reserved.
|
|
|
|
|
|
|
|
|
|
import torch
|
|
|
|
|
from transformers.models.qwen3_moe.configuration_qwen3_moe import Qwen3MoeConfig
|
|
|
|
|
from transformers.models.qwen3_moe.modeling_qwen3_moe import Qwen3MoeSparseMoeBlock
|
|
|
|
|
|
|
|
|
|
@ -1,3 +1,5 @@
|
|
|
|
|
# SPDX-License-Identifier: GNU Affero General Public License v3.0
|
|
|
|
|
# Copyright 2023-present the Unsloth team. All rights reserved.
|
|
|
|
|
|
|
|
|
|
import torch
|
|
|
|
|
import torch.nn.functional as F
|
|
|
|
|
|
|
|
|
|
@ -1,3 +1,6 @@
|
|
|
|
|
# SPDX-License-Identifier: GNU Affero General Public License v3.0
|
|
|
|
|
# Copyright 2023-present the Unsloth team. All rights reserved.
|
|
|
|
|
|
|
|
|
|
import itertools
|
|
|
|
|
from contextlib import contextmanager
|
|
|
|
|
from dataclasses import dataclass, field
|
|
|
|
|
|
|
|
|
|
@ -1,3 +1,6 @@
|
|
|
|
|
# SPDX-License-Identifier: GNU Affero General Public License v3.0
|
|
|
|
|
# Copyright 2023-present the Unsloth team. All rights reserved.
|
|
|
|
|
|
|
|
|
|
from dataclasses import dataclass, fields
|
|
|
|
|
|
|
|
|
|
import torch
|
|
|
|
|
|
|
|
|
|
@ -1,3 +1,6 @@
|
|
|
|
|
# SPDX-License-Identifier: GNU Affero General Public License v3.0
|
|
|
|
|
# Copyright 2023-present the Unsloth team. All rights reserved.
|
|
|
|
|
|
|
|
|
|
from dataclasses import asdict
|
|
|
|
|
|
|
|
|
|
import pytest
|
|
|
|
|
|
|
|
|
|
@ -1,3 +1,6 @@
|
|
|
|
|
# SPDX-License-Identifier: GNU Affero General Public License v3.0
|
|
|
|
|
# Copyright 2023-present the Unsloth team. All rights reserved.
|
|
|
|
|
|
|
|
|
|
import argparse
|
|
|
|
|
import sys
|
|
|
|
|
from contextlib import contextmanager
|
|
|
|
|
|
|
|
|
|
@ -1,3 +1,6 @@
|
|
|
|
|
# SPDX-License-Identifier: GNU Affero General Public License v3.0
|
|
|
|
|
# Copyright 2023-present the Unsloth team. All rights reserved.
|
|
|
|
|
|
|
|
|
|
import argparse
|
|
|
|
|
from contextlib import contextmanager
|
|
|
|
|
|
|
|
|
|
|