perf(dataprep): cache regex and field lists, fix typos (#6714)
* Improve code quality & performance: fix typos, compile regex & cache fields - Fix typos across core files (repeatted → repeated, splitted → split, etc.) - Compile regex patterns once as class attributes in TextPreprocessor - Cache text fields/columns in RawTextDataLoader - Improve comments (re-use → reuse) * Use immutable raw text field constants --------- Co-authored-by: imagineer99 <samleejackson0@gmail.com> Co-authored-by: Lee Jackson <130007945+Imagineer99@users.noreply.github.com>
This commit is contained in:
Muhammad Ikhwan Fathulloh
GitHub
parent
2f8521ed54
commit
02540371c2
5 changed files with 40 additions and 25 deletions
|
|
@ -2276,28 +2276,28 @@ def get_ollama_eos_tokens(tokenizer, extra_eos_tokens = []):
|
|||
if getattr(tokenizer, "bos_token", None) is not None:
|
||||
added_tokens_decoder = [x for x in added_tokens_decoder if x != tokenizer.bos_token]
|
||||
|
||||
repeatted_tokens = []
|
||||
repeated_tokens = []
|
||||
# Join all vocab
|
||||
joined_text = "\x01\x00".join(added_tokens_decoder)
|
||||
for token in added_tokens_decoder:
|
||||
n = len(token)
|
||||
repeatted_counts = joined_text.count(token[:n//2])
|
||||
repeated_counts = joined_text.count(token[:n//2])
|
||||
# Try finding longer than 1/2 of the token in the rest
|
||||
# For eg <|reserved_special_token_0|>, <|reserved_special_token_1|>
|
||||
if repeatted_counts > 2:
|
||||
if repeated_counts > 2:
|
||||
for j in range(n//2+1, n):
|
||||
if joined_text.count(token[:j]) < repeatted_counts:
|
||||
if joined_text.count(token[:j]) < repeated_counts:
|
||||
j -= 1
|
||||
# Remove repeatted tokens to reduce search space
|
||||
# Remove repeated tokens to reduce search space
|
||||
joined_text = joined_text.replace(token[:j], "")
|
||||
repeatted_tokens.append(token[:j])
|
||||
repeated_tokens.append(token[:j])
|
||||
break
|
||||
|
||||
# Remove duplicates
|
||||
splitted = joined_text.split("\x01\x00")
|
||||
final_eos_tokens = [old for old, new in zip(added_tokens_decoder, splitted) if old == new]
|
||||
split = joined_text.split("\x01\x00")
|
||||
final_eos_tokens = [old for old, new in zip(added_tokens_decoder, split) if old == new]
|
||||
final_eos_tokens += extra_eos_tokens
|
||||
final_eos_tokens += repeatted_tokens
|
||||
final_eos_tokens += repeated_tokens
|
||||
|
||||
# Remove new lines, spaces and HTML tags
|
||||
filtered_eos_tokens = []
|
||||
|
|
|
|||
|
|
@ -223,48 +223,63 @@ class RawTextDataLoader:
|
|||
return "\n\n".join(texts)
|
||||
return ""
|
||||
|
||||
# Cache text fields/columns for better performance
|
||||
_TEXT_FIELDS = ("text", "content", "message", "body", "description", "prompt")
|
||||
_TEXT_COLUMNS = _TEXT_FIELDS
|
||||
|
||||
def _extract_text_from_json(self, data):
|
||||
"""Extract text from JSON object using common field names."""
|
||||
text_fields = ["text", "content", "message", "body", "description", "prompt"]
|
||||
for field in text_fields:
|
||||
for field in self._TEXT_FIELDS:
|
||||
if field in data and isinstance(data[field], str):
|
||||
return data[field]
|
||||
return ""
|
||||
|
||||
def _extract_text_from_csv_row(self, row):
|
||||
"""Extract text from CSV row using common column names."""
|
||||
text_columns = ["text", "content", "message", "body", "description", "prompt"]
|
||||
for column in text_columns:
|
||||
for column in self._TEXT_COLUMNS:
|
||||
if column in row and row[column]:
|
||||
return row[column]
|
||||
return ""
|
||||
|
||||
|
||||
class TextPreprocessor:
|
||||
# Compile regex patterns once for better performance
|
||||
_WHITESPACE_PATTERN = re.compile(r"[^\S\n]+")
|
||||
_INVALID_CHARS_PATTERN = re.compile(r"[^\x20-\x7E\n]")
|
||||
_MULTIPLE_SPACES_PATTERN = re.compile(r"[ ]{2,}")
|
||||
_NEWLINE_SPACES_PATTERN = re.compile(r" *\n *")
|
||||
_MULTIPLE_NEWLINES_PATTERN = re.compile(r"\n{3,}")
|
||||
_CHAPTER_PATTERN = re.compile(r"^# (.+)$", re.MULTILINE)
|
||||
_SECTION_PATTERN = re.compile(r"^## (.+)$", re.MULTILINE)
|
||||
_SUBSECTION_PATTERN = re.compile(r"^### (.+)$", re.MULTILINE)
|
||||
_CODE_BLOCK_PATTERN = re.compile(r"```(\w*)\n(.*?)\n```", re.DOTALL)
|
||||
|
||||
def clean_text(self, text):
|
||||
"""Remove unwanted characters, normalize whitespace"""
|
||||
text = text.replace("\r\n", "\n").replace("\r", "\n")
|
||||
text = re.sub(r"[^\S\n]+", " ", text)
|
||||
text = re.sub(r"[^\x20-\x7E\n]", "", text)
|
||||
text = re.sub(r"[ ]{2,}", " ", text)
|
||||
text = re.sub(r" *\n *", "\n", text)
|
||||
text = re.sub(r"\n{3,}", "\n\n", text)
|
||||
text = self._WHITESPACE_PATTERN.sub(" ", text)
|
||||
text = self._INVALID_CHARS_PATTERN.sub("", text)
|
||||
text = self._MULTIPLE_SPACES_PATTERN.sub(" ", text)
|
||||
text = self._NEWLINE_SPACES_PATTERN.sub("\n", text)
|
||||
text = self._MULTIPLE_NEWLINES_PATTERN.sub("\n\n", text)
|
||||
return text.strip()
|
||||
|
||||
def extract_sections(self, text, patterns):
|
||||
"""Extract specific sections (e.g., code blocks, quotes)"""
|
||||
sections = []
|
||||
for pattern in patterns:
|
||||
# Compile pattern on first use and cache? Well, patterns are user-provided,
|
||||
# so just use re.findall with compiled flags
|
||||
matches = re.findall(pattern, text, re.MULTILINE | re.DOTALL)
|
||||
sections.extend(matches)
|
||||
return sections
|
||||
|
||||
def add_structure_tokens(self, text):
|
||||
"""Add special tokens for structure (chapters, sections)"""
|
||||
text = re.sub(r"^# (.+)$", r"<|chapter|>\1<|/chapter|>", text, flags = re.MULTILINE)
|
||||
text = re.sub(r"^## (.+)$", r"<|section|>\1<|/section|>", text, flags = re.MULTILINE)
|
||||
text = re.sub(r"^### (.+)$", r"<|subsection|>\1<|/subsection|>", text, flags = re.MULTILINE)
|
||||
text = re.sub(r"```(\w*)\n(.*?)\n```", r"<|code|\1|>\2<|/code|>", text, flags = re.DOTALL)
|
||||
text = self._CHAPTER_PATTERN.sub(r"<|chapter|>\1<|/chapter|>", text)
|
||||
text = self._SECTION_PATTERN.sub(r"<|section|>\1<|/section|>", text)
|
||||
text = self._SUBSECTION_PATTERN.sub(r"<|subsection|>\1<|/subsection|>", text)
|
||||
text = self._CODE_BLOCK_PATTERN.sub(r"<|code|\1|>\2<|/code|>", text)
|
||||
return text
|
||||
|
||||
def validate_dataset(self, dataset):
|
||||
|
|
|
|||
|
|
@ -97,7 +97,7 @@ def _exact_backward_kernel(
|
|||
e_row = tl.load(e + offsets, mask = mask, other = 0).to(tl.float32)
|
||||
g_row = tl.load(g + offsets, mask = mask, other = 0) # .to(tl.float32)
|
||||
|
||||
# Break e_row away for re-use
|
||||
# Break e_row away for reuse
|
||||
# f = 1/2 * e * (1 + erf(1/sqrt(2) * e))
|
||||
f_partial_row = 0.5 * (tl.math.erf(tl.math.rsqrt(2.0) * e_row) + 1.0)
|
||||
f_row = f_partial_row * e_row
|
||||
|
|
|
|||
|
|
@ -111,7 +111,7 @@ def _grouped_gemm_forward_kernel(
|
|||
while tidx >= processed_tiles and tidx < processed_tiles + num_tiles_per_expert:
|
||||
tile_idx = tidx - processed_tiles
|
||||
|
||||
# Check if L2 cache re-use for this order is optimal
|
||||
# Check if L2 cache reuse for this order is optimal
|
||||
tile_m_idx = tile_idx % num_m_tiles
|
||||
tile_n_idx = tile_idx // num_m_tiles
|
||||
|
||||
|
|
|
|||
|
|
@ -3078,7 +3078,7 @@ class TorchAOConfig:
|
|||
def _untie_input_output_embeddings(model: torch.nn.Module) -> None:
|
||||
"""
|
||||
Utility to untie input/output embeddings in a HuggingFace model.
|
||||
This is useful if we want to quantize the input/ouput embeddings differently.
|
||||
This is useful if we want to quantize the input/output embeddings differently.
|
||||
Model is modified in-place.
|
||||
"""
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue