Write file and template for raw_text dataprep

unsloth/dataprep/raw_text.py

@@ -0,0 +1,76 @@
+# Copyright 2023-present Daniel Han-Chen & the Unsloth team. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os
+import re
+import json
+import csv
+from typing import List, Dict, Any, Union, Optional
+from datasets import Dataset
+from pathlib import Path
+
+class RawTextDataLoader:
+    def __init__(self, tokenizer, chunk_size=2048, stride=512):
+        self.tokenizer = tokenizer
+        self.chunk_size = chunk_size  
+        self.stride = stride
+    
+    def load_from_file(self, file_path):
+        """Load raw text and convert to dataset"""
+        
+    def load_from_files(self, file_paths):
+        """Load multiple text files"""
+        
+    def chunk_text(self, text):
+        """Split text into overlapping chunks"""
+        
+    def create_causal_dataset(self, chunks):
+        """Create dataset for causal language modeling"""
+
+    def smart_chunk_text(self, text, chunk_size, stride):
+        """
+        Intelligent chunking that:
+        1. Respects sentence/paragraph boundaries
+        2. Handles various text formats (.txt, .md, .json, etc.)
+        3. Maintains context with stride overlap
+        4. Adds proper EOS tokens
+        """
+    
+    def tokenize_and_chunk(self, text):
+        """
+        Tokenize first, then chunk by token count:
+        1. More precise length control
+        2. Avoids mid-token splits
+        3. Handles different languages better
+        """
+
+class TextPreprocessor:
+    def clean_text(self, text):
+        """Remove unwanted characters, normalize whitespace"""
+        
+    def extract_sections(self, text, patterns):
+        """Extract specific sections (e.g., code blocks, quotes)"""
+        
+    def add_structure_tokens(self, text):
+        """Add special tokens for structure (chapters, sections)"""
+
+def validate_dataset(self, dataset):
+    """
+    Check for:
+    - Minimum/maximum sequence lengths
+    - Character encoding issues
+    - Repeated content
+    - Empty chunks
+    """
+