Add support for multiple files

unsloth/dataprep/raw_text.py

@@ -41,12 +41,26 @@ class RawTextDataLoader:
     
     def detect_format(self, file_path):
         """Auto-detect file format and parse accordingly"""
+        extension = Path(file_path).suffix.lower()
+        return SUPPORTED_FORMATS.get(extension, 'plain_text')
 
     def load_from_file(self, file_path):
         """Load raw text and convert to dataset"""
+        file_format = self.detect_format(file_path)
+        text_content = self._read_file_by_format(file_path, file_format)
+        chunks = self.smart_chunk_text(text_content, self.chunk_size, self.stride)
+        return self.create_causal_dataset(chunks)
         
     def load_from_files(self, file_paths):
         """Load multiple text files"""
+        all_chunks = []
+        for file_path in file_paths:
+            file_format = self.detect_format(file_path)
+            text_content = self._read_file_by_format(file_path, file_format)
+            chunks = self.smart_chunk_text(text_content, self.chunk_size, self.stride)
+            all_chunks.extend(chunks)
+        return self.create_causal_dataset(all_chunks)
+
         
     def chunk_text(self, text):
         """Split text into overlapping chunks"""
@@ -71,6 +85,48 @@ class RawTextDataLoader:
         3. Handles different languages better
         """
 
+    def _read_file_by_format(self, file_path, file_format):
+        """Read file content based on detected format."""
+        with open(file_path, 'r', encoding='utf-8') as f:
+            if file_format == 'plain_text' or file_format == 'markdown':
+                return f.read()
+            elif file_format == 'json_lines':
+                lines = []
+                for line in f:
+                    try:
+                        data = json.loads(line.strip())
+                        text = self._extract_text_from_json(data)
+                        if text:
+                            lines.append(text)
+                    except json.JSONDecodeError:
+                        continue
+                return '\n\n'.join(lines)
+            elif file_format == 'csv_text_column':
+                reader = csv.DictReader(f)
+                texts = []
+                for row in reader:
+                    text = self._extract_text_from_csv_row(row)
+                    if text:
+                        texts.append(text)
+                return '\n\n'.join(texts)
+        return ""
+    
+    def _extract_text_from_json(self, data):
+        """Extract text from JSON object using common field names."""
+        text_fields = ['text', 'content', 'message', 'body', 'description', 'prompt']
+        for field in text_fields:
+            if field in data and isinstance(data[field], str):
+                return data[field]
+        return ""
+    
+    def _extract_text_from_csv_row(self, row):
+        """Extract text from CSV row using common column names."""
+        text_columns = ['text', 'content', 'message', 'body', 'description', 'prompt']
+        for column in text_columns:
+            if column in row and row[column]:
+                return row[column]
+        return ""
+
 class TextPreprocessor:
     def clean_text(self, text):
         """Remove unwanted characters, normalize whitespace"""