""" ================================================================================ Text Classification Dataset - Text Preprocessing Module ================================================================================ Project: Text Classification Dataset Category: Text Data / NLP Author: Molla Samser Designer & Tester: Rima Khatun Website: https://rskworld.in Email: help@rskworld.in | support@rskworld.in Phone: +91 93305 39277 Copyright (c) 2026 RSK World - All Rights Reserved Content used for educational purposes only. Created: December 2026 ================================================================================ """ import re import string from typing import List, Optional import unicodedata class TextPreprocessor: """ A comprehensive text preprocessing class for NLP tasks. Features: - Lowercase conversion - Punctuation removal - Number handling - Stopword removal - Whitespace normalization - URL and email removal - HTML tag removal - Special character handling Author: Molla Samser | RSK World (https://rskworld.in) """ # Common English stopwords STOPWORDS = { 'a', 'an', 'the', 'and', 'or', 'but', 'in', 'on', 'at', 'to', 'for', 'of', 'with', 'by', 'from', 'as', 'is', 'was', 'are', 'were', 'been', 'be', 'have', 'has', 'had', 'do', 'does', 'did', 'will', 'would', 'could', 'should', 'may', 'might', 'must', 'shall', 'can', 'need', 'it', 'its', "it's", 'this', 'that', 'these', 'those', 'i', 'me', 'my', 'myself', 'we', 'our', 'ours', 'ourselves', 'you', "you're", 'your', 'yours', 'yourself', 'yourselves', 'he', 'him', 'his', 'himself', 'she', "she's", 'her', 'hers', 'herself', 'they', 'them', 'their', 'theirs', 'themselves', 'what', 'which', 'who', 'whom', 'when', 'where', 'why', 'how', 'all', 'each', 'every', 'both', 'few', 'more', 'most', 'other', 'some', 'such', 'no', 'nor', 'not', 'only', 'own', 'same', 'so', 'than', 'too', 'very', 'just', 'about', 'above', 'after', 'again', 'against', 'between', 'into', 'through', 'during', 'before', 'below', 'under', 'over', 'out', 'off', 'down', 'up' } def __init__( self, lowercase: bool = True, remove_punctuation: bool = True, remove_numbers: bool = False, remove_stopwords: bool = False, remove_urls: bool = True, remove_emails: bool = True, remove_html: bool = True, normalize_whitespace: bool = True, custom_stopwords: Optional[List[str]] = None ): """ Initialize the TextPreprocessor with configurable options. Args: lowercase: Convert text to lowercase remove_punctuation: Remove punctuation marks remove_numbers: Remove numeric characters remove_stopwords: Remove common stopwords remove_urls: Remove URLs from text remove_emails: Remove email addresses remove_html: Remove HTML tags normalize_whitespace: Normalize multiple spaces to single space custom_stopwords: Additional stopwords to remove """ self.lowercase = lowercase self.remove_punctuation = remove_punctuation self.remove_numbers = remove_numbers self.remove_stopwords = remove_stopwords self.remove_urls = remove_urls self.remove_emails = remove_emails self.remove_html = remove_html self.normalize_whitespace = normalize_whitespace # Combine default and custom stopwords self.stopwords = self.STOPWORDS.copy() if custom_stopwords: self.stopwords.update(set(custom_stopwords)) # Compile regex patterns for efficiency self.url_pattern = re.compile( r'https?://\S+|www\.\S+' ) self.email_pattern = re.compile( r'\b[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Z|a-z]{2,}\b' ) self.html_pattern = re.compile(r'<[^>]+>') self.whitespace_pattern = re.compile(r'\s+') self.number_pattern = re.compile(r'\d+') def preprocess(self, text: str) -> str: """ Apply all preprocessing steps to the input text. Args: text: Input text string Returns: Preprocessed text string """ if not text or not isinstance(text, str): return "" # Remove HTML tags if self.remove_html: text = self.html_pattern.sub(' ', text) # Remove URLs if self.remove_urls: text = self.url_pattern.sub(' ', text) # Remove emails if self.remove_emails: text = self.email_pattern.sub(' ', text) # Convert to lowercase if self.lowercase: text = text.lower() # Remove numbers if self.remove_numbers: text = self.number_pattern.sub(' ', text) # Remove punctuation if self.remove_punctuation: text = text.translate(str.maketrans('', '', string.punctuation)) # Normalize unicode characters text = unicodedata.normalize('NFKD', text) # Normalize whitespace if self.normalize_whitespace: text = self.whitespace_pattern.sub(' ', text).strip() # Remove stopwords if self.remove_stopwords: words = text.split() words = [w for w in words if w not in self.stopwords] text = ' '.join(words) return text def preprocess_batch(self, texts: List[str]) -> List[str]: """ Apply preprocessing to a batch of texts. Args: texts: List of input text strings Returns: List of preprocessed text strings """ return [self.preprocess(text) for text in texts] def tokenize(self, text: str) -> List[str]: """ Simple whitespace tokenization after preprocessing. Args: text: Input text string Returns: List of tokens """ preprocessed = self.preprocess(text) return preprocessed.split() if preprocessed else [] @staticmethod def remove_special_characters(text: str, keep_spaces: bool = True) -> str: """ Remove all special characters from text. Args: text: Input text string keep_spaces: Whether to keep spaces Returns: Cleaned text string """ if keep_spaces: pattern = r'[^a-zA-Z0-9\s]' else: pattern = r'[^a-zA-Z0-9]' return re.sub(pattern, '', text) def load_and_preprocess(filepath: str, text_column: str = 'text') -> List[str]: """ Load data from CSV and preprocess text column. Args: filepath: Path to CSV file text_column: Name of the text column Returns: List of preprocessed texts Author: Molla Samser | RSK World (https://rskworld.in) """ import pandas as pd # Read CSV file df = pd.read_csv(filepath, comment='#') # Initialize preprocessor preprocessor = TextPreprocessor( lowercase=True, remove_punctuation=True, remove_urls=True, remove_stopwords=False # Keep stopwords for transformer models ) # Preprocess texts preprocessed_texts = preprocessor.preprocess_batch(df[text_column].tolist()) return preprocessed_texts if __name__ == "__main__": # Example usage sample_texts = [ "Apple unveils revolutionary new iPhone! Check it out at https://apple.com", "Manchester United wins Premier League title in dramatic fashion!", "Contact us at help@rskworld.in for more information." ] print("=" * 60) print("Text Preprocessing Demo - RSK World") print("Author: Molla Samser | Website: https://rskworld.in") print("=" * 60) preprocessor = TextPreprocessor( lowercase=True, remove_punctuation=True, remove_urls=True, remove_emails=True, remove_html=True ) for i, text in enumerate(sample_texts, 1): print(f"\nOriginal {i}: {text}") print(f"Processed {i}: {preprocessor.preprocess(text)}") print("\n" + "=" * 60) print("Preprocessing complete!")