"""
Text Preprocessing Module
Handles text cleaning, tokenization, and normalization

Developer: RSK World
Website: https://rskworld.in
Email: help@rskworld.in
Phone: +91 93305 39277
Year: 2026
"""

import re
from nltk.tokenize import word_tokenize, sent_tokenize
from nltk.corpus import stopwords
from nltk.stem import WordNetLemmatizer
import nltk

# Download required NLTK data
try:
    nltk.data.find('tokenizers/punkt')
except LookupError:
    nltk.download('punkt', quiet=True)

try:
    nltk.data.find('corpora/stopwords')
except LookupError:
    nltk.download('stopwords', quiet=True)

try:
    nltk.data.find('corpora/wordnet')
except LookupError:
    nltk.download('wordnet', quiet=True)

try:
    nltk.data.find('taggers/averaged_perceptron_tagger')
except LookupError:
    nltk.download('averaged_perceptron_tagger', quiet=True)

class TextPreprocessor:
    """
    Text preprocessing class for cleaning and normalizing text
    Developer: RSK World - https://rskworld.in
    """
    
    def __init__(self):
        """Initialize preprocessor with stopwords and lemmatizer"""
        self.stop_words = set(stopwords.words('english'))
        self.lemmatizer = WordNetLemmatizer()
    
    def clean_text(self, text):
        """
        Clean text by removing special characters and normalizing
        
        Args:
            text (str): Raw text input
            
        Returns:
            str: Cleaned text
        """
        # Convert to lowercase
        text = text.lower()
        
        # Remove URLs
        text = re.sub(r'http\S+|www\S+|https\S+', '', text, flags=re.MULTILINE)
        
        # Remove email addresses
        text = re.sub(r'\S+@\S+', '', text)
        
        # Remove special characters but keep punctuation for sentence detection
        text = re.sub(r'[^a-zA-Z\s]', '', text)
        
        # Remove extra whitespace
        text = re.sub(r'\s+', ' ', text).strip()
        
        return text
    
    def tokenize(self, text):
        """
        Tokenize text into words and sentences
        
        Args:
            text (str): Input text
            
        Returns:
            dict: Tokenized words and sentences
        """
        words = word_tokenize(text)
        sentences = sent_tokenize(text)
        
        return {
            'words': words,
            'sentences': sentences
        }
    
    def remove_stopwords(self, tokens):
        """
        Remove stopwords from token list
        
        Args:
            tokens (list): List of word tokens
            
        Returns:
            list: Tokens without stopwords
        """
        return [token for token in tokens if token.lower() not in self.stop_words]
    
    def lemmatize(self, tokens):
        """
        Lemmatize tokens to their root form
        
        Args:
            tokens (list): List of word tokens
            
        Returns:
            list: Lemmatized tokens
        """
        return [self.lemmatizer.lemmatize(token) for token in tokens]
    
    def preprocess(self, text):
        """
        Complete preprocessing pipeline
        
        Args:
            text (str): Raw input text
            
        Returns:
            dict: Preprocessed text data
        """
        # Clean text
        cleaned_text = self.clean_text(text)
        
        # Tokenize
        tokenized = self.tokenize(cleaned_text)
        
        # Remove stopwords
        filtered_tokens = self.remove_stopwords(tokenized['words'])
        
        # Lemmatize
        lemmatized_tokens = self.lemmatize(filtered_tokens)
        
        return {
            'original_text': text,
            'cleaned_text': cleaned_text,
            'tokens': tokenized['words'],
            'filtered_tokens': filtered_tokens,
            'lemmatized_tokens': lemmatized_tokens,
            'sentences': tokenized['sentences'],
            'word_count': len(tokenized['words']),
            'sentence_count': len(tokenized['sentences']),
            'unique_words': len(set(tokenized['words'])),
            'vocabulary_richness': len(set(tokenized['words'])) / len(tokenized['words']) if tokenized['words'] else 0
        }