help@rskworld.in +91 93305 39277
RSK World
  • Home
  • Development
    • Web Development
    • Mobile Apps
    • Software
    • Games
    • Project
  • Technologies
    • Data Science
    • AI Development
    • Cloud Development
    • Blockchain
    • Cyber Security
    • Dev Tools
    • Testing Tools
  • Blog
  • About
  • Contact

Theme Settings

Color Scheme
Display Options
Font Size
100%
Back to Project
RSK World
text-classification
/
scripts
RSK World
text-classification
Text Classification Dataset - NLP + Multi-Class Classification + Machine Learning
scripts
  • __init__.py2.3 KB
  • active_learning.py26.8 KB
  • api_server.py12.7 KB
  • batch_processor.py16.4 KB
  • data_augmentation.py18.2 KB
  • data_quality.py20 KB
  • deep_learning.py24.2 KB
  • hyperparameter_tuning.py22.5 KB
  • model_explainability.py17.9 KB
  • preprocessing.py8.7 KB
  • train_classifier.py13.8 KB
  • train_transformers.py12.5 KB
  • visualizations.py19 KB
generate_data_standalone.pypreprocessing.pyapi_server.py
scripts/preprocessing.py
Raw Download
Find: Go to:
"""
================================================================================
Text Classification Dataset - Text Preprocessing Module
================================================================================
Project: Text Classification Dataset
Category: Text Data / NLP

Author: Molla Samser
Designer & Tester: Rima Khatun
Website: https://rskworld.in
Email: help@rskworld.in | support@rskworld.in
Phone: +91 93305 39277

Copyright (c) 2026 RSK World - All Rights Reserved
Content used for educational purposes only.

Created: December 2026
================================================================================
"""

import re
import string
from typing import List, Optional
import unicodedata


class TextPreprocessor:
    """
    A comprehensive text preprocessing class for NLP tasks.
    
    Features:
    - Lowercase conversion
    - Punctuation removal
    - Number handling
    - Stopword removal
    - Whitespace normalization
    - URL and email removal
    - HTML tag removal
    - Special character handling
    
    Author: Molla Samser | RSK World (https://rskworld.in)
    """
    
    # Common English stopwords
    STOPWORDS = {
        'a', 'an', 'the', 'and', 'or', 'but', 'in', 'on', 'at', 'to', 'for',
        'of', 'with', 'by', 'from', 'as', 'is', 'was', 'are', 'were', 'been',
        'be', 'have', 'has', 'had', 'do', 'does', 'did', 'will', 'would',
        'could', 'should', 'may', 'might', 'must', 'shall', 'can', 'need',
        'it', 'its', "it's", 'this', 'that', 'these', 'those', 'i', 'me',
        'my', 'myself', 'we', 'our', 'ours', 'ourselves', 'you', "you're",
        'your', 'yours', 'yourself', 'yourselves', 'he', 'him', 'his',
        'himself', 'she', "she's", 'her', 'hers', 'herself', 'they', 'them',
        'their', 'theirs', 'themselves', 'what', 'which', 'who', 'whom',
        'when', 'where', 'why', 'how', 'all', 'each', 'every', 'both', 'few',
        'more', 'most', 'other', 'some', 'such', 'no', 'nor', 'not', 'only',
        'own', 'same', 'so', 'than', 'too', 'very', 'just', 'about', 'above',
        'after', 'again', 'against', 'between', 'into', 'through', 'during',
        'before', 'below', 'under', 'over', 'out', 'off', 'down', 'up'
    }
    
    def __init__(
        self,
        lowercase: bool = True,
        remove_punctuation: bool = True,
        remove_numbers: bool = False,
        remove_stopwords: bool = False,
        remove_urls: bool = True,
        remove_emails: bool = True,
        remove_html: bool = True,
        normalize_whitespace: bool = True,
        custom_stopwords: Optional[List[str]] = None
    ):
        """
        Initialize the TextPreprocessor with configurable options.
        
        Args:
            lowercase: Convert text to lowercase
            remove_punctuation: Remove punctuation marks
            remove_numbers: Remove numeric characters
            remove_stopwords: Remove common stopwords
            remove_urls: Remove URLs from text
            remove_emails: Remove email addresses
            remove_html: Remove HTML tags
            normalize_whitespace: Normalize multiple spaces to single space
            custom_stopwords: Additional stopwords to remove
        """
        self.lowercase = lowercase
        self.remove_punctuation = remove_punctuation
        self.remove_numbers = remove_numbers
        self.remove_stopwords = remove_stopwords
        self.remove_urls = remove_urls
        self.remove_emails = remove_emails
        self.remove_html = remove_html
        self.normalize_whitespace = normalize_whitespace
        
        # Combine default and custom stopwords
        self.stopwords = self.STOPWORDS.copy()
        if custom_stopwords:
            self.stopwords.update(set(custom_stopwords))
        
        # Compile regex patterns for efficiency
        self.url_pattern = re.compile(
            r'https?://\S+|www\.\S+'
        )
        self.email_pattern = re.compile(
            r'\b[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Z|a-z]{2,}\b'
        )
        self.html_pattern = re.compile(r'<[^>]+>')
        self.whitespace_pattern = re.compile(r'\s+')
        self.number_pattern = re.compile(r'\d+')
    
    def preprocess(self, text: str) -> str:
        """
        Apply all preprocessing steps to the input text.
        
        Args:
            text: Input text string
            
        Returns:
            Preprocessed text string
        """
        if not text or not isinstance(text, str):
            return ""
        
        # Remove HTML tags
        if self.remove_html:
            text = self.html_pattern.sub(' ', text)
        
        # Remove URLs
        if self.remove_urls:
            text = self.url_pattern.sub(' ', text)
        
        # Remove emails
        if self.remove_emails:
            text = self.email_pattern.sub(' ', text)
        
        # Convert to lowercase
        if self.lowercase:
            text = text.lower()
        
        # Remove numbers
        if self.remove_numbers:
            text = self.number_pattern.sub(' ', text)
        
        # Remove punctuation
        if self.remove_punctuation:
            text = text.translate(str.maketrans('', '', string.punctuation))
        
        # Normalize unicode characters
        text = unicodedata.normalize('NFKD', text)
        
        # Normalize whitespace
        if self.normalize_whitespace:
            text = self.whitespace_pattern.sub(' ', text).strip()
        
        # Remove stopwords
        if self.remove_stopwords:
            words = text.split()
            words = [w for w in words if w not in self.stopwords]
            text = ' '.join(words)
        
        return text
    
    def preprocess_batch(self, texts: List[str]) -> List[str]:
        """
        Apply preprocessing to a batch of texts.
        
        Args:
            texts: List of input text strings
            
        Returns:
            List of preprocessed text strings
        """
        return [self.preprocess(text) for text in texts]
    
    def tokenize(self, text: str) -> List[str]:
        """
        Simple whitespace tokenization after preprocessing.
        
        Args:
            text: Input text string
            
        Returns:
            List of tokens
        """
        preprocessed = self.preprocess(text)
        return preprocessed.split() if preprocessed else []
    
    @staticmethod
    def remove_special_characters(text: str, keep_spaces: bool = True) -> str:
        """
        Remove all special characters from text.
        
        Args:
            text: Input text string
            keep_spaces: Whether to keep spaces
            
        Returns:
            Cleaned text string
        """
        if keep_spaces:
            pattern = r'[^a-zA-Z0-9\s]'
        else:
            pattern = r'[^a-zA-Z0-9]'
        return re.sub(pattern, '', text)


def load_and_preprocess(filepath: str, text_column: str = 'text') -> List[str]:
    """
    Load data from CSV and preprocess text column.
    
    Args:
        filepath: Path to CSV file
        text_column: Name of the text column
        
    Returns:
        List of preprocessed texts
        
    Author: Molla Samser | RSK World (https://rskworld.in)
    """
    import pandas as pd
    
    # Read CSV file
    df = pd.read_csv(filepath, comment='#')
    
    # Initialize preprocessor
    preprocessor = TextPreprocessor(
        lowercase=True,
        remove_punctuation=True,
        remove_urls=True,
        remove_stopwords=False  # Keep stopwords for transformer models
    )
    
    # Preprocess texts
    preprocessed_texts = preprocessor.preprocess_batch(df[text_column].tolist())
    
    return preprocessed_texts


if __name__ == "__main__":
    # Example usage
    sample_texts = [
        "Apple unveils revolutionary new iPhone! Check it out at https://apple.com",
        "Manchester United wins Premier League title in <b>dramatic</b> fashion!",
        "Contact us at help@rskworld.in for more information."
    ]
    
    print("=" * 60)
    print("Text Preprocessing Demo - RSK World")
    print("Author: Molla Samser | Website: https://rskworld.in")
    print("=" * 60)
    
    preprocessor = TextPreprocessor(
        lowercase=True,
        remove_punctuation=True,
        remove_urls=True,
        remove_emails=True,
        remove_html=True
    )
    
    for i, text in enumerate(sample_texts, 1):
        print(f"\nOriginal {i}: {text}")
        print(f"Processed {i}: {preprocessor.preprocess(text)}")
    
    print("\n" + "=" * 60)
    print("Preprocessing complete!")

269 lines•8.7 KB
python
scripts/api_server.py
Raw Download
Find: Go to:
"""
================================================================================
Text Classification Dataset - REST API Server
================================================================================
Project: Text Classification Dataset
Category: Text Data / NLP

Author: Molla Samser
Designer & Tester: Rima Khatun
Website: https://rskworld.in
Email: help@rskworld.in | support@rskworld.in
Phone: +91 93305 39277

Copyright (c) 2026 RSK World - All Rights Reserved
Content used for educational purposes only.

Features:
- Single text classification
- Batch classification
- Model info endpoint
- Category listing
- Health check

Usage:
    python api_server.py --model model.joblib --port 5000

API Endpoints:
    GET  /                  - API info
    GET  /health            - Health check
    GET  /categories        - List categories
    POST /predict           - Classify single text
    POST /predict/batch     - Classify multiple texts

Created: December 2026
================================================================================
"""

import os
import json
import argparse
from datetime import datetime
from typing import Dict, List, Optional

try:
    from flask import Flask, request, jsonify
    from flask_cors import CORS
except ImportError:
    print("Please install Flask: pip install flask flask-cors")
    exit(1)

import joblib
import numpy as np

# Project information
__author__ = "Molla Samser"
__website__ = "https://rskworld.in"
__email__ = "help@rskworld.in"
__version__ = "1.0.0"

# Initialize Flask app
app = Flask(__name__)
CORS(app)

# Global model storage
model_data = None
classifier = None
vectorizer = None

# Category mapping
CATEGORIES = {
    0: 'Technology',
    1: 'Sports',
    2: 'Politics',
    3: 'Entertainment',
    4: 'Business',
    5: 'Science'
}

CATEGORY_DESCRIPTIONS = {
    'Technology': 'Tech news, gadgets, software, AI, innovation',
    'Sports': 'Athletics, competitions, leagues, tournaments',
    'Politics': 'Government, policy, elections, international relations',
    'Entertainment': 'Movies, music, TV shows, celebrities, pop culture',
    'Business': 'Finance, markets, economy, corporate news',
    'Science': 'Research, discoveries, space, health, environment'
}


def load_model(model_path: str):
    """Load the trained model and vectorizer."""
    global model_data, classifier, vectorizer
    
    if not os.path.exists(model_path):
        raise FileNotFoundError(f"Model not found: {model_path}")
    
    model_data = joblib.load(model_path)
    
    # Extract pipeline components
    pipeline = model_data.get('pipeline')
    if pipeline:
        vectorizer = pipeline.named_steps.get('vectorizer')
        classifier = pipeline.named_steps.get('classifier')
    
    print(f"Model loaded successfully from: {model_path}")
    print(f"Algorithm: {model_data.get('algorithm', 'unknown')}")


def preprocess_text(text: str) -> str:
    """Basic text preprocessing."""
    import re
    import string
    
    # Lowercase
    text = text.lower()
    # Remove URLs
    text = re.sub(r'https?://\S+|www\.\S+', '', text)
    # Remove punctuation
    text = text.translate(str.maketrans('', '', string.punctuation))
    # Normalize whitespace
    text = ' '.join(text.split())
    
    return text


def predict_text(text: str) -> Dict:
    """
    Classify a single text.
    
    Args:
        text: Input text to classify
        
    Returns:
        Dictionary with prediction results
    """
    if model_data is None:
        return {'error': 'Model not loaded'}
    
    # Preprocess
    processed_text = preprocess_text(text)
    
    # Get prediction
    pipeline = model_data['pipeline']
    prediction = pipeline.predict([processed_text])[0]
    
    # Get probabilities if available
    probabilities = None
    if hasattr(pipeline, 'predict_proba'):
        try:
            probs = pipeline.predict_proba([processed_text])[0]
            probabilities = {CATEGORIES[i]: float(p) for i, p in enumerate(probs)}
        except:
            pass
    
    result = {
        'text': text[:200] + '...' if len(text) > 200 else text,
        'predicted_label': int(prediction),
        'predicted_category': CATEGORIES[prediction],
        'category_description': CATEGORY_DESCRIPTIONS[CATEGORIES[prediction]],
        'confidence': max(probabilities.values()) if probabilities else None,
        'probabilities': probabilities
    }
    
    return result


# ===== API Routes =====

@app.route('/')
def home():
    """API information endpoint."""
    return jsonify({
        'name': 'Text Classification API',
        'version': __version__,
        'author': __author__,
        'website': __website__,
        'email': __email__,
        'description': 'Multi-class text classification API for news categorization',
        'endpoints': {
            'GET /': 'API information',
            'GET /health': 'Health check',
            'GET /categories': 'List all categories',
            'GET /model/info': 'Model information',
            'POST /predict': 'Classify single text',
            'POST /predict/batch': 'Classify multiple texts'
        },
        'copyright': 'Copyright (c) 2026 RSK World - All Rights Reserved'
    })


@app.route('/health')
def health():
    """Health check endpoint."""
    return jsonify({
        'status': 'healthy',
        'model_loaded': model_data is not None,
        'timestamp': datetime.now().isoformat(),
        'version': __version__
    })


@app.route('/categories')
def categories():
    """List all categories."""
    return jsonify({
        'categories': [
            {
                'id': label,
                'name': name,
                'description': CATEGORY_DESCRIPTIONS[name]
            }
            for label, name in CATEGORIES.items()
        ],
        'total': len(CATEGORIES)
    })


@app.route('/model/info')
def model_info():
    """Get model information."""
    if model_data is None:
        return jsonify({'error': 'Model not loaded'}), 500
    
    return jsonify({
        'algorithm': model_data.get('algorithm', 'unknown'),
        'max_features': model_data.get('max_features', 'unknown'),
        'ngram_range': model_data.get('ngram_range', 'unknown'),
        'training_history': model_data.get('training_history', {}),
        'metadata': model_data.get('metadata', {})
    })


@app.route('/predict', methods=['POST'])
def predict():
    """
    Classify a single text.
    
    Request body:
        {"text": "Your text here"}
        
    Returns:
        Classification result with category and confidence
    """
    if model_data is None:
        return jsonify({'error': 'Model not loaded'}), 500
    
    data = request.get_json()
    
    if not data or 'text' not in data:
        return jsonify({
            'error': 'Missing required field: text',
            'usage': {'text': 'Your text to classify'}
        }), 400
    
    text = data['text']
    
    if not text or len(text.strip()) == 0:
        return jsonify({'error': 'Text cannot be empty'}), 400
    
    result = predict_text(text)
    result['request_timestamp'] = datetime.now().isoformat()
    
    return jsonify(result)


@app.route('/predict/batch', methods=['POST'])
def predict_batch():
    """
    Classify multiple texts.
    
    Request body:
        {"texts": ["Text 1", "Text 2", ...]}
        
    Returns:
        List of classification results
    """
    if model_data is None:
        return jsonify({'error': 'Model not loaded'}), 500
    
    data = request.get_json()
    
    if not data or 'texts' not in data:
        return jsonify({
            'error': 'Missing required field: texts',
            'usage': {'texts': ['Text 1', 'Text 2']}
        }), 400
    
    texts = data['texts']
    
    if not isinstance(texts, list):
        return jsonify({'error': 'texts must be a list'}), 400
    
    if len(texts) == 0:
        return jsonify({'error': 'texts list cannot be empty'}), 400
    
    if len(texts) > 100:
        return jsonify({'error': 'Maximum 100 texts per batch'}), 400
    
    results = [predict_text(text) for text in texts]
    
    return jsonify({
        'results': results,
        'total': len(results),
        'request_timestamp': datetime.now().isoformat()
    })


@app.route('/analyze', methods=['POST'])
def analyze():
    """
    Detailed text analysis with statistics.
    
    Request body:
        {"text": "Your text here"}
        
    Returns:
        Classification + text statistics
    """
    if model_data is None:
        return jsonify({'error': 'Model not loaded'}), 500
    
    data = request.get_json()
    
    if not data or 'text' not in data:
        return jsonify({'error': 'Missing required field: text'}), 400
    
    text = data['text']
    
    # Get prediction
    result = predict_text(text)
    
    # Add text statistics
    words = text.split()
    result['statistics'] = {
        'character_count': len(text),
        'word_count': len(words),
        'sentence_count': text.count('.') + text.count('!') + text.count('?'),
        'average_word_length': sum(len(w) for w in words) / len(words) if words else 0,
        'unique_words': len(set(w.lower() for w in words))
    }
    
    result['request_timestamp'] = datetime.now().isoformat()
    
    return jsonify(result)


@app.errorhandler(404)
def not_found(error):
    """Handle 404 errors."""
    return jsonify({
        'error': 'Endpoint not found',
        'message': 'Please check the API documentation at /',
        'website': __website__
    }), 404


@app.errorhandler(500)
def server_error(error):
    """Handle 500 errors."""
    return jsonify({
        'error': 'Internal server error',
        'message': 'Please try again or contact support',
        'email': __email__
    }), 500


def create_demo_model():
    """Create a demo model for testing without training."""
    from sklearn.feature_extraction.text import TfidfVectorizer
    from sklearn.linear_model import LogisticRegression
    from sklearn.pipeline import Pipeline
    
    # Simple demo data
    texts = [
        "Apple launches new iPhone with AI features",
        "Manchester United wins Premier League",
        "Congress passes new legislation",
        "Taylor Swift breaks streaming records",
        "Stock market reaches all-time high",
        "NASA discovers new exoplanet"
    ]
    labels = [0, 1, 2, 3, 4, 5]
    
    # Create pipeline
    pipeline = Pipeline([
        ('vectorizer', TfidfVectorizer(max_features=1000)),
        ('classifier', LogisticRegression(max_iter=1000))
    ])
    
    pipeline.fit(texts, labels)
    
    return {
        'pipeline': pipeline,
        'algorithm': 'logistic_regression',
        'max_features': 1000,
        'ngram_range': (1, 1),
        'metadata': {
            'author': __author__,
            'website': __website__,
            'demo': True
        }
    }


def main():
    """Main entry point."""
    parser = argparse.ArgumentParser(
        description='Text Classification API - RSK World'
    )
    parser.add_argument(
        '--model', type=str, default='model.joblib',
        help='Path to trained model file'
    )
    parser.add_argument(
        '--port', type=int, default=5000,
        help='Port to run the server on'
    )
    parser.add_argument(
        '--host', type=str, default='0.0.0.0',
        help='Host to bind to'
    )
    parser.add_argument(
        '--debug', action='store_true',
        help='Run in debug mode'
    )
    parser.add_argument(
        '--demo', action='store_true',
        help='Run with demo model (no training required)'
    )
    
    args = parser.parse_args()
    
    print(f"\n{'='*60}")
    print("Text Classification API - RSK World")
    print(f"Author: {__author__}")
    print(f"Website: {__website__}")
    print(f"Email: {__email__}")
    print(f"{'='*60}\n")
    
    global model_data
    
    if args.demo:
        print("Creating demo model...")
        model_data = create_demo_model()
        print("Demo model created successfully!")
    else:
        try:
            load_model(args.model)
        except FileNotFoundError:
            print(f"Model not found at: {args.model}")
            print("Creating demo model instead...")
            model_data = create_demo_model()
    
    print(f"\nStarting server on http://{args.host}:{args.port}")
    print("Press Ctrl+C to stop\n")
    
    app.run(host=args.host, port=args.port, debug=args.debug)


if __name__ == '__main__':
    main()

462 lines•12.7 KB
python
🚀 Support RSK World

Subscribe to our YouTube channel for latest tutorials & updates!



Click subscribe & support our work ❤️

About RSK World

Founded by Molla Samser, with Designer & Tester Rima Khatun, RSK World is your one-stop destination for free programming resources, source code, and development tools.

Founder: Molla Samser
Designer & Tester: Rima Khatun

Development

  • Game Development
  • Web Development
  • Mobile Development
  • AI Development
  • Development Tools

Legal

  • Terms & Conditions
  • Privacy Policy
  • Disclaimer

Contact Info

Nutanhat, Mongolkote
Purba Burdwan, West Bengal
India, 713147

+91 93305 39277

hello@rskworld.in
support@rskworld.in

© 2026 RSK World. All rights reserved.

Content used for educational purposes only. View Disclaimer