"""
Customer Churn Dataset - Feature Selection Script
=================================================
Provided by: RSK World
Website: https://rskworld.in/
Email: help@rskworld.in
Phone: +91 93305 39277
Contact Page: https://rskworld.in/contact.php

This script performs advanced feature selection techniques.
"""

import pandas as pd
import numpy as np
from sklearn.feature_selection import (SelectKBest, f_classif, mutual_info_classif,
                                       chi2, RFE, SelectFromModel)
from sklearn.ensemble import RandomForestClassifier
from sklearn.preprocessing import LabelEncoder
import matplotlib.pyplot as plt
import seaborn as sns
import warnings
warnings.filterwarnings('ignore')

def load_data(file_path='../data/customer_churn_preprocessed.csv'):
    """
    Load preprocessed dataset
    """
    try:
        df = pd.read_csv(file_path)
        return df
    except FileNotFoundError:
        print("Preprocessed data not found. Please run data_preprocessing.py first.")
        return None

def prepare_data(df):
    """
    Prepare data for feature selection
    """
    exclude_cols = ['CustomerID', 'LastLogin', 'ChurnDate', 'Churn']
    feature_cols = [col for col in df.columns if col not in exclude_cols]
    
    X = df[feature_cols]
    y = df['Churn']
    
    return X, y, feature_cols

def univariate_feature_selection(X, y, k=10):
    """
    Perform univariate feature selection using f-test and mutual information
    """
    print("\n" + "="*60)
    print("UNIVARIATE FEATURE SELECTION")
    print("="*60)
    
    # F-test (ANOVA)
    selector_f = SelectKBest(score_func=f_classif, k=k)
    selector_f.fit(X, y)
    
    # Mutual Information
    selector_mi = SelectKBest(score_func=mutual_info_classif, k=k)
    selector_mi.fit(X, y)
    
    # Get feature scores
    f_scores = pd.DataFrame({
        'Feature': X.columns,
        'F_Score': selector_f.scores_,
        'MI_Score': selector_mi.scores_
    }).sort_values('F_Score', ascending=False)
    
    print("\nTop features by F-Score:")
    print(f_scores.head(k))
    
    # Visualize scores
    fig, axes = plt.subplots(2, 1, figsize=(12, 10))
    
    top_k_f = f_scores.nlargest(k, 'F_Score')
    axes[0].barh(range(len(top_k_f)), top_k_f['F_Score'])
    axes[0].set_yticks(range(len(top_k_f)))
    axes[0].set_yticklabels(top_k_f['Feature'])
    axes[0].set_xlabel('F-Score')
    axes[0].set_title(f'Top {k} Features by F-Score')
    axes[0].invert_yaxis()
    
    top_k_mi = f_scores.nlargest(k, 'MI_Score')
    axes[1].barh(range(len(top_k_mi)), top_k_mi['MI_Score'])
    axes[1].set_yticks(range(len(top_k_mi)))
    axes[1].set_yticklabels(top_k_mi['Feature'])
    axes[1].set_xlabel('Mutual Information Score')
    axes[1].set_title(f'Top {k} Features by Mutual Information')
    axes[1].invert_yaxis()
    
    plt.tight_layout()
    plt.savefig('../output/feature_selection_univariate.png')
    plt.close()
    
    return selector_f, selector_mi, f_scores

def recursive_feature_elimination(X, y, n_features=10):
    """
    Perform Recursive Feature Elimination
    """
    print("\n" + "="*60)
    print("RECURSIVE FEATURE ELIMINATION (RFE)")
    print("="*60)
    
    estimator = RandomForestClassifier(n_estimators=100, random_state=42, n_jobs=-1)
    selector = RFE(estimator, n_features_to_select=n_features, step=1)
    selector = selector.fit(X, y)
    
    selected_features = X.columns[selector.support_]
    rankings = pd.DataFrame({
        'Feature': X.columns,
        'Ranking': selector.ranking_,
        'Selected': selector.support_
    }).sort_values('Ranking')
    
    print(f"\nTop {n_features} selected features:")
    print(selected_features.tolist())
    
    print("\nFeature rankings:")
    print(rankings)
    
    return selector, selected_features

def model_based_feature_selection(X, y):
    """
    Perform model-based feature selection
    """
    print("\n" + "="*60)
    print("MODEL-BASED FEATURE SELECTION")
    print("="*60)
    
    model = RandomForestClassifier(n_estimators=100, random_state=42, n_jobs=-1)
    model.fit(X, y)
    
    # Get feature importances
    importances = pd.DataFrame({
        'Feature': X.columns,
        'Importance': model.feature_importances_
    }).sort_values('Importance', ascending=False)
    
    print("\nFeature importances:")
    print(importances)
    
    # Select features with importance > threshold
    threshold = importances['Importance'].quantile(0.5)
    selector = SelectFromModel(model, threshold=threshold, prefit=True)
    selected_features = X.columns[selector.get_support()]
    
    print(f"\nFeatures selected (importance > {threshold:.4f}):")
    print(selected_features.tolist())
    
    # Visualize feature importances
    plt.figure(figsize=(12, 8))
    top_features = importances.head(15)
    plt.barh(range(len(top_features)), top_features['Importance'])
    plt.yticks(range(len(top_features)), top_features['Feature'])
    plt.xlabel('Importance')
    plt.title('Top 15 Feature Importances (Random Forest)')
    plt.gca().invert_yaxis()
    plt.tight_layout()
    plt.savefig('../output/feature_selection_model_based.png')
    plt.close()
    
    return selector, selected_features, importances

def main():
    """
    Main function to run feature selection pipeline
    """
    print("Customer Churn Dataset - Feature Selection")
    print("Provided by: RSK World (https://rskworld.in/)")
    print("="*60)
    
    # Load data
    df = load_data()
    if df is None:
        return
    
    # Prepare data
    X, y, feature_names = prepare_data(df)
    print(f"\nDataset shape: {X.shape}")
    print(f"Number of features: {len(feature_names)}")
    
    # Univariate feature selection
    selector_f, selector_mi, f_scores = univariate_feature_selection(X, y, k=15)
    
    # Recursive Feature Elimination
    rfe_selector, rfe_features = recursive_feature_elimination(X, y, n_features=15)
    
    # Model-based feature selection
    model_selector, model_features, importances = model_based_feature_selection(X, y)
    
    # Compare selected features
    print("\n" + "="*60)
    print("FEATURE SELECTION SUMMARY")
    print("="*60)
    
    all_selected = set(rfe_features) | set(model_features)
    common_features = set(rfe_features) & set(model_features)
    
    print(f"\nFeatures selected by RFE: {len(rfe_features)}")
    print(f"Features selected by Model-based: {len(model_features)}")
    print(f"Common features: {len(common_features)}")
    print(f"\nCommon important features:")
    print(sorted(common_features))
    
    # Save results
    results = {
        'f_test_features': selector_f.get_support(),
        'mi_features': selector_mi.get_support(),
        'rfe_features': rfe_selector.support_,
        'model_features': model_selector.get_support()
    }
    
    results_df = pd.DataFrame(results, index=X.columns)
    results_df['Selected_Count'] = results_df.sum(axis=1)
    results_df = results_df.sort_values('Selected_Count', ascending=False)
    results_df.to_csv('../output/feature_selection_results.csv')
    
    print("\nFeature selection results saved to '../output/feature_selection_results.csv'")
    
    print("\n" + "="*60)
    print("FEATURE SELECTION COMPLETE")
    print("="*60)
    print("\nFor more information, visit: https://rskworld.in/")
    print("Contact: help@rskworld.in | +91 93305 39277")

if __name__ == "__main__":
    main()