RSK World - Statsmodels Statistical Modeling - Project Files | RSK World

.gitignore index.html panel_data_analysis.py

.gitignore

# Python
__pycache__/
*.py[cod]
*$py.class
*.so
.Python
build/
develop-eggs/
dist/
downloads/
eggs/
.eggs/
lib/
lib64/
parts/
sdist/
var/
wheels/
*.egg-info/
.installed.cfg
*.egg

# Virtual Environment
venv/
env/
ENV/
.venv

# Jupyter Notebook
.ipynb_checkpoints
*.ipynb_checkpoints/

# IDE
.vscode/
.idea/
*.swp
*.swo
*~

# OS
.DS_Store
Thumbs.db

# Project specific
*.log
.pytest_cache/
.coverage
htmlcov/

50 lines•458 B

text

index.html

Raw Download

<!DOCTYPE html>
<html lang="en">
<head>
    <!--
    Author: RSK World
    Website: https://rskworld.in
    Email: help@rskworld.in
    Phone: +91 93305 39277
    Description: Statsmodels Statistical Modeling Project Demo
    -->
    <meta charset="UTF-8">
    <meta name="viewport" content="width=device-width, initial-scale=1.0">
    <title>Statsmodels Statistical Modeling - RSK World</title>
    <style>
        * {
            margin: 0;
            padding: 0;
            box-sizing: border-box;
        }
        
        body {
            font-family: 'Segoe UI', Tahoma, Geneva, Verdana, sans-serif;
            line-height: 1.6;
            color: #333;
            background: linear-gradient(135deg, #667eea 0%, #764ba2 100%);
            min-height: 100vh;
            padding: 20px;
        }
        
        .container {
            max-width: 1200px;
            margin: 0 auto;
            background: white;
            border-radius: 10px;
            box-shadow: 0 10px 30px rgba(0,0,0,0.3);
            overflow: hidden;
        }
        
        header {
            background: linear-gradient(135deg, #667eea 0%, #764ba2 100%);
            color: white;
            padding: 40px;
            text-align: center;
        }
        
        header h1 {
            font-size: 2.5em;
            margin-bottom: 10px;
        }
        
        header p {
            font-size: 1.2em;
            opacity: 0.9;
        }
        
        .content {
            padding: 40px;
        }
        
        .section {
            margin-bottom: 40px;
        }
        
        .section h2 {
            color: #667eea;
            border-bottom: 3px solid #667eea;
            padding-bottom: 10px;
            margin-bottom: 20px;
        }
        
        .features {
            display: grid;
            grid-template-columns: repeat(auto-fit, minmax(250px, 1fr));
            gap: 20px;
            margin-top: 20px;
        }
        
        .feature-card {
            background: #f8f9fa;
            padding: 20px;
            border-radius: 8px;
            border-left: 4px solid #667eea;
            transition: transform 0.3s;
        }
        
        .feature-card:hover {
            transform: translateY(-5px);
            box-shadow: 0 5px 15px rgba(0,0,0,0.1);
        }
        
        .feature-card h3 {
            color: #667eea;
            margin-bottom: 10px;
        }
        
        .tech-stack {
            display: flex;
            flex-wrap: wrap;
            gap: 10px;
            margin-top: 15px;
        }
        
        .tech-badge {
            background: #667eea;
            color: white;
            padding: 8px 15px;
            border-radius: 20px;
            font-size: 0.9em;
        }
        
        .code-block {
            background: #2d2d2d;
            color: #f8f8f2;
            padding: 20px;
            border-radius: 5px;
            overflow-x: auto;
            margin: 20px 0;
        }
        
        .code-block code {
            font-family: 'Courier New', monospace;
        }
        
        .btn {
            display: inline-block;
            padding: 12px 30px;
            background: #667eea;
            color: white;
            text-decoration: none;
            border-radius: 5px;
            transition: background 0.3s;
            margin: 10px 10px 10px 0;
        }
        
        .btn:hover {
            background: #5568d3;
        }
        
        footer {
            background: #2d2d2d;
            color: white;
            padding: 30px;
            text-align: center;
        }
        
        footer a {
            color: #667eea;
            text-decoration: none;
        }
        
        footer a:hover {
            text-decoration: underline;
        }
        
        .author-info {
            background: #f8f9fa;
            padding: 20px;
            border-radius: 8px;
            margin-top: 30px;
        }
        
        .author-info h3 {
            color: #667eea;
            margin-bottom: 15px;
        }
        
        .author-info p {
            margin: 5px 0;
        }
    </style>
</head>
<body>
    <div class="container">
        <header>
            <h1><i class="fas fa-microscope"></i> Statsmodels Statistical Modeling</h1>
            <p>Advanced Statistical Analysis and Econometric Modeling</p>
        </header>
        
        <div class="content">
            <div class="section">
                <h2>About This Project</h2>
                <p>This project demonstrates Statsmodels, a powerful library for statistical modeling and econometrics in Python. It covers linear and generalized linear models, time series analysis, hypothesis testing, statistical tests, and diagnostic tools. Perfect for statistical analysis and econometric modeling.</p>
            </div>
            
            <div class="section">
                <h2>Features</h2>
                <div class="features">
                    <div class="feature-card">
                        <h3>Linear & GLM Regression</h3>
                        <p>Comprehensive regression analysis with diagnostic tools and model validation.</p>
                    </div>
                    <div class="feature-card">
                        <h3>Time Series Analysis</h3>
                        <p>ARIMA, SARIMA models, decomposition, forecasting, and stationarity testing.</p>
                    </div>
                    <div class="feature-card">
                        <h3>Advanced Time Series</h3>
                        <p>Auto ARIMA selection, SARIMA models, comprehensive stationarity tests.</p>
                    </div>
                    <div class="feature-card">
                        <h3>Hypothesis Testing</h3>
                        <p>T-tests, ANOVA, chi-square, normality tests, and non-parametric tests.</p>
                    </div>
                    <div class="feature-card">
                        <h3>Statistical Diagnostics</h3>
                        <p>Comprehensive model diagnostics including multicollinearity, heteroscedasticity, and autocorrelation checks.</p>
                    </div>
                    <div class="feature-card">
                        <h3>Econometric Modeling</h3>
                        <p>VAR, VARMAX models, cointegration tests, impulse response functions, and Granger causality.</p>
                    </div>
                    <div class="feature-card">
                        <h3>Model Selection</h3>
                        <p>Stepwise selection, model comparison, information criteria (AIC, BIC, HQIC).</p>
                    </div>
                    <div class="feature-card">
                        <h3>Model Evaluation</h3>
                        <p>Cross-validation, time series CV, multiple metrics (MSE, MAE, R², MAPE), learning curves.</p>
                    </div>
                    <div class="feature-card">
                        <h3>Feature Selection</h3>
                        <p>VIF-based removal, correlation filtering, automated feature selection.</p>
                    </div>
                    <div class="feature-card">
                        <h3>Data Preprocessing</h3>
                        <p>Missing value handling, outlier detection/removal, scaling, stationarity transformation.</p>
                    </div>
                    <div class="feature-card">
                        <h3>Visualization</h3>
                        <p>Comprehensive plotting functions for all analyses including residuals, forecasts, and comparisons.</p>
                    </div>
                </div>
            </div>
            
            <div class="section">
                <h2>Technologies</h2>
                <div class="tech-stack">
                    <span class="tech-badge">Python</span>
                    <span class="tech-badge">Statsmodels</span>
                    <span class="tech-badge">Pandas</span>
                    <span class="tech-badge">NumPy</span>
                    <span class="tech-badge">Matplotlib</span>
                    <span class="tech-badge">Seaborn</span>
                    <span class="tech-badge">Scikit-learn</span>
                    <span class="tech-badge">SciPy</span>
                    <span class="tech-badge">Jupyter Notebook</span>
                </div>
            </div>
            
            <div class="section">
                <h2>Quick Start</h2>
                <div class="code-block">
                    <code>
# Install dependencies<br>
pip install -r requirements.txt<br><br>
# Run regression example<br>
python examples/regression_example.py<br><br>
# Run time series example<br>
python examples/time_series_example.py<br><br>
# Run hypothesis testing example<br>
python examples/hypothesis_testing_example.py
                    </code>
                </div>
            </div>
            
            <div class="section">
                <h2>Project Structure</h2>
                <div class="code-block">
                    <code>
statsmodels-statistical/<br>
├── regression_analysis.py<br>
├── time_series_analysis.py<br>
├── advanced_time_series.py<br>
├── hypothesis_testing.py<br>
├── statistical_diagnostics.py<br>
├── econometric_modeling.py<br>
├── model_selection.py<br>
├── model_evaluation.py<br>
├── data_preprocessing.py<br>
├── visualization_utils.py<br>
├── notebooks/<br>
│   ├── 01_linear_regression.ipynb<br>
│   ├── 02_time_series.ipynb<br>
│   ├── 03_hypothesis_testing.ipynb<br>
│   └── 04_econometric_modeling.ipynb<br>
├── examples/<br>
│   ├── regression_example.py<br>
│   ├── time_series_example.py<br>
│   ├── hypothesis_testing_example.py<br>
│   ├── model_selection_example.py<br>
│   └── advanced_time_series_example.py<br>
└── data/<br>
    └── sample_data.csv
                    </code>
                </div>
            </div>
            
            <div class="author-info">
                <h3>Project Author</h3>
                <p><strong>RSK World</strong></p>
                <p>Website: <a href="https://rskworld.in" target="_blank">https://rskworld.in</a></p>
                <p>Email: <a href="mailto:help@rskworld.in">help@rskworld.in</a></p>
                <p>Phone: +91 93305 39277</p>
            </div>
        </div>
        
        <footer>
            <p>&copy; 2025 RSK World. All rights reserved.</p>
            <p>Visit <a href="https://rskworld.in" target="_blank">rskworld.in</a> for more programming resources and source code.</p>
        </footer>
    </div>
</body>
</html>

315 lines•10.8 KB

markup

panel_data_analysis.py

Raw Download

"""
Panel Data Analysis

Author: RSK World
Website: https://rskworld.in
Email: help@rskworld.in
Phone: +91 93305 39277
"""

import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
from scipy import stats
from statsmodels.regression.linear_model import OLS
from statsmodels.tools.tools import add_constant
import warnings
warnings.filterwarnings('ignore')


class PanelDataAnalysis:
    """
    Panel Data Analysis Tools
    
    Author: RSK World
    Website: https://rskworld.in
    Email: help@rskworld.in
    Phone: +91 93305 39277
    """
    
    def __init__(self):
        self.data = None
        self.entity_col = None
        self.time_col = None
    
    def prepare_panel_data(self, data, entity_col, time_col, value_cols):
        """
        Prepare data for panel analysis
        
        Parameters:
        -----------
        data : DataFrame
            Panel data
        entity_col : str
            Entity identifier column
        time_col : str
            Time identifier column
        value_cols : list
            Value columns to analyze
        """
        self.data = data.copy()
        self.entity_col = entity_col
        self.time_col = time_col
        
        # Set multi-index
        if not isinstance(data.index, pd.MultiIndex):
            self.data = self.data.set_index([entity_col, time_col])
        
        return self.data
    
    def fixed_effects_regression(self, y_col, X_cols):
        """
        Fixed Effects Panel Regression
        
        Parameters:
        -----------
        y_col : str
            Dependent variable column
        X_cols : list
            Independent variable columns
        """
        if self.data is None:
            raise ValueError("Data not prepared. Call prepare_panel_data() first.")
        
        # Create entity dummies
        entities = self.data.index.get_level_values(0).unique()
        entity_dummies = pd.get_dummies(self.data.index.get_level_values(0), 
                                       prefix='entity')
        entity_dummies.index = self.data.index
        
        # Combine features
        X = pd.concat([self.data[X_cols], entity_dummies], axis=1)
        y = self.data[y_col]
        
        # Fit model
        model = OLS(y, add_constant(X)).fit()
        
        print("Fixed Effects Panel Regression:")
        print("=" * 70)
        print(model.summary())
        
        return model
    
    def random_effects_regression(self, y_col, X_cols):
        """
        Random Effects Panel Regression
        
        Parameters:
        -----------
        y_col : str
            Dependent variable column
        X_cols : list
            Independent variable columns
        """
        if self.data is None:
            raise ValueError("Data not prepared. Call prepare_panel_data() first.")
        
        # Group by entity and calculate means
        entity_means = self.data.groupby(level=0)[X_cols + [y_col]].mean()
        
        # Demean data
        data_demeaned = self.data.copy()
        for col in X_cols + [y_col]:
            entity_mean = entity_means[col]
            data_demeaned[col] = data_demeaned[col] - data_demeaned.index.get_level_values(0).map(entity_mean)
        
        # Fit model on demeaned data
        X = data_demeaned[X_cols]
        y = data_demeaned[y_col]
        
        model = OLS(y, add_constant(X)).fit()
        
        print("Random Effects Panel Regression:")
        print("=" * 70)
        print(model.summary())
        
        return model
    
    def hausman_test(self, y_col, X_cols):
        """
        Hausman Test for Fixed vs Random Effects
        
        Parameters:
        -----------
        y_col : str
            Dependent variable column
        X_cols : list
            Independent variable columns
        """
        # Fixed effects
        fe_model = self.fixed_effects_regression(y_col, X_cols)
        
        # Random effects
        re_model = self.random_effects_regression(y_col, X_cols)
        
        # Extract coefficients
        fe_coef = fe_model.params[X_cols]
        re_coef = re_model.params[X_cols]
        
        # Calculate test statistic
        diff = fe_coef - re_coef
        fe_cov = fe_model.cov_params().loc[X_cols, X_cols]
        re_cov = re_model.cov_params().loc[X_cols, X_cols]
        cov_diff = fe_cov - re_cov
        
        try:
            test_stat = diff.T @ np.linalg.inv(cov_diff) @ diff
            df = len(X_cols)
            p_value = 1 - stats.chi2.cdf(test_stat, df)
            
            print("\nHausman Test:")
            print("=" * 70)
            print(f"Test Statistic: {test_stat:.4f}")
            print(f"Degrees of Freedom: {df}")
            print(f"p-value: {p_value:.4f}")
            
            if p_value < 0.05:
                print("Result: Reject null hypothesis - Use Fixed Effects")
            else:
                print("Result: Fail to reject null hypothesis - Use Random Effects")
            
            return {'test_statistic': test_stat, 'p_value': p_value, 'df': df}
        except:
            print("Hausman test could not be computed (singular matrix)")
            return None
    
    def plot_panel_data(self, y_col, entity_subset=None):
        """Plot panel data over time"""
        if self.data is None:
            raise ValueError("Data not prepared. Call prepare_panel_data() first.")
        
        if entity_subset is None:
            entities = self.data.index.get_level_values(0).unique()[:10]  # First 10
        else:
            entities = entity_subset
        
        plt.figure(figsize=(12, 6))
        
        for entity in entities:
            entity_data = self.data.loc[entity, y_col]
            if isinstance(entity_data, pd.Series):
                plt.plot(entity_data.index, entity_data.values, 
                        label=f'Entity {entity}', alpha=0.7)
        
        plt.xlabel('Time')
        plt.ylabel(y_col)
        plt.title('Panel Data Over Time')
        plt.legend(bbox_to_anchor=(1.05, 1), loc='upper left')
        plt.grid(True, alpha=0.3)
        plt.tight_layout()
        plt.show()


if __name__ == "__main__":
    # Example usage
    print("Panel Data Analysis Example")
    print("=" * 70)
    
    # Generate sample panel data
    np.random.seed(42)
    n_entities = 5
    n_periods = 10
    
    data_list = []
    for entity in range(n_entities):
        for period in range(n_periods):
            data_list.append({
                'entity': entity,
                'time': period,
                'X1': np.random.randn(),
                'X2': np.random.randn(),
                'y': 2 + 1.5 * np.random.randn() + 0.8 * np.random.randn() + np.random.randn() * 0.5
            })
    
    df = pd.DataFrame(data_list)
    
    # Create panel analysis
    panel = PanelDataAnalysis()
    panel.prepare_panel_data(df, 'entity', 'time', ['X1', 'X2', 'y'])
    
    # Fixed effects
    fe_model = panel.fixed_effects_regression('y', ['X1', 'X2'])
    
    # Random effects
    re_model = panel.random_effects_regression('y', ['X1', 'X2'])
    
    # Hausman test
    panel.hausman_test('y', ['X1', 'X2'])

239 lines•7.3 KB

python

Theme Settings

Color Scheme

Display Options

Font Size

Theme Settings

Color Scheme

Display Options

Font Size

🚀 Support RSK World