TSLA.csv index.html advanced_analysis.py style.css .gitignore README.md exercising_003.mp4 .gitkeep anomaly_detection.py

index.html

<!DOCTYPE html>
<html lang="en">
<head>
    <!--
    Project: Energy Consumption Dataset
    Author: RSK World
    Website: https://rskworld.in
    Email: help@rskworld.in
    Phone: +91 93305 39277
    -->
    <meta charset="UTF-8">
    <meta name="viewport" content="width=device-width, initial-scale=1.0">
    <title>Energy Consumption Dataset - RSK World</title>
    <link href="https://cdn.jsdelivr.net/npm/bootstrap@5.3.0/dist/css/bootstrap.min.css" rel="stylesheet">
    <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/6.4.0/css/all.min.css">
    <script src="https://cdn.jsdelivr.net/npm/chart.js@4.4.0/dist/chart.umd.min.js"></script>
    <style>
        :root {
            --primary-color: #2ecc71;
            --secondary-color: #3498db;
            --dark-color: #2c3e50;
        }
        body {
            font-family: 'Segoe UI', Tahoma, Geneva, Verdana, sans-serif;
            background: linear-gradient(135deg, #667eea 0%, #764ba2 100%);
            min-height: 100vh;
            padding: 20px 0;
        }
        .container {
            max-width: 1200px;
        }
        .card {
            border: none;
            border-radius: 15px;
            box-shadow: 0 10px 30px rgba(0,0,0,0.2);
            margin-bottom: 30px;
            overflow: hidden;
        }
        .card-header {
            background: linear-gradient(135deg, var(--primary-color), var(--secondary-color));
            color: white;
            padding: 20px;
            border: none;
        }
        .card-body {
            padding: 30px;
            background: white;
        }
        .icon-large {
            font-size: 3rem;
            margin-bottom: 20px;
            color: var(--primary-color);
        }
        .feature-list {
            list-style: none;
            padding: 0;
        }
        .feature-list li {
            padding: 10px 0;
            border-bottom: 1px solid #eee;
        }
        .feature-list li:last-child {
            border-bottom: none;
        }
        .feature-list li i {
            color: var(--primary-color);
            margin-right: 10px;
        }
        .btn-primary {
            background: linear-gradient(135deg, var(--primary-color), var(--secondary-color));
            border: none;
            padding: 12px 30px;
            border-radius: 25px;
            font-weight: 600;
        }
        .btn-primary:hover {
            transform: translateY(-2px);
            box-shadow: 0 5px 15px rgba(0,0,0,0.3);
        }
        .stats-card {
            text-align: center;
            padding: 20px;
            background: white;
            border-radius: 10px;
            margin: 10px 0;
        }
        .stats-number {
            font-size: 2.5rem;
            font-weight: bold;
            color: var(--primary-color);
        }
        .stats-label {
            color: #666;
            font-size: 0.9rem;
            margin-top: 5px;
        }
        .chart-container {
            position: relative;
            height: 400px;
            margin: 20px 0;
        }
        .footer {
            text-align: center;
            padding: 20px;
            color: white;
            margin-top: 40px;
        }
        .footer a {
            color: white;
            text-decoration: none;
        }
        .footer a:hover {
            text-decoration: underline;
        }
    </style>
</head>
<body>
    <div class="container">
        <!-- Header -->
        <div class="card">
            <div class="card-header text-center">
                <i class="fas fa-clock icon-large"></i>
                <h1 class="mb-0">Energy Consumption Dataset</h1>
                <p class="mb-0 mt-2">Smart meter energy consumption dataset with hourly electricity usage patterns</p>
            </div>
        </div>

        <!-- Description -->
        <div class="card">
            <div class="card-body">
                <h3><i class="fas fa-info-circle text-success"></i> About This Dataset</h3>
                <p class="lead">
                    This dataset contains hourly energy consumption data from smart meters with seasonal patterns, 
                    peak hours, and consumption trends. Perfect for energy demand forecasting, load prediction, 
                    and smart grid applications.
                </p>
                
                <h4 class="mt-4"><i class="fas fa-star text-warning"></i> Key Features</h4>
                <ul class="feature-list">
                    <li><i class="fas fa-check-circle"></i> Hourly consumption data</li>
                    <li><i class="fas fa-check-circle"></i> Seasonal patterns</li>
                    <li><i class="fas fa-check-circle"></i> Peak hour identification</li>
                    <li><i class="fas fa-check-circle"></i> Multiple households</li>
                    <li><i class="fas fa-check-circle"></i> Time series ready format</li>
                </ul>
            </div>
        </div>

        <!-- Statistics -->
        <div class="row">
            <div class="col-md-3">
                <div class="stats-card">
                    <div class="stats-number" id="totalRecords">-</div>
                    <div class="stats-label">Total Records</div>
                </div>
            </div>
            <div class="col-md-3">
                <div class="stats-card">
                    <div class="stats-number" id="households">-</div>
                    <div class="stats-label">Households</div>
                </div>
            </div>
            <div class="col-md-3">
                <div class="stats-card">
                    <div class="stats-number" id="avgConsumption">-</div>
                    <div class="stats-label">Avg Consumption (kWh)</div>
                </div>
            </div>
            <div class="col-md-3">
                <div class="stats-card">
                    <div class="stats-number" id="dateRange">-</div>
                    <div class="stats-label">Days of Data</div>
                </div>
            </div>
        </div>

        <!-- Visualizations -->
        <div class="card">
            <div class="card-body">
                <h3><i class="fas fa-chart-line text-primary"></i> Data Visualizations</h3>
                <div class="chart-container">
                    <canvas id="consumptionChart"></canvas>
                </div>
            </div>
        </div>

        <div class="card">
            <div class="card-body">
                <h3><i class="fas fa-chart-bar text-info"></i> Hourly Patterns</h3>
                <div class="chart-container">
                    <canvas id="hourlyChart"></canvas>
                </div>
            </div>
        </div>

        <!-- Seasonal Patterns -->
        <div class="card">
            <div class="card-body">
                <h3><i class="fas fa-calendar-alt text-warning"></i> Seasonal Patterns</h3>
                <div class="chart-container">
                    <canvas id="seasonalChart"></canvas>
                </div>
            </div>
        </div>

        <!-- Household Comparison -->
        <div class="card">
            <div class="card-body">
                <h3><i class="fas fa-home text-danger"></i> Household Comparison</h3>
                <div class="chart-container">
                    <canvas id="householdChart"></canvas>
                </div>
            </div>
        </div>

        <!-- Advanced Features Section -->
        <div class="card">
            <div class="card-header text-center">
                <h3><i class="fas fa-rocket"></i> Advanced Features</h3>
            </div>
            <div class="card-body">
                <div class="row">
                    <div class="col-md-4 mb-3">
                        <div class="card h-100 border-primary">
                            <div class="card-body text-center">
                                <i class="fas fa-brain fa-3x text-primary mb-3"></i>
                                <h5>Machine Learning</h5>
                                <p class="small">Forecasting models including Linear Regression and Random Forest</p>
                            </div>
                        </div>
                    </div>
                    <div class="col-md-4 mb-3">
                        <div class="card h-100 border-warning">
                            <div class="card-body text-center">
                                <i class="fas fa-exclamation-triangle fa-3x text-warning mb-3"></i>
                                <h5>Anomaly Detection</h5>
                                <p class="small">Multiple detection methods: IQR, Z-score, Isolation Forest, Time Series</p>
                            </div>
                        </div>
                    </div>
                    <div class="col-md-4 mb-3">
                        <div class="card h-100 border-success">
                            <div class="card-body text-center">
                                <i class="fas fa-chart-line fa-3x text-success mb-3"></i>
                                <h5>Time Series Analysis</h5>
                                <p class="small">Decomposition, autocorrelation, trend detection, and stationarity tests</p>
                            </div>
                        </div>
                    </div>
                </div>
            </div>
        </div>

        <!-- Technologies -->
        <div class="card">
            <div class="card-body">
                <h3><i class="fas fa-tools text-secondary"></i> Technologies</h3>
                <div class="d-flex flex-wrap gap-2">
                    <span class="badge bg-primary p-2">CSV</span>
                    <span class="badge bg-primary p-2">JSON</span>
                    <span class="badge bg-primary p-2">Pandas</span>
                    <span class="badge bg-primary p-2">Time Series</span>
                    <span class="badge bg-primary p-2">Python</span>
                    <span class="badge bg-primary p-2">Data Analysis</span>
                </div>
            </div>
        </div>

        <!-- Download Section -->
        <div class="card">
            <div class="card-body text-center">
                <h3><i class="fas fa-download text-success"></i> Download Dataset</h3>
                <p>Get the complete dataset in CSV or JSON format</p>
                <div class="d-flex justify-content-center gap-3 flex-wrap">
                    <a href="energy_consumption.csv" class="btn btn-primary" download>
                        <i class="fas fa-file-csv"></i> Download CSV
                    </a>
                    <a href="energy_consumption.json" class="btn btn-primary" download>
                        <i class="fas fa-file-code"></i> Download JSON
                    </a>
                </div>
                <hr class="my-4">
                <h5 class="mb-3">Advanced Scripts</h5>
                <div class="d-flex justify-content-center gap-2 flex-wrap">
                    <span class="badge bg-info p-2">forecasting.py - ML Models</span>
                    <span class="badge bg-warning p-2">anomaly_detection.py</span>
                    <span class="badge bg-success p-2">advanced_analysis.py</span>
                    <span class="badge bg-secondary p-2">preprocessing.py</span>
                </div>
            </div>
        </div>

        <!-- Usage Example -->
        <div class="card">
            <div class="card-body">
                <h3><i class="fas fa-code text-dark"></i> Usage Example</h3>
                <pre class="bg-dark text-light p-3 rounded"><code>import pandas as pd

# Load the dataset
df = pd.read_csv('energy_consumption.csv')

# Basic statistics
print(df.describe())

# Peak hour analysis
hourly_avg = df.groupby('hour')['consumption_kwh'].mean()
print(hourly_avg.sort_values(ascending=False).head())</code></pre>
            </div>
        </div>

        <!-- Footer -->
        <div class="footer">
            <p><strong>Energy Consumption Dataset</strong></p>
            <p>Project by <a href="https://rskworld.in" target="_blank">RSK World</a></p>
            <p>
                <a href="mailto:help@rskworld.in">help@rskworld.in</a> | 
                <a href="tel:+919330539277">+91 93305 39277</a>
            </p>
            <p><small>&copy; 2026 RSK World. Free to use for educational and research purposes.</small></p>
        </div>
    </div>

    <script src="https://cdn.jsdelivr.net/npm/bootstrap@5.3.0/dist/js/bootstrap.bundle.min.js"></script>
    <script>
        // Load and display data
        async function loadData() {
            try {
                const response = await fetch('energy_consumption.json');
                const data = await response.json();
                
                // Calculate statistics
                const totalRecords = data.length;
                const households = new Set(data.map(d => d.household_id)).size;
                const avgConsumption = (data.reduce((sum, d) => sum + d.consumption_kwh, 0) / totalRecords).toFixed(2);
                const dates = data.map(d => new Date(d.timestamp));
                const dateRange = Math.ceil((Math.max(...dates) - Math.min(...dates)) / (1000 * 60 * 60 * 24));
                
                // Update stats
                document.getElementById('totalRecords').textContent = totalRecords.toLocaleString();
                document.getElementById('households').textContent = households;
                document.getElementById('avgConsumption').textContent = avgConsumption;
                document.getElementById('dateRange').textContent = dateRange;
                
                // Sample data for charts (first 1000 records for performance)
                const sampleData = data.slice(0, 1000);
                
                // Time series chart
                const ctx1 = document.getElementById('consumptionChart').getContext('2d');
                new Chart(ctx1, {
                    type: 'line',
                    data: {
                        labels: sampleData.map(d => new Date(d.timestamp).toLocaleDateString()),
                        datasets: [{
                            label: 'Consumption (kWh)',
                            data: sampleData.map(d => d.consumption_kwh),
                            borderColor: '#2ecc71',
                            backgroundColor: 'rgba(46, 204, 113, 0.1)',
                            tension: 0.4
                        }]
                    },
                    options: {
                        responsive: true,
                        maintainAspectRatio: false,
                        plugins: {
                            title: {
                                display: true,
                                text: 'Energy Consumption Over Time'
                            }
                        },
                        scales: {
                            y: {
                                beginAtZero: true
                            }
                        }
                    }
                });
                
                // Hourly patterns chart
                const hourlyData = {};
                data.forEach(d => {
                    const hour = d.hour;
                    if (!hourlyData[hour]) {
                        hourlyData[hour] = [];
                    }
                    hourlyData[hour].push(d.consumption_kwh);
                });
                
                const hourlyAvg = Object.keys(hourlyData).sort((a, b) => a - b).map(hour => {
                    const values = hourlyData[hour];
                    return values.reduce((sum, v) => sum + v, 0) / values.length;
                });
                
                const ctx2 = document.getElementById('hourlyChart').getContext('2d');
                new Chart(ctx2, {
                    type: 'bar',
                    data: {
                        labels: Array.from({length: 24}, (_, i) => i + ':00'),
                        datasets: [{
                            label: 'Average Consumption (kWh)',
                            data: hourlyAvg,
                            backgroundColor: '#3498db',
                            borderColor: '#2980b9',
                            borderWidth: 1
                        }]
                    },
                    options: {
                        responsive: true,
                        maintainAspectRatio: false,
                        plugins: {
                            title: {
                                display: true,
                                text: 'Average Consumption by Hour of Day'
                            }
                        },
                        scales: {
                            y: {
                                beginAtZero: true
                            }
                        }
                    }
                });
                
                // Seasonal patterns chart
                const monthlyData = {};
                data.forEach(d => {
                    const date = new Date(d.timestamp);
                    const month = date.getMonth();
                    if (!monthlyData[month]) {
                        monthlyData[month] = [];
                    }
                    monthlyData[month].push(d.consumption_kwh);
                });
                
                const months = ['Jan', 'Feb', 'Mar', 'Apr', 'May', 'Jun', 'Jul', 'Aug', 'Sep', 'Oct', 'Nov', 'Dec'];
                const monthlyAvg = months.map((_, month) => {
                    const values = monthlyData[month] || [];
                    return values.length > 0 ? values.reduce((sum, v) => sum + v, 0) / values.length : 0;
                });
                
                const ctx3 = document.getElementById('seasonalChart').getContext('2d');
                new Chart(ctx3, {
                    type: 'line',
                    data: {
                        labels: months,
                        datasets: [{
                            label: 'Average Consumption (kWh)',
                            data: monthlyAvg,
                            borderColor: '#e74c3c',
                            backgroundColor: 'rgba(231, 76, 60, 0.1)',
                            tension: 0.4,
                            fill: true
                        }]
                    },
                    options: {
                        responsive: true,
                        maintainAspectRatio: false,
                        plugins: {
                            title: {
                                display: true,
                                text: 'Average Consumption by Month'
                            }
                        },
                        scales: {
                            y: {
                                beginAtZero: true
                            }
                        }
                    }
                });
                
                // Household comparison chart
                const householdData = {};
                data.forEach(d => {
                    const household = d.household_id;
                    if (!householdData[household]) {
                        householdData[household] = [];
                    }
                    householdData[household].push(d.consumption_kwh);
                });
                
                const householdAvg = Object.keys(householdData).sort().map(household => {
                    const values = householdData[household];
                    return values.reduce((sum, v) => sum + v, 0) / values.length;
                });
                
                const ctx4 = document.getElementById('householdChart').getContext('2d');
                new Chart(ctx4, {
                    type: 'doughnut',
                    data: {
                        labels: Object.keys(householdData).sort(),
                        datasets: [{
                            label: 'Average Consumption (kWh)',
                            data: householdAvg,
                            backgroundColor: [
                                '#3498db',
                                '#2ecc71',
                                '#e74c3c',
                                '#f39c12',
                                '#9b59b6'
                            ],
                            borderWidth: 2
                        }]
                    },
                    options: {
                        responsive: true,
                        maintainAspectRatio: false,
                        plugins: {
                            title: {
                                display: true,
                                text: 'Average Consumption by Household'
                            },
                            legend: {
                                position: 'bottom'
                            }
                        }
                    }
                });
                
            } catch (error) {
                console.error('Error loading data:', error);
                document.getElementById('totalRecords').textContent = 'Error';
            }
        }
        
        // Load data on page load
        loadData();
    </script>
</body>
</html>

531 lines•21.4 KB

markup

advanced_analysis.py

Raw Download

"""
Energy Consumption Dataset - Advanced Time Series Analysis

Project: Energy Consumption Dataset
Author: RSK World
Website: https://rskworld.in
Email: help@rskworld.in
Phone: +91 93305 39277

Advanced time series analysis including decomposition, autocorrelation, and trend analysis.
"""

import pandas as pd
import numpy as np
from scipy import stats
from scipy.stats import pearsonr
import warnings
warnings.filterwarnings('ignore')

class AdvancedTimeSeriesAnalysis:
    """
    Advanced time series analysis for energy consumption data.
    """
    
    def __init__(self, df):
        """
        Initialize analyzer with data.
        
        Args:
            df: pandas.DataFrame containing energy consumption data
        """
        self.df = df.copy()
        self.df['timestamp'] = pd.to_datetime(self.df['timestamp'])
        self.df = self.df.sort_values('timestamp').reset_index(drop=True)
    
    def decompose_time_series(self, household_id=None, period=24):
        """
        Decompose time series into trend, seasonal, and residual components.
        
        Args:
            household_id: Specific household to analyze (None for all)
            period: Seasonal period (24 for hourly data)
        
        Returns:
            dict: Decomposed components
        """
        if household_id:
            data = self.df[self.df['household_id'] == household_id]['consumption_kwh'].values
        else:
            data = self.df.groupby('timestamp')['consumption_kwh'].mean().values
        
        # Simple moving average for trend
        window = min(period * 7, len(data) // 10)  # 7 days or 10% of data
        if window % 2 == 0:
            window += 1
        
        trend = pd.Series(data).rolling(window=window, center=True).mean().values
        
        # Detrend
        detrended = data - trend
        
        # Seasonal component (average pattern)
        n_seasons = len(data) // period
        seasonal_pattern = []
        for i in range(period):
            indices = range(i, len(data), period)
            if len(indices) > 0:
                seasonal_pattern.append(np.mean([detrended[j] for j in indices if j < len(detrended)]))
            else:
                seasonal_pattern.append(0)
        
        # Extend seasonal pattern
        seasonal = np.tile(seasonal_pattern, n_seasons + 1)[:len(data)]
        
        # Residual
        residual = detrended - seasonal
        
        return {
            'original': data,
            'trend': trend,
            'seasonal': seasonal,
            'residual': residual
        }
    
    def calculate_autocorrelation(self, household_id=None, max_lags=48):
        """
        Calculate autocorrelation function.
        
        Args:
            household_id: Specific household to analyze
            max_lags: Maximum number of lags to calculate
        
        Returns:
            dict: Autocorrelation values
        """
        if household_id:
            data = self.df[self.df['household_id'] == household_id]['consumption_kwh'].values
        else:
            data = self.df.groupby('timestamp')['consumption_kwh'].mean().values
        
        autocorrs = {}
        for lag in range(1, min(max_lags + 1, len(data) // 2)):
            if lag < len(data):
                corr, _ = pearsonr(data[lag:], data[:-lag])
                autocorrs[lag] = corr
        
        return autocorrs
    
    def detect_trend(self, household_id=None):
        """
        Detect trend in the time series using Mann-Kendall test.
        
        Args:
            household_id: Specific household to analyze
        
        Returns:
            dict: Trend analysis results
        """
        if household_id:
            data = self.df[self.df['household_id'] == household_id]['consumption_kwh'].values
        else:
            data = self.df.groupby('timestamp')['consumption_kwh'].mean().values
        
        # Mann-Kendall test
        n = len(data)
        s = 0
        
        for i in range(n - 1):
            for j in range(i + 1, n):
                s += np.sign(data[j] - data[i])
        
        # Calculate variance
        var_s = n * (n - 1) * (2 * n + 5) / 18
        
        # Z-score
        if s > 0:
            z = (s - 1) / np.sqrt(var_s)
        elif s < 0:
            z = (s + 1) / np.sqrt(var_s)
        else:
            z = 0
        
        # P-value (two-tailed)
        p_value = 2 * (1 - stats.norm.cdf(abs(z)))
        
        # Determine trend direction
        if z > 0:
            trend_direction = "Increasing"
        elif z < 0:
            trend_direction = "Decreasing"
        else:
            trend_direction = "No trend"
        
        return {
            'z_score': z,
            'p_value': p_value,
            'trend_direction': trend_direction,
            'significant': p_value < 0.05
        }
    
    def calculate_stationarity(self, household_id=None):
        """
        Test for stationarity using Augmented Dickey-Fuller test approximation.
        
        Args:
            household_id: Specific household to analyze
        
        Returns:
            dict: Stationarity test results
        """
        if household_id:
            data = self.df[self.df['household_id'] == household_id]['consumption_kwh'].values
        else:
            data = self.df.groupby('timestamp')['consumption_kwh'].mean().values
        
        # Simple variance ratio test
        n = len(data)
        first_half = data[:n//2]
        second_half = data[n//2:]
        
        var_first = np.var(first_half)
        var_second = np.var(second_half)
        
        variance_ratio = var_second / var_first if var_first > 0 else 1
        
        # Mean difference
        mean_diff = np.abs(np.mean(second_half) - np.mean(first_half))
        mean_avg = np.mean(data)
        mean_change_pct = (mean_diff / mean_avg * 100) if mean_avg > 0 else 0
        
        # Stationary if variance ratio close to 1 and small mean change
        is_stationary = (0.8 < variance_ratio < 1.2) and (mean_change_pct < 10)
        
        return {
            'variance_ratio': variance_ratio,
            'mean_change_percent': mean_change_pct,
            'is_stationary': is_stationary
        }
    
    def calculate_seasonality_strength(self, household_id=None):
        """
        Calculate strength of seasonality.
        
        Args:
            household_id: Specific household to analyze
        
        Returns:
            dict: Seasonality metrics
        """
        if household_id:
            data = self.df[self.df['household_id'] == household_id]
        else:
            data = self.df.copy()
        
        # Group by hour and calculate variance
        hourly_variance = data.groupby('hour')['consumption_kwh'].var().mean()
        overall_variance = data['consumption_kwh'].var()
        
        # Group by day of week
        daily_variance = data.groupby('day_of_week')['consumption_kwh'].var().mean()
        
        # Group by month
        data['month'] = data['timestamp'].dt.month
        monthly_variance = data.groupby('month')['consumption_kwh'].var().mean()
        
        seasonality_strength = {
            'hourly_seasonality': hourly_variance / overall_variance if overall_variance > 0 else 0,
            'daily_seasonality': daily_variance / overall_variance if overall_variance > 0 else 0,
            'monthly_seasonality': monthly_variance / overall_variance if overall_variance > 0 else 0
        }
        
        return seasonality_strength
    
    def comprehensive_analysis(self, household_id=None):
        """
        Run comprehensive time series analysis.
        
        Args:
            household_id: Specific household to analyze
        
        Returns:
            dict: Complete analysis results
        """
        print(f"Running comprehensive analysis{' for ' + household_id if household_id else ''}...")
        
        results = {
            'decomposition': self.decompose_time_series(household_id),
            'autocorrelation': self.calculate_autocorrelation(household_id),
            'trend': self.detect_trend(household_id),
            'stationarity': self.calculate_stationarity(household_id),
            'seasonality': self.calculate_seasonality_strength(household_id)
        }
        
        return results

def main():
    """
    Main function to demonstrate advanced analysis.
    """
    print("\n" + "=" * 60)
    print("ENERGY CONSUMPTION DATASET - ADVANCED TIME SERIES ANALYSIS")
    print("=" * 60)
    print("Project: Energy Consumption Dataset")
    print("Author: RSK World")
    print("Website: https://rskworld.in")
    print("=" * 60 + "\n")
    
    # Load data
    try:
        df = pd.read_csv('energy_consumption.csv')
        df['timestamp'] = pd.to_datetime(df['timestamp'])
        print(f"Loaded {len(df):,} records")
    except FileNotFoundError:
        print("Error: energy_consumption.csv not found. Please generate data first.")
        return
    
    # Initialize analyzer
    analyzer = AdvancedTimeSeriesAnalysis(df)
    
    # Run comprehensive analysis
    results = analyzer.comprehensive_analysis()
    
    print("\n" + "=" * 60)
    print("ADVANCED ANALYSIS RESULTS")
    print("=" * 60)
    
    # Trend analysis
    print("\nTREND ANALYSIS:")
    print(f"  Direction: {results['trend']['trend_direction']}")
    print(f"  Z-Score: {results['trend']['z_score']:.3f}")
    print(f"  P-Value: {results['trend']['p_value']:.4f}")
    print(f"  Significant: {'Yes' if results['trend']['significant'] else 'No'}")
    
    # Stationarity
    print("\nSTATIONARITY TEST:")
    print(f"  Variance Ratio: {results['stationarity']['variance_ratio']:.3f}")
    print(f"  Mean Change: {results['stationarity']['mean_change_percent']:.2f}%")
    print(f"  Stationary: {'Yes' if results['stationarity']['is_stationary'] else 'No'}")
    
    # Seasonality
    print("\nSEASONALITY STRENGTH:")
    print(f"  Hourly: {results['seasonality']['hourly_seasonality']:.3f}")
    print(f"  Daily: {results['seasonality']['daily_seasonality']:.3f}")
    print(f"  Monthly: {results['seasonality']['monthly_seasonality']:.3f}")
    
    # Autocorrelation (show first 10 lags)
    print("\nAUTOCORRELATION (First 10 Lags):")
    for lag in sorted(list(results['autocorrelation'].keys()))[:10]:
        print(f"  Lag {lag:2d}: {results['autocorrelation'][lag]:.3f}")
    
    print("\n" + "=" * 60)
    print("Advanced analysis complete!")
    print("For more information, visit: https://rskworld.in")

if __name__ == "__main__":
    main()

318 lines•10.7 KB

python

.gitignore

Raw Download

# Energy Consumption Dataset - Git Ignore
# Project: Energy Consumption Dataset
# Author: RSK World
# Website: https://rskworld.in
# Email: help@rskworld.in
# Phone: +91 93305 39277

# Python
__pycache__/
*.py[cod]
*$py.class
*.so
.Python
env/
venv/
ENV/
build/
dist/
*.egg-info/

# IDE
.vscode/
.idea/
*.swp
*.swo
*~

# OS
.DS_Store
Thumbs.db

# Generated plots
*.png
!energy-consumption.png

36 lines•429 B

text

README.md

Raw Download

README.md

# Energy Consumption Dataset



Smart meter energy consumption dataset with hourly electricity usage patterns for demand forecasting and energy analytics.

## Description

This dataset contains hourly energy consumption data from smart meters with seasonal patterns, peak hours, and consumption trends. Perfect for energy demand forecasting, load prediction, and smart grid applications.

## Features

- Hourly consumption data
- Seasonal patterns
- Peak hour identification
- Multiple households
- Time series ready format

## Technologies

- CSV
- JSON
- Pandas
- Time Series Analysis

## Difficulty Level

Intermediate

## Dataset Structure

The dataset includes:
- `energy_consumption.csv` - Main dataset in CSV format
- `energy_consumption.json` - Dataset in JSON format
- `index.html` - Interactive demo page with visualizations

### Python Scripts

**Basic Analysis:**
- `generate_data.py` - Generate synthetic energy consumption data
- `analysis.py` - Basic statistical analysis and insights
- `visualization.py` - Create charts and visualizations

**Advanced Features:**
- `forecasting.py` - Machine learning forecasting models (Linear Regression, Random Forest)
- `anomaly_detection.py` - Multiple anomaly detection methods (IQR, Z-score, Isolation Forest, Time Series)
- `advanced_analysis.py` - Advanced time series analysis (decomposition, autocorrelation, trend detection)
- `preprocessing.py` - Data preprocessing and feature engineering utilities
- `model_evaluation.py` - Comprehensive model evaluation metrics and comparison

## Usage

### Basic Analysis

```python
import pandas as pd

# Load the dataset
df = pd.read_csv('energy_consumption.csv')

# Basic statistics
print(df.describe())

# Run analysis script
python analysis.py

# Generate visualizations
python visualization.py
```

### Advanced Features

**Machine Learning Forecasting:**
```python
python forecasting.py
```
- Linear Regression model
- Random Forest model
- 24-hour future forecasting
- Model comparison metrics

**Anomaly Detection:**
```python
python anomaly_detection.py
```
- IQR method
- Z-score method
- Isolation Forest
- Time series anomaly detection

**Advanced Time Series Analysis:**
```python
python advanced_analysis.py
```
- Time series decomposition
- Autocorrelation analysis
- Trend detection (Mann-Kendall test)
- Stationarity testing
- Seasonality strength calculation

**Data Preprocessing:**
```python
python preprocessing.py
```
- Missing value handling
- Outlier removal
- Data normalization
- Feature engineering
- Lag and rolling features

**Model Evaluation:**
```python
python model_evaluation.py
```
- Comprehensive metrics (MAE, RMSE, R², MAPE, MBE)
- Model comparison
- Prediction plots
- Evaluation reports

### Data Format

- **timestamp**: Date and time of measurement
- **household_id**: Unique identifier for each household
- **consumption_kwh**: Energy consumption in kilowatt-hours
- **temperature**: Outdoor temperature (for correlation analysis)
- **hour**: Hour of the day (0-23)
- **day_of_week**: Day of the week (0-6, Monday=0)

## Installation

```bash
# Install all dependencies
pip install -r requirements.txt

# Or install individually
pip install pandas numpy matplotlib seaborn scikit-learn scipy
```

## Advanced Features

### 1. Machine Learning Forecasting
- **Linear Regression**: Fast and interpretable forecasting model
- **Random Forest**: Ensemble method for better accuracy
- **Feature Engineering**: Time-based features, lag features, rolling statistics
- **Future Forecasting**: Predict consumption for next 24 hours

### 2. Anomaly Detection
- **Statistical Methods**: IQR and Z-score based detection
- **Isolation Forest**: Machine learning based anomaly detection
- **Time Series Methods**: Rolling window based anomaly detection
- **Comprehensive Detection**: Combines multiple methods for robust detection

### 3. Advanced Time Series Analysis
- **Decomposition**: Trend, seasonal, and residual components
- **Autocorrelation**: Identify patterns and dependencies
- **Trend Detection**: Mann-Kendall test for trend significance
- **Stationarity Testing**: Variance ratio and mean change analysis
- **Seasonality Analysis**: Strength of hourly, daily, and monthly patterns

### 4. Data Preprocessing
- **Missing Value Handling**: Forward fill, backward fill, mean, median, or drop
- **Outlier Removal**: IQR or Z-score based methods
- **Normalization**: Min-max or standard scaling
- **Feature Engineering**: Time features, cyclical encoding, lag features, rolling statistics

### 5. Model Evaluation
- **Comprehensive Metrics**: MAE, RMSE, R², MAPE, MBE, CV(RMSE)
- **Visual Comparisons**: Prediction plots, residual plots, model comparison charts
- **Evaluation Reports**: Detailed text reports with recommendations

## License

This project is licensed under the MIT License - see the [LICENSE](LICENSE) file for details.

Free to use for educational and research purposes.

## Contact

For questions or support:
- Website: https://rskworld.in
- Email: help@rskworld.in
- Phone: +91 93305 39277

anomaly_detection.py

Raw Download

"""
Energy Consumption Dataset - Anomaly Detection

Project: Energy Consumption Dataset
Author: RSK World
Website: https://rskworld.in
Email: help@rskworld.in
Phone: +91 93305 39277

Advanced anomaly detection for energy consumption patterns.
"""

import pandas as pd
import numpy as np
from sklearn.ensemble import IsolationForest
from sklearn.preprocessing import StandardScaler
import warnings
warnings.filterwarnings('ignore')

class AnomalyDetector:
    """
    Anomaly detection class for energy consumption data.
    """
    
    def __init__(self, df):
        """
        Initialize anomaly detector with data.
        
        Args:
            df: pandas.DataFrame containing energy consumption data
        """
        self.df = df.copy()
        self.df['timestamp'] = pd.to_datetime(self.df['timestamp'])
        self.anomalies = None
        self.scaler = StandardScaler()
    
    def detect_statistical_outliers(self, method='iqr', threshold=3):
        """
        Detect outliers using statistical methods.
        
        Args:
            method: 'iqr' for Interquartile Range or 'zscore' for Z-score
            threshold: Threshold for z-score method
        
        Returns:
            pandas.DataFrame: Data with anomaly flags
        """
        df_result = self.df.copy()
        
        if method == 'iqr':
            Q1 = df_result['consumption_kwh'].quantile(0.25)
            Q3 = df_result['consumption_kwh'].quantile(0.75)
            IQR = Q3 - Q1
            lower_bound = Q1 - 1.5 * IQR
            upper_bound = Q3 + 1.5 * IQR
            
            df_result['is_anomaly_iqr'] = (
                (df_result['consumption_kwh'] < lower_bound) | 
                (df_result['consumption_kwh'] > upper_bound)
            )
            
        elif method == 'zscore':
            mean = df_result['consumption_kwh'].mean()
            std = df_result['consumption_kwh'].std()
            z_scores = np.abs((df_result['consumption_kwh'] - mean) / std)
            df_result['is_anomaly_zscore'] = z_scores > threshold
        
        return df_result
    
    def detect_isolation_forest(self, contamination=0.1, random_state=42):
        """
        Detect anomalies using Isolation Forest algorithm.
        
        Args:
            contamination: Expected proportion of anomalies
            random_state: Random state for reproducibility
        
        Returns:
            pandas.DataFrame: Data with anomaly flags
        """
        df_result = self.df.copy()
        
        # Prepare features
        features = ['consumption_kwh', 'hour', 'day_of_week', 'temperature']
        features = [f for f in features if f in df_result.columns]
        
        X = df_result[features].values
        X_scaled = self.scaler.fit_transform(X)
        
        # Train Isolation Forest
        iso_forest = IsolationForest(
            contamination=contamination,
            random_state=random_state,
            n_jobs=-1
        )
        df_result['is_anomaly_iso'] = iso_forest.fit_predict(X_scaled) == -1
        df_result['anomaly_score'] = iso_forest.score_samples(X_scaled)
        
        return df_result
    
    def detect_time_series_anomalies(self, window=24, threshold=2):
        """
        Detect anomalies based on time series patterns.
        
        Args:
            window: Rolling window size for moving average
            threshold: Number of standard deviations for threshold
        
        Returns:
            pandas.DataFrame: Data with anomaly flags
        """
        df_result = self.df.copy()
        df_result = df_result.sort_values('timestamp').reset_index(drop=True)
        
        # Calculate rolling statistics
        df_result['rolling_mean'] = df_result.groupby('household_id')['consumption_kwh'].transform(
            lambda x: x.rolling(window=window, min_periods=1).mean()
        )
        df_result['rolling_std'] = df_result.groupby('household_id')['consumption_kwh'].transform(
            lambda x: x.rolling(window=window, min_periods=1).std()
        )
        
        # Detect anomalies
        df_result['upper_bound'] = df_result['rolling_mean'] + threshold * df_result['rolling_std']
        df_result['lower_bound'] = df_result['rolling_mean'] - threshold * df_result['rolling_std']
        
        df_result['is_anomaly_ts'] = (
            (df_result['consumption_kwh'] > df_result['upper_bound']) |
            (df_result['consumption_kwh'] < df_result['lower_bound'])
        )
        
        return df_result
    
    def comprehensive_detection(self):
        """
        Run all detection methods and combine results.
        
        Returns:
            pandas.DataFrame: Data with comprehensive anomaly flags
        """
        print("Running comprehensive anomaly detection...")
        
        # Run all methods
        df_iqr = self.detect_statistical_outliers(method='iqr')
        df_zscore = self.detect_statistical_outliers(method='zscore', threshold=3)
        df_iso = self.detect_isolation_forest(contamination=0.1)
        df_ts = self.detect_time_series_anomalies(window=24, threshold=2)
        
        # Combine results
        df_result = self.df.copy()
        df_result['is_anomaly_iqr'] = df_iqr['is_anomaly_iqr'] if 'is_anomaly_iqr' in df_iqr.columns else False
        df_result['is_anomaly_zscore'] = df_zscore['is_anomaly_zscore'] if 'is_anomaly_zscore' in df_zscore.columns else False
        df_result['is_anomaly_iso'] = df_iso['is_anomaly_iso'] if 'is_anomaly_iso' in df_iso.columns else False
        df_result['is_anomaly_ts'] = df_ts['is_anomaly_ts'] if 'is_anomaly_ts' in df_ts.columns else False
        df_result['anomaly_score'] = df_iso['anomaly_score'] if 'anomaly_score' in df_iso.columns else 0
        
        # Combined flag (anomaly if detected by at least 2 methods)
        anomaly_cols = ['is_anomaly_iqr', 'is_anomaly_zscore', 'is_anomaly_iso', 'is_anomaly_ts']
        df_result['is_anomaly'] = df_result[anomaly_cols].sum(axis=1) >= 2
        
        self.anomalies = df_result[df_result['is_anomaly']]
        
        return df_result
    
    def get_anomaly_summary(self):
        """
        Get summary statistics of detected anomalies.
        
        Returns:
            dict: Summary statistics
        """
        if self.anomalies is None:
            return None
        
        summary = {
            'total_anomalies': len(self.anomalies),
            'anomaly_percentage': (len(self.anomalies) / len(self.df)) * 100,
            'avg_anomaly_consumption': self.anomalies['consumption_kwh'].mean(),
            'avg_normal_consumption': self.df[~self.df['is_anomaly']]['consumption_kwh'].mean() if 'is_anomaly' in self.df.columns else self.df['consumption_kwh'].mean(),
            'anomalies_by_household': self.anomalies.groupby('household_id').size().to_dict(),
            'anomalies_by_hour': self.anomalies.groupby('hour').size().to_dict()
        }
        
        return summary

def main():
    """
    Main function to demonstrate anomaly detection.
    """
    print("\n" + "=" * 60)
    print("ENERGY CONSUMPTION DATASET - ANOMALY DETECTION")
    print("=" * 60)
    print("Project: Energy Consumption Dataset")
    print("Author: RSK World")
    print("Website: https://rskworld.in")
    print("=" * 60 + "\n")
    
    # Load data
    try:
        df = pd.read_csv('energy_consumption.csv')
        df['timestamp'] = pd.to_datetime(df['timestamp'])
        print(f"Loaded {len(df):,} records")
    except FileNotFoundError:
        print("Error: energy_consumption.csv not found. Please generate data first.")
        return
    
    # Initialize detector
    detector = AnomalyDetector(df)
    
    # Run comprehensive detection
    df_with_anomalies = detector.comprehensive_detection()
    
    # Get summary
    summary = detector.get_anomaly_summary()
    
    print("\n" + "=" * 60)
    print("ANOMALY DETECTION RESULTS")
    print("=" * 60)
    print(f"\nTotal Anomalies Detected: {summary['total_anomalies']:,}")
    print(f"Anomaly Percentage: {summary['anomaly_percentage']:.2f}%")
    print(f"\nAverage Consumption:")
    print(f"  Normal: {summary['avg_normal_consumption']:.2f} kWh")
    print(f"  Anomalies: {summary['avg_anomaly_consumption']:.2f} kWh")
    
    print("\nAnomalies by Household:")
    for household, count in summary['anomalies_by_household'].items():
        print(f"  {household}: {count:,} anomalies")
    
    # Save results
    df_with_anomalies.to_csv('energy_consumption_with_anomalies.csv', index=False)
    print("\nResults saved to energy_consumption_with_anomalies.csv")
    
    # Save only anomalies
    detector.anomalies.to_csv('anomalies_only.csv', index=False)
    print("Anomalies saved to anomalies_only.csv")
    
    print("\n" + "=" * 60)
    print("Anomaly detection complete!")
    print("For more information, visit: https://rskworld.in")

if __name__ == "__main__":
    main()

244 lines•9 KB

python

Theme Settings

Color Scheme

Display Options

Font Size

README.md

Theme Settings

Color Scheme

Display Options

Font Size

README.md

🚀 Support RSK World