RSK World - Polars Fast DataFrames - Project Files | RSK World - Free Programming Resources & Source Code

performance_comparison.py transformers.py generate_data_standalone.py main.py classification_metadata.json transfer_learning.py index.html

scripts/performance_comparison.py

Raw Download

"""
Performance Comparison: Polars vs Pandas
Compares performance of Polars and Pandas for various operations

Author: RSK World
Website: https://rskworld.in
Email: help@rskworld.in
Phone: +91 93305 39277
"""

import polars as pl
import pandas as pd
import numpy as np
import time
from datetime import datetime

def compare_filtering(num_rows=1000000):
    """Compare filtering performance"""
    print("=" * 60)
    print("FILTERING PERFORMANCE COMPARISON")
    print("=" * 60)
    
    print(f"\nGenerating {num_rows:,} rows of test data...")
    np.random.seed(42)
    data = {
        'id': range(1, num_rows + 1),
        'category': np.random.choice(['A', 'B', 'C', 'D', 'E'], num_rows),
        'value1': np.random.randn(num_rows) * 100,
        'value2': np.random.randn(num_rows) * 50,
        'value3': np.random.randint(1, 1000, num_rows)
    }
    
    df_polars = pl.DataFrame(data)
    df_pandas = pd.DataFrame(data)
    
    # Polars filtering
    print("\n1. Filtering (value1 > 50):")
    start = time.time()
    result_polars = df_polars.filter(pl.col('value1') > 50)
    polars_time = time.time() - start
    print(f"   Polars time: {polars_time:.4f} seconds")
    print(f"   Result shape: {result_polars.shape}")
    
    # Pandas filtering
    start = time.time()
    result_pandas = df_pandas[df_pandas['value1'] > 50]
    pandas_time = time.time() - start
    print(f"   Pandas time: {pandas_time:.4f} seconds")
    print(f"   Result shape: {result_pandas.shape}")
    
    speedup = pandas_time / polars_time if polars_time > 0 else 0
    print(f"\n   Speedup: {speedup:.2f}x faster with Polars")
    
    return polars_time, pandas_time

def compare_groupby(num_rows=1000000):
    """Compare group by performance"""
    print("\n" + "=" * 60)
    print("GROUP BY PERFORMANCE COMPARISON")
    print("=" * 60)
    
    print(f"\nGenerating {num_rows:,} rows of test data...")
    np.random.seed(42)
    data = {
        'id': range(1, num_rows + 1),
        'category': np.random.choice(['A', 'B', 'C', 'D', 'E'], num_rows),
        'value1': np.random.randn(num_rows) * 100,
        'value2': np.random.randn(num_rows) * 50,
        'value3': np.random.randint(1, 1000, num_rows)
    }
    
    df_polars = pl.DataFrame(data)
    df_pandas = pd.DataFrame(data)
    
    # Polars group by
    print("\n1. Group By and Aggregate:")
    start = time.time()
    result_polars = df_polars.group_by('category').agg([
        pl.col('value1').mean().alias('avg_value1'),
        pl.col('value2').mean().alias('avg_value2'),
        pl.col('value3').sum().alias('sum_value3'),
        pl.count().alias('count')
    ])
    polars_time = time.time() - start
    print(f"   Polars time: {polars_time:.4f} seconds")
    print(result_polars)
    
    # Pandas group by
    start = time.time()
    result_pandas = df_pandas.groupby('category').agg({
        'value1': 'mean',
        'value2': 'mean',
        'value3': 'sum'
    }).reset_index()
    result_pandas['count'] = df_pandas.groupby('category').size().values
    pandas_time = time.time() - start
    print(f"\n   Pandas time: {pandas_time:.4f} seconds")
    print(result_pandas)
    
    speedup = pandas_time / polars_time if polars_time > 0 else 0
    print(f"\n   Speedup: {speedup:.2f}x faster with Polars")
    
    return polars_time, pandas_time

def compare_lazy_evaluation(num_rows=1000000):
    """Compare lazy evaluation performance"""
    print("\n" + "=" * 60)
    print("LAZY EVALUATION PERFORMANCE COMPARISON")
    print("=" * 60)
    
    print(f"\nGenerating {num_rows:,} rows of test data...")
    np.random.seed(42)
    data = {
        'id': range(1, num_rows + 1),
        'category': np.random.choice(['A', 'B', 'C', 'D', 'E'], num_rows),
        'value1': np.random.randn(num_rows) * 100,
        'value2': np.random.randn(num_rows) * 50,
        'value3': np.random.randint(1, 1000, num_rows)
    }
    
    df_polars = pl.DataFrame(data)
    df_pandas = pd.DataFrame(data)
    
    # Polars with lazy evaluation
    print("\n1. Complex Query with Lazy Evaluation:")
    start = time.time()
    result_polars_lazy = (df_polars.lazy()
        .filter(pl.col('value1') > 50)
        .filter(pl.col('value2') < 20)
        .group_by('category')
        .agg([
            pl.col('value1').mean().alias('avg_value1'),
            pl.count().alias('count')
        ])
        .collect()
    )
    polars_lazy_time = time.time() - start
    print(f"   Polars lazy time: {polars_lazy_time:.4f} seconds")
    print(result_polars_lazy)
    
    # Pandas equivalent
    start = time.time()
    filtered = df_pandas[(df_pandas['value1'] > 50) & (df_pandas['value2'] < 20)]
    result_pandas = filtered.groupby('category').agg({
        'value1': 'mean'
    }).reset_index()
    result_pandas['count'] = filtered.groupby('category').size().values
    pandas_time = time.time() - start
    print(f"\n   Pandas time: {pandas_time:.4f} seconds")
    print(result_pandas)
    
    speedup = pandas_time / polars_lazy_time if polars_lazy_time > 0 else 0
    print(f"\n   Speedup: {speedup:.2f}x faster with Polars lazy evaluation")
    
    return polars_lazy_time, pandas_time

def compare_memory_usage(num_rows=1000000):
    """Compare memory usage"""
    print("\n" + "=" * 60)
    print("MEMORY USAGE COMPARISON")
    print("=" * 60)
    
    print(f"\nGenerating {num_rows:,} rows of test data...")
    np.random.seed(42)
    data = {
        'id': range(1, num_rows + 1),
        'category': np.random.choice(['A', 'B', 'C', 'D', 'E'], num_rows),
        'value1': np.random.randn(num_rows) * 100,
        'value2': np.random.randn(num_rows) * 50,
        'value3': np.random.randint(1, 1000, num_rows)
    }
    
    df_polars = pl.DataFrame(data)
    df_pandas = pd.DataFrame(data)
    
    # Memory usage
    polars_memory = df_polars.estimated_size() / (1024 * 1024)  # MB
    pandas_memory = df_pandas.memory_usage(deep=True).sum() / (1024 * 1024)  # MB
    
    print(f"\nPolars memory usage: {polars_memory:.2f} MB")
    print(f"Pandas memory usage: {pandas_memory:.2f} MB")
    
    if pandas_memory > 0:
        efficiency = (1 - polars_memory / pandas_memory) * 100
        print(f"Memory efficiency: {efficiency:.1f}% less memory with Polars")

def run_all_comparisons():
    """Run all performance comparisons"""
    print("\n" + "=" * 60)
    print("POLARS VS PANDAS PERFORMANCE COMPARISON")
    print("=" * 60)
    print("\nNote: Using 1,000,000 rows for testing")
    print("Adjust num_rows parameter for different dataset sizes\n")
    
    # Run comparisons with smaller dataset for faster execution
    num_rows = 100000  # Reduced for faster demo
    
    compare_filtering(num_rows)
    compare_groupby(num_rows)
    compare_lazy_evaluation(num_rows)
    compare_memory_usage(num_rows)
    
    print("\n" + "=" * 60)
    print("Performance comparison complete!")
    print("=" * 60)
    print("\nKey Takeaways:")
    print("- Polars is typically 5-30x faster than Pandas")
    print("- Polars uses less memory due to Apache Arrow format")
    print("- Lazy evaluation provides additional optimization")
    print("- Polars is ideal for large-scale data processing")

if __name__ == "__main__":
    run_all_comparisons()

215 lines•7.3 KB

python

index.html

Raw Download

<!DOCTYPE html>
<html lang="en">
<head>
    <!--
    Author: RSK World
    Website: https://rskworld.in
    Email: help@rskworld.in
    Phone: +91 93305 39277
    -->
    <meta charset="UTF-8">
    <meta name="viewport" content="width=device-width, initial-scale=1.0">
    <title>Polars Fast DataFrames - RSK World</title>
    <style>
        * {
            margin: 0;
            padding: 0;
            box-sizing: border-box;
        }
        
        body {
            font-family: 'Segoe UI', Tahoma, Geneva, Verdana, sans-serif;
            line-height: 1.6;
            color: #333;
            background: linear-gradient(135deg, #667eea 0%, #764ba2 100%);
            min-height: 100vh;
            padding: 20px;
        }
        
        .container {
            max-width: 1200px;
            margin: 0 auto;
            background: white;
            border-radius: 10px;
            box-shadow: 0 10px 30px rgba(0,0,0,0.3);
            overflow: hidden;
        }
        
        header {
            background: linear-gradient(135deg, #667eea 0%, #764ba2 100%);
            color: white;
            padding: 40px;
            text-align: center;
        }
        
        header h1 {
            font-size: 2.5em;
            margin-bottom: 10px;
        }
        
        header p {
            font-size: 1.2em;
            opacity: 0.9;
        }
        
        .content {
            padding: 40px;
        }
        
        .section {
            margin-bottom: 40px;
        }
        
        .section h2 {
            color: #667eea;
            margin-bottom: 20px;
            font-size: 1.8em;
            border-bottom: 2px solid #667eea;
            padding-bottom: 10px;
        }
        
        .features {
            display: grid;
            grid-template-columns: repeat(auto-fit, minmax(250px, 1fr));
            gap: 20px;
            margin-top: 20px;
        }
        
        .feature-card {
            background: #f8f9fa;
            padding: 20px;
            border-radius: 8px;
            border-left: 4px solid #667eea;
            transition: transform 0.3s;
        }
        
        .feature-card:hover {
            transform: translateY(-5px);
            box-shadow: 0 5px 15px rgba(0,0,0,0.1);
        }
        
        .feature-card h3 {
            color: #667eea;
            margin-bottom: 10px;
        }
        
        .tech-stack {
            display: flex;
            flex-wrap: wrap;
            gap: 10px;
            margin-top: 20px;
        }
        
        .tech-badge {
            background: #667eea;
            color: white;
            padding: 8px 16px;
            border-radius: 20px;
            font-size: 0.9em;
        }
        
        .code-block {
            background: #2d2d2d;
            color: #f8f8f2;
            padding: 20px;
            border-radius: 8px;
            overflow-x: auto;
            margin: 20px 0;
        }
        
        .code-block code {
            font-family: 'Courier New', monospace;
            font-size: 0.9em;
        }
        
        .btn {
            display: inline-block;
            padding: 12px 24px;
            background: #667eea;
            color: white;
            text-decoration: none;
            border-radius: 5px;
            transition: background 0.3s;
            margin: 10px 10px 10px 0;
        }
        
        .btn:hover {
            background: #764ba2;
        }
        
        .btn-secondary {
            background: #6c757d;
        }
        
        .btn-secondary:hover {
            background: #5a6268;
        }
        
        footer {
            background: #2d2d2d;
            color: white;
            padding: 30px;
            text-align: center;
        }
        
        footer a {
            color: #667eea;
            text-decoration: none;
        }
        
        footer a:hover {
            text-decoration: underline;
        }
        
        .author-info {
            background: #f8f9fa;
            padding: 20px;
            border-radius: 8px;
            margin-top: 30px;
        }
        
        .author-info h3 {
            color: #667eea;
            margin-bottom: 15px;
        }
        
        .author-info p {
            margin: 5px 0;
        }
    </style>
</head>
<body>
    <div class="container">
        <header>
            <h1>⚡ Polars Fast DataFrames</h1>
            <p>High-performance DataFrame library for fast data processing, querying, and analysis</p>
        </header>
        
        <div class="content">
            <div class="section">
                <h2>📋 Description</h2>
                <p>This project demonstrates <strong>Polars</strong>, a blazingly fast DataFrame library written in Rust. It covers DataFrame operations, lazy evaluation, query optimization, and performance comparisons. Perfect for high-performance data processing and analysis on large datasets.</p>
            </div>
            
            <div class="section">
                <h2>✨ Features</h2>
                <div class="features">
                    <div class="feature-card">
                        <h3>⚡ Fast Operations</h3>
                        <p>Lightning-fast DataFrame operations optimized for performance</p>
                    </div>
                    <div class="feature-card">
                        <h3>🔄 Lazy Evaluation</h3>
                        <p>Query optimization through lazy evaluation and query planning</p>
                    </div>
                    <div class="feature-card">
                        <h3>💾 Memory Efficient</h3>
                        <p>Memory-efficient processing using Apache Arrow columnar format</p>
                    </div>
                    <div class="feature-card">
                        <h3>🎯 Query Optimization</h3>
                        <p>Automatic query optimization for better performance</p>
                    </div>
                    <div class="feature-card">
                        <h3>🔗 Pandas Compatible</h3>
                        <p>Easy integration with existing Pandas workflows</p>
                    </div>
                </div>
            </div>
            
            <div class="section">
                <h2>🛠️ Technologies</h2>
                <div class="tech-stack">
                    <span class="tech-badge">Python</span>
                    <span class="tech-badge">Polars</span>
                    <span class="tech-badge">Pandas</span>
                    <span class="tech-badge">Jupyter Notebook</span>
                    <span class="tech-badge">NumPy</span>
                    <span class="tech-badge">Matplotlib</span>
                </div>
            </div>
            
            <div class="section">
                <h2>📚 Project Structure</h2>
                <div class="code-block">
                    <code>
polars-fastdataframes/<br>
├── README.md<br>
├── requirements.txt<br>
├── notebooks/<br>
│   ├── 01_basic_operations.ipynb<br>
│   ├── 02_lazy_evaluation.ipynb<br>
│   ├── 03_performance_comparison.ipynb<br>
│   └── 04_advanced_queries.ipynb<br>
├── scripts/<br>
│   ├── basic_operations.py<br>
│   ├── lazy_evaluation.py<br>
│   ├── performance_comparison.py<br>
│   └── data_generator.py<br>
└── data/<br>
    └── sample_data.csv
                    </code>
                </div>
            </div>
            
            <div class="section">
                <h2>🚀 Quick Start</h2>
                <h3>Installation</h3>
                <div class="code-block">
                    <code>pip install -r requirements.txt</code>
                </div>
                
                <h3>Run Scripts</h3>
                <div class="code-block">
                    <code>python scripts/basic_operations.py<br>python scripts/lazy_evaluation.py<br>python scripts/performance_comparison.py</code>
                </div>
                
                <h3>Jupyter Notebooks</h3>
                <div class="code-block">
                    <code>jupyter notebook</code>
                </div>
            </div>
            
            <div class="section">
                <h2>📊 Performance</h2>
                <p>Polars is designed for speed and efficiency. It uses:</p>
                <ul style="margin-left: 20px; margin-top: 10px;">
                    <li>Apache Arrow columnar memory format</li>
                    <li>Query optimization through lazy evaluation</li>
                    <li>Parallel processing capabilities</li>
                    <li>Zero-copy reads</li>
                </ul>
                <p style="margin-top: 15px;"><strong>Typical Performance:</strong> Polars is 5-30x faster than Pandas for most operations.</p>
            </div>
            
            <div class="author-info">
                <h3>👤 Author Information</h3>
                <p><strong>Author:</strong> RSK World</p>
                <p><strong>Website:</strong> <a href="https://rskworld.in" target="_blank">https://rskworld.in</a></p>
                <p><strong>Email:</strong> <a href="mailto:help@rskworld.in">help@rskworld.in</a></p>
                <p><strong>Phone:</strong> +91 93305 39277</p>
            </div>
        </div>
        
        <footer>
            <p>&copy; 2024 RSK World. All rights reserved.</p>
            <p>
                <a href="https://rskworld.in" target="_blank">Website</a> | 
                <a href="mailto:help@rskworld.in">Contact</a> | 
                <a href="https://github.com/rskworld/polars-fastdataframes" target="_blank">GitHub</a>
            </p>
        </footer>
    </div>
</body>
</html>

306 lines•9.9 KB

markup

Theme Settings

Color Scheme

Display Options

Font Size

Theme Settings

Color Scheme

Display Options

Font Size

🚀 Support RSK World