{
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "# Performance Comparison: Polars vs Pandas\n",
        "\n",
        "<!--\n",
        "Author: RSK World\n",
        "Website: https://rskworld.in\n",
        "Email: help@rskworld.in\n",
        "Phone: +91 93305 39277\n",
        "-->\n",
        "\n",
        "This notebook compares the performance of Polars and Pandas for various DataFrame operations on large datasets.\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {},
      "outputs": [],
      "source": [
        "# Author: RSK World\n",
        "# Website: https://rskworld.in\n",
        "# Email: help@rskworld.in\n",
        "# Phone: +91 93305 39277\n",
        "\n",
        "import polars as pl\n",
        "import pandas as pd\n",
        "import numpy as np\n",
        "import time\n",
        "from datetime import datetime\n",
        "\n",
        "print(\"Polars version:\", pl.__version__)\n",
        "print(\"Pandas version:\", pd.__version__)\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "## 1. Generate Test Data\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {},
      "outputs": [],
      "source": [
        "# Generate a large dataset for comparison\n",
        "num_rows = 1000000\n",
        "print(f\"Generating {num_rows:,} rows of test data...\")\n",
        "\n",
        "np.random.seed(42)\n",
        "data = {\n",
        "    'id': range(1, num_rows + 1),\n",
        "    'category': np.random.choice(['A', 'B', 'C', 'D', 'E'], num_rows),\n",
        "    'value1': np.random.randn(num_rows) * 100,\n",
        "    'value2': np.random.randn(num_rows) * 50,\n",
        "    'value3': np.random.randint(1, 1000, num_rows)\n",
        "}\n",
        "\n",
        "# Create both Polars and Pandas DataFrames\n",
        "df_polars = pl.DataFrame(data)\n",
        "df_pandas = pd.DataFrame(data)\n",
        "\n",
        "print(f\"Polars DataFrame shape: {df_polars.shape}\")\n",
        "print(f\"Pandas DataFrame shape: {df_pandas.shape}\")\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "## 2. Filtering Performance\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {},
      "outputs": [],
      "source": [
        "# Polars filtering\n",
        "start = time.time()\n",
        "result_polars = df_polars.filter(pl.col('value1') > 50)\n",
        "polars_time = time.time() - start\n",
        "print(f\"Polars filtering time: {polars_time:.4f} seconds\")\n",
        "print(f\"Result shape: {result_polars.shape}\")\n",
        "\n",
        "# Pandas filtering\n",
        "start = time.time()\n",
        "result_pandas = df_pandas[df_pandas['value1'] > 50]\n",
        "pandas_time = time.time() - start\n",
        "print(f\"\\nPandas filtering time: {pandas_time:.4f} seconds\")\n",
        "print(f\"Result shape: {result_pandas.shape}\")\n",
        "\n",
        "print(f\"\\nSpeedup: {pandas_time / polars_time:.2f}x faster with Polars\")\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "## 3. Group By and Aggregate Performance\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {},
      "outputs": [],
      "source": [
        "# Polars group by\n",
        "start = time.time()\n",
        "result_polars = df_polars.group_by('category').agg([\n",
        "    pl.col('value1').mean().alias('avg_value1'),\n",
        "    pl.col('value2').mean().alias('avg_value2'),\n",
        "    pl.col('value3').sum().alias('sum_value3'),\n",
        "    pl.count().alias('count')\n",
        "])\n",
        "polars_time = time.time() - start\n",
        "print(f\"Polars group by time: {polars_time:.4f} seconds\")\n",
        "print(result_polars)\n",
        "\n",
        "# Pandas group by\n",
        "start = time.time()\n",
        "result_pandas = df_pandas.groupby('category').agg({\n",
        "    'value1': 'mean',\n",
        "    'value2': 'mean',\n",
        "    'value3': 'sum'\n",
        "}).reset_index()\n",
        "result_pandas['count'] = df_pandas.groupby('category').size().values\n",
        "pandas_time = time.time() - start\n",
        "print(f\"\\nPandas group by time: {pandas_time:.4f} seconds\")\n",
        "print(result_pandas)\n",
        "\n",
        "print(f\"\\nSpeedup: {pandas_time / polars_time:.2f}x faster with Polars\")\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "## 4. Lazy Evaluation Performance\n",
        "\n",
        "Using lazy evaluation can provide additional performance benefits.\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {},
      "outputs": [],
      "source": [
        "# Polars with lazy evaluation\n",
        "start = time.time()\n",
        "result_polars_lazy = (df_polars.lazy()\n",
        "    .filter(pl.col('value1') > 50)\n",
        "    .filter(pl.col('value2') < 20)\n",
        "    .group_by('category')\n",
        "    .agg([\n",
        "        pl.col('value1').mean().alias('avg_value1'),\n",
        "        pl.count().alias('count')\n",
        "    ])\n",
        "    .collect()\n",
        ")\n",
        "polars_lazy_time = time.time() - start\n",
        "print(f\"Polars lazy evaluation time: {polars_lazy_time:.4f} seconds\")\n",
        "print(result_polars_lazy)\n",
        "\n",
        "# Pandas equivalent\n",
        "start = time.time()\n",
        "result_pandas = (df_pandas[df_pandas['value1'] > 50]\n",
        "    [df_pandas['value2'] < 20]\n",
        "    .groupby('category')\n",
        "    .agg({\n",
        "        'value1': 'mean'\n",
        "    })\n",
        "    .reset_index()\n",
        ")\n",
        "result_pandas['count'] = (df_pandas[df_pandas['value1'] > 50]\n",
        "    [df_pandas['value2'] < 20]\n",
        "    .groupby('category')\n",
        "    .size()\n",
        "    .values\n",
        ")\n",
        "pandas_time = time.time() - start\n",
        "print(f\"\\nPandas time: {pandas_time:.4f} seconds\")\n",
        "print(result_pandas)\n",
        "\n",
        "print(f\"\\nSpeedup: {pandas_time / polars_lazy_time:.2f}x faster with Polars lazy evaluation\")\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "## 5. Memory Usage Comparison\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {},
      "outputs": [],
      "source": [
        "# Memory usage\n",
        "polars_memory = df_polars.estimated_size() / (1024 * 1024)  # MB\n",
        "pandas_memory = df_pandas.memory_usage(deep=True).sum() / (1024 * 1024)  # MB\n",
        "\n",
        "print(f\"Polars memory usage: {polars_memory:.2f} MB\")\n",
        "print(f\"Pandas memory usage: {pandas_memory:.2f} MB\")\n",
        "print(f\"Memory efficiency: {(1 - polars_memory / pandas_memory) * 100:.1f}% less memory with Polars\")\n"
      ]
    }
  ],
  "metadata": {
    "language_info": {
      "name": "python"
    }
  },
  "nbformat": 4,
  "nbformat_minor": 2
}