{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "from pandas import DataFrame, Series\n",
    "from numpy import nan\n",
    "import pathlib\n",
    "import matplotlib.pyplot as plt\n",
    "plt.rcParams[\"figure.figsize\"] = (30,5)\n",
    "import seaborn as sns\n",
    "sns.set()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>TEST</th>\n",
       "      <th>MAP</th>\n",
       "      <th>50000</th>\n",
       "      <th>100000</th>\n",
       "      <th>150000</th>\n",
       "      <th>200000</th>\n",
       "      <th>250000</th>\n",
       "      <th>300000</th>\n",
       "      <th>350000</th>\n",
       "      <th>400000</th>\n",
       "      <th>...</th>\n",
       "      <th>9000000</th>\n",
       "      <th>10000000</th>\n",
       "      <th>15000000</th>\n",
       "      <th>20000000</th>\n",
       "      <th>25000000</th>\n",
       "      <th>30000000</th>\n",
       "      <th>35000000</th>\n",
       "      <th>40000000</th>\n",
       "      <th>45000000</th>\n",
       "      <th>50000000</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Unnamed: 0</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>int_delete</td>\n",
       "      <td>absl::flat_hash_map</td>\n",
       "      <td>9</td>\n",
       "      <td>14</td>\n",
       "      <td>11</td>\n",
       "      <td>14</td>\n",
       "      <td>11</td>\n",
       "      <td>12</td>\n",
       "      <td>13</td>\n",
       "      <td>15</td>\n",
       "      <td>...</td>\n",
       "      <td>21</td>\n",
       "      <td>21</td>\n",
       "      <td>24</td>\n",
       "      <td>28</td>\n",
       "      <td>25</td>\n",
       "      <td>25</td>\n",
       "      <td>26</td>\n",
       "      <td>25</td>\n",
       "      <td>26</td>\n",
       "      <td>26</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>int_delete</td>\n",
       "      <td>absl::flat_hash_map</td>\n",
       "      <td>9</td>\n",
       "      <td>14</td>\n",
       "      <td>11</td>\n",
       "      <td>14</td>\n",
       "      <td>11</td>\n",
       "      <td>12</td>\n",
       "      <td>13</td>\n",
       "      <td>16</td>\n",
       "      <td>...</td>\n",
       "      <td>20</td>\n",
       "      <td>23</td>\n",
       "      <td>24</td>\n",
       "      <td>23</td>\n",
       "      <td>27</td>\n",
       "      <td>26</td>\n",
       "      <td>26</td>\n",
       "      <td>26</td>\n",
       "      <td>26</td>\n",
       "      <td>26</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>int_delete</td>\n",
       "      <td>absl::flat_hash_map</td>\n",
       "      <td>9</td>\n",
       "      <td>14</td>\n",
       "      <td>11</td>\n",
       "      <td>14</td>\n",
       "      <td>11</td>\n",
       "      <td>12</td>\n",
       "      <td>13</td>\n",
       "      <td>15</td>\n",
       "      <td>...</td>\n",
       "      <td>20</td>\n",
       "      <td>21</td>\n",
       "      <td>24</td>\n",
       "      <td>24</td>\n",
       "      <td>25</td>\n",
       "      <td>26</td>\n",
       "      <td>26</td>\n",
       "      <td>30</td>\n",
       "      <td>26</td>\n",
       "      <td>27</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>int_delete</td>\n",
       "      <td>absl::flat_hash_map</td>\n",
       "      <td>9</td>\n",
       "      <td>14</td>\n",
       "      <td>11</td>\n",
       "      <td>14</td>\n",
       "      <td>11</td>\n",
       "      <td>12</td>\n",
       "      <td>13</td>\n",
       "      <td>15</td>\n",
       "      <td>...</td>\n",
       "      <td>21</td>\n",
       "      <td>21</td>\n",
       "      <td>24</td>\n",
       "      <td>24</td>\n",
       "      <td>25</td>\n",
       "      <td>25</td>\n",
       "      <td>26</td>\n",
       "      <td>29</td>\n",
       "      <td>26</td>\n",
       "      <td>26</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>int_delete</td>\n",
       "      <td>absl::flat_hash_map</td>\n",
       "      <td>9</td>\n",
       "      <td>14</td>\n",
       "      <td>11</td>\n",
       "      <td>14</td>\n",
       "      <td>11</td>\n",
       "      <td>12</td>\n",
       "      <td>13</td>\n",
       "      <td>15</td>\n",
       "      <td>...</td>\n",
       "      <td>20</td>\n",
       "      <td>20</td>\n",
       "      <td>24</td>\n",
       "      <td>24</td>\n",
       "      <td>25</td>\n",
       "      <td>26</td>\n",
       "      <td>26</td>\n",
       "      <td>26</td>\n",
       "      <td>26</td>\n",
       "      <td>27</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 33 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                  TEST                  MAP  50000  100000  150000  200000  \\\n",
       "Unnamed: 0                                                                   \n",
       "0           int_delete  absl::flat_hash_map      9      14      11      14   \n",
       "1           int_delete  absl::flat_hash_map      9      14      11      14   \n",
       "2           int_delete  absl::flat_hash_map      9      14      11      14   \n",
       "3           int_delete  absl::flat_hash_map      9      14      11      14   \n",
       "4           int_delete  absl::flat_hash_map      9      14      11      14   \n",
       "\n",
       "            250000  300000  350000  400000  ...  9000000  10000000  15000000  \\\n",
       "Unnamed: 0                                  ...                                \n",
       "0               11      12      13      15  ...       21        21        24   \n",
       "1               11      12      13      16  ...       20        23        24   \n",
       "2               11      12      13      15  ...       20        21        24   \n",
       "3               11      12      13      15  ...       21        21        24   \n",
       "4               11      12      13      15  ...       20        20        24   \n",
       "\n",
       "            20000000  25000000  30000000  35000000  40000000  45000000  \\\n",
       "Unnamed: 0                                                               \n",
       "0                 28        25        25        26        25        26   \n",
       "1                 23        27        26        26        26        26   \n",
       "2                 24        25        26        26        30        26   \n",
       "3                 24        25        25        26        29        26   \n",
       "4                 24        25        26        26        26        26   \n",
       "\n",
       "            50000000  \n",
       "Unnamed: 0            \n",
       "0                 26  \n",
       "1                 26  \n",
       "2                 27  \n",
       "3                 26  \n",
       "4                 27  \n",
       "\n",
       "[5 rows x 33 columns]"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cols = [\n",
    "\"TEST\",\"MAP\",\n",
    "50000, 100000, 150000, 200000, 250000, 300000, 350000, 400000, 500000,\n",
    "600000, 700000, 800000, 900000, 1000000,\n",
    "2000000, 3000000, 4000000, 5000000, 6000000, 7000000, 8000000, 9000000, 10000000,\n",
    "15000000, 20000000, 25000000, 30000000, 35000000, 40000000, 45000000, 50000000\n",
    "]\n",
    "if not pathlib.Path(\"./sorted.csv\").is_file():\n",
    "    data = pd.read_csv(\"results.csv\", quotechar=\"'\", header=None)\n",
    "    data.columns= cols\n",
    "    data = data.sort_values(by=['TEST', 'MAP'], ignore_index=True)\n",
    "    data.to_csv(\"sorted.csv\")\n",
    "\n",
    "else:\n",
    "    data = pd.read_csv(\"sorted.csv\")\n",
    "    data.set_index(\"Unnamed: 0\", inplace=True)\n",
    "data.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "def remove_with_iqr(data):\n",
    "    stats = data.describe()\n",
    "    IQR = stats[\"75%\"] - stats[\"25%\"]\n",
    "    min_cutoff = stats[\"25%\"] - 1.5 * IQR\n",
    "    max_cutoff = stats[\"75%\"] + 1.5 * IQR\n",
    "    lower_max = data <= max_cutoff\n",
    "    higher_min = data >= min_cutoff\n",
    "    data = data * lower_max * higher_min\n",
    "    data = data.replace(0, nan)\n",
    "    return data\n",
    "    \n",
    "def remove_with_zscore(data, cutoff=3.5):\n",
    "    stats = data.describe()\n",
    "    if not stats[\"std\"]:\n",
    "        return data\n",
    "    z = abs((data - stats[\"mean\"]) / stats[\"std\"])\n",
    "    cut = z <= cutoff\n",
    "    data = data * cut\n",
    "    data = data.replace(0, nan)\n",
    "    return data \n",
    "\n",
    "def remove_with_modified_z_score(data, treshold=3.5):\n",
    "    stats = data.describe()\n",
    "    median_absolute_deviation = abs(data - data.median()).median()\n",
    "    if not median_absolute_deviation:\n",
    "        return data\n",
    "    modified_z_scores = abs(0.6745 * (data - data.median()) / median_absolute_deviation)\n",
    "    cutoff = modified_z_scores <= treshold\n",
    "    data = data * cutoff\n",
    "    data = data.replace(0, nan)\n",
    "    return data\n",
    "\n",
    "def max_val(hmap, test):\n",
    "    return groups_mean.loc[test, hmap].max()\n",
    "\n",
    "def sort_maps(test):\n",
    "    maps = list(groups_mean.loc[test].index)\n",
    "    new = [(max_val(i, test), i) for i in maps]\n",
    "    new.sort()\n",
    "    new = [i[1] for i in new]\n",
    "    return new\n",
    "\n",
    "def plot_test(test, include_error=True):\n",
    "    sizes = [50000, 100000, 150000, 200000, 250000, 300000, 350000, 400000, 500000,\n",
    "        600000, 700000, 800000, 900000, 1000000,\n",
    "        2000000, 3000000, 4000000, 5000000, 6000000, 7000000, 8000000, 9000000, 10000000,\n",
    "        15000000, 20000000, 25000000, 30000000, 35000000, 40000000, 45000000, 50000000]\n",
    "    maps = sort_maps(test)\n",
    "    count = 16\n",
    "    repeats = [0, 5, 11]\n",
    "    while count > -1:\n",
    "        if not count and count not in repeats:\n",
    "            break\n",
    "        mp = maps[count]\n",
    "        else:\n",
    "            sns.lineplot(x=sizes, y=groups_mean.loc[test, mp], label=mp, order=2)\n",
    "        if count in repeats:\n",
    "            plt.xscale(\"log\")\n",
    "            plt.ylabel(\"{} time (ns)\".format(test))\n",
    "            plt.legend()\n",
    "            plt.title(test)\n",
    "            plt.show()\n",
    "            repeats.pop(repeats.index(count))\n",
    "        else:\n",
    "            count -=1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "iqr_data = data.copy(True)\n",
    "# modz_data = data.copy(True)\n",
    "# z_data = data.copy(True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "for col in range(2,33):\n",
    "    for row in range(8 * 17):\n",
    "        start = row * 30\n",
    "        end = (row + 1 ) * 30\n",
    "        temp = data.iloc[start : end,col]\n",
    "        iqr_data.iloc[start : end,col] = remove_with_iqr(temp)\n",
    "#         modz_data.iloc[start : end,col] = remove_with_modified_z_score(temp)\n",
    "#         z_data.iloc[start : end,col] = remove_with_zscore(temp)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 70376 characters]\n",
      "text/plain": [
       "<Figure size 2160x360 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "[encoded data, 146172 characters]\n",
      "text/plain": [
       "<Figure size 2160x360 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "[encoded data, 121892 characters]\n",
      "text/plain": [
       "<Figure size 2160x360 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "tester = iqr_data # use removed outliers data\n",
    "groups_modz = tester.groupby([\"TEST\", \"MAP\"])\n",
    "groups_mean = groups_modz.mean()\n",
    "groups_std = groups_modz.std()\n",
    "groups_std.columns = [int(i) for i in groups_std.columns]\n",
    "groups_mean.columns = groups_std.columns\n",
    "plot_test(\"int_insert\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 49940 characters]\n",
      "text/plain": [
       "<Figure size 2160x360 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "[encoded data, 118028 characters]\n",
      "text/plain": [
       "<Figure size 2160x360 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "[encoded data, 113056 characters]\n",
      "text/plain": [
       "<Figure size 2160x360 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "tester = data # use original data\n",
    "groups_modz = tester.groupby([\"TEST\", \"MAP\"])\n",
    "groups_mean = groups_modz.mean()\n",
    "groups_std = groups_modz.std()\n",
    "groups_std.columns = [int(i) for i in groups_std.columns]\n",
    "groups_mean.columns = groups_std.columns\n",
    "plot_test(\"int_insert\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 49940 characters]\n",
      "text/plain": [
       "<Figure size 2160x360 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "[encoded data, 118028 characters]\n",
      "text/plain": [
       "<Figure size 2160x360 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "[encoded data, 113056 characters]\n",
      "text/plain": [
       "<Figure size 2160x360 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "tests = data[\"TEST\"].unique()\n",
    "plot_test(tests[1])\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 54516 characters]\n",
      "text/plain": [
       "<Figure size 2160x360 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "[encoded data, 104116 characters]\n",
      "text/plain": [
       "<Figure size 2160x360 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "[encoded data, 122172 characters]\n",
      "text/plain": [
       "<Figure size 2160x360 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plot_test(tests[2])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 100912 characters]\n",
      "text/plain": [
       "<Figure size 2160x360 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "[encoded data, 119224 characters]\n",
      "text/plain": [
       "<Figure size 2160x360 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "[encoded data, 118524 characters]\n",
      "text/plain": [
       "<Figure size 2160x360 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plot_test(tests[3])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 145856 characters]\n",
      "text/plain": [
       "<Figure size 2160x360 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "[encoded data, 174236 characters]\n",
      "text/plain": [
       "<Figure size 2160x360 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "[encoded data, 145656 characters]\n",
      "text/plain": [
       "<Figure size 2160x360 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plot_test(tests[4])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}
