{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Importing libs and setting some defaults like the plot size and forcing matplotlib to use seaborn color schemes with softer, more palatable colors "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "from pandas import DataFrame, Series\n",
    "from numpy import nan\n",
    "import matplotlib.pyplot as plt\n",
    "import statsmodels.api as sm\n",
    "import statsmodels.formula.api as smf\n",
    "import numpy as np\n",
    "plt.rcParams[\"figure.figsize\"] = (24,5)\n",
    "import seaborn as sns\n",
    "sns.set()\n",
    "sns.set(font_scale=1.75)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Some default styles that will be used for some functions such as tickmarks, labels, etc"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "cols = [\n",
    "    \"TEST\",\"MAP\",\"SIZE\", \"TIME\"\n",
    "]\n",
    "onmodded = pd.read_csv(\"new_repr.csv\", quotechar=\"'\", header=None)\n",
    "\n",
    "onmodded = onmodded.drop([0],1)\n",
    "onmodded.columns= cols\n",
    "onmodded = onmodded.drop([0],0)\n",
    "onmodded.iloc[:,2:] = onmodded.iloc[:,2:].astype(\"int32\")\n",
    "\n",
    "# used for plotter and plotter3, but we can't choose nice looking and distinct enough colors to plot 17 different lines.\n",
    "# ultimately abandoned in favor of the default styles from seaborn\n",
    "styles = {'absl::flat_hash_map': [\"#0000cc\"],  # blue\n",
    "          \"absl::node_hash_map'\": [\"#3366ff\"],\n",
    "          'absl::node_hash_map': [\"#99ccff\"],\n",
    "\n",
    "          'google::dense_hash_map': [\"#ff0000\"],  # reds\n",
    "          'google::sparse_hash_map': [\"#ff6666\"],\n",
    "          'phmap::parallel_flat_hash_map': [\"#ff0066\"],\n",
    "\n",
    "          'ska::bytell_hash_map': [\"#009933\"],  # greens\n",
    "          'ska::flat_hash_map': [\"#33cc33\"],\n",
    "          'ska::unordered_map': [\"#99ff66\"],\n",
    "\n",
    "          'tsl::hopscotch_map': [\"#9900cc\"],  # purples\n",
    "          'tsl::robin_map': [\"#cc33ff\"],\n",
    "          'tsl::sparse_map': [\"#cc99ff\"],\n",
    "\n",
    "          'robin_hood::unordered_flat_map': [\"#ffcc99\"],\n",
    "          'robin_hood::unordered_node_map': [\"#ccff66\"],\n",
    "\n",
    "          'boost::unordered::unordered_map': [\"#663300\"],  # brown\n",
    "\n",
    "          'emilib::HashMap': [\"#9900cc\"],  # purple\n",
    "\n",
    "          # weird orange\n",
    "          'phmap::parallel_node_hash_map': [\"#ffcc66\", \"solid\"],\n",
    "\n",
    "          'std::unordered_map': [\"#000000\", \"solid\"],  # black\n",
    "         }\n",
    "# where to place the x tickmarks and how to label them\n",
    "ticks = [50000, 100000, 150000, 200000, 250000, 300000, 350000, 400000, 500000,\n",
    "600000, 700000, 800000, 900000, 1000000,\n",
    "2000000, 3000000, 4000000, 5000000, 6000000, 7000000, 8000000, 9000000, 10000000,\n",
    "15000000, 20000000, 25000000, 30000000, 35000000, 40000000, 45000000, 50000000]\n",
    "ticklabels = ['50 K', '100 K',\n",
    " '150 K', '200 K', '250 K', '300 K',\n",
    " '350 K', '400 K', '0.5 M', '0.6 M',\n",
    " '0.7 M', '0.8 M', '0.9 M', '1 M',\n",
    " '2 M', '3 M', '4 M', '5 M',\n",
    " '6 M', '7 M', '8 M', '9 M',\n",
    " '10 M', '15 M', '20 M', '25 M',\n",
    " '30 M', '35 M', '40 M', '45 M', '50 M']\n",
    "# convenient way to label the plots for later\n",
    "labels = {\n",
    "    'int_delete' : [\"mean int deletion time\", \"deletion time (ns)\"],\n",
    "    'int_insert' : [\"mean int insertion time\", \"insertion time(ns)\"],\n",
    "    'int_nosucc_lookup' : [\"mean int unsucessful lookup time\", \"unsucessful lookup time (ns)\"],\n",
    "    'int_succ_lookup' : [\"mean int succesful lookup time\", \"succesful lookup time (ns)\"],\n",
    "    'string_delete' : [\"mean string deletion time\", \"deletion time (ns)\"],\n",
    "    'string_insert' : [\"mean string insertion time\", \"insertion time(ns)\"], \n",
    "    'string_nosucc_lookup' : [\"mean string unsucessful lookup time\", \"unsucessful lookup time (ns)\"],\n",
    "    'string_succ_lookup' : [\"mean string succesful lookup time\", \"succesful lookup time (ns)\"]\n",
    "}"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Function definitions\n",
    "Some functions that accept and delete outliers in a range  \n",
    "A function that removes outliers with the functions mentioned above  \n",
    "Some helper functions  \n",
    "Some plotter functions  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# outlier testing functions\n",
    "def remove_with_modified_z_score(data, treshold=3.5):\n",
    "    # https://www.itl.nist.gov/div898/handbook/eda/section3/eda35h.htm\n",
    "    data = data.astype(int)\n",
    "    stats = data.describe()\n",
    "    median_absolute_deviation = abs(data - data.median()).median()\n",
    "    if not median_absolute_deviation:\n",
    "        return data\n",
    "    modified_z_scores = abs(0.6745 * (data - data.median()) / median_absolute_deviation)\n",
    "    cutoff = modified_z_scores <= treshold\n",
    "    data = data * cutoff\n",
    "    data = data.replace(0, nan)\n",
    "    return data\n",
    "\n",
    "#function that takes one of the outlier testers and data, and removes outliers\n",
    "# originally, we had multiple outlier tests we tried out and this made it convenient to check the differences\n",
    "# now it's just the modified_z_score, which we chose\n",
    "def remove_outlier(data, function):\n",
    "    new_data = data.copy(True)\n",
    "    new_data[\"TIME\"] = new_data[\"TIME\"].astype(int)\n",
    "    new_data[\"SIZE\"] = new_data[\"SIZE\"].astype(int)\n",
    "    new_data\n",
    "    for i in range(4216):\n",
    "        start = i * 30\n",
    "        end = start+30\n",
    "        new_data.loc[start:end, \"TIME\"] = function(data.loc[start:end, \"TIME\"])\n",
    "        if not i % 420:\n",
    "            print(i / 42 , \"% done\")\n",
    "    return new_data\n",
    "\n",
    "\n",
    "# helpers for plot functions\n",
    "def sort_maps(test):\n",
    "    maps = data[data[\"TEST\"]== test][\"MAP\"].unique()\n",
    "    new = [(gr_max.loc[test, i][\"TIME\"], i) for i in maps]\n",
    "    new.sort()\n",
    "    new = [i[1] for i in new]\n",
    "    return new\n",
    "\n",
    "def divider(df, maplist):\n",
    "    filters = df['MAP'].isin(maplist)\n",
    "    return df[filters]\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "# These are in no way optimized\n",
    "# way too slow for normal use\n",
    "# And not particularly reusable\n",
    "\n",
    "def plotter(test, groups):\n",
    "    maps = sort_maps(test)\n",
    "    plot_now = [5,11,16]\n",
    "#     plot_now = [2,5,8,11,14,16]\n",
    "\n",
    "    for mp in range(0,17):\n",
    "        map_data = groups.get_group(('int_insert', maps[mp]))        \n",
    "        sns.lineplot(x=\"SIZE\", y=\"TIME\", data=map_data, err_style=\"bars\", color=styles[maps[mp]][0], label=maps[mp], linestyle=\"--\")\n",
    "#         sns.lmplot(x=map_data[\"SIZE\"], y=map_data[\"TIME\"], color=styles[maps[mp]][0], label=maps[mp])\n",
    "        if mp in plot_now:\n",
    "            plt.xscale(\"log\")\n",
    "            plt.xticks(ticks, ticklabels)\n",
    "            plt.set_xticklabels(rotation=45)\n",
    "            plt.ylabel(labels[test][1])\n",
    "            plt.legend()\n",
    "            plt.title(labels[test][0])\n",
    "            plt.show()\n",
    "\n",
    "def plotter2(test, data):\n",
    "    mydata = data[data[\"TEST\"] == test]\n",
    "    maps = sort_maps(test)\n",
    "    set1 = divider(mydata, maps[:5])\n",
    "    set3 = divider(mydata, maps[12:])\n",
    "    # ^ updated this to only do the top 5 and bottom 5\n",
    "    plot = sns.lineplot(x=\"SIZE\", y=\"TIME\", hue=\"MAP\", data=set1, ci=95) # < updated to use 95% CI\n",
    "    plt.xscale(\"log\")\n",
    "    plt.xticks(ticks, ticklabels)\n",
    "    plot.set_xticklabels(\n",
    "        plot.get_xticklabels(), \n",
    "        rotation=55, \n",
    "        horizontalalignment='center',\n",
    "        fontweight='light',\n",
    "    )\n",
    "    plt.ylabel(labels[test][1])\n",
    "    plt.legend()\n",
    "    plt.title(labels[test][0])\n",
    "    plt.show()\n",
    "    \n",
    "    plot = sns.lineplot(x=\"SIZE\", y=\"TIME\", hue=\"MAP\", data=set3, ci=95) # < updated to use 95% CI\n",
    "    plt.xscale(\"log\")\n",
    "    plt.xticks(ticks, ticklabels)\n",
    "    plot.set_xticklabels(\n",
    "        plot.get_xticklabels(), \n",
    "        rotation=55, \n",
    "        horizontalalignment='center',\n",
    "        fontweight='light',\n",
    "    )\n",
    "    plt.ylabel(labels[test][1])\n",
    "    plt.legend()\n",
    "    plt.title(labels[test][0])\n",
    "    plt.show()\n",
    "\n",
    "    \n",
    "def plotter3(test, groups):\n",
    "    maps = sort_maps(test)\n",
    "    plot_now = [5,11,16]\n",
    "#     plot_now = [2,5,8,11,14,16]\n",
    "    for mp in range(0,17):\n",
    "        map_data = groups.get_group(('int_insert', maps[mp]))    \n",
    "        sns.lineplot(x=\"SIZE\", y=\"TIME\", data=map_data, err_style=\"band\", color=styles[maps[mp]][0], label=maps[mp], linestyle=\"--\")\n",
    "#         sns.regplot(x=\"SIZE\", y=\"TIME\", color=styles[maps[mp]][0],label=maps[mp], data=map_data)\n",
    "        if mp in plot_now:\n",
    "            plt.xscale(\"log\")\n",
    "            plt.xticks(ticks, ticklabels)\n",
    "            plt.ylabel(labels[test][1])\n",
    "            plt.legend()\n",
    "#             plt.set_xticklabels(rotation=45)\n",
    "            plt.title(labels[test][0])\n",
    "            plt.show()\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "* Remove the outliers\n",
    "* group the data by test and map, in case plotter or plotter3 is used\n",
    "* compute means and max of the map"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.0 % done\n",
      "10.0 % done\n",
      "20.0 % done\n",
      "30.0 % done\n",
      "40.0 % done\n",
      "50.0 % done\n",
      "60.0 % done\n",
      "70.0 % done\n",
      "80.0 % done\n",
      "90.0 % done\n",
      "100.0 % done\n"
     ]
    }
   ],
   "source": [
    "data = remove_outlier(onmodded, remove_with_modified_z_score)\n",
    "groups = data.groupby([\"TEST\", \"MAP\"])\n",
    "gr_max = groups.max()\n",
    "gr_mean = groups.mean()\n",
    "tests = data[\"TEST\"].unique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 223720 characters]\n",
      "text/plain": [
       "<Figure size 1728x360 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "[encoded data, 172372 characters]\n",
      "text/plain": [
       "<Figure size 1728x360 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plotter2(tests[0], data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 159452 characters]\n",
      "text/plain": [
       "<Figure size 1728x360 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "[encoded data, 121872 characters]\n",
      "text/plain": [
       "<Figure size 1728x360 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plotter2(tests[1], data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 165400 characters]\n",
      "text/plain": [
       "<Figure size 1728x360 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "[encoded data, 139644 characters]\n",
      "text/plain": [
       "<Figure size 1728x360 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plotter2(tests[2], data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 178628 characters]\n",
      "text/plain": [
       "<Figure size 1728x360 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "[encoded data, 162192 characters]\n",
      "text/plain": [
       "<Figure size 1728x360 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plotter2(tests[3], data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 206852 characters]\n",
      "text/plain": [
       "<Figure size 1728x360 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "[encoded data, 179336 characters]\n",
      "text/plain": [
       "<Figure size 1728x360 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plotter2(tests[4], data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 151028 characters]\n",
      "text/plain": [
       "<Figure size 1728x360 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "[encoded data, 160196 characters]\n",
      "text/plain": [
       "<Figure size 1728x360 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plotter2(tests[5], data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 158312 characters]\n",
      "text/plain": [
       "<Figure size 1728x360 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "[encoded data, 157604 characters]\n",
      "text/plain": [
       "<Figure size 1728x360 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plotter2(tests[6], data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 139220 characters]\n",
      "text/plain": [
       "<Figure size 1728x360 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "[encoded data, 153284 characters]\n",
      "text/plain": [
       "<Figure size 1728x360 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plotter2(tests[7], data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array(['absl::flat_hash_map', 'absl::node_hash_map',\n",
       "       'boost::unordered::unordered_map', 'emilib::HashMap',\n",
       "       'google::dense_hash_map', 'google::sparse_hash_map',\n",
       "       'phmap::parallel_flat_hash_map', 'phmap::parallel_node_hash_map',\n",
       "       'robin_hood::unordered_flat_map', 'robin_hood::unordered_node_map',\n",
       "       'ska::bytell_hash_map', 'ska::flat_hash_map', 'ska::unordered_map',\n",
       "       'std::unordered_map', 'tsl::hopscotch_map', 'tsl::robin_map',\n",
       "       'tsl::sparse_map'], dtype=object)"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data[\"MAP\"].unique()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Quick way of getting the mean time of each operation and map (for all sizes)  \n",
    "This number is totally meaningless on it's own, and the absolute values (in ns) should just not be mentioned\n",
    "I think it's meaningless to mention that the mean insertion time of a map is 50ns between sizes 50K and 50M.\n",
    "It \"loses\" so much info.  \n",
    "But it is a decent use to rank the maps on.  \n",
    "Eventhough it is somewhat biased towards maps that perform decently at large sizes. Or in other words, this ranking makes the fact that a function is blazing fast for small sizes matter way less.  \n",
    "This is due to the following:\n",
    "lets say there are 5 measuring points below 1M, and 20 measuring points between 1M and 50M\n",
    "when that is averaged, way more points will be >1M, so they will weight more for the mean.\n",
    "\n",
    "But the graphs are pretty hard to get a quick overview of after a quick glance, so this is my best attempt to create a summary of the relative rankings.\n",
    "And I don't have the slightest idea how you would rank them besides a mean value. A max value would be a good idea, except that there's random spiky behavour which have higher values in the middle than at 50M. And it's even less representative of the times as a whole compared to the mean. So we have this. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "11\n",
      "9\n",
      "13\n",
      "4\n",
      "6\n",
      "17\n",
      "12\n",
      "14\n",
      "7\n",
      "1\n",
      "8\n",
      "3\n",
      "10\n",
      "15\n",
      "5\n",
      "2\n",
      "16\n"
     ]
    }
   ],
   "source": [
    "test = \"string_insert\"\n",
    "alpha_maps = gr_mean.loc[test][\"TIME\"].index\n",
    "rank = list(gr_mean.loc[test][\"TIME\"].sort_values().index) # sort by mean\n",
    "rank = [rank.index(i) for i in alpha_maps] # get the index for each map in alphamaps\n",
    "for i in rank:\n",
    "    print(i + 1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Index(['absl::flat_hash_map', 'absl::node_hash_map',\n",
       "       'boost::unordered::unordered_map', 'emilib::HashMap',\n",
       "       'google::dense_hash_map', 'google::sparse_hash_map',\n",
       "       'phmap::parallel_flat_hash_map', 'phmap::parallel_node_hash_map',\n",
       "       'robin_hood::unordered_flat_map', 'robin_hood::unordered_node_map',\n",
       "       'ska::bytell_hash_map', 'ska::flat_hash_map', 'ska::unordered_map',\n",
       "       'std::unordered_map', 'tsl::hopscotch_map', 'tsl::robin_map',\n",
       "       'tsl::sparse_map'],\n",
       "      dtype='object', name='MAP')"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "alpha_maps"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}
