{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 71,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "from pandas import DataFrame, Series\n",
    "from numpy import nan\n",
    "import matplotlib.pyplot as plt\n",
    "import statsmodels.api as sm\n",
    "import statsmodels.formula.api as smf\n",
    "import numpy as np\n",
    "plt.rcParams[\"figure.figsize\"] = (24,5)\n",
    "import seaborn as sns\n",
    "sns.set()\n",
    "sns.set(font_scale=1.75)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "cols = [\n",
    "    \"TEST\",\"MAP\",\"SIZE\", \"TIME\"\n",
    "]\n",
    "onmodded = pd.read_csv(\"new_repr.csv\", quotechar=\"'\", header=None)\n",
    "\n",
    "onmodded = onmodded.drop([0],1)\n",
    "onmodded.columns= cols\n",
    "onmodded = onmodded.drop([0],0)\n",
    "onmodded.iloc[:,2:] = onmodded.iloc[:,2:].astype(\"int32\")\n",
    "\n",
    "\n",
    "styles = {'absl::flat_hash_map': [\"#0000cc\"],  # blue\n",
    "          \"absl::node_hash_map'\": [\"#3366ff\"],\n",
    "          'absl::node_hash_map': [\"#99ccff\"],\n",
    "\n",
    "          'google::dense_hash_map': [\"#ff0000\"],  # reds\n",
    "          'google::sparse_hash_map': [\"#ff6666\"],\n",
    "          'phmap::parallel_flat_hash_map': [\"#ff0066\"],\n",
    "\n",
    "          'ska::bytell_hash_map': [\"#009933\"],  # greens\n",
    "          'ska::flat_hash_map': [\"#33cc33\"],\n",
    "          'ska::unordered_map': [\"#99ff66\"],\n",
    "\n",
    "          'tsl::hopscotch_map': [\"#9900cc\"],  # purples\n",
    "          'tsl::robin_map': [\"#cc33ff\"],\n",
    "          'tsl::sparse_map': [\"#cc99ff\"],\n",
    "\n",
    "          'robin_hood::unordered_flat_map': [\"#ffcc99\"],\n",
    "          'robin_hood::unordered_node_map': [\"#ccff66\"],\n",
    "\n",
    "          'boost::unordered::unordered_map': [\"#663300\"],  # brown\n",
    "\n",
    "          'emilib::HashMap': [\"#9900cc\"],  # purple\n",
    "\n",
    "          # weird orange\n",
    "          'phmap::parallel_node_hash_map': [\"#ffcc66\", \"solid\"],\n",
    "\n",
    "          'std::unordered_map': [\"#000000\", \"solid\"],  # black\n",
    "         }\n",
    "ticks = [50000, 100000, 150000, 200000, 250000, 300000, 350000, 400000, 500000,\n",
    "600000, 700000, 800000, 900000, 1000000,\n",
    "2000000, 3000000, 4000000, 5000000, 6000000, 7000000, 8000000, 9000000, 10000000,\n",
    "15000000, 20000000, 25000000, 30000000, 35000000, 40000000, 45000000, 50000000]\n",
    "ticklabels = ['50 K', '100 K',\n",
    " '150 K', '200 K', '250 K', '300 K',\n",
    " '350 K', '400 K', '0.5 M', '0.6 M',\n",
    " '0.7 M', '0.8 M', '0.9 M', '1 M',\n",
    " '2 M', '3 M', '4 M', '5 M',\n",
    " '6 M', '7 M', '8 M', '9 M',\n",
    " '10 M', '15 M', '20 M', '25 M',\n",
    " '30 M', '35 M', '40 M', '45 M', '50 M']\n",
    "labels = {\n",
    "    'int_delete' : [\"mean int deletion time\", \"deletion time (ns)\"],\n",
    "    'int_insert' : [\"mean int insertion time\", \"insertion time(ns)\"],\n",
    "    'int_nosucc_lookup' : [\"mean int unsucessful lookup time\", \"unsucessful lookup time (ns)\"],\n",
    "    'int_succ_lookup' : [\"mean int succesful lookup time\", \"succesful lookup time (ns)\"],\n",
    "    'string_delete' : [\"mean string deletion time\", \"deletion time (ns)\"],\n",
    "    'string_insert' : [\"mean string insertion time\", \"insertion time(ns)\"], \n",
    "    'string_nosucc_lookup' : [\"mean string unsucessful lookup time\", \"unsucessful lookup time (ns)\"],\n",
    "    'string_succ_lookup' : [\"mean string succesful lookup time\", \"succesful lookup time (ns)\"]\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# outlier testing functions\n",
    "def remove_with_modified_z_score(data, treshold=3.5):\n",
    "    # https://www.itl.nist.gov/div898/handbook/eda/section3/eda35h.htm\n",
    "    data = data.astype(int)\n",
    "    stats = data.describe()\n",
    "    median_absolute_deviation = abs(data - data.median()).median()\n",
    "    if not median_absolute_deviation:\n",
    "        return data\n",
    "    modified_z_scores = abs(0.6745 * (data - data.median()) / median_absolute_deviation)\n",
    "    cutoff = modified_z_scores <= treshold\n",
    "    data = data * cutoff\n",
    "    data = data.replace(0, nan)\n",
    "    return data\n",
    "\n",
    "#function that takes one of the outlier testers and data, and removes outliers\n",
    "def remove_outlier(data, function):\n",
    "    new_data = data.copy(True)\n",
    "    new_data[\"TIME\"] = new_data[\"TIME\"].astype(int)\n",
    "    new_data[\"SIZE\"] = new_data[\"SIZE\"].astype(int)\n",
    "    new_data\n",
    "    for i in range(4216):\n",
    "        start = i * 30\n",
    "        end = start+30\n",
    "        new_data.loc[start:end, \"TIME\"] = function(data.loc[start:end, \"TIME\"])\n",
    "        if not i % 420:\n",
    "            print(i / 42 , \"% done\")\n",
    "    return new_data\n",
    "\n",
    "\n",
    "# helpers for plot functions\n",
    "def sort_maps(test):\n",
    "    maps = data[data[\"TEST\"]== test][\"MAP\"].unique()\n",
    "    new = [(groups_max.loc[test, i][\"TIME\"], i) for i in maps]\n",
    "    new.sort()\n",
    "    new = [i[1] for i in new]\n",
    "    return new\n",
    "\n",
    "def divider(df, maplist):\n",
    "    filters = df['MAP'].isin(maplist)\n",
    "    return df[filters]\n",
    "\n",
    "\n",
    "def plotter(test, groups):\n",
    "    maps = sort_maps(test)\n",
    "    plot_now = [5,11,16]\n",
    "#     plot_now = [2,5,8,11,14,16]\n",
    "\n",
    "    for mp in range(0,17):\n",
    "        map_data = groups.get_group(('int_insert', maps[mp]))        \n",
    "        sns.lineplot(x=\"SIZE\", y=\"TIME\", data=map_data, err_style=\"bars\", color=styles[maps[mp]][0], label=maps[mp], linestyle=\"--\")\n",
    "#         sns.regplot(x=map_data[\"SIZE\"], y=map_data[\"TIME\"], color=styles[maps[mp]][0], label=maps[mp])\n",
    "        if mp in plot_now:\n",
    "            plt.xscale(\"log\")\n",
    "            plt.xticks(ticks, ticklabels)\n",
    "            plt.set_xticklabels(rotation=45)\n",
    "            plt.ylabel(labels[test][1])\n",
    "            plt.legend()\n",
    "            plt.title(labels[test][0])\n",
    "            plt.show()\n",
    "\n",
    "def plotter2(test, data):\n",
    "    mydata = data[data[\"TEST\"] == test]\n",
    "    maps = sort_maps(test)\n",
    "    set1 = divider(mydata, maps[:5])\n",
    "    set2 = divider(mydata, maps[5:11])\n",
    "    set3 = divider(mydata, maps[11:])\n",
    "    \n",
    "    plot = sns.lineplot(x=\"SIZE\", y=\"TIME\", hue=\"MAP\", data=set1)\n",
    "    plt.xscale(\"log\")\n",
    "    plt.xticks(ticks, ticklabels)\n",
    "    plot.set_xticklabels(\n",
    "        plot.get_xticklabels(), \n",
    "        rotation=55, \n",
    "        horizontalalignment='center',\n",
    "        fontweight='light',\n",
    "    )\n",
    "    plt.ylabel(labels[test][1])\n",
    "    plt.legend()\n",
    "    plt.title(labels[test][0])\n",
    "    plt.show()\n",
    "    \n",
    "    plot = sns.lineplot(x=\"SIZE\", y=\"TIME\", hue=\"MAP\", data=set2)\n",
    "    plt.xscale(\"log\")\n",
    "    plt.xticks(ticks, ticklabels)\n",
    "    plot.set_xticklabels(\n",
    "        plot.get_xticklabels(), \n",
    "        rotation=55, \n",
    "        horizontalalignment='center',\n",
    "        fontweight='light',\n",
    "        )\n",
    "    plt.ylabel(labels[test][1])\n",
    "    plt.legend()\n",
    "    plt.title(labels[test][0])\n",
    "    plt.show()\n",
    "    \n",
    "    plot = sns.lineplot(x=\"SIZE\", y=\"TIME\", hue=\"MAP\", data=set3)\n",
    "    plt.xscale(\"log\")\n",
    "    plt.xticks(ticks, ticklabels)\n",
    "    plot.set_xticklabels(\n",
    "        plot.get_xticklabels(), \n",
    "        rotation=55, \n",
    "        horizontalalignment='center',\n",
    "        fontweight='light',\n",
    "    )\n",
    "    plt.ylabel(labels[test][1])\n",
    "    plt.legend()\n",
    "    plt.title(labels[test][0])\n",
    "    plt.show()\n",
    "\n",
    "    \n",
    "def plotter3(test, groups):\n",
    "    maps = sort_maps(test)\n",
    "    plot_now = [5,11,16]\n",
    "#     plot_now = [2,5,8,11,14,16]\n",
    "\n",
    "    for mp in range(0,17):\n",
    "        map_data = groups.get_group(('int_insert', maps[mp]))        \n",
    "#         sns.lineplot(x=\"SIZE\", y=\"TIME\", data=map_data, err_style=\"band\", color=styles[maps[mp]][0], label=maps[mp], linestyle=\"--\")\n",
    "        sns.regplot(x=map_data[\"SIZE\"], y=map_data[\"TIME\"], color=styles[maps[mp]][0], label=maps[mp])\n",
    "        if mp in plot_now:\n",
    "            plt.xscale(\"log\")\n",
    "            plt.xticks(ticks, ticklabels)\n",
    "            plt.ylabel(labels[test][1])\n",
    "            plt.legend()\n",
    "            plt.set_xticklabels(rotation=45)\n",
    "            plt.title(labels[test][0])\n",
    "            plt.show()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.0 % done\n",
      "10.0 % done\n",
      "20.0 % done\n",
      "30.0 % done\n",
      "40.0 % done\n",
      "50.0 % done\n",
      "60.0 % done\n",
      "70.0 % done\n",
      "80.0 % done\n",
      "90.0 % done\n",
      "100.0 % done\n"
     ]
    }
   ],
   "source": [
    "data = remove_outlier(onmodded, remove_with_modified_z_score)\n",
    "groups = data.groupby([\"TEST\", \"MAP\"])\n",
    "gr_max = groups.max()\n",
    "gr_mean = groups.mean()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "tests = data[\"TEST\"].unique()\n",
    "# for i in tests:\n",
    "#     plotter2(i, data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# for i in tests:\n",
    "#     plotter(i, groups)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "# for i in tests:\n",
    "#     plotter3(i, groups)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 222960 characters]\n",
      "text/plain": [
       "<Figure size 1728x360 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "[encoded data, 219028 characters]\n",
      "text/plain": [
       "<Figure size 1728x360 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "[encoded data, 185808 characters]\n",
      "text/plain": [
       "<Figure size 1728x360 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plotter2(tests[0], data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 158964 characters]\n",
      "text/plain": [
       "<Figure size 1728x360 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "[encoded data, 202420 characters]\n",
      "text/plain": [
       "<Figure size 1728x360 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "[encoded data, 128672 characters]\n",
      "text/plain": [
       "<Figure size 1728x360 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plotter2(tests[1], data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 165472 characters]\n",
      "text/plain": [
       "<Figure size 1728x360 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "[encoded data, 219352 characters]\n",
      "text/plain": [
       "<Figure size 1728x360 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "[encoded data, 146128 characters]\n",
      "text/plain": [
       "<Figure size 1728x360 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plotter2(tests[2], data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 178704 characters]\n",
      "text/plain": [
       "<Figure size 1728x360 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "[encoded data, 181008 characters]\n",
      "text/plain": [
       "<Figure size 1728x360 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "[encoded data, 178940 characters]\n",
      "text/plain": [
       "<Figure size 1728x360 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plotter2(tests[3], data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 207028 characters]\n",
      "text/plain": [
       "<Figure size 1728x360 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "[encoded data, 221300 characters]\n",
      "text/plain": [
       "<Figure size 1728x360 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "[encoded data, 206216 characters]\n",
      "text/plain": [
       "<Figure size 1728x360 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plotter2(tests[4], data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 151308 characters]\n",
      "text/plain": [
       "<Figure size 1728x360 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "[encoded data, 191808 characters]\n",
      "text/plain": [
       "<Figure size 1728x360 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "[encoded data, 179424 characters]\n",
      "text/plain": [
       "<Figure size 1728x360 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plotter2(tests[5], data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 158316 characters]\n",
      "text/plain": [
       "<Figure size 1728x360 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "[encoded data, 202812 characters]\n",
      "text/plain": [
       "<Figure size 1728x360 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "[encoded data, 163216 characters]\n",
      "text/plain": [
       "<Figure size 1728x360 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plotter2(tests[6], data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 139672 characters]\n",
      "text/plain": [
       "<Figure size 1728x360 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "[encoded data, 161284 characters]\n",
      "text/plain": [
       "<Figure size 1728x360 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "[encoded data, 173044 characters]\n",
      "text/plain": [
       "<Figure size 1728x360 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plotter2(tests[7], data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array(['absl::flat_hash_map', \"absl::node_hash_map'\",\n",
       "       'boost::unordered::unordered_map', 'emilib::HashMap',\n",
       "       'google::dense_hash_map', 'google::sparse_hash_map',\n",
       "       'phmap::parallel_flat_hash_map', 'phmap::parallel_node_hash_map',\n",
       "       'robin_hood::unordered_flat_map', 'robin_hood::unordered_node_map',\n",
       "       'ska::bytell_hash_map', 'ska::flat_hash_map', 'ska::unordered_map',\n",
       "       'std::unordered_map', 'tsl::hopscotch_map', 'tsl::robin_map',\n",
       "       'tsl::sparse_map', 'absl::node_hash_map'], dtype=object)"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data[\"MAP\"].unique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "11\n",
      "9\n",
      "13\n",
      "4\n",
      "6\n",
      "17\n",
      "12\n",
      "14\n",
      "7\n",
      "1\n",
      "8\n",
      "3\n",
      "10\n",
      "15\n",
      "5\n",
      "2\n",
      "16\n"
     ]
    }
   ],
   "source": [
    "test = \"string_insert\"\n",
    "alpha_maps = gr_mean.loc[test][\"TIME\"].index\n",
    "rank = list(gr_mean.loc[test][\"TIME\"].sort_values().index)\n",
    "rank = [rank.index(i) for i in alpha_maps]\n",
    "for i in rank:\n",
    "    print(i + 1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array(['int_delete', 'int_insert', 'int_nosucc_lookup', 'int_succ_lookup',\n",
       "       'string_delete', 'string_insert', 'string_nosucc_lookup',\n",
       "       'string_succ_lookup'], dtype=object)"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "MAP\n",
      "google::dense_hash_map              25\n",
      "tsl::robin_map                      25\n",
      "ska::flat_hash_map                  28\n",
      "emilib::HashMap                     32\n",
      "tsl::hopscotch_map                  43\n",
      "robin_hood::unordered_node_map      50\n",
      "robin_hood::unordered_flat_map      54\n",
      "ska::bytell_hash_map                64\n",
      "absl::flat_hash_map                 65\n",
      "phmap::parallel_flat_hash_map       74\n",
      "ska::unordered_map                  84\n",
      "absl::node_hash_map                 94\n",
      "tsl::sparse_map                    108\n",
      "phmap::parallel_node_hash_map      110\n",
      "boost::unordered::unordered_map    115\n",
      "google::sparse_hash_map            123\n",
      "std::unordered_map                 130\n",
      "dtype: int64\n"
     ]
    }
   ],
   "source": [
    "tests = data[\"TEST\"].unique()\n",
    "alpha_maps = gr_mean.loc[test][\"TIME\"].index\n",
    "total_rank = pd.Series([0 for i in range(17)]) \n",
    "\n",
    "for test in tests:\n",
    "    alpha_maps = gr_mean.loc[test][\"TIME\"].index\n",
    "    rank = list(gr_mean.loc[test][\"TIME\"].sort_values().index)\n",
    "    rank = Series([rank.index(i) + 1 for i in alpha_maps])\n",
    "    total_rank += rank\n",
    "total_rank.index = alpha_maps\n",
    "print(total_rank.sort_values())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 89,
   "metadata": {},
   "outputs": [],
   "source": [
    "from statsmodels.sandbox.regression.predstd import wls_prediction_std"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 116,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "                                 OLS Regression Results                                \n",
      "=======================================================================================\n",
      "Dep. Variable:                   TIME   R-squared (uncentered):                   0.547\n",
      "Model:                            OLS   Adj. R-squared (uncentered):              0.547\n",
      "Method:                 Least Squares   F-statistic:                              4423.\n",
      "Date:                Tue, 14 Apr 2020   Prob (F-statistic):                        0.00\n",
      "Time:                        11:06:23   Log-Likelihood:                         -41641.\n",
      "No. Observations:                7331   AIC:                                  8.329e+04\n",
      "Df Residuals:                    7329   BIC:                                  8.330e+04\n",
      "Df Model:                           2                                                  \n",
      "Covariance Type:            nonrobust                                                  \n",
      "==============================================================================\n",
      "                 coef    std err          t      P>|t|      [0.025      0.975]\n",
      "------------------------------------------------------------------------------\n",
      "SIZE        4.907e-07   6.19e-08      7.923      0.000    3.69e-07    6.12e-07\n",
      "SIZE2          4.8407      0.075     64.882      0.000       4.694       4.987\n",
      "==============================================================================\n",
      "Omnibus:                     2601.169   Durbin-Watson:                   0.002\n",
      "Prob(Omnibus):                  0.000   Jarque-Bera (JB):              918.695\n",
      "Skew:                           0.693   Prob(JB):                    3.22e-200\n",
      "Kurtosis:                       1.957   Cond. No.                     1.61e+06\n",
      "==============================================================================\n",
      "\n",
      "Warnings:\n",
      "[1] Standard Errors assume that the covariance matrix of the errors is correctly specified.\n",
      "[2] The condition number is large, 1.61e+06. This might indicate that there are\n",
      "strong multicollinearity or other numerical problems.\n"
     ]
    }
   ],
   "source": [
    "test = data[data[\"TEST\"] == \"int_delete\"]\n",
    "test = data[data[\"MAP\"] == \"google::dense_hash_map\" ]\n",
    "test = test.dropna()\n",
    "test[\"TIME\"] = test[\"TIME\"].astype(int)\n",
    "test[\"SIZE\"] = test[\"SIZE\"].astype(int)\n",
    "test[\"SIZE2\"] = np.log(test[\"SIZE\"])\n",
    "\n",
    "# test[\"SIZE2\"] = test[\"SIZE\"]\n",
    "# model = smf.ols(\"TIME~SIZE + SIZE2\",data=test)\n",
    "\n",
    "model = sm.OLS(test[\"TIME\"], test[[\"SIZE\", \"SIZE2\"]])\n",
    "results = model.fit()\n",
    "print(results.summary())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 115,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 101420 characters]\n",
      "text/plain": [
       "<Figure size 864x576 with 4 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 120,
   "metadata": {},
   "outputs": [
    {
     "ename": "PatsyError",
     "evalue": "Error evaluating factor: NameError: name 'SIZE3' is not defined\n    SIZE3\n    ^^^^^",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[1;32mc:\\python3.8\\lib\\site-packages\\patsy\\compat.py\u001b[0m in \u001b[0;36mcall_and_wrap_exc\u001b[1;34m(msg, origin, f, *args, **kwargs)\u001b[0m\n\u001b[0;32m     35\u001b[0m     \u001b[1;32mtry\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 36\u001b[1;33m         \u001b[1;32mreturn\u001b[0m \u001b[0mf\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m*\u001b[0m\u001b[0margs\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m     37\u001b[0m     \u001b[1;32mexcept\u001b[0m \u001b[0mException\u001b[0m \u001b[1;32mas\u001b[0m \u001b[0me\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mc:\\python3.8\\lib\\site-packages\\patsy\\eval.py\u001b[0m in \u001b[0;36meval\u001b[1;34m(self, expr, source_name, inner_namespace)\u001b[0m\n\u001b[0;32m    164\u001b[0m         \u001b[0mcode\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mcompile\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mexpr\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0msource_name\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;34m\"eval\"\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mflags\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;32mFalse\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 165\u001b[1;33m         return eval(code, {}, VarLookupDict([inner_namespace]\n\u001b[0m\u001b[0;32m    166\u001b[0m                                             + self._namespaces))\n",
      "\u001b[1;32m<string>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m\u001b[0m\n",
      "\u001b[1;31mNameError\u001b[0m: name 'SIZE3' is not defined",
      "\nThe above exception was the direct cause of the following exception:\n",
      "\u001b[1;31mPatsyError\u001b[0m                                Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-120-7b7e59571e48>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m\u001b[0m\n\u001b[1;32m----> 1\u001b[1;33m \u001b[0msm\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mgraphics\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mplot_partregress\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;34m\"SIZE\"\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;34m\"SIZE2\"\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;34m\"SIZE3\"\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mdata\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mtest\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mobs_labels\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;32mFalse\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[1;32mc:\\python3.8\\lib\\site-packages\\statsmodels\\graphics\\regressionplots.py\u001b[0m in \u001b[0;36mplot_partregress\u001b[1;34m(endog, exog_i, exog_others, data, title_kwargs, obs_labels, label_kwargs, ax, ret_coords, **kwargs)\u001b[0m\n\u001b[0;32m    390\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    391\u001b[0m     \u001b[1;32mif\u001b[0m \u001b[0misinstance\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mexog_others\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mstr\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 392\u001b[1;33m         \u001b[0mRHS\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mdmatrix\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mexog_others\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mdata\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    393\u001b[0m     \u001b[1;32melif\u001b[0m \u001b[0misinstance\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mexog_others\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mlist\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    394\u001b[0m         \u001b[0mRHS\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;34m\"+\"\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mjoin\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mexog_others\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mc:\\python3.8\\lib\\site-packages\\patsy\\highlevel.py\u001b[0m in \u001b[0;36mdmatrix\u001b[1;34m(formula_like, data, eval_env, NA_action, return_type)\u001b[0m\n\u001b[0;32m    288\u001b[0m     \"\"\"\n\u001b[0;32m    289\u001b[0m     \u001b[0meval_env\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mEvalEnvironment\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mcapture\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0meval_env\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mreference\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;36m1\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 290\u001b[1;33m     (lhs, rhs) = _do_highlevel_design(formula_like, data, eval_env,\n\u001b[0m\u001b[0;32m    291\u001b[0m                                       NA_action, return_type)\n\u001b[0;32m    292\u001b[0m     \u001b[1;32mif\u001b[0m \u001b[0mlhs\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mshape\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;36m1\u001b[0m\u001b[1;33m]\u001b[0m \u001b[1;33m!=\u001b[0m \u001b[1;36m0\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mc:\\python3.8\\lib\\site-packages\\patsy\\highlevel.py\u001b[0m in \u001b[0;36m_do_highlevel_design\u001b[1;34m(formula_like, data, eval_env, NA_action, return_type)\u001b[0m\n\u001b[0;32m    162\u001b[0m     \u001b[1;32mdef\u001b[0m \u001b[0mdata_iter_maker\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    163\u001b[0m         \u001b[1;32mreturn\u001b[0m \u001b[0miter\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m[\u001b[0m\u001b[0mdata\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 164\u001b[1;33m     design_infos = _try_incr_builders(formula_like, data_iter_maker, eval_env,\n\u001b[0m\u001b[0;32m    165\u001b[0m                                       NA_action)\n\u001b[0;32m    166\u001b[0m     \u001b[1;32mif\u001b[0m \u001b[0mdesign_infos\u001b[0m \u001b[1;32mis\u001b[0m \u001b[1;32mnot\u001b[0m \u001b[1;32mNone\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mc:\\python3.8\\lib\\site-packages\\patsy\\highlevel.py\u001b[0m in \u001b[0;36m_try_incr_builders\u001b[1;34m(formula_like, data_iter_maker, eval_env, NA_action)\u001b[0m\n\u001b[0;32m     64\u001b[0m     \u001b[1;32mif\u001b[0m \u001b[0misinstance\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mformula_like\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mModelDesc\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     65\u001b[0m         \u001b[1;32massert\u001b[0m \u001b[0misinstance\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0meval_env\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mEvalEnvironment\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 66\u001b[1;33m         return design_matrix_builders([formula_like.lhs_termlist,\n\u001b[0m\u001b[0;32m     67\u001b[0m                                        formula_like.rhs_termlist],\n\u001b[0;32m     68\u001b[0m                                       \u001b[0mdata_iter_maker\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mc:\\python3.8\\lib\\site-packages\\patsy\\build.py\u001b[0m in \u001b[0;36mdesign_matrix_builders\u001b[1;34m(termlists, data_iter_maker, eval_env, NA_action)\u001b[0m\n\u001b[0;32m    691\u001b[0m     \u001b[1;31m# on some data to find out what type of data they return.\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    692\u001b[0m     (num_column_counts,\n\u001b[1;32m--> 693\u001b[1;33m      \u001b[0mcat_levels_contrasts\u001b[0m\u001b[1;33m)\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0m_examine_factor_types\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mall_factors\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    694\u001b[0m                                                    \u001b[0mfactor_states\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    695\u001b[0m                                                    \u001b[0mdata_iter_maker\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mc:\\python3.8\\lib\\site-packages\\patsy\\build.py\u001b[0m in \u001b[0;36m_examine_factor_types\u001b[1;34m(factors, factor_states, data_iter_maker, NA_action)\u001b[0m\n\u001b[0;32m    441\u001b[0m     \u001b[1;32mfor\u001b[0m \u001b[0mdata\u001b[0m \u001b[1;32min\u001b[0m \u001b[0mdata_iter_maker\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    442\u001b[0m         \u001b[1;32mfor\u001b[0m \u001b[0mfactor\u001b[0m \u001b[1;32min\u001b[0m \u001b[0mlist\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mexamine_needed\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 443\u001b[1;33m             \u001b[0mvalue\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mfactor\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0meval\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mfactor_states\u001b[0m\u001b[1;33m[\u001b[0m\u001b[0mfactor\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mdata\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    444\u001b[0m             \u001b[1;32mif\u001b[0m \u001b[0mfactor\u001b[0m \u001b[1;32min\u001b[0m \u001b[0mcat_sniffers\u001b[0m \u001b[1;32mor\u001b[0m \u001b[0mguess_categorical\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mvalue\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    445\u001b[0m                 \u001b[1;32mif\u001b[0m \u001b[0mfactor\u001b[0m \u001b[1;32mnot\u001b[0m \u001b[1;32min\u001b[0m \u001b[0mcat_sniffers\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mc:\\python3.8\\lib\\site-packages\\patsy\\eval.py\u001b[0m in \u001b[0;36meval\u001b[1;34m(self, memorize_state, data)\u001b[0m\n\u001b[0;32m    562\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    563\u001b[0m     \u001b[1;32mdef\u001b[0m \u001b[0meval\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mmemorize_state\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mdata\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 564\u001b[1;33m         return self._eval(memorize_state[\"eval_code\"],\n\u001b[0m\u001b[0;32m    565\u001b[0m                           \u001b[0mmemorize_state\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    566\u001b[0m                           data)\n",
      "\u001b[1;32mc:\\python3.8\\lib\\site-packages\\patsy\\eval.py\u001b[0m in \u001b[0;36m_eval\u001b[1;34m(self, code, memorize_state, data)\u001b[0m\n\u001b[0;32m    545\u001b[0m     \u001b[1;32mdef\u001b[0m \u001b[0m_eval\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mcode\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mmemorize_state\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mdata\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    546\u001b[0m         \u001b[0minner_namespace\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mVarLookupDict\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m[\u001b[0m\u001b[0mdata\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mmemorize_state\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;34m\"transforms\"\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 547\u001b[1;33m         return call_and_wrap_exc(\"Error evaluating factor\",\n\u001b[0m\u001b[0;32m    548\u001b[0m                                  \u001b[0mself\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    549\u001b[0m                                  \u001b[0mmemorize_state\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;34m\"eval_env\"\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0meval\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mc:\\python3.8\\lib\\site-packages\\patsy\\compat.py\u001b[0m in \u001b[0;36mcall_and_wrap_exc\u001b[1;34m(msg, origin, f, *args, **kwargs)\u001b[0m\n\u001b[0;32m     41\u001b[0m                                  origin)\n\u001b[0;32m     42\u001b[0m             \u001b[1;31m# Use 'exec' to hide this syntax from the Python 2 parser:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 43\u001b[1;33m             \u001b[0mexec\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;34m\"raise new_exc from e\"\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m     44\u001b[0m         \u001b[1;32melse\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     45\u001b[0m             \u001b[1;31m# In python 2, we just let the original exception escape -- better\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mc:\\python3.8\\lib\\site-packages\\patsy\\compat.py\u001b[0m in \u001b[0;36m<module>\u001b[1;34m\u001b[0m\n",
      "\u001b[1;31mPatsyError\u001b[0m: Error evaluating factor: NameError: name 'SIZE3' is not defined\n    SIZE3\n    ^^^^^"
     ]
    },
    {
     "data": {
      "image/png": "[encoded data, 13428 characters]\n",
      "text/plain": [
       "<Figure size 1728x360 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}
