diff --git a/extra/Getting_Started/ElasticsearchConnection.ipynb b/extra/Getting_Started/ElasticsearchConnection.ipynb
new file mode 100755
index 0000000..c860001
--- /dev/null
+++ b/extra/Getting_Started/ElasticsearchConnection.ipynb
@@ -0,0 +1,1243 @@
+{
+ "cells": [
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "# Elasticsearch Data Analytics\n",
+ "\n",
+ "This notebook provides sample code to fetch Elasticsearch Data into and analyze it."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 1,
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/html": [
+ "![](\"https://www.antaresnet.com/wp-content/uploads/2018/07/Elasticsearch-Logo-Color-V.png\"/)
"
+ ],
+ "text/plain": [
+ ""
+ ]
+ },
+ "execution_count": 1,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "from IPython.display import Image\n",
+ "from IPython.core.display import HTML \n",
+ "Image(url= \"https://www.antaresnet.com/wp-content/uploads/2018/07/Elasticsearch-Logo-Color-V.png\")"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "## Loading modules and connect to the Elastic Stack"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 2,
+ "metadata": {
+ "pycharm": {
+ "is_executing": false
+ }
+ },
+ "outputs": [],
+ "source": [
+ "import sys\n",
+ "import json\n",
+ "import requests\n",
+ "import numpy as np\n",
+ "import pandas as pd\n",
+ "import seaborn as sns\n",
+ "import pytz\n",
+ "from datetime import datetime, timedelta\n",
+ "from dateutil import tz\n",
+ "\n",
+ "import matplotlib.pyplot as plt\n",
+ "sns.set(style=\"darkgrid\")\n",
+ "plt.rcParams[\"figure.figsize\"] = (18,10)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 3,
+ "metadata": {
+ "pycharm": {
+ "is_executing": false
+ }
+ },
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Last run: 2019-12-20 08:12:58.766840 UTC, status: 7.0604683677036 %\n"
+ ]
+ },
+ {
+ "data": {
+ "text/plain": [
+ "{'name': 'c185f3ed577c',\n",
+ " 'cluster_name': 'il.es.cluster',\n",
+ " 'cluster_uuid': 'sBgbgyRXTvKta2cEJCczKQ',\n",
+ " 'version': {'number': '6.2.2',\n",
+ " 'build_hash': '10b1edd',\n",
+ " 'build_date': '2018-02-16T19:01:30.685723Z',\n",
+ " 'build_snapshot': False,\n",
+ " 'lucene_version': '7.2.1',\n",
+ " 'minimum_wire_compatibility_version': '5.6.0',\n",
+ " 'minimum_index_compatibility_version': '5.0.0'},\n",
+ " 'tagline': 'You Know, for Search'}"
+ ]
+ },
+ "execution_count": 3,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "# connect to our cluster\n",
+ "from elasticsearch import Elasticsearch\n",
+ "es = Elasticsearch([{'host': 'elasticsearch', 'port': 9200}])\n",
+ "print('Last run: {} UTC, status: {} %'.format(\n",
+ " datetime.utcnow(),\n",
+ " es.cluster.health()['active_shards_percent_as_number']))\n",
+ "es.info()"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "Display our indices and document types saved in elasticsearch.\n",
+ "\n",
+ "Update the elasticsearch package."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 5,
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Requirement already satisfied: elasticsearch<8.0.0,>=7.0.0 in /opt/conda/lib/python3.7/site-packages (7.1.0)\n",
+ "Requirement already satisfied: urllib3>=1.21.1 in /opt/conda/lib/python3.7/site-packages (from elasticsearch<8.0.0,>=7.0.0) (1.25.7)\n"
+ ]
+ }
+ ],
+ "source": [
+ "!sudo pip install \"elasticsearch>=7.0.0,<8.0.0\""
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "## Defining useful functions"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 6,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# %load scroller.py\n",
+ "def scroller(index, quantity, timerange=timedelta(days=0), startdt=\"\", enddt=\"\"):\n",
+ " print(\"Starting to scroll\", end='')\n",
+ " # Retrieve the datetimes, note that timerange has a higher priority\n",
+ " if timerange.total_seconds() > 0:\n",
+ " now = datetime.utcnow().replace(tzinfo=pytz.UTC)\n",
+ " startdt = (now - timerange).isoformat()\n",
+ " enddt = now.isoformat()\n",
+ " \n",
+ " # search the first page and write the result to data\n",
+ " response = es.search(\n",
+ " index=index,\n",
+ " body={\n",
+ " \"query\": {\n",
+ " \"bool\": {\n",
+ " \"must\": [\n",
+ " {\"range\" : {\n",
+ " \"phenomenonTime\" : {\n",
+ " #\"gte\": \"2018-02-20T09:08:34.230693+00:00\", \n",
+ " \"gte\": startdt,\n",
+ " \"lte\": enddt, \n",
+ " \"time_zone\": \"+01:00\"\n",
+ " }\n",
+ " }},\n",
+ " {\n",
+ " \"match_phrase\": {\n",
+ " \"Datastream.name.keyword\": quantity\n",
+ " }\n",
+ " }\n",
+ " ]\n",
+ " }\n",
+ " }\n",
+ " },\n",
+ " scroll='10m'\n",
+ " )\n",
+ " data = [[row[\"_source\"][\"phenomenonTime\"], row[\"_source\"][\"result\"]] for row in response['hits']['hits']]\n",
+ "\n",
+ " # Append new pages until there aren't any left\n",
+ " while len(response['hits']['hits']):\n",
+ " print(\".\", end='')\n",
+ " # process results\n",
+ " # print([item[\"_id\"] for item in response[\"hits\"][\"hits\"]])\n",
+ " response = es.scroll(scroll_id=response['_scroll_id'], scroll='10m')\n",
+ " data += [[row[\"_source\"][\"phenomenonTime\"], row[\"_source\"][\"result\"]] for row in response['hits']['hits']]\n",
+ " \n",
+ " # Convert data to a DataFrame and return it\n",
+ " df = pd.DataFrame(data, columns=[\"phenomenonTime\", quantity])\n",
+ " # df.index = pd.to_datetime(df[\"phenomenonTime\"].map(lambda t: t.split(\".\")[0]), utc=True)\n",
+ " df.index = pd.to_datetime(df[\"phenomenonTime\"].map(lambda t: roundto(t, 1)), utc=True)\n",
+ " df = df.drop([\"phenomenonTime\"], axis=1)\n",
+ " print(\"\\nFetched {} tuples.\".format(df.shape[0]))\n",
+ " return df\n",
+ "\n",
+ "def roundto(string, n):\n",
+ " base = string.split(\".\")[0]\n",
+ " if n > 0:\n",
+ " base += \".\" + string.split(\".\")[1][:n]\n",
+ " return base\n"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "# Gather data from Elasticsearch\n",
+ "\n",
+ "It is supposed that in the Elasticsearch instance, there is data with the Datastream.name \"at.srfg.iot-iot4cps-wp5.CarFleet1.car_1.Air Temperature\" within the index name \"at.srfg.iot-iot4cps-wp5.infraprov.internal-*\""
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 7,
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Starting to scroll\n",
+ "Fetched 0 tuples.\n"
+ ]
+ },
+ {
+ "data": {
+ "text/html": [
+ "\n",
+ "\n",
+ "
\n",
+ " \n",
+ " \n",
+ " | \n",
+ " at.srfg.iot-iot4cps-wp5.CarFleet1.car_1.Air Temperature | \n",
+ "
\n",
+ " \n",
+ " | phenomenonTime | \n",
+ " | \n",
+ "
\n",
+ " \n",
+ " \n",
+ " \n",
+ "
\n",
+ "
\n",
+ "\n",
+ "
\n",
+ " \n",
+ " \n",
+ " | \n",
+ " at.srfg.iot-iot4cps-wp5.CarFleet1.car_1.Air Temperature | \n",
+ "
\n",
+ " \n",
+ " | phenomenonTime | \n",
+ " | \n",
+ "
\n",
+ " \n",
+ " \n",
+ " \n",
+ "
\n",
+ "
"
+ ]
+ },
+ "metadata": {},
+ "output_type": "display_data"
+ }
+ ],
+ "source": [
+ "# Plot the extracted data using pandas and seaborn\n",
+ "df.plot()\n",
+ "plt.show()"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 52,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# Get multiple quantities and (outer) join them to a single DataFrame.\n",
+ "# There can be a lot of missing values\n",
+ "used_quantities = [\"at.srfg.iot-iot4cps-wp5.CarFleet1.car_1.Air Temperature\", \n",
+ " \"at.srfg.iot-iot4cps-wp5.CarFleet2.car_2.Air Temperature\"]"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 53,
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Starting to scroll.....\n",
+ "Fetched 48 tuples.\n",
+ "at.srfg.iot-iot4cps-wp5.CarFleet2.car_2.Air Temperature\n",
+ "Starting to scroll\n",
+ "Fetched 0 tuples.\n"
+ ]
+ }
+ ],
+ "source": [
+ "df = scroller(\"at.srfg.iot-iot4cps-wp5.infraprov.internal-*\",\n",
+ " used_quantities[0],\n",
+ " timerange=timedelta(days=10))\n",
+ "for q in used_quantities[1:]:\n",
+ " print(q)\n",
+ " df = df.join(scroller(\"at.srfg.iot-iot4cps-wp5.infraprov.internal-*\", q,\n",
+ " timerange=timedelta(days=10)),\n",
+ " how=\"outer\")"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 54,
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/html": [
+ "\n",
+ "\n",
+ "
\n",
+ " \n",
+ " \n",
+ " | \n",
+ " at.srfg.iot-iot4cps-wp5.CarFleet1.car_1.Air Temperature | \n",
+ " at.srfg.iot-iot4cps-wp5.CarFleet2.car_2.Air Temperature | \n",
+ "
\n",
+ " \n",
+ " | phenomenonTime | \n",
+ " | \n",
+ " | \n",
+ "
\n",
+ " \n",
+ " \n",
+ " \n",
+ " | 2019-08-07 09:32:50.600000+00:00 | \n",
+ " -1.396915 | \n",
+ " NaN | \n",
+ "
\n",
+ " \n",
+ " | 2019-08-07 09:33:40.700000+00:00 | \n",
+ " -2.559881 | \n",
+ " NaN | \n",
+ "
\n",
+ " \n",
+ " | 2019-08-07 09:34:10.700000+00:00 | \n",
+ " -3.360251 | \n",
+ " NaN | \n",
+ "
\n",
+ " \n",
+ " | 2019-08-07 09:32:10.600000+00:00 | \n",
+ " -0.112741 | \n",
+ " NaN | \n",
+ "
\n",
+ " \n",
+ " | 2019-08-07 09:32:40.600000+00:00 | \n",
+ " -0.956904 | \n",
+ " NaN | \n",
+ "
\n",
+ " \n",
+ "
\n",
+ "
\n",
+ "\n",
+ "
\n",
+ " \n",
+ " \n",
+ " | \n",
+ " at.srfg.iot-iot4cps-wp5.CarFleet1.car_1.Air Temperature | \n",
+ " at.srfg.iot-iot4cps-wp5.CarFleet2.car_2.Air Temperature | \n",
+ "
\n",
+ " \n",
+ " | phenomenonTime | \n",
+ " | \n",
+ " | \n",
+ "
\n",
+ " \n",
+ " \n",
+ " \n",
+ " | 2019-08-07 09:39:01.100000+00:00 | \n",
+ " -3.039692 | \n",
+ " NaN | \n",
+ "
\n",
+ " \n",
+ " | 2019-08-07 09:39:51.100000+00:00 | \n",
+ " -1.599475 | \n",
+ " NaN | \n",
+ "
\n",
+ " \n",
+ " | 2019-08-07 09:39:21.100000+00:00 | \n",
+ " -2.488179 | \n",
+ " NaN | \n",
+ "
\n",
+ " \n",
+ " | 2019-08-07 09:39:31.100000+00:00 | \n",
+ " -2.259640 | \n",
+ " NaN | \n",
+ "
\n",
+ " \n",
+ " | 2019-08-07 09:40:01.200000+00:00 | \n",
+ " -1.246612 | \n",
+ " NaN | \n",
+ "
\n",
+ " \n",
+ "
\n",
+ "
\n",
+ "\n",
+ "
\n",
+ " \n",
+ " \n",
+ " | \n",
+ " car1_temp | \n",
+ " car2_temp | \n",
+ "
\n",
+ " \n",
+ " | time | \n",
+ " | \n",
+ " | \n",
+ "
\n",
+ " \n",
+ " \n",
+ " \n",
+ " | 2019-08-07 09:32:50.600000+00:00 | \n",
+ " -1.396915 | \n",
+ " NaN | \n",
+ "
\n",
+ " \n",
+ " | 2019-08-07 09:33:40.700000+00:00 | \n",
+ " -2.559881 | \n",
+ " NaN | \n",
+ "
\n",
+ " \n",
+ " | 2019-08-07 09:34:10.700000+00:00 | \n",
+ " -3.360251 | \n",
+ " NaN | \n",
+ "
\n",
+ " \n",
+ " | 2019-08-07 09:32:10.600000+00:00 | \n",
+ " -0.112741 | \n",
+ " NaN | \n",
+ "
\n",
+ " \n",
+ " | 2019-08-07 09:32:40.600000+00:00 | \n",
+ " -0.956904 | \n",
+ " NaN | \n",
+ "
\n",
+ " \n",
+ "
\n",
+ "
\n",
+ "\n",
+ "
\n",
+ " \n",
+ " \n",
+ " | \n",
+ " car1_temp | \n",
+ " car2_temp | \n",
+ "
\n",
+ " \n",
+ " \n",
+ " \n",
+ " | count | \n",
+ " 0.0 | \n",
+ " 0.0 | \n",
+ "
\n",
+ " \n",
+ " | mean | \n",
+ " NaN | \n",
+ " NaN | \n",
+ "
\n",
+ " \n",
+ " | std | \n",
+ " NaN | \n",
+ " NaN | \n",
+ "
\n",
+ " \n",
+ " | min | \n",
+ " NaN | \n",
+ " NaN | \n",
+ "
\n",
+ " \n",
+ " | 25% | \n",
+ " NaN | \n",
+ " NaN | \n",
+ "
\n",
+ " \n",
+ " | 50% | \n",
+ " NaN | \n",
+ " NaN | \n",
+ "
\n",
+ " \n",
+ " | 75% | \n",
+ " NaN | \n",
+ " NaN | \n",
+ "
\n",
+ " \n",
+ " | max | \n",
+ " NaN | \n",
+ " NaN | \n",
+ "
\n",
+ " \n",
+ "
\n",
+ "
\n",
+ "\n",
+ "
\n",
+ " \n",
+ " \n",
+ " | \n",
+ " car1_temp | \n",
+ " car2_temp | \n",
+ "
\n",
+ " \n",
+ " | time | \n",
+ " | \n",
+ " | \n",
+ "
\n",
+ " \n",
+ " \n",
+ " \n",
+ "
\n",
+ "