Installing recommended packages and update conda, sample code with performance test
This commit is contained in:
Christoph Schranz
1243
extra/Getting_Started/ElasticsearchConnection.ipynb
Executable file
1243
extra/Getting_Started/ElasticsearchConnection.ipynb
Executable file
File diff suppressed because one or more lines are too long
417
extra/Getting_Started/GPU-processing.ipynb
Normal file
417
extra/Getting_Started/GPU-processing.ipynb
Normal file
@@ -0,0 +1,417 @@
|
||||
{
|
||||
"cells": [
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"# GPU-Jupyter\n",
|
||||
"\n",
|
||||
"This Jupyterlab Instance is connected to the GPU via CUDA drivers. In this notebook, we test the installation and perform some basic operations on the GPU."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## Test GPU connection\n",
|
||||
"\n",
|
||||
"#### Using the following command, your GPU type and its NVIDIA-SMI driver version should be listed:"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 1,
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"Fri Dec 20 09:42:29 2019 \n",
|
||||
"+-----------------------------------------------------------------------------+\n",
|
||||
"| NVIDIA-SMI 440.26 Driver Version: 440.26 CUDA Version: 10.2 |\n",
|
||||
"|-------------------------------+----------------------+----------------------+\n",
|
||||
"| GPU Name Persistence-M| Bus-Id Disp.A | Volatile Uncorr. ECC |\n",
|
||||
"| Fan Temp Perf Pwr:Usage/Cap| Memory-Usage | GPU-Util Compute M. |\n",
|
||||
"|===============================+======================+======================|\n",
|
||||
"| 0 GeForce RTX 207... Off | 00000000:01:00.0 Off | N/A |\n",
|
||||
"| 0% 54C P0 38W / 215W | 204MiB / 7974MiB | 0% Default |\n",
|
||||
"+-------------------------------+----------------------+----------------------+\n",
|
||||
" \n",
|
||||
"+-----------------------------------------------------------------------------+\n",
|
||||
"| Processes: GPU Memory |\n",
|
||||
"| GPU PID Type Process name Usage |\n",
|
||||
"|=============================================================================|\n",
|
||||
"+-----------------------------------------------------------------------------+\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"!nvidia-smi"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"#### Now, test if PyTorch can access the GPU via CUDA:"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 2,
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"data": {
|
||||
"text/plain": [
|
||||
"True"
|
||||
]
|
||||
},
|
||||
"execution_count": 2,
|
||||
"metadata": {},
|
||||
"output_type": "execute_result"
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"import torch\n",
|
||||
"torch.cuda.is_available()"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 3,
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"data": {
|
||||
"text/plain": [
|
||||
"[name: \"/device:CPU:0\"\n",
|
||||
" device_type: \"CPU\"\n",
|
||||
" memory_limit: 268435456\n",
|
||||
" locality {\n",
|
||||
" }\n",
|
||||
" incarnation: 891330946073693377, name: \"/device:XLA_CPU:0\"\n",
|
||||
" device_type: \"XLA_CPU\"\n",
|
||||
" memory_limit: 17179869184\n",
|
||||
" locality {\n",
|
||||
" }\n",
|
||||
" incarnation: 9415777875944419380\n",
|
||||
" physical_device_desc: \"device: XLA_CPU device\"]"
|
||||
]
|
||||
},
|
||||
"execution_count": 3,
|
||||
"metadata": {},
|
||||
"output_type": "execute_result"
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"from tensorflow.python.client import device_lib\n",
|
||||
"device_lib.list_local_devices()"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 4,
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"data": {
|
||||
"text/plain": [
|
||||
"tensor([[0.8722, 0.5115, 0.9504],\n",
|
||||
" [0.7723, 0.2860, 0.5793],\n",
|
||||
" [0.5388, 0.5681, 0.4295],\n",
|
||||
" [0.5269, 0.5165, 0.7475],\n",
|
||||
" [0.4882, 0.8255, 0.6498]])"
|
||||
]
|
||||
},
|
||||
"execution_count": 4,
|
||||
"metadata": {},
|
||||
"output_type": "execute_result"
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"from __future__ import print_function\n",
|
||||
"import numpy as np\n",
|
||||
"import torch\n",
|
||||
"a = torch.rand(5, 3)\n",
|
||||
"a"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## Performance test\n",
|
||||
"\n",
|
||||
"#### Now we want to know how much faster a typical operation is using GPU. Therefore we do the same operation in numpy, PyTorch and PyTorch with CUDA. The test operation is the calculation of the prediction matrix that is done in a linear regression."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"### 1) Numpy"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 5,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"x = np.random.rand(10000, 256)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 6,
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"590 ms ± 41.6 ms per loop (mean ± std. dev. of 7 runs, 1 loop each)\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"%%timeit\n",
|
||||
"H = x.dot(np.linalg.inv(x.transpose().dot(x))).dot(x.transpose())"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"### 2) PyTorch"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 7,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"x = torch.rand(10000, 256)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 8,
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"853 ms ± 16.6 ms per loop (mean ± std. dev. of 7 runs, 10 loops each)\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"%%timeit\n",
|
||||
"# Calculate the projection matrix of x\n",
|
||||
"H = x.mm( (x.t().mm(x)).inverse() ).mm(x.t())"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"### 3) PyTorch on GPU via CUDA"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 9,
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"tensor([[0.1054, 0.3291, 0.7729, 0.6005, 0.2372],\n",
|
||||
" [0.1022, 0.4534, 0.3964, 0.9174, 0.2610],\n",
|
||||
" [0.3969, 0.5472, 0.3876, 0.1979, 0.4063],\n",
|
||||
" [0.3630, 0.6374, 0.4176, 0.4804, 0.0396],\n",
|
||||
" [0.8256, 0.2289, 0.2265, 0.4388, 0.6070]], device='cuda:0')\n",
|
||||
"tensor([[0.1054, 0.3291, 0.7729, 0.6005, 0.2372],\n",
|
||||
" [0.1022, 0.4534, 0.3964, 0.9174, 0.2610],\n",
|
||||
" [0.3969, 0.5472, 0.3876, 0.1979, 0.4063],\n",
|
||||
" [0.3630, 0.6374, 0.4176, 0.4804, 0.0396],\n",
|
||||
" [0.8256, 0.2289, 0.2265, 0.4388, 0.6070]], dtype=torch.float64)\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"# let us run this cell only if CUDA is available\n",
|
||||
"# We will use ``torch.device`` objects to move tensors in and out of GPU\n",
|
||||
"if torch.cuda.is_available():\n",
|
||||
" device = torch.device(\"cuda\") # a CUDA device object\n",
|
||||
" x = torch.rand(10000, 256, device=device) # directly create a tensor on GPU\n",
|
||||
" y = x.to(device) # or just use strings ``.to(\"cuda\")``\n",
|
||||
" print(x[0:5, 0:5])\n",
|
||||
" print(y.to(\"cpu\", torch.double)[0:5, 0:5])"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 10,
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"11.3 ms ± 60.3 µs per loop (mean ± std. dev. of 7 runs, 1 loop each)\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"%%timeit\n",
|
||||
"H = x.mm( (x.t().mm(x)).inverse() ).mm(x.t())"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": []
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## Exhaustive Testing on GPU"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 11,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# let us run this cell only if CUDA is available\n",
|
||||
"# We will use ``torch.device`` objects to move tensors in and out of GPU\n",
|
||||
"import torch\n",
|
||||
"if torch.cuda.is_available():\n",
|
||||
" device = torch.device(\"cuda\") # a CUDA device object\n",
|
||||
" x = torch.rand(10000, 10, device=device) # directly create a tensor on GPU"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 12,
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"tensor([[0.3112, 0.7480, 0.1882, 0.8453, 0.8198],\n",
|
||||
" [0.5953, 0.8401, 0.3126, 0.6025, 0.5252],\n",
|
||||
" [0.1902, 0.5610, 0.7968, 0.1463, 0.7154],\n",
|
||||
" [0.7979, 0.2161, 0.6176, 0.2951, 0.1980],\n",
|
||||
" [0.6451, 0.3837, 0.5305, 0.2740, 0.3330]], device='cuda:0')\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"if torch.cuda.is_available():\n",
|
||||
" y = x.to(device) # or just use strings ``.to(\"cuda\")``\n",
|
||||
" print(x[0:5, 0:5])"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 13,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"if torch.cuda.is_available():\n",
|
||||
" # Here is the memory of the GPU a border. \n",
|
||||
" # A matrix with 100000 lines requires 37 GB, but only 8 GB are available.\n",
|
||||
" H = x.mm( (x.t().mm(x)).inverse() ).mm(x.t())"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 14,
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"tensor([[ 1.2748e-03, 5.3656e-04, 1.7376e-04, 3.3888e-06, -1.7049e-04],\n",
|
||||
" [ 5.3656e-04, 6.3624e-04, 2.5957e-05, 3.3281e-04, -1.6239e-05],\n",
|
||||
" [ 1.7376e-04, 2.5957e-05, 7.6328e-04, 7.7603e-05, 1.8272e-04],\n",
|
||||
" [ 3.3888e-06, 3.3281e-04, 7.7603e-05, 9.6281e-04, 1.2375e-04],\n",
|
||||
" [-1.7049e-04, -1.6239e-05, 1.8272e-04, 1.2375e-04, 6.9231e-04]],\n",
|
||||
" device='cuda:0')\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"if torch.cuda.is_available():\n",
|
||||
" print(H[0:5, 0:5])"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 15,
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"tensor([[ 1.2748e-03, 5.3656e-04, 1.7376e-04, 3.3888e-06, -1.7049e-04],\n",
|
||||
" [ 5.3656e-04, 6.3624e-04, 2.5957e-05, 3.3281e-04, -1.6239e-05],\n",
|
||||
" [ 1.7376e-04, 2.5957e-05, 7.6328e-04, 7.7603e-05, 1.8272e-04],\n",
|
||||
" [ 3.3888e-06, 3.3281e-04, 7.7603e-05, 9.6281e-04, 1.2375e-04],\n",
|
||||
" [-1.7049e-04, -1.6239e-05, 1.8272e-04, 1.2375e-04, 6.9231e-04]],\n",
|
||||
" dtype=torch.float64)\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"if torch.cuda.is_available():\n",
|
||||
" # This operation is difficult, as an symmetric matrix is transferred \n",
|
||||
" # back to the CPU. Is possible up to 30000 rows.\n",
|
||||
" print(H.to(\"cpu\", torch.double)[0:5, 0:5])"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": []
|
||||
}
|
||||
],
|
||||
"metadata": {
|
||||
"kernelspec": {
|
||||
"display_name": "Python 3",
|
||||
"language": "python",
|
||||
"name": "python3"
|
||||
},
|
||||
"language_info": {
|
||||
"codemirror_mode": {
|
||||
"name": "ipython",
|
||||
"version": 3
|
||||
},
|
||||
"file_extension": ".py",
|
||||
"mimetype": "text/x-python",
|
||||
"name": "python",
|
||||
"nbconvert_exporter": "python",
|
||||
"pygments_lexer": "ipython3",
|
||||
"version": "3.7.3"
|
||||
}
|
||||
},
|
||||
"nbformat": 4,
|
||||
"nbformat_minor": 4
|
||||
}
|
||||
5742
extra/Getting_Started/JuliaQuickstart.ipynb
Executable file
5742
extra/Getting_Started/JuliaQuickstart.ipynb
Executable file
File diff suppressed because it is too large
Load Diff
337
extra/Getting_Started/JupyterBasics.ipynb
Executable file
337
extra/Getting_Started/JupyterBasics.ipynb
Executable file
@@ -0,0 +1,337 @@
|
||||
{
|
||||
"cells": [
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"# Getting Started\n",
|
||||
"\n",
|
||||
"**Markdown** is a method to write documentations and even embed `HTML` and `Latex` formulas.\n",
|
||||
"\n",
|
||||
"### Jupyter tutorial\n",
|
||||
"\n",
|
||||
"[](https://www.youtube.com/watch?v=CwFq3YDU6_Y?rel=0&showinfo=0)\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"### Bayesian Rule\n",
|
||||
"$$P(A \\mid B) = \\frac{P(B \\mid A)P(A)}{P(B)}$$"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## Coding and magic commands"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 1,
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"data": {
|
||||
"text/plain": [
|
||||
"[1, 8, 27, 64, 125, 216, 343]"
|
||||
]
|
||||
},
|
||||
"execution_count": 1,
|
||||
"metadata": {},
|
||||
"output_type": "execute_result"
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"[x**3 for x in range(1,8)]"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 2,
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"data": {
|
||||
"application/json": {
|
||||
"cell": {
|
||||
"!": "OSMagics",
|
||||
"HTML": "Other",
|
||||
"SVG": "Other",
|
||||
"bash": "Other",
|
||||
"capture": "ExecutionMagics",
|
||||
"debug": "ExecutionMagics",
|
||||
"file": "Other",
|
||||
"html": "DisplayMagics",
|
||||
"javascript": "DisplayMagics",
|
||||
"js": "DisplayMagics",
|
||||
"latex": "DisplayMagics",
|
||||
"markdown": "DisplayMagics",
|
||||
"perl": "Other",
|
||||
"prun": "ExecutionMagics",
|
||||
"pypy": "Other",
|
||||
"python": "Other",
|
||||
"python2": "Other",
|
||||
"python3": "Other",
|
||||
"ruby": "Other",
|
||||
"script": "ScriptMagics",
|
||||
"sh": "Other",
|
||||
"svg": "DisplayMagics",
|
||||
"sx": "OSMagics",
|
||||
"system": "OSMagics",
|
||||
"time": "ExecutionMagics",
|
||||
"timeit": "ExecutionMagics",
|
||||
"writefile": "OSMagics"
|
||||
},
|
||||
"line": {
|
||||
"alias": "OSMagics",
|
||||
"alias_magic": "BasicMagics",
|
||||
"autoawait": "AsyncMagics",
|
||||
"autocall": "AutoMagics",
|
||||
"automagic": "AutoMagics",
|
||||
"autosave": "KernelMagics",
|
||||
"bookmark": "OSMagics",
|
||||
"cat": "Other",
|
||||
"cd": "OSMagics",
|
||||
"clear": "KernelMagics",
|
||||
"colors": "BasicMagics",
|
||||
"conda": "PackagingMagics",
|
||||
"config": "ConfigMagics",
|
||||
"connect_info": "KernelMagics",
|
||||
"cp": "Other",
|
||||
"debug": "ExecutionMagics",
|
||||
"dhist": "OSMagics",
|
||||
"dirs": "OSMagics",
|
||||
"doctest_mode": "BasicMagics",
|
||||
"ed": "Other",
|
||||
"edit": "KernelMagics",
|
||||
"env": "OSMagics",
|
||||
"gui": "BasicMagics",
|
||||
"hist": "Other",
|
||||
"history": "HistoryMagics",
|
||||
"killbgscripts": "ScriptMagics",
|
||||
"ldir": "Other",
|
||||
"less": "KernelMagics",
|
||||
"lf": "Other",
|
||||
"lk": "Other",
|
||||
"ll": "Other",
|
||||
"load": "CodeMagics",
|
||||
"load_ext": "ExtensionMagics",
|
||||
"loadpy": "CodeMagics",
|
||||
"logoff": "LoggingMagics",
|
||||
"logon": "LoggingMagics",
|
||||
"logstart": "LoggingMagics",
|
||||
"logstate": "LoggingMagics",
|
||||
"logstop": "LoggingMagics",
|
||||
"ls": "Other",
|
||||
"lsmagic": "BasicMagics",
|
||||
"lx": "Other",
|
||||
"macro": "ExecutionMagics",
|
||||
"magic": "BasicMagics",
|
||||
"man": "KernelMagics",
|
||||
"matplotlib": "PylabMagics",
|
||||
"mkdir": "Other",
|
||||
"more": "KernelMagics",
|
||||
"mv": "Other",
|
||||
"notebook": "BasicMagics",
|
||||
"page": "BasicMagics",
|
||||
"pastebin": "CodeMagics",
|
||||
"pdb": "ExecutionMagics",
|
||||
"pdef": "NamespaceMagics",
|
||||
"pdoc": "NamespaceMagics",
|
||||
"pfile": "NamespaceMagics",
|
||||
"pinfo": "NamespaceMagics",
|
||||
"pinfo2": "NamespaceMagics",
|
||||
"pip": "PackagingMagics",
|
||||
"popd": "OSMagics",
|
||||
"pprint": "BasicMagics",
|
||||
"precision": "BasicMagics",
|
||||
"prun": "ExecutionMagics",
|
||||
"psearch": "NamespaceMagics",
|
||||
"psource": "NamespaceMagics",
|
||||
"pushd": "OSMagics",
|
||||
"pwd": "OSMagics",
|
||||
"pycat": "OSMagics",
|
||||
"pylab": "PylabMagics",
|
||||
"qtconsole": "KernelMagics",
|
||||
"quickref": "BasicMagics",
|
||||
"recall": "HistoryMagics",
|
||||
"rehashx": "OSMagics",
|
||||
"reload_ext": "ExtensionMagics",
|
||||
"rep": "Other",
|
||||
"rerun": "HistoryMagics",
|
||||
"reset": "NamespaceMagics",
|
||||
"reset_selective": "NamespaceMagics",
|
||||
"rm": "Other",
|
||||
"rmdir": "Other",
|
||||
"run": "ExecutionMagics",
|
||||
"save": "CodeMagics",
|
||||
"sc": "OSMagics",
|
||||
"set_env": "OSMagics",
|
||||
"store": "StoreMagics",
|
||||
"sx": "OSMagics",
|
||||
"system": "OSMagics",
|
||||
"tb": "ExecutionMagics",
|
||||
"time": "ExecutionMagics",
|
||||
"timeit": "ExecutionMagics",
|
||||
"unalias": "OSMagics",
|
||||
"unload_ext": "ExtensionMagics",
|
||||
"who": "NamespaceMagics",
|
||||
"who_ls": "NamespaceMagics",
|
||||
"whos": "NamespaceMagics",
|
||||
"xdel": "NamespaceMagics",
|
||||
"xmode": "BasicMagics"
|
||||
}
|
||||
},
|
||||
"text/plain": [
|
||||
"Available line magics:\n",
|
||||
"%alias %alias_magic %autoawait %autocall %automagic %autosave %bookmark %cat %cd %clear %colors %conda %config %connect_info %cp %debug %dhist %dirs %doctest_mode %ed %edit %env %gui %hist %history %killbgscripts %ldir %less %lf %lk %ll %load %load_ext %loadpy %logoff %logon %logstart %logstate %logstop %ls %lsmagic %lx %macro %magic %man %matplotlib %mkdir %more %mv %notebook %page %pastebin %pdb %pdef %pdoc %pfile %pinfo %pinfo2 %pip %popd %pprint %precision %prun %psearch %psource %pushd %pwd %pycat %pylab %qtconsole %quickref %recall %rehashx %reload_ext %rep %rerun %reset %reset_selective %rm %rmdir %run %save %sc %set_env %store %sx %system %tb %time %timeit %unalias %unload_ext %who %who_ls %whos %xdel %xmode\n",
|
||||
"\n",
|
||||
"Available cell magics:\n",
|
||||
"%%! %%HTML %%SVG %%bash %%capture %%debug %%file %%html %%javascript %%js %%latex %%markdown %%perl %%prun %%pypy %%python %%python2 %%python3 %%ruby %%script %%sh %%svg %%sx %%system %%time %%timeit %%writefile\n",
|
||||
"\n",
|
||||
"Automagic is ON, % prefix IS NOT needed for line magics."
|
||||
]
|
||||
},
|
||||
"execution_count": 2,
|
||||
"metadata": {},
|
||||
"output_type": "execute_result"
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"# List all magic commands\n",
|
||||
"%lsmagic"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 3,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"import numpy as np \n",
|
||||
"x = np.random.normal(0, 1, 1_000_000)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 4,
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"1.03 ms ± 4.93 µs per loop (mean ± std. dev. of 7 runs, 1000 loops each)\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"%%timeit \n",
|
||||
"x.mean()**2 - (x**2).mean()"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## Send variables back and fourth in bash\n",
|
||||
"\n",
|
||||
"The strings will be saved as files in the working directory"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 5,
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"string_0\n",
|
||||
"string_1\n",
|
||||
"string_2\n",
|
||||
"string_3\n",
|
||||
"string_4\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"for i in range(5):\n",
|
||||
" string = \"string_{}\".format(i)\n",
|
||||
" print(string)\n",
|
||||
" !touch $string"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 6,
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"-rw-r--r-- 1 jovyan users 0 Dec 19 08:44 string_0\n",
|
||||
"-rw-r--r-- 1 jovyan users 0 Dec 19 08:44 string_1\n",
|
||||
"-rw-r--r-- 1 jovyan users 0 Dec 19 08:44 string_2\n",
|
||||
"-rw-r--r-- 1 jovyan users 0 Dec 19 08:44 string_3\n",
|
||||
"-rw-r--r-- 1 jovyan users 0 Dec 19 08:44 string_4\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"!ls -l | grep string_"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 7,
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"data": {
|
||||
"text/plain": [
|
||||
"['string_0', 'string_1', 'string_2', 'string_3', 'string_4']"
|
||||
]
|
||||
},
|
||||
"execution_count": 7,
|
||||
"metadata": {},
|
||||
"output_type": "execute_result"
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"files = !ls -1 string_*\n",
|
||||
"!rm string_*\n",
|
||||
"files"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": []
|
||||
}
|
||||
],
|
||||
"metadata": {
|
||||
"kernelspec": {
|
||||
"display_name": "Python 3",
|
||||
"language": "python",
|
||||
"name": "python3"
|
||||
},
|
||||
"language_info": {
|
||||
"codemirror_mode": {
|
||||
"name": "ipython",
|
||||
"version": 3
|
||||
},
|
||||
"file_extension": ".py",
|
||||
"mimetype": "text/x-python",
|
||||
"name": "python",
|
||||
"nbconvert_exporter": "python",
|
||||
"pygments_lexer": "ipython3",
|
||||
"version": "3.7.3"
|
||||
}
|
||||
},
|
||||
"nbformat": 4,
|
||||
"nbformat_minor": 4
|
||||
}
|
||||
142
extra/Getting_Started/MultiInterpreterNotebook.ipynb
Executable file
142
extra/Getting_Started/MultiInterpreterNotebook.ipynb
Executable file
File diff suppressed because one or more lines are too long
BIN
extra/Getting_Started/jupyterlab-overview.png
Executable file
BIN
extra/Getting_Started/jupyterlab-overview.png
Executable file
Binary file not shown.
|
After Width: | Height: | Size: 118 KiB |
57
extra/Getting_Started/scroller.py
Executable file
57
extra/Getting_Started/scroller.py
Executable file
@@ -0,0 +1,57 @@
|
||||
def scroller(index, quantity, timerange=timedelta(days=0), startdt="", enddt=""):
|
||||
print("Starting to scroll", end='')
|
||||
# Retrieve the datetimes, note that timerange has a higher priority
|
||||
if timerange.total_seconds() > 0:
|
||||
now = datetime.utcnow().replace(tzinfo=pytz.UTC)
|
||||
startdt = (now - timerange).isoformat()
|
||||
enddt = now.isoformat()
|
||||
|
||||
# search the first page and write the result to data
|
||||
response = es.search(
|
||||
index=index,
|
||||
body={
|
||||
"query": {
|
||||
"bool": {
|
||||
"must": [
|
||||
{"range" : {
|
||||
"phenomenonTime" : {
|
||||
#"gte": "2018-02-20T09:08:34.230693+00:00",
|
||||
"gte": startdt,
|
||||
"lte": enddt,
|
||||
"time_zone": "+01:00"
|
||||
}
|
||||
}},
|
||||
{
|
||||
"match_phrase": {
|
||||
"Datastream.name.keyword": quantity
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
},
|
||||
scroll='10m'
|
||||
)
|
||||
data = [[row["_source"]["phenomenonTime"], row["_source"]["result"]] for row in response['hits']['hits']]
|
||||
|
||||
# Append new pages until there aren't any left
|
||||
while len(response['hits']['hits']):
|
||||
print(".", end='')
|
||||
# process results
|
||||
# print([item["_id"] for item in response["hits"]["hits"]])
|
||||
response = es.scroll(scroll_id=response['_scroll_id'], scroll='10m')
|
||||
data += [[row["_source"]["phenomenonTime"], row["_source"]["result"]] for row in response['hits']['hits']]
|
||||
|
||||
# Convert data to a DataFrame and return it
|
||||
df = pd.DataFrame(data, columns=["phenomenonTime", quantity])
|
||||
# df.index = pd.to_datetime(df["phenomenonTime"].map(lambda t: t.split(".")[0]), utc=True)
|
||||
df.index = pd.to_datetime(df["phenomenonTime"].map(lambda t: roundto(t, 1)), utc=True)
|
||||
df = df.drop(["phenomenonTime"], axis=1)
|
||||
print("\nFetched {} tuples.".format(df.shape[0]))
|
||||
return df
|
||||
|
||||
def roundto(string, n):
|
||||
base = string.split(".")[0]
|
||||
if n > 0:
|
||||
base += "." + string.split(".")[1][:n]
|
||||
return base
|
||||
BIN
extra/performance-test.png
Normal file
BIN
extra/performance-test.png
Normal file
Binary file not shown.
|
After Width: | Height: | Size: 346 KiB |
Reference in New Issue
Block a user