From fbfc7bae6336a5503ba2a83847b5988a61c29dcf Mon Sep 17 00:00:00 2001 From: Andreas Motl Date: Tue, 5 Dec 2023 02:04:56 +0100 Subject: [PATCH 1/2] Timeseries: Add notebook about using CrateDB Cloud with Datashader --- topic/timeseries/explore/README.md | 28 +- .../timeseries/explore/cloud-datashader.ipynb | 664 ++++++++++++++++++ 2 files changed, 686 insertions(+), 6 deletions(-) create mode 100644 topic/timeseries/explore/cloud-datashader.ipynb diff --git a/topic/timeseries/explore/README.md b/topic/timeseries/explore/README.md index 15aa5f57..700a3d68 100644 --- a/topic/timeseries/explore/README.md +++ b/topic/timeseries/explore/README.md @@ -1,7 +1,9 @@ -# Time Series with CrateDB +# CrateDB Time Series Exploration and Visualization -This folder provides examples, tutorials and runnable code on how to use CrateDB -for time series use cases. +This folder provides examples, tutorials and runnable code on how to use +[CrateDB] for time series use cases, together with software packages from +the [PyViz] ecosystem, to analyze and explore data on behalf of visually +stunning data plots. The tutorials and examples focus on being easy to understand and use. They are a good starting point for your own projects. @@ -17,13 +19,27 @@ This folder provides guidelines and runnable code to get started with time serie walkthrough about how to get started with time series and CrateDB, and guides you to corresponding example programs. +- [requirements.txt](requirements.txt): For pulling the required dependencies to + run the example programs. + +- `cloud-datashader.ipynb` [![Open on GitHub](https://img.shields.io/badge/Open%20on-GitHub-lightgray?logo=GitHub)](cloud-datashader.ipynb) [![Open in Collab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/crate/cratedb-examples/blob/main/topic/timeseries/explore/cloud-datashader.ipynb) + + This notebook explores the [HoloViews] and [Datashader] frameworks and outlines + how to use them to plot the venerable NYC Taxi dataset, after importing it into + a CrateDB Cloud database cluster. + - `timeseries-queries-and-visualization.ipynb` [![Open on GitHub](https://img.shields.io/badge/Open%20on-GitHub-lightgray?logo=GitHub)](timeseries-queries-and-visualization.ipynb) [![Open in Collab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/crate/cratedb-examples/blob/main/topic/timeseries/explore/timeseries-queries-and-visualization.ipynb) This notebook explores how to access timeseries data from CrateDB via SQL, - load it into pandas data frames, and visulaize it via plotly. + load it into pandas DataFrames, and visualize it using [Plotly]. - It also demonstrates more advanced time series queries in SQL, e.g. aggregations, + It also demonstrates advanced time series queries in SQL, e.g. aggregations, window functions, interpolation of missing data, common table expressions, - moving averages, JOINs and the handling of JSON data. + moving averages, JOINs, and document store details about handling of JSON data. + [CrateDB]: https://github.com/crate/crate +[Datashader]: https://datashader.org/ +[HoloViews]: https://www.holoviews.org/ +[Plotly]: https://plotly.com/python/ +[PyViz]: https://pyviz.org/ diff --git a/topic/timeseries/explore/cloud-datashader.ipynb b/topic/timeseries/explore/cloud-datashader.ipynb new file mode 100644 index 00000000..fe3551e4 --- /dev/null +++ b/topic/timeseries/explore/cloud-datashader.ipynb @@ -0,0 +1,664 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "source": [ + "# CrateDB Cloud Import\n", + "\n", + "This is an example notebook demonstrating how to load data from\n", + "files using the [Import API] interface of [CrateDB Cloud] into\n", + "a [CrateDB Cloud Cluster].\n", + "\n", + "The supported file types are CSV, JSON, Parquet, optionally with\n", + "gzip compression. They can be acquired from the local filesystem,\n", + "or from remote HTTP and AWS S3 resources.\n", + "\n", + "[CrateDB Cloud]: https://cratedb.com/docs/cloud/\n", + "[CrateDB Cloud Cluster]: https://cratedb.com/docs/cloud/en/latest/reference/services.html\n", + "[Import API]: https://community.cratedb.com/t/importing-data-to-cratedb-cloud-clusters/1467\n", + "\n", + "## Setup\n", + "\n", + "To install the client SDK, use `pip`." + ], + "metadata": { + "collapsed": false + } + }, + { + "cell_type": "code", + "execution_count": null, + "outputs": [], + "source": [ + "#!pip install 'cratedb-toolkit'" + ], + "metadata": { + "collapsed": false + } + }, + { + "cell_type": "markdown", + "source": [ + "## Configuration\n", + "\n", + "The notebook assumes you are appropriately authenticated to the CrateDB Cloud\n", + "platform, for example using `croud login --idp azuread`. To inspect the list\n", + "of available clusters, run `croud clusters list`.\n", + "\n", + "For addressing a database cluster, and obtaining corresponding credentials,\n", + "the program uses environment variables, which you can define interactively,\n", + "or store them within a `.env` file.\n", + "\n", + "You can use those configuration snippet as a blueprint. Please adjust the\n", + "individual settings accordingly.\n", + "```shell\n", + "CRATEDB_CLOUD_CLUSTER_NAME=Hotzenplotz\n", + "CRATEDB_USERNAME='admin'\n", + "CRATEDB_PASSWORD='H3IgNXNvQBJM3CiElOiVHuSp6CjXMCiQYhB4I9dLccVHGvvvitPSYr1vTpt4'\n", + "```" + ], + "metadata": { + "collapsed": false + } + }, + { + "cell_type": "markdown", + "source": [ + "## Acquire Database Cluster\n", + "\n", + "As a first measure, acquire a resource handle, which manages a CrateDB Cloud\n", + "cluster instance.\n", + "\n", + "For effortless configuration, it will obtain configuration settings from\n", + "environment variables as defined above." + ], + "metadata": { + "collapsed": false + } + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [ + "import cratedb_toolkit as ctk\n", + "\n", + "# FIXME: HEADLESS!\n", + "cluster = ctk.ManagedCluster.from_env().start()" + ] + }, + { + "cell_type": "markdown", + "source": [ + "## Import Data\n", + "\n", + "The venerable New York City [TLC Trip Record Data] dataset includes taxi trip\n", + "data capturing pick-up and drop-off dates/times, pick-up and drop-off locations,\n", + "trip distances, itemized fares, rate types, payment types, and driver-reported\n", + "passenger counts.\n", + "\n", + "The next code cell will import 15 million records worth of data from a Parquet\n", + "file using the [CrateDB Cloud Import] API.\n", + "\n", + "[CrateDB Cloud Import]: https://community.cratedb.com/t/importing-data-to-cratedb-cloud-clusters/1467\n", + "[TLC Trip Record Data]: https://www.nyc.gov/site/tlc/about/tlc-trip-record-data.page" + ], + "metadata": { + "collapsed": false + } + }, + { + "cell_type": "code", + "execution_count": 21, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\u001B[36m==> Info: \u001B[0mStatus: REGISTERED (Your import job was received and is pending processing.)\n", + "\u001B[36m==> Info: \u001B[0mStatus: SENT (Your creation request was sent to the region.)\n", + "\u001B[36m==> Info: \u001B[0mStatus: IN_PROGRESS (Import in progress)\n", + "\u001B[36m==> Info: \u001B[0mimporting... 270.00K records (1.78%) imported so far.\n", + "\u001B[36m==> Info: \u001B[0mimporting... 660.00K records (4.36%) imported so far.\n", + "\u001B[36m==> Info: \u001B[0mimporting... 880.00K records (5.81%) imported so far.\n", + "\u001B[36m==> Info: \u001B[0mimporting... 1.25M records (8.25%) imported so far.\n", + "\u001B[36m==> Info: \u001B[0mimporting... 1.61M records (10.63%) imported so far.\n", + "\u001B[36m==> Info: \u001B[0mimporting... 1.98M records (13.07%) imported so far.\n", + "\u001B[36m==> Info: \u001B[0mimporting... 2.34M records (15.45%) imported so far.\n", + "\u001B[36m==> Info: \u001B[0mimporting... 2.71M records (17.89%) imported so far.\n", + "\u001B[36m==> Info: \u001B[0mimporting... 3.07M records (20.27%) imported so far.\n", + "\u001B[36m==> Info: \u001B[0mimporting... 3.45M records (22.78%) imported so far.\n", + "\u001B[36m==> Info: \u001B[0mimporting... 3.94M records (26.02%) imported so far.\n", + "\u001B[36m==> Info: \u001B[0mimporting... 4.31M records (28.46%) imported so far.\n", + "\u001B[36m==> Info: \u001B[0mimporting... 4.66M records (30.77%) imported so far.\n", + "\u001B[36m==> Info: \u001B[0mimporting... 5.03M records (33.21%) imported so far.\n", + "\u001B[36m==> Info: \u001B[0mimporting... 5.41M records (35.72%) imported so far.\n", + "\u001B[36m==> Info: \u001B[0mimporting... 5.79M records (38.23%) imported so far.\n", + "\u001B[36m==> Info: \u001B[0mimporting... 6.01M records (39.68%) imported so far.\n", + "\u001B[36m==> Info: \u001B[0mimporting... 6.25M records (41.27%) imported so far.\n", + "\u001B[36m==> Info: \u001B[0mimporting... 6.61M records (43.64%) imported so far.\n", + "\u001B[36m==> Info: \u001B[0mimporting... 6.99M records (46.15%) imported so far.\n", + "\u001B[36m==> Info: \u001B[0mimporting... 7.35M records (48.53%) imported so far.\n", + "\u001B[36m==> Info: \u001B[0mimporting... 7.74M records (51.11%) imported so far.\n", + "\u001B[36m==> Info: \u001B[0mimporting... 8.11M records (53.55%) imported so far.\n", + "\u001B[36m==> Info: \u001B[0mimporting... 8.60M records (56.78%) imported so far.\n", + "\u001B[36m==> Info: \u001B[0mimporting... 8.94M records (59.03%) imported so far.\n", + "\u001B[36m==> Info: \u001B[0mimporting... 9.20M records (60.75%) imported so far.\n", + "\u001B[36m==> Info: \u001B[0mimporting... 9.48M records (62.59%) imported so far.\n", + "\u001B[36m==> Info: \u001B[0mimporting... 9.81M records (64.77%) imported so far.\n", + "\u001B[36m==> Info: \u001B[0mimporting... 10.18M records (67.22%) imported so far.\n", + "\u001B[36m==> Info: \u001B[0mimporting... 10.56M records (69.73%) imported so far.\n", + "\u001B[36m==> Info: \u001B[0mimporting... 11.05M records (72.96%) imported so far.\n", + "\u001B[36m==> Info: \u001B[0mimporting... 11.43M records (75.47%) imported so far.\n", + "\u001B[36m==> Info: \u001B[0mimporting... 11.80M records (77.91%) imported so far.\n", + "\u001B[36m==> Info: \u001B[0mimporting... 12.17M records (80.36%) imported so far.\n", + "\u001B[36m==> Info: \u001B[0mimporting... 12.51M records (82.60%) imported so far.\n", + "\u001B[36m==> Info: \u001B[0mimporting... 12.76M records (84.25%) imported so far.\n", + "\u001B[36m==> Info: \u001B[0mimporting... 13.05M records (86.17%) imported so far.\n", + "\u001B[36m==> Info: \u001B[0mimporting... 13.50M records (89.14%) imported so far.\n", + "\u001B[36m==> Info: \u001B[0mimporting... 13.87M records (91.58%) imported so far.\n", + "\u001B[36m==> Info: \u001B[0mimporting... 14.25M records (94.09%) imported so far.\n", + "\u001B[36m==> Info: \u001B[0mimporting... 14.62M records (96.53%) imported so far.\n", + "\u001B[36m==> Info: \u001B[0mimporting... 15.01M records (99.11%) imported so far.\n", + "\u001B[36m==> Info: \u001B[0mDone importing 15.14M records\n", + "\u001B[32m==> Success: \u001B[0mOperation completed.\n" + ] + }, + { + "data": { + "text/plain": "CloudJob(info={'cluster_id': '11cbcb3d-7057-4f80-adc3-38673838b38e', 'compression': 'none', 'dc': {'created': '2023-11-18T03:04:18.497000+00:00', 'modified': '2023-11-18T03:04:18.497000+00:00'}, 'destination': {'create_table': True, 'table': '\"testdrive\".\"yellow_tripdata_2010-04\"'}, 'file': None, 'format': 'parquet', 'id': 'b4d4711b-7912-4590-a4a6-bfca1dee2d4b', 'progress': {'bytes': 0, 'details': {'create_table_sql': 'CREATE TABLE \"testdrive\".\"yellow_tripdata_2010-04\" (\"vendor_id\" VARCHAR,\"pickup_datetime\" VARCHAR,\"dropoff_datetime\" VARCHAR,\"passenger_count\" BIGINT,\"trip_distance\" DOUBLE,\"pickup_longitude\" DOUBLE,\"pickup_latitude\" DOUBLE,\"rate_code\" VARCHAR,\"store_and_fwd_flag\" VARCHAR,\"dropoff_longitude\" DOUBLE,\"dropoff_latitude\" DOUBLE,\"payment_type\" VARCHAR,\"fare_amount\" DOUBLE,\"surcharge\" DOUBLE,\"mta_tax\" DOUBLE,\"tip_amount\" DOUBLE,\"tolls_amount\" DOUBLE,\"total_amount\" DOUBLE)'}, 'failed_files': 0, 'failed_records': 0, 'message': 'Import succeeded', 'percent': 100.0, 'processed_files': 1, 'records': 15144990, 'total_files': 1, 'total_records': 15144990}, 'schema': {'type': 'parquet'}, 'status': 'SUCCEEDED', 'type': 'url', 'url': {'url': 'https://d37ci6vzurychx.cloudfront.net/trip-data/yellow_tripdata_2010-04.parquet'}}, found=True, _custom_status=None, _custom_message=None)" + }, + "execution_count": 21, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "# Define data source.\n", + "url = \"https://d37ci6vzurychx.cloudfront.net/trip-data/yellow_tripdata_2010-04.parquet\"\n", + "source = ctk.InputOutputResource(url=url)\n", + "target = ctk.TableAddress(schema=\"testdrive\")\n", + "\n", + "# Invoke import job. Without `target` argument, the destination\n", + "# table name will be derived from the input file name.\n", + "cluster.load_table(source=source, target=target)" + ], + "metadata": { + "collapsed": false + } + }, + { + "cell_type": "markdown", + "source": [ + "## Load Data" + ], + "metadata": { + "collapsed": false + } + }, + { + "cell_type": "code", + "execution_count": 13, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Number of records: 15144990\n" + ] + }, + { + "data": { + "text/plain": " dropoff_longitude dropoff_latitude pickup_longitude pickup_latitude \\\n0 -73.995568 40.733463 -73.990177 40.756135 \n1 -73.975875 40.734825 -73.988605 40.736607 \n2 -73.945578 40.815380 -73.981063 40.733482 \n3 -73.965268 40.684530 -74.007440 40.713240 \n4 -74.006438 40.739648 -73.980638 40.761542 \n... ... ... ... ... \n4995 -73.972765 40.757685 -73.999252 40.723053 \n4996 -73.931530 40.757507 -73.936367 40.758273 \n4997 -73.952612 40.765850 -73.947745 40.783605 \n4998 -73.989458 40.752847 -73.972528 40.747457 \n4999 -73.974220 40.754883 -74.003170 40.734293 \n\n passenger_count \n0 1 \n1 1 \n2 2 \n3 1 \n4 3 \n... ... \n4995 3 \n4996 1 \n4997 2 \n4998 1 \n4999 2 \n\n[5000 rows x 5 columns]", + "text/html": "
\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n
dropoff_longitudedropoff_latitudepickup_longitudepickup_latitudepassenger_count
0-73.99556840.733463-73.99017740.7561351
1-73.97587540.734825-73.98860540.7366071
2-73.94557840.815380-73.98106340.7334822
3-73.96526840.684530-74.00744040.7132401
4-74.00643840.739648-73.98063840.7615423
..................
4995-73.97276540.757685-73.99925240.7230533
4996-73.93153040.757507-73.93636740.7582731
4997-73.95261240.765850-73.94774540.7836052
4998-73.98945840.752847-73.97252840.7474571
4999-73.97422040.754883-74.00317040.7342932
\n

5000 rows × 5 columns

\n
" + }, + "execution_count": 13, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "import dask.dataframe as dd\n", + "import pandas as pd\n", + "\n", + "# Acquire database clients.\n", + "client_bundle = cluster.get_client_bundle()\n", + "\n", + "# Quickly inspect the number of records.\n", + "# TODO: Provide `count_records` functions directly on `cluster` object?\n", + "total_count = client_bundle.adapter.count_records('\"testdrive\".\"yellow_tripdata_2010-04\"')\n", + "print(f\"Number of records: {total_count}\")\n", + "\n", + "# Query data from database.\n", + "sql = 'SELECT dropoff_longitude, dropoff_latitude, pickup_longitude, pickup_latitude, passenger_count ' \\\n", + " 'FROM \"testdrive\".\"yellow_tripdata_2010-04\" LIMIT 5000;'\n", + "df = pd.read_sql(sql=sql, con=client_bundle.sqlalchemy)\n", + "\n", + "# TODO: Use Dask for reading from database?\n", + "# df = dd.read_sql(sql=sql, con=client_bundle.adapter.dburi)\n", + "\n", + "# Inspect DataFrame.\n", + "df" + ], + "metadata": { + "collapsed": false + } + }, + { + "cell_type": "markdown", + "source": [ + "## Data Refinement\n", + "\n", + "Before visualizing the DataFrame loaded from the database, it will be filtered\n", + "by a bounding box limiter, and transformed into a categorical DataFrame, while\n", + "unifying names of previously different geolocation column names into canonical\n", + "`longitude` / `latitude` columns." + ], + "metadata": { + "collapsed": false + } + }, + { + "cell_type": "code", + "execution_count": 8, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Number of records: 973068\n" + ] + } + ], + "source": [ + "# Limit data to bounding box around NYC.\n", + "# TODO: Reformulate into WHERE clause of SQL expression.\n", + "df = df.loc[\n", + " (df.dropoff_longitude > -74.1) & (df.dropoff_longitude < -73.7) &\n", + " (df.dropoff_latitude > 40.6) & (df.dropoff_latitude < 40.9) &\n", + " (df.pickup_longitude > -74.1) & (df.pickup_longitude < -73.7) &\n", + " (df.pickup_latitude > 40.6) & (df.pickup_latitude < 40.9)\n", + "]\n", + "\n", + "# Create a DataFrame with `{dropoff,pickup}_{longitude,latitude}` converged\n", + "# into `longitude`/`latitude` and a `journey_type` categorical column.\n", + "# TODO: Can this be reformulated into SQL as well?\n", + "\n", + "# Get a DataFrame with just drop-off locations.\n", + "df_drop = df[[\"dropoff_longitude\", \"dropoff_latitude\", \"passenger_count\"]]\n", + "df_drop.loc[:, (\"journey_type\",)] = \"dropoff\"\n", + "df_drop = df_drop.rename(columns={\"dropoff_longitude\": \"longitude\", \"dropoff_latitude\": \"latitude\"})\n", + "\n", + "# Get a DataFrame with just pickup locations.\n", + "df_pick = df[[\"pickup_longitude\", \"pickup_latitude\", \"passenger_count\"]]\n", + "df_pick.loc[:, (\"journey_type\",)] = \"pickup\"\n", + "df_pick = df_pick.rename(columns={\"pickup_longitude\": \"longitude\", \"pickup_latitude\": \"latitude\"})\n", + "\n", + "# Create unified categorized DataFrame suitable for plotting.\n", + "df_plot = dd.concat([df_drop, df_pick])\n", + "df_plot = df_plot.astype({\"journey_type\": \"category\"})\n", + "df_plot[\"journey_type\"] = df_plot[\"journey_type\"].cat.set_categories([\"dropoff\", \"pickup\"])\n", + "\n", + "# Partitions are small, it is better to repartition.\n", + "df_plot = df_plot.persist()\n", + "df_plot = df_plot.repartition(partition_size=\"256MiB\").persist()\n", + "\n", + "print(\"Number of records:\", len(df_plot))" + ], + "metadata": { + "collapsed": false + } + }, + { + "cell_type": "markdown", + "source": [ + "## Visualize\n", + "\n", + "In order to visualize the DataFrame prepared above, let's use the excellent [hvPlot]\n", + "and [Datashader] packages, based on [HoloViews], from the family of [HoloViz] packages\n", + "of [PyViz] fame.\n", + "\n", + "With Datashader, we can choose what we want to plot on the x and y axes and see the full data immediately, with no parameter tweaking, magic numbers, subsampling, or approximation, up to the resolution of the display\n", + "\n", + "[Datashader]: https://datashader.org/\n", + "[HoloViews]: https://www.holoviews.org/\n", + "[HoloViz]: https://holoviz.org/\n", + "[hvPlot]: https://hvplot.holoviz.org/\n", + "[PyViz]: https://pyviz.org/" + ], + "metadata": { + "collapsed": false + } + }, + { + "cell_type": "code", + "execution_count": 9, + "outputs": [ + { + "data": { + "application/javascript": "(function(root) {\n function now() {\n return new Date();\n }\n\n var force = true;\n var py_version = '3.3.1'.replace('rc', '-rc.').replace('.dev', '-dev.');\n var is_dev = py_version.indexOf(\"+\") !== -1 || py_version.indexOf(\"-\") !== -1;\n var reloading = false;\n var Bokeh = root.Bokeh;\n var bokeh_loaded = Bokeh != null && (Bokeh.version === py_version || (Bokeh.versions !== undefined && Bokeh.versions.has(py_version)));\n\n if (typeof (root._bokeh_timeout) === \"undefined\" || force) {\n root._bokeh_timeout = Date.now() + 5000;\n root._bokeh_failed_load = false;\n }\n\n function run_callbacks() {\n try {\n root._bokeh_onload_callbacks.forEach(function(callback) {\n if (callback != null)\n callback();\n });\n } finally {\n delete root._bokeh_onload_callbacks;\n }\n console.debug(\"Bokeh: all callbacks have finished\");\n }\n\n function load_libs(css_urls, js_urls, js_modules, js_exports, callback) {\n if (css_urls == null) css_urls = [];\n if (js_urls == null) js_urls = [];\n if (js_modules == null) js_modules = [];\n if (js_exports == null) js_exports = {};\n\n root._bokeh_onload_callbacks.push(callback);\n\n if (root._bokeh_is_loading > 0) {\n console.debug(\"Bokeh: BokehJS is being loaded, scheduling callback at\", now());\n return null;\n }\n if (js_urls.length === 0 && js_modules.length === 0 && Object.keys(js_exports).length === 0) {\n run_callbacks();\n return null;\n }\n if (!reloading) {\n console.debug(\"Bokeh: BokehJS not loaded, scheduling load and callback at\", now());\n }\n\n function on_load() {\n root._bokeh_is_loading--;\n if (root._bokeh_is_loading === 0) {\n console.debug(\"Bokeh: all BokehJS libraries/stylesheets loaded\");\n run_callbacks()\n }\n }\n window._bokeh_on_load = on_load\n\n function on_error() {\n console.error(\"failed to load \" + url);\n }\n\n var skip = [];\n if (window.requirejs) {\n window.requirejs.config({'packages': {}, 'paths': {'jspanel': 'https://cdn.jsdelivr.net/npm/jspanel4@4.12.0/dist/jspanel', 'jspanel-modal': 'https://cdn.jsdelivr.net/npm/jspanel4@4.12.0/dist/extensions/modal/jspanel.modal', 'jspanel-tooltip': 'https://cdn.jsdelivr.net/npm/jspanel4@4.12.0/dist/extensions/tooltip/jspanel.tooltip', 'jspanel-hint': 'https://cdn.jsdelivr.net/npm/jspanel4@4.12.0/dist/extensions/hint/jspanel.hint', 'jspanel-layout': 'https://cdn.jsdelivr.net/npm/jspanel4@4.12.0/dist/extensions/layout/jspanel.layout', 'jspanel-contextmenu': 'https://cdn.jsdelivr.net/npm/jspanel4@4.12.0/dist/extensions/contextmenu/jspanel.contextmenu', 'jspanel-dock': 'https://cdn.jsdelivr.net/npm/jspanel4@4.12.0/dist/extensions/dock/jspanel.dock', 'gridstack': 'https://cdn.jsdelivr.net/npm/gridstack@7.2.3/dist/gridstack-all', 'notyf': 'https://cdn.jsdelivr.net/npm/notyf@3/notyf.min'}, 'shim': {'jspanel': {'exports': 'jsPanel'}, 'gridstack': {'exports': 'GridStack'}}});\n require([\"jspanel\"], function(jsPanel) {\n\twindow.jsPanel = jsPanel\n\ton_load()\n })\n require([\"jspanel-modal\"], function() {\n\ton_load()\n })\n require([\"jspanel-tooltip\"], function() {\n\ton_load()\n })\n require([\"jspanel-hint\"], function() {\n\ton_load()\n })\n require([\"jspanel-layout\"], function() {\n\ton_load()\n })\n require([\"jspanel-contextmenu\"], function() {\n\ton_load()\n })\n require([\"jspanel-dock\"], function() {\n\ton_load()\n })\n require([\"gridstack\"], function(GridStack) {\n\twindow.GridStack = GridStack\n\ton_load()\n })\n require([\"notyf\"], function() {\n\ton_load()\n })\n root._bokeh_is_loading = css_urls.length + 9;\n } else {\n root._bokeh_is_loading = css_urls.length + js_urls.length + js_modules.length + Object.keys(js_exports).length;\n }\n\n var existing_stylesheets = []\n var links = document.getElementsByTagName('link')\n for (var i = 0; i < links.length; i++) {\n var link = links[i]\n if (link.href != null) {\n\texisting_stylesheets.push(link.href)\n }\n }\n for (var i = 0; i < css_urls.length; i++) {\n var url = css_urls[i];\n if (existing_stylesheets.indexOf(url) !== -1) {\n\ton_load()\n\tcontinue;\n }\n const element = document.createElement(\"link\");\n element.onload = on_load;\n element.onerror = on_error;\n element.rel = \"stylesheet\";\n element.type = \"text/css\";\n element.href = url;\n console.debug(\"Bokeh: injecting link tag for BokehJS stylesheet: \", url);\n document.body.appendChild(element);\n } if (((window['jsPanel'] !== undefined) && (!(window['jsPanel'] instanceof HTMLElement))) || window.requirejs) {\n var urls = ['https://cdn.holoviz.org/panel/1.3.1/dist/bundled/floatpanel/jspanel4@4.12.0/dist/jspanel.js', 'https://cdn.holoviz.org/panel/1.3.1/dist/bundled/floatpanel/jspanel4@4.12.0/dist/extensions/modal/jspanel.modal.js', 'https://cdn.holoviz.org/panel/1.3.1/dist/bundled/floatpanel/jspanel4@4.12.0/dist/extensions/tooltip/jspanel.tooltip.js', 'https://cdn.holoviz.org/panel/1.3.1/dist/bundled/floatpanel/jspanel4@4.12.0/dist/extensions/hint/jspanel.hint.js', 'https://cdn.holoviz.org/panel/1.3.1/dist/bundled/floatpanel/jspanel4@4.12.0/dist/extensions/layout/jspanel.layout.js', 'https://cdn.holoviz.org/panel/1.3.1/dist/bundled/floatpanel/jspanel4@4.12.0/dist/extensions/contextmenu/jspanel.contextmenu.js', 'https://cdn.holoviz.org/panel/1.3.1/dist/bundled/floatpanel/jspanel4@4.12.0/dist/extensions/dock/jspanel.dock.js'];\n for (var i = 0; i < urls.length; i++) {\n skip.push(urls[i])\n }\n } if (((window['GridStack'] !== undefined) && (!(window['GridStack'] instanceof HTMLElement))) || window.requirejs) {\n var urls = ['https://cdn.holoviz.org/panel/1.3.1/dist/bundled/gridstack/gridstack@7.2.3/dist/gridstack-all.js'];\n for (var i = 0; i < urls.length; i++) {\n skip.push(urls[i])\n }\n } if (((window['Notyf'] !== undefined) && (!(window['Notyf'] instanceof HTMLElement))) || window.requirejs) {\n var urls = ['https://cdn.holoviz.org/panel/1.3.1/dist/bundled/notificationarea/notyf@3/notyf.min.js'];\n for (var i = 0; i < urls.length; i++) {\n skip.push(urls[i])\n }\n } var existing_scripts = []\n var scripts = document.getElementsByTagName('script')\n for (var i = 0; i < scripts.length; i++) {\n var script = scripts[i]\n if (script.src != null) {\n\texisting_scripts.push(script.src)\n }\n }\n for (var i = 0; i < js_urls.length; i++) {\n var url = js_urls[i];\n if (skip.indexOf(url) !== -1 || existing_scripts.indexOf(url) !== -1) {\n\tif (!window.requirejs) {\n\t on_load();\n\t}\n\tcontinue;\n }\n var element = document.createElement('script');\n element.onload = on_load;\n element.onerror = on_error;\n element.async = false;\n element.src = url;\n console.debug(\"Bokeh: injecting script tag for BokehJS library: \", url);\n document.head.appendChild(element);\n }\n for (var i = 0; i < js_modules.length; i++) {\n var url = js_modules[i];\n if (skip.indexOf(url) !== -1 || existing_scripts.indexOf(url) !== -1) {\n\tif (!window.requirejs) {\n\t on_load();\n\t}\n\tcontinue;\n }\n var element = document.createElement('script');\n element.onload = on_load;\n element.onerror = on_error;\n element.async = false;\n element.src = url;\n element.type = \"module\";\n console.debug(\"Bokeh: injecting script tag for BokehJS library: \", url);\n document.head.appendChild(element);\n }\n for (const name in js_exports) {\n var url = js_exports[name];\n if (skip.indexOf(url) >= 0 || root[name] != null) {\n\tif (!window.requirejs) {\n\t on_load();\n\t}\n\tcontinue;\n }\n var element = document.createElement('script');\n element.onerror = on_error;\n element.async = false;\n element.type = \"module\";\n console.debug(\"Bokeh: injecting script tag for BokehJS library: \", url);\n element.textContent = `\n import ${name} from \"${url}\"\n window.${name} = ${name}\n window._bokeh_on_load()\n `\n document.head.appendChild(element);\n }\n if (!js_urls.length && !js_modules.length) {\n on_load()\n }\n };\n\n function inject_raw_css(css) {\n const element = document.createElement(\"style\");\n element.appendChild(document.createTextNode(css));\n document.body.appendChild(element);\n }\n\n var js_urls = [\"https://cdn.bokeh.org/bokeh/release/bokeh-3.3.1.min.js\", \"https://cdn.bokeh.org/bokeh/release/bokeh-gl-3.3.1.min.js\", \"https://cdn.bokeh.org/bokeh/release/bokeh-widgets-3.3.1.min.js\", \"https://cdn.bokeh.org/bokeh/release/bokeh-tables-3.3.1.min.js\", \"https://cdn.holoviz.org/panel/1.3.1/dist/panel.min.js\"];\n var js_modules = [];\n var js_exports = {};\n var css_urls = [];\n var inline_js = [ function(Bokeh) {\n Bokeh.set_log_level(\"info\");\n },\nfunction(Bokeh) {} // ensure no trailing comma for IE\n ];\n\n function run_inline_js() {\n if ((root.Bokeh !== undefined) || (force === true)) {\n for (var i = 0; i < inline_js.length; i++) {\n inline_js[i].call(root, root.Bokeh);\n }\n // Cache old bokeh versions\n if (Bokeh != undefined && !reloading) {\n\tvar NewBokeh = root.Bokeh;\n\tif (Bokeh.versions === undefined) {\n\t Bokeh.versions = new Map();\n\t}\n\tif (NewBokeh.version !== Bokeh.version) {\n\t Bokeh.versions.set(NewBokeh.version, NewBokeh)\n\t}\n\troot.Bokeh = Bokeh;\n }} else if (Date.now() < root._bokeh_timeout) {\n setTimeout(run_inline_js, 100);\n } else if (!root._bokeh_failed_load) {\n console.log(\"Bokeh: BokehJS failed to load within specified timeout.\");\n root._bokeh_failed_load = true;\n }\n root._bokeh_is_initializing = false\n }\n\n function load_or_wait() {\n // Implement a backoff loop that tries to ensure we do not load multiple\n // versions of Bokeh and its dependencies at the same time.\n // In recent versions we use the root._bokeh_is_initializing flag\n // to determine whether there is an ongoing attempt to initialize\n // bokeh, however for backward compatibility we also try to ensure\n // that we do not start loading a newer (Panel>=1.0 and Bokeh>3) version\n // before older versions are fully initialized.\n if (root._bokeh_is_initializing && Date.now() > root._bokeh_timeout) {\n root._bokeh_is_initializing = false;\n root._bokeh_onload_callbacks = undefined;\n console.log(\"Bokeh: BokehJS was loaded multiple times but one version failed to initialize.\");\n load_or_wait();\n } else if (root._bokeh_is_initializing || (typeof root._bokeh_is_initializing === \"undefined\" && root._bokeh_onload_callbacks !== undefined)) {\n setTimeout(load_or_wait, 100);\n } else {\n Bokeh = root.Bokeh;\n bokeh_loaded = Bokeh != null && (Bokeh.version === py_version || (Bokeh.versions !== undefined && Bokeh.versions.has(py_version)));\n root._bokeh_is_initializing = true\n root._bokeh_onload_callbacks = []\n if (!reloading && (!bokeh_loaded || is_dev)) {\n\troot.Bokeh = undefined;\n }\n load_libs(css_urls, js_urls, js_modules, js_exports, function() {\n\tconsole.debug(\"Bokeh: BokehJS plotting callback run at\", now());\n\trun_inline_js();\n });\n }\n }\n // Give older versions of the autoload script a head-start to ensure\n // they initialize before we start loading newer version.\n setTimeout(load_or_wait, 100)\n}(window));", + "application/vnd.holoviews_load.v0+json": "(function(root) {\n function now() {\n return new Date();\n }\n\n var force = true;\n var py_version = '3.3.1'.replace('rc', '-rc.').replace('.dev', '-dev.');\n var is_dev = py_version.indexOf(\"+\") !== -1 || py_version.indexOf(\"-\") !== -1;\n var reloading = false;\n var Bokeh = root.Bokeh;\n var bokeh_loaded = Bokeh != null && (Bokeh.version === py_version || (Bokeh.versions !== undefined && Bokeh.versions.has(py_version)));\n\n if (typeof (root._bokeh_timeout) === \"undefined\" || force) {\n root._bokeh_timeout = Date.now() + 5000;\n root._bokeh_failed_load = false;\n }\n\n function run_callbacks() {\n try {\n root._bokeh_onload_callbacks.forEach(function(callback) {\n if (callback != null)\n callback();\n });\n } finally {\n delete root._bokeh_onload_callbacks;\n }\n console.debug(\"Bokeh: all callbacks have finished\");\n }\n\n function load_libs(css_urls, js_urls, js_modules, js_exports, callback) {\n if (css_urls == null) css_urls = [];\n if (js_urls == null) js_urls = [];\n if (js_modules == null) js_modules = [];\n if (js_exports == null) js_exports = {};\n\n root._bokeh_onload_callbacks.push(callback);\n\n if (root._bokeh_is_loading > 0) {\n console.debug(\"Bokeh: BokehJS is being loaded, scheduling callback at\", now());\n return null;\n }\n if (js_urls.length === 0 && js_modules.length === 0 && Object.keys(js_exports).length === 0) {\n run_callbacks();\n return null;\n }\n if (!reloading) {\n console.debug(\"Bokeh: BokehJS not loaded, scheduling load and callback at\", now());\n }\n\n function on_load() {\n root._bokeh_is_loading--;\n if (root._bokeh_is_loading === 0) {\n console.debug(\"Bokeh: all BokehJS libraries/stylesheets loaded\");\n run_callbacks()\n }\n }\n window._bokeh_on_load = on_load\n\n function on_error() {\n console.error(\"failed to load \" + url);\n }\n\n var skip = [];\n if (window.requirejs) {\n window.requirejs.config({'packages': {}, 'paths': {'jspanel': 'https://cdn.jsdelivr.net/npm/jspanel4@4.12.0/dist/jspanel', 'jspanel-modal': 'https://cdn.jsdelivr.net/npm/jspanel4@4.12.0/dist/extensions/modal/jspanel.modal', 'jspanel-tooltip': 'https://cdn.jsdelivr.net/npm/jspanel4@4.12.0/dist/extensions/tooltip/jspanel.tooltip', 'jspanel-hint': 'https://cdn.jsdelivr.net/npm/jspanel4@4.12.0/dist/extensions/hint/jspanel.hint', 'jspanel-layout': 'https://cdn.jsdelivr.net/npm/jspanel4@4.12.0/dist/extensions/layout/jspanel.layout', 'jspanel-contextmenu': 'https://cdn.jsdelivr.net/npm/jspanel4@4.12.0/dist/extensions/contextmenu/jspanel.contextmenu', 'jspanel-dock': 'https://cdn.jsdelivr.net/npm/jspanel4@4.12.0/dist/extensions/dock/jspanel.dock', 'gridstack': 'https://cdn.jsdelivr.net/npm/gridstack@7.2.3/dist/gridstack-all', 'notyf': 'https://cdn.jsdelivr.net/npm/notyf@3/notyf.min'}, 'shim': {'jspanel': {'exports': 'jsPanel'}, 'gridstack': {'exports': 'GridStack'}}});\n require([\"jspanel\"], function(jsPanel) {\n\twindow.jsPanel = jsPanel\n\ton_load()\n })\n require([\"jspanel-modal\"], function() {\n\ton_load()\n })\n require([\"jspanel-tooltip\"], function() {\n\ton_load()\n })\n require([\"jspanel-hint\"], function() {\n\ton_load()\n })\n require([\"jspanel-layout\"], function() {\n\ton_load()\n })\n require([\"jspanel-contextmenu\"], function() {\n\ton_load()\n })\n require([\"jspanel-dock\"], function() {\n\ton_load()\n })\n require([\"gridstack\"], function(GridStack) {\n\twindow.GridStack = GridStack\n\ton_load()\n })\n require([\"notyf\"], function() {\n\ton_load()\n })\n root._bokeh_is_loading = css_urls.length + 9;\n } else {\n root._bokeh_is_loading = css_urls.length + js_urls.length + js_modules.length + Object.keys(js_exports).length;\n }\n\n var existing_stylesheets = []\n var links = document.getElementsByTagName('link')\n for (var i = 0; i < links.length; i++) {\n var link = links[i]\n if (link.href != null) {\n\texisting_stylesheets.push(link.href)\n }\n }\n for (var i = 0; i < css_urls.length; i++) {\n var url = css_urls[i];\n if (existing_stylesheets.indexOf(url) !== -1) {\n\ton_load()\n\tcontinue;\n }\n const element = document.createElement(\"link\");\n element.onload = on_load;\n element.onerror = on_error;\n element.rel = \"stylesheet\";\n element.type = \"text/css\";\n element.href = url;\n console.debug(\"Bokeh: injecting link tag for BokehJS stylesheet: \", url);\n document.body.appendChild(element);\n } if (((window['jsPanel'] !== undefined) && (!(window['jsPanel'] instanceof HTMLElement))) || window.requirejs) {\n var urls = ['https://cdn.holoviz.org/panel/1.3.1/dist/bundled/floatpanel/jspanel4@4.12.0/dist/jspanel.js', 'https://cdn.holoviz.org/panel/1.3.1/dist/bundled/floatpanel/jspanel4@4.12.0/dist/extensions/modal/jspanel.modal.js', 'https://cdn.holoviz.org/panel/1.3.1/dist/bundled/floatpanel/jspanel4@4.12.0/dist/extensions/tooltip/jspanel.tooltip.js', 'https://cdn.holoviz.org/panel/1.3.1/dist/bundled/floatpanel/jspanel4@4.12.0/dist/extensions/hint/jspanel.hint.js', 'https://cdn.holoviz.org/panel/1.3.1/dist/bundled/floatpanel/jspanel4@4.12.0/dist/extensions/layout/jspanel.layout.js', 'https://cdn.holoviz.org/panel/1.3.1/dist/bundled/floatpanel/jspanel4@4.12.0/dist/extensions/contextmenu/jspanel.contextmenu.js', 'https://cdn.holoviz.org/panel/1.3.1/dist/bundled/floatpanel/jspanel4@4.12.0/dist/extensions/dock/jspanel.dock.js'];\n for (var i = 0; i < urls.length; i++) {\n skip.push(urls[i])\n }\n } if (((window['GridStack'] !== undefined) && (!(window['GridStack'] instanceof HTMLElement))) || window.requirejs) {\n var urls = ['https://cdn.holoviz.org/panel/1.3.1/dist/bundled/gridstack/gridstack@7.2.3/dist/gridstack-all.js'];\n for (var i = 0; i < urls.length; i++) {\n skip.push(urls[i])\n }\n } if (((window['Notyf'] !== undefined) && (!(window['Notyf'] instanceof HTMLElement))) || window.requirejs) {\n var urls = ['https://cdn.holoviz.org/panel/1.3.1/dist/bundled/notificationarea/notyf@3/notyf.min.js'];\n for (var i = 0; i < urls.length; i++) {\n skip.push(urls[i])\n }\n } var existing_scripts = []\n var scripts = document.getElementsByTagName('script')\n for (var i = 0; i < scripts.length; i++) {\n var script = scripts[i]\n if (script.src != null) {\n\texisting_scripts.push(script.src)\n }\n }\n for (var i = 0; i < js_urls.length; i++) {\n var url = js_urls[i];\n if (skip.indexOf(url) !== -1 || existing_scripts.indexOf(url) !== -1) {\n\tif (!window.requirejs) {\n\t on_load();\n\t}\n\tcontinue;\n }\n var element = document.createElement('script');\n element.onload = on_load;\n element.onerror = on_error;\n element.async = false;\n element.src = url;\n console.debug(\"Bokeh: injecting script tag for BokehJS library: \", url);\n document.head.appendChild(element);\n }\n for (var i = 0; i < js_modules.length; i++) {\n var url = js_modules[i];\n if (skip.indexOf(url) !== -1 || existing_scripts.indexOf(url) !== -1) {\n\tif (!window.requirejs) {\n\t on_load();\n\t}\n\tcontinue;\n }\n var element = document.createElement('script');\n element.onload = on_load;\n element.onerror = on_error;\n element.async = false;\n element.src = url;\n element.type = \"module\";\n console.debug(\"Bokeh: injecting script tag for BokehJS library: \", url);\n document.head.appendChild(element);\n }\n for (const name in js_exports) {\n var url = js_exports[name];\n if (skip.indexOf(url) >= 0 || root[name] != null) {\n\tif (!window.requirejs) {\n\t on_load();\n\t}\n\tcontinue;\n }\n var element = document.createElement('script');\n element.onerror = on_error;\n element.async = false;\n element.type = \"module\";\n console.debug(\"Bokeh: injecting script tag for BokehJS library: \", url);\n element.textContent = `\n import ${name} from \"${url}\"\n window.${name} = ${name}\n window._bokeh_on_load()\n `\n document.head.appendChild(element);\n }\n if (!js_urls.length && !js_modules.length) {\n on_load()\n }\n };\n\n function inject_raw_css(css) {\n const element = document.createElement(\"style\");\n element.appendChild(document.createTextNode(css));\n document.body.appendChild(element);\n }\n\n var js_urls = [\"https://cdn.bokeh.org/bokeh/release/bokeh-3.3.1.min.js\", \"https://cdn.bokeh.org/bokeh/release/bokeh-gl-3.3.1.min.js\", \"https://cdn.bokeh.org/bokeh/release/bokeh-widgets-3.3.1.min.js\", \"https://cdn.bokeh.org/bokeh/release/bokeh-tables-3.3.1.min.js\", \"https://cdn.holoviz.org/panel/1.3.1/dist/panel.min.js\"];\n var js_modules = [];\n var js_exports = {};\n var css_urls = [];\n var inline_js = [ function(Bokeh) {\n Bokeh.set_log_level(\"info\");\n },\nfunction(Bokeh) {} // ensure no trailing comma for IE\n ];\n\n function run_inline_js() {\n if ((root.Bokeh !== undefined) || (force === true)) {\n for (var i = 0; i < inline_js.length; i++) {\n inline_js[i].call(root, root.Bokeh);\n }\n // Cache old bokeh versions\n if (Bokeh != undefined && !reloading) {\n\tvar NewBokeh = root.Bokeh;\n\tif (Bokeh.versions === undefined) {\n\t Bokeh.versions = new Map();\n\t}\n\tif (NewBokeh.version !== Bokeh.version) {\n\t Bokeh.versions.set(NewBokeh.version, NewBokeh)\n\t}\n\troot.Bokeh = Bokeh;\n }} else if (Date.now() < root._bokeh_timeout) {\n setTimeout(run_inline_js, 100);\n } else if (!root._bokeh_failed_load) {\n console.log(\"Bokeh: BokehJS failed to load within specified timeout.\");\n root._bokeh_failed_load = true;\n }\n root._bokeh_is_initializing = false\n }\n\n function load_or_wait() {\n // Implement a backoff loop that tries to ensure we do not load multiple\n // versions of Bokeh and its dependencies at the same time.\n // In recent versions we use the root._bokeh_is_initializing flag\n // to determine whether there is an ongoing attempt to initialize\n // bokeh, however for backward compatibility we also try to ensure\n // that we do not start loading a newer (Panel>=1.0 and Bokeh>3) version\n // before older versions are fully initialized.\n if (root._bokeh_is_initializing && Date.now() > root._bokeh_timeout) {\n root._bokeh_is_initializing = false;\n root._bokeh_onload_callbacks = undefined;\n console.log(\"Bokeh: BokehJS was loaded multiple times but one version failed to initialize.\");\n load_or_wait();\n } else if (root._bokeh_is_initializing || (typeof root._bokeh_is_initializing === \"undefined\" && root._bokeh_onload_callbacks !== undefined)) {\n setTimeout(load_or_wait, 100);\n } else {\n Bokeh = root.Bokeh;\n bokeh_loaded = Bokeh != null && (Bokeh.version === py_version || (Bokeh.versions !== undefined && Bokeh.versions.has(py_version)));\n root._bokeh_is_initializing = true\n root._bokeh_onload_callbacks = []\n if (!reloading && (!bokeh_loaded || is_dev)) {\n\troot.Bokeh = undefined;\n }\n load_libs(css_urls, js_urls, js_modules, js_exports, function() {\n\tconsole.debug(\"Bokeh: BokehJS plotting callback run at\", now());\n\trun_inline_js();\n });\n }\n }\n // Give older versions of the autoload script a head-start to ensure\n // they initialize before we start loading newer version.\n setTimeout(load_or_wait, 100)\n}(window));" + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "application/vnd.holoviews_load.v0+json": "\nif ((window.PyViz === undefined) || (window.PyViz instanceof HTMLElement)) {\n window.PyViz = {comms: {}, comm_status:{}, kernels:{}, receivers: {}, plot_index: []}\n}\n\n\n function JupyterCommManager() {\n }\n\n JupyterCommManager.prototype.register_target = function(plot_id, comm_id, msg_handler) {\n if (window.comm_manager || ((window.Jupyter !== undefined) && (Jupyter.notebook.kernel != null))) {\n var comm_manager = window.comm_manager || Jupyter.notebook.kernel.comm_manager;\n comm_manager.register_target(comm_id, function(comm) {\n comm.on_msg(msg_handler);\n });\n } else if ((plot_id in window.PyViz.kernels) && (window.PyViz.kernels[plot_id])) {\n window.PyViz.kernels[plot_id].registerCommTarget(comm_id, function(comm) {\n comm.onMsg = msg_handler;\n });\n } else if (typeof google != 'undefined' && google.colab.kernel != null) {\n google.colab.kernel.comms.registerTarget(comm_id, (comm) => {\n var messages = comm.messages[Symbol.asyncIterator]();\n function processIteratorResult(result) {\n var message = result.value;\n console.log(message)\n var content = {data: message.data, comm_id};\n var buffers = []\n for (var buffer of message.buffers || []) {\n buffers.push(new DataView(buffer))\n }\n var metadata = message.metadata || {};\n var msg = {content, buffers, metadata}\n msg_handler(msg);\n return messages.next().then(processIteratorResult);\n }\n return messages.next().then(processIteratorResult);\n })\n }\n }\n\n JupyterCommManager.prototype.get_client_comm = function(plot_id, comm_id, msg_handler) {\n if (comm_id in window.PyViz.comms) {\n return window.PyViz.comms[comm_id];\n } else if (window.comm_manager || ((window.Jupyter !== undefined) && (Jupyter.notebook.kernel != null))) {\n var comm_manager = window.comm_manager || Jupyter.notebook.kernel.comm_manager;\n var comm = comm_manager.new_comm(comm_id, {}, {}, {}, comm_id);\n if (msg_handler) {\n comm.on_msg(msg_handler);\n }\n } else if ((plot_id in window.PyViz.kernels) && (window.PyViz.kernels[plot_id])) {\n var comm = window.PyViz.kernels[plot_id].connectToComm(comm_id);\n comm.open();\n if (msg_handler) {\n comm.onMsg = msg_handler;\n }\n } else if (typeof google != 'undefined' && google.colab.kernel != null) {\n var comm_promise = google.colab.kernel.comms.open(comm_id)\n comm_promise.then((comm) => {\n window.PyViz.comms[comm_id] = comm;\n if (msg_handler) {\n var messages = comm.messages[Symbol.asyncIterator]();\n function processIteratorResult(result) {\n var message = result.value;\n var content = {data: message.data};\n var metadata = message.metadata || {comm_id};\n var msg = {content, metadata}\n msg_handler(msg);\n return messages.next().then(processIteratorResult);\n }\n return messages.next().then(processIteratorResult);\n }\n }) \n var sendClosure = (data, metadata, buffers, disposeOnDone) => {\n return comm_promise.then((comm) => {\n comm.send(data, metadata, buffers, disposeOnDone);\n });\n };\n var comm = {\n send: sendClosure\n };\n }\n window.PyViz.comms[comm_id] = comm;\n return comm;\n }\n window.PyViz.comm_manager = new JupyterCommManager();\n \n\n\nvar JS_MIME_TYPE = 'application/javascript';\nvar HTML_MIME_TYPE = 'text/html';\nvar EXEC_MIME_TYPE = 'application/vnd.holoviews_exec.v0+json';\nvar CLASS_NAME = 'output';\n\n/**\n * Render data to the DOM node\n */\nfunction render(props, node) {\n var div = document.createElement(\"div\");\n var script = document.createElement(\"script\");\n node.appendChild(div);\n node.appendChild(script);\n}\n\n/**\n * Handle when a new output is added\n */\nfunction handle_add_output(event, handle) {\n var output_area = handle.output_area;\n var output = handle.output;\n if ((output.data == undefined) || (!output.data.hasOwnProperty(EXEC_MIME_TYPE))) {\n return\n }\n var id = output.metadata[EXEC_MIME_TYPE][\"id\"];\n var toinsert = output_area.element.find(\".\" + CLASS_NAME.split(' ')[0]);\n if (id !== undefined) {\n var nchildren = toinsert.length;\n var html_node = toinsert[nchildren-1].children[0];\n html_node.innerHTML = output.data[HTML_MIME_TYPE];\n var scripts = [];\n var nodelist = html_node.querySelectorAll(\"script\");\n for (var i in nodelist) {\n if (nodelist.hasOwnProperty(i)) {\n scripts.push(nodelist[i])\n }\n }\n\n scripts.forEach( function (oldScript) {\n var newScript = document.createElement(\"script\");\n var attrs = [];\n var nodemap = oldScript.attributes;\n for (var j in nodemap) {\n if (nodemap.hasOwnProperty(j)) {\n attrs.push(nodemap[j])\n }\n }\n attrs.forEach(function(attr) { newScript.setAttribute(attr.name, attr.value) });\n newScript.appendChild(document.createTextNode(oldScript.innerHTML));\n oldScript.parentNode.replaceChild(newScript, oldScript);\n });\n if (JS_MIME_TYPE in output.data) {\n toinsert[nchildren-1].children[1].textContent = output.data[JS_MIME_TYPE];\n }\n output_area._hv_plot_id = id;\n if ((window.Bokeh !== undefined) && (id in Bokeh.index)) {\n window.PyViz.plot_index[id] = Bokeh.index[id];\n } else {\n window.PyViz.plot_index[id] = null;\n }\n } else if (output.metadata[EXEC_MIME_TYPE][\"server_id\"] !== undefined) {\n var bk_div = document.createElement(\"div\");\n bk_div.innerHTML = output.data[HTML_MIME_TYPE];\n var script_attrs = bk_div.children[0].attributes;\n for (var i = 0; i < script_attrs.length; i++) {\n toinsert[toinsert.length - 1].childNodes[1].setAttribute(script_attrs[i].name, script_attrs[i].value);\n }\n // store reference to server id on output_area\n output_area._bokeh_server_id = output.metadata[EXEC_MIME_TYPE][\"server_id\"];\n }\n}\n\n/**\n * Handle when an output is cleared or removed\n */\nfunction handle_clear_output(event, handle) {\n var id = handle.cell.output_area._hv_plot_id;\n var server_id = handle.cell.output_area._bokeh_server_id;\n if (((id === undefined) || !(id in PyViz.plot_index)) && (server_id !== undefined)) { return; }\n var comm = window.PyViz.comm_manager.get_client_comm(\"hv-extension-comm\", \"hv-extension-comm\", function () {});\n if (server_id !== null) {\n comm.send({event_type: 'server_delete', 'id': server_id});\n return;\n } else if (comm !== null) {\n comm.send({event_type: 'delete', 'id': id});\n }\n delete PyViz.plot_index[id];\n if ((window.Bokeh !== undefined) & (id in window.Bokeh.index)) {\n var doc = window.Bokeh.index[id].model.document\n doc.clear();\n const i = window.Bokeh.documents.indexOf(doc);\n if (i > -1) {\n window.Bokeh.documents.splice(i, 1);\n }\n }\n}\n\n/**\n * Handle kernel restart event\n */\nfunction handle_kernel_cleanup(event, handle) {\n delete PyViz.comms[\"hv-extension-comm\"];\n window.PyViz.plot_index = {}\n}\n\n/**\n * Handle update_display_data messages\n */\nfunction handle_update_output(event, handle) {\n handle_clear_output(event, {cell: {output_area: handle.output_area}})\n handle_add_output(event, handle)\n}\n\nfunction register_renderer(events, OutputArea) {\n function append_mime(data, metadata, element) {\n // create a DOM node to render to\n var toinsert = this.create_output_subarea(\n metadata,\n CLASS_NAME,\n EXEC_MIME_TYPE\n );\n this.keyboard_manager.register_events(toinsert);\n // Render to node\n var props = {data: data, metadata: metadata[EXEC_MIME_TYPE]};\n render(props, toinsert[0]);\n element.append(toinsert);\n return toinsert\n }\n\n events.on('output_added.OutputArea', handle_add_output);\n events.on('output_updated.OutputArea', handle_update_output);\n events.on('clear_output.CodeCell', handle_clear_output);\n events.on('delete.Cell', handle_clear_output);\n events.on('kernel_ready.Kernel', handle_kernel_cleanup);\n\n OutputArea.prototype.register_mime_type(EXEC_MIME_TYPE, append_mime, {\n safe: true,\n index: 0\n });\n}\n\nif (window.Jupyter !== undefined) {\n try {\n var events = require('base/js/events');\n var OutputArea = require('notebook/js/outputarea').OutputArea;\n if (OutputArea.prototype.mime_types().indexOf(EXEC_MIME_TYPE) == -1) {\n register_renderer(events, OutputArea);\n }\n } catch(err) {\n }\n}\n", + "application/javascript": "\nif ((window.PyViz === undefined) || (window.PyViz instanceof HTMLElement)) {\n window.PyViz = {comms: {}, comm_status:{}, kernels:{}, receivers: {}, plot_index: []}\n}\n\n\n function JupyterCommManager() {\n }\n\n JupyterCommManager.prototype.register_target = function(plot_id, comm_id, msg_handler) {\n if (window.comm_manager || ((window.Jupyter !== undefined) && (Jupyter.notebook.kernel != null))) {\n var comm_manager = window.comm_manager || Jupyter.notebook.kernel.comm_manager;\n comm_manager.register_target(comm_id, function(comm) {\n comm.on_msg(msg_handler);\n });\n } else if ((plot_id in window.PyViz.kernels) && (window.PyViz.kernels[plot_id])) {\n window.PyViz.kernels[plot_id].registerCommTarget(comm_id, function(comm) {\n comm.onMsg = msg_handler;\n });\n } else if (typeof google != 'undefined' && google.colab.kernel != null) {\n google.colab.kernel.comms.registerTarget(comm_id, (comm) => {\n var messages = comm.messages[Symbol.asyncIterator]();\n function processIteratorResult(result) {\n var message = result.value;\n console.log(message)\n var content = {data: message.data, comm_id};\n var buffers = []\n for (var buffer of message.buffers || []) {\n buffers.push(new DataView(buffer))\n }\n var metadata = message.metadata || {};\n var msg = {content, buffers, metadata}\n msg_handler(msg);\n return messages.next().then(processIteratorResult);\n }\n return messages.next().then(processIteratorResult);\n })\n }\n }\n\n JupyterCommManager.prototype.get_client_comm = function(plot_id, comm_id, msg_handler) {\n if (comm_id in window.PyViz.comms) {\n return window.PyViz.comms[comm_id];\n } else if (window.comm_manager || ((window.Jupyter !== undefined) && (Jupyter.notebook.kernel != null))) {\n var comm_manager = window.comm_manager || Jupyter.notebook.kernel.comm_manager;\n var comm = comm_manager.new_comm(comm_id, {}, {}, {}, comm_id);\n if (msg_handler) {\n comm.on_msg(msg_handler);\n }\n } else if ((plot_id in window.PyViz.kernels) && (window.PyViz.kernels[plot_id])) {\n var comm = window.PyViz.kernels[plot_id].connectToComm(comm_id);\n comm.open();\n if (msg_handler) {\n comm.onMsg = msg_handler;\n }\n } else if (typeof google != 'undefined' && google.colab.kernel != null) {\n var comm_promise = google.colab.kernel.comms.open(comm_id)\n comm_promise.then((comm) => {\n window.PyViz.comms[comm_id] = comm;\n if (msg_handler) {\n var messages = comm.messages[Symbol.asyncIterator]();\n function processIteratorResult(result) {\n var message = result.value;\n var content = {data: message.data};\n var metadata = message.metadata || {comm_id};\n var msg = {content, metadata}\n msg_handler(msg);\n return messages.next().then(processIteratorResult);\n }\n return messages.next().then(processIteratorResult);\n }\n }) \n var sendClosure = (data, metadata, buffers, disposeOnDone) => {\n return comm_promise.then((comm) => {\n comm.send(data, metadata, buffers, disposeOnDone);\n });\n };\n var comm = {\n send: sendClosure\n };\n }\n window.PyViz.comms[comm_id] = comm;\n return comm;\n }\n window.PyViz.comm_manager = new JupyterCommManager();\n \n\n\nvar JS_MIME_TYPE = 'application/javascript';\nvar HTML_MIME_TYPE = 'text/html';\nvar EXEC_MIME_TYPE = 'application/vnd.holoviews_exec.v0+json';\nvar CLASS_NAME = 'output';\n\n/**\n * Render data to the DOM node\n */\nfunction render(props, node) {\n var div = document.createElement(\"div\");\n var script = document.createElement(\"script\");\n node.appendChild(div);\n node.appendChild(script);\n}\n\n/**\n * Handle when a new output is added\n */\nfunction handle_add_output(event, handle) {\n var output_area = handle.output_area;\n var output = handle.output;\n if ((output.data == undefined) || (!output.data.hasOwnProperty(EXEC_MIME_TYPE))) {\n return\n }\n var id = output.metadata[EXEC_MIME_TYPE][\"id\"];\n var toinsert = output_area.element.find(\".\" + CLASS_NAME.split(' ')[0]);\n if (id !== undefined) {\n var nchildren = toinsert.length;\n var html_node = toinsert[nchildren-1].children[0];\n html_node.innerHTML = output.data[HTML_MIME_TYPE];\n var scripts = [];\n var nodelist = html_node.querySelectorAll(\"script\");\n for (var i in nodelist) {\n if (nodelist.hasOwnProperty(i)) {\n scripts.push(nodelist[i])\n }\n }\n\n scripts.forEach( function (oldScript) {\n var newScript = document.createElement(\"script\");\n var attrs = [];\n var nodemap = oldScript.attributes;\n for (var j in nodemap) {\n if (nodemap.hasOwnProperty(j)) {\n attrs.push(nodemap[j])\n }\n }\n attrs.forEach(function(attr) { newScript.setAttribute(attr.name, attr.value) });\n newScript.appendChild(document.createTextNode(oldScript.innerHTML));\n oldScript.parentNode.replaceChild(newScript, oldScript);\n });\n if (JS_MIME_TYPE in output.data) {\n toinsert[nchildren-1].children[1].textContent = output.data[JS_MIME_TYPE];\n }\n output_area._hv_plot_id = id;\n if ((window.Bokeh !== undefined) && (id in Bokeh.index)) {\n window.PyViz.plot_index[id] = Bokeh.index[id];\n } else {\n window.PyViz.plot_index[id] = null;\n }\n } else if (output.metadata[EXEC_MIME_TYPE][\"server_id\"] !== undefined) {\n var bk_div = document.createElement(\"div\");\n bk_div.innerHTML = output.data[HTML_MIME_TYPE];\n var script_attrs = bk_div.children[0].attributes;\n for (var i = 0; i < script_attrs.length; i++) {\n toinsert[toinsert.length - 1].childNodes[1].setAttribute(script_attrs[i].name, script_attrs[i].value);\n }\n // store reference to server id on output_area\n output_area._bokeh_server_id = output.metadata[EXEC_MIME_TYPE][\"server_id\"];\n }\n}\n\n/**\n * Handle when an output is cleared or removed\n */\nfunction handle_clear_output(event, handle) {\n var id = handle.cell.output_area._hv_plot_id;\n var server_id = handle.cell.output_area._bokeh_server_id;\n if (((id === undefined) || !(id in PyViz.plot_index)) && (server_id !== undefined)) { return; }\n var comm = window.PyViz.comm_manager.get_client_comm(\"hv-extension-comm\", \"hv-extension-comm\", function () {});\n if (server_id !== null) {\n comm.send({event_type: 'server_delete', 'id': server_id});\n return;\n } else if (comm !== null) {\n comm.send({event_type: 'delete', 'id': id});\n }\n delete PyViz.plot_index[id];\n if ((window.Bokeh !== undefined) & (id in window.Bokeh.index)) {\n var doc = window.Bokeh.index[id].model.document\n doc.clear();\n const i = window.Bokeh.documents.indexOf(doc);\n if (i > -1) {\n window.Bokeh.documents.splice(i, 1);\n }\n }\n}\n\n/**\n * Handle kernel restart event\n */\nfunction handle_kernel_cleanup(event, handle) {\n delete PyViz.comms[\"hv-extension-comm\"];\n window.PyViz.plot_index = {}\n}\n\n/**\n * Handle update_display_data messages\n */\nfunction handle_update_output(event, handle) {\n handle_clear_output(event, {cell: {output_area: handle.output_area}})\n handle_add_output(event, handle)\n}\n\nfunction register_renderer(events, OutputArea) {\n function append_mime(data, metadata, element) {\n // create a DOM node to render to\n var toinsert = this.create_output_subarea(\n metadata,\n CLASS_NAME,\n EXEC_MIME_TYPE\n );\n this.keyboard_manager.register_events(toinsert);\n // Render to node\n var props = {data: data, metadata: metadata[EXEC_MIME_TYPE]};\n render(props, toinsert[0]);\n element.append(toinsert);\n return toinsert\n }\n\n events.on('output_added.OutputArea', handle_add_output);\n events.on('output_updated.OutputArea', handle_update_output);\n events.on('clear_output.CodeCell', handle_clear_output);\n events.on('delete.Cell', handle_clear_output);\n events.on('kernel_ready.Kernel', handle_kernel_cleanup);\n\n OutputArea.prototype.register_mime_type(EXEC_MIME_TYPE, append_mime, {\n safe: true,\n index: 0\n });\n}\n\nif (window.Jupyter !== undefined) {\n try {\n var events = require('base/js/events');\n var OutputArea = require('notebook/js/outputarea').OutputArea;\n if (OutputArea.prototype.mime_types().indexOf(EXEC_MIME_TYPE) == -1) {\n register_renderer(events, OutputArea);\n }\n } catch(err) {\n }\n}\n" + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/html": "" + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/html": "
\n
\n
\n", + "application/vnd.holoviews_exec.v0+json": "" + }, + "metadata": { + "application/vnd.holoviews_exec.v0+json": { + "id": "p1006" + } + }, + "output_type": "display_data" + }, + { + "data": { + "text/html": "\n
\n\n\n\n \n \n\n\n\n\n
\n" + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "import colorcet as cc\n", + "import datashader as ds\n", + "import holoviews as hv\n", + "import holoviews.operation.datashader as hd\n", + "import hvplot.dask\n", + "\n", + "hv.extension(\"bokeh\")\n", + "\n", + "plot_width = int(750)\n", + "plot_height = int(plot_width//1.33)\n", + "\n", + "plot_options = hv.Options(\n", + " width=plot_width,\n", + " height=plot_height,\n", + " xaxis=None,\n", + " yaxis=None,\n", + " bgcolor=\"black\",\n", + " border=0,\n", + " legend_position=\"bottom_right\",\n", + ")" + ], + "metadata": { + "collapsed": false + } + }, + { + "cell_type": "markdown", + "source": [ + "### Scatterplot\n", + "\n", + "First, let's do a quick scatter plot displaying both pickup and dropoff locations." + ], + "metadata": { + "collapsed": false + } + }, + { + "cell_type": "code", + "execution_count": 25, + "outputs": [ + { + "data": {}, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/html": "
\n
\n
\n", + "application/vnd.holoviews_exec.v0+json": "", + "text/plain": ":DynamicMap []\n :RGB [longitude,latitude] (R,G,B,A)" + }, + "execution_count": 25, + "metadata": { + "application/vnd.holoviews_exec.v0+json": { + "id": "p2092" + } + }, + "output_type": "execute_result" + } + ], + "source": [ + "color_key = {\"pickup\": \"#EF1561\", \"dropoff\": \"#1F5AFF\"}\n", + "df_plot.hvplot.scatter(\n", + " x=\"longitude\",\n", + " y=\"latitude\",\n", + " aggregator=ds.by(\"journey_type\"),\n", + " datashade=True,\n", + " cnorm=\"eq_hist\",\n", + " color_key=color_key,\n", + ").opts(plot_options).opts(bgcolor=\"white\")" + ], + "metadata": { + "collapsed": false + } + }, + { + "cell_type": "markdown", + "source": [ + "### Colormapping\n", + "\n", + "Datashader's `datashade()` convenience function applies the aggregate and shade\n", + "operations, aggregating all elements in the supplied object and then applying\n", + "normalization and color-mapping the aggregated data, returning RGB elements.\n", + "\n", + "This example uses the passenger count value for the aggregation part of the\n", + "datashade operation.\n", + "\n", + " With Datashader, we can choose what we want to plot on the x and y axes and see the full data immediately, with no parameter tweaking, magic numbers, subsampling, or approximation, up to the resolution of the display:" + ], + "metadata": { + "collapsed": false + } + }, + { + "cell_type": "code", + "execution_count": 26, + "outputs": [ + { + "data": {}, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/html": "
\n
\n
\n", + "application/vnd.holoviews_exec.v0+json": "", + "text/plain": ":DynamicMap []\n :RGB [longitude,latitude] (R,G,B,A)" + }, + "execution_count": 26, + "metadata": { + "application/vnd.holoviews_exec.v0+json": { + "id": "p2168" + } + }, + "output_type": "execute_result" + } + ], + "source": [ + "points = hv.Points(df_plot, [\"longitude\", \"latitude\"])\n", + "shaded = hd.datashade(points, cmap=ds.colors.viridis, aggregator=ds.count(\"passenger_count\"))\n", + "plot = hd.dynspread(shaded, threshold=0.5, max_px=4)\n", + "\n", + "plot.opts(plot_options)" + ], + "metadata": { + "collapsed": false + } + }, + { + "cell_type": "markdown", + "source": [ + "## Visualize on a Map\n", + "\n", + "Overlay the data on digital geographic maps, by using open map tiles, for\n", + "example from OSM or ESRI.\n", + "\n", + "Note that HoloViews/Bokeh works with Web Mercator coordinates, so you will\n", + "need to convert geolocation positions from original longitude / latitude\n", + "values." + ], + "metadata": { + "collapsed": false + } + }, + { + "cell_type": "code", + "execution_count": 0, + "outputs": [], + "source": [ + "df_map_plot = df_plot.copy()\n", + "df_map_plot.longitude, df_map_plot.latitude = hv.Tiles.lon_lat_to_easting_northing(df_map_plot.longitude, df_map_plot.latitude)" + ], + "metadata": { + "collapsed": false + } + }, + { + "cell_type": "markdown", + "source": [ + "### Overlay on a streets map\n", + "\n", + "This examples uses map tiles from Open Street Maps (OSM)." + ], + "metadata": { + "collapsed": false + } + }, + { + "cell_type": "code", + "execution_count": 62, + "outputs": [ + { + "data": {}, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/html": "
\n
\n
\n", + "application/vnd.holoviews_exec.v0+json": "", + "text/plain": ":DynamicMap []\n :Overlay\n .Tiles.I :Tiles [x,y]\n .RGB.I :RGB [longitude,latitude] (R,G,B,A)" + }, + "execution_count": 62, + "metadata": { + "application/vnd.holoviews_exec.v0+json": { + "id": "p4712" + } + }, + "output_type": "execute_result" + } + ], + "source": [ + "# Create a map tiles layer, and a points layer.\n", + "# HoloViews defines overlays using the star operator.\n", + "map_tiles = hv.element.OSM()\n", + "points = hv.Points(df_map_plot, [\"longitude\", \"latitude\"])\n", + "shaded = hd.datashade(points, x_sampling=1, y_sampling=1)\n", + "plot = map_tiles * shaded\n", + "\n", + "plot.opts(plot_options)" + ], + "metadata": { + "collapsed": false + } + }, + { + "cell_type": "markdown", + "source": [ + "### Overlay on a map with satellite imagery\n", + "\n", + "This example uses map tiles from the [Esri World Imagery Map], colormapping to\n", + "a contrast-rich color palette.\n", + "\n", + "[Esri World Imagery Map]: https://www.arcgis.com/home/item.html?id=50c23e4987a44de4ab163e1baeab4a46" + ], + "metadata": { + "collapsed": false + } + }, + { + "cell_type": "code", + "execution_count": 49, + "outputs": [ + { + "data": {}, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/html": "
\n
\n
\n", + "application/vnd.holoviews_exec.v0+json": "", + "text/plain": ":DynamicMap []\n :Overlay\n .Tiles.I :Tiles [x,y]\n .RGB.I :RGB [longitude,latitude] (R,G,B,A)" + }, + "execution_count": 49, + "metadata": { + "application/vnd.holoviews_exec.v0+json": { + "id": "p3831" + } + }, + "output_type": "execute_result" + } + ], + "source": [ + "map_tiles = hv.element.EsriImagery().opts(alpha=0.5)\n", + "points = hv.Points(df_map_plot, [\"longitude\", \"latitude\"])\n", + "#shaded = hd.datashade(points, x_sampling=1, y_sampling=1, cmap=cc.kbc)\n", + "#shaded = hd.datashade(points, x_sampling=1, y_sampling=1, cmap=cc.CET_L20)\n", + "shaded = hd.datashade(points, x_sampling=1, y_sampling=1, cmap=cc.CET_R4)\n", + "#shaded = hd.datashade(points, cmap=cc.CET_R4)\n", + "plot = map_tiles * shaded\n", + "\n", + "plot.opts(plot_options)" + ], + "metadata": { + "collapsed": false + } + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 2 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython2", + "version": "2.7.6" + } + }, + "nbformat": 4, + "nbformat_minor": 0 +} From 79361c83d14afcf6bb9b33ea6d0bf9083fe29ffe Mon Sep 17 00:00:00 2001 From: Andreas Motl Date: Tue, 5 Dec 2023 02:07:23 +0100 Subject: [PATCH 2/2] Timeseries: Add software tests for CrateDB Cloud / Datashader notebook --- .github/workflows/test-timeseries.yml | 72 +++++++++++++++++++ .gitignore | 4 +- topic/timeseries/explore/pyproject.toml | 31 ++++++++ topic/timeseries/explore/requirements-dev.txt | 11 +++ topic/timeseries/explore/requirements.txt | 8 +++ topic/timeseries/explore/test.py | 38 ++++++++++ .../explore/yellow_tripdata_sample.csv | 3 + 7 files changed, 166 insertions(+), 1 deletion(-) create mode 100644 .github/workflows/test-timeseries.yml create mode 100644 topic/timeseries/explore/pyproject.toml create mode 100644 topic/timeseries/explore/requirements-dev.txt create mode 100644 topic/timeseries/explore/requirements.txt create mode 100644 topic/timeseries/explore/test.py create mode 100644 topic/timeseries/explore/yellow_tripdata_sample.csv diff --git a/.github/workflows/test-timeseries.yml b/.github/workflows/test-timeseries.yml new file mode 100644 index 00000000..831fb696 --- /dev/null +++ b/.github/workflows/test-timeseries.yml @@ -0,0 +1,72 @@ +name: "Timeseries » Explore" + +on: + pull_request: + branches: ~ + paths: + - '.github/workflows/test-timeseries.yml' + - 'topic/timeseries/explore/**' + - 'requirements.txt' + push: + branches: [ main ] + paths: + - '.github/workflows/test-timeseries.yml' + - 'topic/timeseries/explore/**' + - 'requirements.txt' + + # Allow job to be triggered manually. + workflow_dispatch: + + # Run job each night after CrateDB nightly has been published. + schedule: + - cron: '0 3 * * *' + +# Cancel in-progress jobs when pushing to the same branch. +concurrency: + cancel-in-progress: true + group: ${{ github.workflow }}-${{ github.ref }} + +jobs: + test: + name: " + Python: ${{ matrix.python-version }} + CrateDB: ${{ matrix.cratedb-version }} + on ${{ matrix.os }}" + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + os: [ 'ubuntu-latest' ] + python-version: [ '3.11' ] + cratedb-version: [ 'nightly' ] + + services: + cratedb: + image: crate/crate:nightly + ports: + - 4200:4200 + - 5432:5432 + + steps: + + - name: Acquire sources + uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v4 + with: + python-version: ${{ matrix.python-version }} + architecture: x64 + cache: 'pip' + cache-dependency-path: | + requirements.txt + topic/timeseries/explore/requirements.txt + topic/timeseries/explore/requirements-dev.txt + + - name: Install utilities + run: | + pip install -r requirements.txt + + - name: Validate topic/timeseries/explore + run: | + ngr test --accept-no-venv topic/timeseries/explore diff --git a/.gitignore b/.gitignore index 44639d77..c775111b 100644 --- a/.gitignore +++ b/.gitignore @@ -1,8 +1,10 @@ +.env .idea +.pytest_cache .venv* __pycache__ .coverage coverage.xml mlruns/ archive/ -logs.log \ No newline at end of file +logs.log diff --git a/topic/timeseries/explore/pyproject.toml b/topic/timeseries/explore/pyproject.toml new file mode 100644 index 00000000..b4e2ecc5 --- /dev/null +++ b/topic/timeseries/explore/pyproject.toml @@ -0,0 +1,31 @@ +[tool.pytest.ini_options] +minversion = "2.0" +addopts = """ + -rfEX -p pytester --strict-markers --verbosity=3 --capture=no + """ +env = [ + "CRATEDB_CLOUD_CLUSTER_NAME=testcluster", + "CRATEDB_CONNECTION_STRING=crate://crate@localhost/?schema=testdrive", + "PYDEVD_DISABLE_FILE_VALIDATION=1", +] + +#log_level = "DEBUG" +#log_cli_level = "DEBUG" + +testpaths = [ + "*.py", +] +xfail_strict = true +markers = [ +] + +[tool.coverage.run] +branch = false + +[tool.coverage.report] +fail_under = 0 +show_missing = true +omit = [ + "conftest.py", + "test*.py", +] diff --git a/topic/timeseries/explore/requirements-dev.txt b/topic/timeseries/explore/requirements-dev.txt new file mode 100644 index 00000000..c27837f7 --- /dev/null +++ b/topic/timeseries/explore/requirements-dev.txt @@ -0,0 +1,11 @@ +# Real. +# pueblo[testing] + +# Development. +pueblo[notebook,testing] @ git+https://github.com/pyveci/pueblo.git@main +#pytest-mock<4 +#testbook<0.5 + +# Workstation. +# --editable=/Users/amo/dev/crate/ecosystem/cratedb-retentions[io] +# --editable=/Users/amo/dev/pyveci/sources/pueblo[testing] diff --git a/topic/timeseries/explore/requirements.txt b/topic/timeseries/explore/requirements.txt new file mode 100644 index 00000000..1f2a74fe --- /dev/null +++ b/topic/timeseries/explore/requirements.txt @@ -0,0 +1,8 @@ +# Real. +crate[sqlalchemy] +# cratedb-toolkit +datashader<0.17 +hvplot<0.10 + +# Development. +cratedb-toolkit @ git+https://github.com/crate-workbench/cratedb-toolkit.git@amo/cloud-second diff --git a/topic/timeseries/explore/test.py b/topic/timeseries/explore/test.py new file mode 100644 index 00000000..1c80f021 --- /dev/null +++ b/topic/timeseries/explore/test.py @@ -0,0 +1,38 @@ +""" +## About + +Test cases for timeseries exploration and analysis examples with CrateDB and PyViz. + + +## Synopsis + +Run all test cases. +``` +pytest +``` + +Run individual test cases. +``` +pytest -k file +pytest -k notebook +``` +""" +from pathlib import Path +from testbook import testbook + +HERE = Path(__file__).parent + + +def test_cloud_datashader(): + + notebook = HERE / "cloud-datashader.ipynb" + with testbook(str(notebook)) as tb: + tb.inject( + """ + df = pd.read_csv("yellow_tripdata_sample.csv") + """, + run=False, + after=9, + ) + with tb.patch("cratedb_toolkit.ManagedCluster") as mock_cluster: + tb.execute() diff --git a/topic/timeseries/explore/yellow_tripdata_sample.csv b/topic/timeseries/explore/yellow_tripdata_sample.csv new file mode 100644 index 00000000..4322ce98 --- /dev/null +++ b/topic/timeseries/explore/yellow_tripdata_sample.csv @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:97362475ef6180215252441e0725ea417850fa3ca0a7420480fc874f775aee16 +size 2546