04-dask.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "7fb02db3-b9d4-4ef7-b439-10df710cd527",
   "metadata": {},
   "source": [
    "# Scale computations\n",
    "\n",
    "Scale your workflows to Terabytes of data with Dask's distributed clusters."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "eabe89e4-ed15-4f5a-8d61-080b2e96e80f",
   "metadata": {},
   "outputs": [],
   "source": [
    "from dask_gateway import Gateway\n",
    "import dask.dataframe as dd\n",
    "\n",
    "import hvplot\n",
    "import hvplot.dask"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "7df8b981-cf76-4cd5-b075-e3aa5cb47252",
   "metadata": {},
   "outputs": [],
   "source": [
    "gateway = Gateway()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "14286cc5-4433-45c4-b0ca-df34f70e4049",
   "metadata": {},
   "outputs": [],
   "source": [
    "options = gateway.cluster_options()\n",
    "options"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "70554f55-57f0-4b37-a9be-da04a78d13b2",
   "metadata": {},
   "outputs": [],
   "source": [
    "cluster = gateway.new_cluster(options)\n",
    "cluster"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "53724ff5-1563-431c-a535-7a8873e83ffe",
   "metadata": {},
   "outputs": [],
   "source": [
    "client = cluster.get_client()\n",
    "client"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "530dcbdb-5a4f-497e-9d8b-c00a9bb10f60",
   "metadata": {},
   "outputs": [],
   "source": [
    "ddf = dd.read_parquet(\"gcs://quansight-datasets/airline-ontime-performance/sorted/full_dataset.parquet\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "01d5d2d8-54c9-4f47-9b94-49f8f0605572",
   "metadata": {},
   "outputs": [],
   "source": [
    "fligths_per_day = ddf.groupby(\"FL_DATE\", sort=True)[\"FLIGHTS\"].count().persist()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "68434394-929a-4503-a149-6bd78bb96ab1",
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "hvplot.extension('bokeh')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "8b90d2f8-5d26-4953-98b4-7213a9d36d97",
   "metadata": {},
   "outputs": [],
   "source": [
    "fligths_per_day.hvplot('FL_DATE', 'FLIGHTS')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e4668ffa-de80-482d-9867-82731835eeb6",
   "metadata": {},
   "outputs": [],
   "source": [
    "client.close()\n",
    "cluster.shutdown()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "4a204c3b-263d-4f80-ad02-160aeaf71b3c",
   "metadata": {},
   "source": [
    "---\n",
    "\n",
    "Next: [Leverage GPUs →](./05-gpus.ipynb)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "988c772a-2911-4c28-94f0-3c41fcbf29cc",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "global-global-demo-dask",
   "language": "python",
   "name": "conda-env-global-global-demo-dask-py"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.14"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}