From 0e5aae64d2318abab5970daff7bfba7eade37d76 Mon Sep 17 00:00:00 2001 From: Michael Wellington Date: Mon, 25 Nov 2024 04:57:21 +0000 Subject: [PATCH] datasets notebook --- Datasets/WaPOR.ipynb | 878 +++++++++++++++++++++++++++++ Tools/deafrica_tools/load_wapor.py | 141 +++++ 2 files changed, 1019 insertions(+) create mode 100644 Datasets/WaPOR.ipynb create mode 100644 Tools/deafrica_tools/load_wapor.py diff --git a/Datasets/WaPOR.ipynb b/Datasets/WaPOR.ipynb new file mode 100644 index 00000000..608fde74 --- /dev/null +++ b/Datasets/WaPOR.ipynb @@ -0,0 +1,878 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "4b6fe63d-d62c-403f-9634-6afe55d7e541", + "metadata": {}, + "source": [ + "# Introduction to WaPOR and data loading\n", + "\n", + "* **Products used:** WaPOR" + ] + }, + { + "cell_type": "raw", + "id": "69bcac84-6289-4591-b102-1b61853c2242", + "metadata": { + "raw_mimetype": "text/restructuredtext" + }, + "source": [ + "**Keywords** :index:`data used; wapor` :index:`data used; crop mask`" + ] + }, + { + "cell_type": "markdown", + "id": "2569ec9d-4353-41c0-957f-5c13387bc9bc", + "metadata": {}, + "source": [ + "## Background\n", + "The **Wa**ter **P**roductivity through **O**pen access of **R**emotely sensed derived data (WaPOR) monitors and reports on agricultural water productivity through biophysical measures with a focus on Africa and the Near East. This information assists partner countries improve land and water productivity in both rainfed and irrigated agriculture ([Peiser et al. 2017](https://openknowledge.fao.org/items/cdacf817-c13a-4859-b681-a4bdc20c065c)). \n", + "\n", + "WaPOR provides numerous datasets related to vegetation productivity and water consumption, and associated meteorological and physical conditions such as soil moisture and precipitation. These datasets can be combined with Digital Earth Africa products, services, and workflows for numerous applications including:\n", + " * Monitoring drought conditions\n", + " * Monitoring the water use efficiency of crops\n", + " * Mapping irrigated areas\n", + " * Estimating crop water requirements\n", + " * Irrigation scheduling and budgeting" + ] + }, + { + "cell_type": "markdown", + "id": "7001cb3f-c7b5-431b-b546-f7d853b4b950", + "metadata": {}, + "source": [ + "## Description\n", + "This notebook provides an introduction to WaPOR data and nomenclature, and demonstrates loading and plotting.\n", + "\n", + "1. First, we explore the datasets available and how they are labelled.\n", + "2. Then, we download and plot annual evapotranspiration.\n", + "3. Finally, we download and plot dekadal (10 day temporal frequency) data.\n", + "\n", + "***" + ] + }, + { + "cell_type": "markdown", + "id": "206d1d08-66fd-4b77-8216-0e605b327a1f", + "metadata": {}, + "source": [ + "## Getting started\n", + "\n", + "To run this analysis, run all the cells in the notebook, starting with the \"Load packages\" cell. " + ] + }, + { + "cell_type": "markdown", + "id": "61f074de-f1aa-46cf-acbe-2d93c0a3d9ee", + "metadata": {}, + "source": [ + "### Load packages\n", + "Import Python packages that are used for the analysis.\n", + "\n", + "Use standard import commands; some are shown below. \n", + "Begin with any `iPython` magic commands, followed by standard Python packages, then any additional functionality you need from the `Tools` package." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "84431ef8-9d48-4681-b49c-71cf30eb0ec1", + "metadata": {}, + "outputs": [], + "source": [ + "#!pip3 install wapordl==0.9\n", + "\n", + "# require lower version because wapordl 0.12.2 requires numpy<2,>=1.15, but sandbox has numpy 2.1.3 which is incompatible." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "7a39679f-7772-4424-b0e1-8a56e80b91de", + "metadata": {}, + "outputs": [], + "source": [ + "#!pip uninstall Tools -y\n", + "#!pip install ../Tools" + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "id": "f67d797a-211f-4366-8240-6dcd8c4647be", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Collecting GDAL==3.8.4\n", + " Downloading GDAL-3.8.4.tar.gz (802 kB)\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m802.5/802.5 kB\u001b[0m \u001b[31m44.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[?25h Preparing metadata (setup.py) ... \u001b[?25ldone\n", + "\u001b[?25hBuilding wheels for collected packages: GDAL\n", + " Building wheel for GDAL (setup.py) ... \u001b[?25ldone\n", + "\u001b[?25h Created wheel for GDAL: filename=GDAL-3.8.4-cp310-cp310-linux_x86_64.whl size=3650634 sha256=fdf063b25f1d6dedad1772150ebc4b62edb557cffd415cb5b1fac002906bc919\n", + " Stored in directory: /home/jovyan/.cache/pip/wheels/67/f4/aa/ad21e29df3d3124c18bc1282d3441730fb386c84497b29e250\n", + "Successfully built GDAL\n", + "Installing collected packages: GDAL\n", + " Attempting uninstall: GDAL\n", + " Found existing installation: GDAL 3.6.2\n", + " Uninstalling GDAL-3.6.2:\n", + " Successfully uninstalled GDAL-3.6.2\n", + "Successfully installed GDAL-3.8.4\n", + "\n", + "\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m A new release of pip is available: \u001b[0m\u001b[31;49m24.2\u001b[0m\u001b[39;49m -> \u001b[0m\u001b[32;49m24.3.1\u001b[0m\n", + "\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m To update, run: \u001b[0m\u001b[32;49mpip install --upgrade pip\u001b[0m\n" + ] + } + ], + "source": [ + "!pip3 install GDAL==3.8.4" + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "id": "78d0f46b-a5de-470b-aa56-663298e68518", + "metadata": {}, + "outputs": [], + "source": [ + "import datetime\n", + "import datacube\n", + "\n", + "from deafrica_tools.load_wapor import get_all_WaPORv3_mapsets, get_WaPORv3_info, load_wapor_ds\n", + "from deafrica_tools.plotting import display_map\n", + "from wapordl import wapor_map" + ] + }, + { + "cell_type": "markdown", + "id": "ca9edf94-2f9c-4efe-a89a-aa5f6c3328d9", + "metadata": {}, + "source": [ + "### Connect to the datacube\n", + "\n", + "Connect to the datacube so we can access DE Africa data.\n", + "The `app` parameter is a unique name for the analysis which is based on the notebook file name." + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "id": "154ddcd1-1443-4795-b7d3-dabcc8ace7ea", + "metadata": {}, + "outputs": [], + "source": [ + "dc = datacube.Datacube(app=\"WaPOR\")" + ] + }, + { + "cell_type": "markdown", + "id": "21d416c1-e3a0-4860-8166-e2c5df3e083a", + "metadata": {}, + "source": [ + "### WaPOR Data\n", + "\n", + "WaPOR data has three levels:\n", + "1. Global 300m resolution\n", + "2. National 100m resolution\n", + "3. Sub-national 20m resolution\n", + "\n", + "The table below covers L1 and L2 datasets. L3 datasets can be viewed in the [WaPOR maps platform](https://data.apps.fao.org/wapor) which is built with the same software as [Digital Earth Africa Maps](https://maps.digitalearth.africa/). L3 datasets cover several regions of interest in northern and eastern Africa. This notebook loads level 3 20m data for Egypt. It is recommended that the [WaPOR maps platform](https://data.apps.fao.org/wapor) is inspected to check the availability of level, variable, and temporal frequency combinations for your area of interest. The maps platform also shows map codes in the data description.\n", + "\n", + "Mapset codes are structured as `level-variable-temporal frequency` as shown below. The temporal frequencies available are:\n", + "* A - annual\n", + "* M - monthly\n", + "* D - dekadal (10 days)\n", + "\n", + "So, for level 3 net primary productivity at dekadal intervals the code would be `L3-NPP-D`." + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "id": "5e9c35e3-0d5b-4bfe-9c8a-c4e86ab6b451", + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
Mapset CodeMapset Description
0L1-AETI-AActual EvapoTranspiration and Interception (Gl...
1L1-AETI-DActual EvapoTranspiration and Interception (Gl...
2L1-AETI-MActual EvapoTranspiration and Interception (Gl...
3L1-E-AEvaporation (Global - Annual - 300m)
4L1-E-DEvaporation (Global - Dekadal - 300m)
5L1-GBWP-AGross biomass water productivity (Annual - 300m)
6L1-I-AInterception (Global - Annual - 300m)
7L1-I-DInterception (Global - Dekadal - 300m)
8L1-NBWP-ANet biomass water productivity (Annual - 300m)
9L1-NPP-DNet Primary Production (Global - Dekadal - 300m)
10L1-NPP-MNet Primary Production (Global - Monthly - 300m)
11L1-PCP-APrecipitation (Global - Annual - Approximately...
12L1-PCP-DPrecipitation (Global - Dekadal - Approximatel...
13L1-PCP-EPrecipitation (Global - Daily - Approximately ...
14L1-PCP-MPrecipitation (Global - Monthly - Approximatel...
15L1-QUAL-LST-DQuality land surface temperature (Global - Dek...
16L1-QUAL-NDVI-DQuality of Normalized Difference Vegetation In...
17L1-RET-AReference Evapotranspiration (Global - Annual ...
18L1-RET-DReference Evapotranspiration (Global - Dekadal...
19L1-RET-EReference Evapotranspiration (Global - Daily -...
20L1-RET-MReference Evapotranspiration (Global - Monthly...
21L1-RSM-DRelative Soil Moisture (Global - Dekadal - 300m)
22L1-T-ATranspiration (Global - Annual - 300m)
23L1-T-DTranspiration (Global - Dekadal - 300m)
24L1-TBP-ATotal Biomass Production (Global - Annual - 300m)
25L2-AETI-AActual EvapoTranspiration and Interception (Na...
26L2-AETI-DActual EvapoTranspiration and Interception (Na...
27L2-AETI-MActual EvapoTranspiration and Interception (Na...
28L2-E-AEvaporation (National - Annual - 100m)
29L2-E-DEvaporation (National - Dekadal - 100m)
30L2-GBWP-AGross biomass water productivity (Annual - 100m)
31L2-I-AInterception (National - Annual - 100m)
32L2-I-DInterception (National - Dekadal - 100m)
33L2-NBWP-ANet biomass water productivity (Annual - 100m)
34L2-NPP-DNet Primary Production (National - Dekadal - 1...
35L2-NPP-MNet Primary Production (National - Monthly - 1...
36L2-QUAL-NDVI-DQuality of normalized difference vegetation in...
37L2-RSM-DRelative Soil Moisture (National - Dekadal - 1...
38L2-T-ATranspiration (National - Annual - 100m)
39L2-T-DTranspiration (National - Dekadal - 100m)
40L2-TBP-ATotal Biomass Production (Global - Annual - 100m)
\n", + "
" + ], + "text/plain": [ + " Mapset Code Mapset Description\n", + "0 L1-AETI-A Actual EvapoTranspiration and Interception (Gl...\n", + "1 L1-AETI-D Actual EvapoTranspiration and Interception (Gl...\n", + "2 L1-AETI-M Actual EvapoTranspiration and Interception (Gl...\n", + "3 L1-E-A Evaporation (Global - Annual - 300m)\n", + "4 L1-E-D Evaporation (Global - Dekadal - 300m)\n", + "5 L1-GBWP-A Gross biomass water productivity (Annual - 300m)\n", + "6 L1-I-A Interception (Global - Annual - 300m)\n", + "7 L1-I-D Interception (Global - Dekadal - 300m)\n", + "8 L1-NBWP-A Net biomass water productivity (Annual - 300m)\n", + "9 L1-NPP-D Net Primary Production (Global - Dekadal - 300m)\n", + "10 L1-NPP-M Net Primary Production (Global - Monthly - 300m)\n", + "11 L1-PCP-A Precipitation (Global - Annual - Approximately...\n", + "12 L1-PCP-D Precipitation (Global - Dekadal - Approximatel...\n", + "13 L1-PCP-E Precipitation (Global - Daily - Approximately ...\n", + "14 L1-PCP-M Precipitation (Global - Monthly - Approximatel...\n", + "15 L1-QUAL-LST-D Quality land surface temperature (Global - Dek...\n", + "16 L1-QUAL-NDVI-D Quality of Normalized Difference Vegetation In...\n", + "17 L1-RET-A Reference Evapotranspiration (Global - Annual ...\n", + "18 L1-RET-D Reference Evapotranspiration (Global - Dekadal...\n", + "19 L1-RET-E Reference Evapotranspiration (Global - Daily -...\n", + "20 L1-RET-M Reference Evapotranspiration (Global - Monthly...\n", + "21 L1-RSM-D Relative Soil Moisture (Global - Dekadal - 300m)\n", + "22 L1-T-A Transpiration (Global - Annual - 300m)\n", + "23 L1-T-D Transpiration (Global - Dekadal - 300m)\n", + "24 L1-TBP-A Total Biomass Production (Global - Annual - 300m)\n", + "25 L2-AETI-A Actual EvapoTranspiration and Interception (Na...\n", + "26 L2-AETI-D Actual EvapoTranspiration and Interception (Na...\n", + "27 L2-AETI-M Actual EvapoTranspiration and Interception (Na...\n", + "28 L2-E-A Evaporation (National - Annual - 100m)\n", + "29 L2-E-D Evaporation (National - Dekadal - 100m)\n", + "30 L2-GBWP-A Gross biomass water productivity (Annual - 100m)\n", + "31 L2-I-A Interception (National - Annual - 100m)\n", + "32 L2-I-D Interception (National - Dekadal - 100m)\n", + "33 L2-NBWP-A Net biomass water productivity (Annual - 100m)\n", + "34 L2-NPP-D Net Primary Production (National - Dekadal - 1...\n", + "35 L2-NPP-M Net Primary Production (National - Monthly - 1...\n", + "36 L2-QUAL-NDVI-D Quality of normalized difference vegetation in...\n", + "37 L2-RSM-D Relative Soil Moisture (National - Dekadal - 1...\n", + "38 L2-T-A Transpiration (National - Annual - 100m)\n", + "39 L2-T-D Transpiration (National - Dekadal - 100m)\n", + "40 L2-TBP-A Total Biomass Production (Global - Annual - 100m)" + ] + }, + "execution_count": 3, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "get_all_WaPORv3_mapsets()" + ] + }, + { + "cell_type": "markdown", + "id": "436aa0a4-5a3a-4575-b171-2cdcc035c810", + "metadata": {}, + "source": [ + "### Analysis parameters\n", + "\n", + "The cell below specifies:\n", + "* The area of interest. This can also be a `.geojson` file\n", + "* The folder where the downloaded data will be stored. If you are using this script repeatedly, it is recommended you empty this folder from time to time to manage storage. It is advised that any data required for repeated or future analysis is stored outside the sandbox.\n", + "* The variable of interest in the form of a mapset code, explained above.\n", + "* The period of interest." + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "id": "b56ab843-7592-4389-a532-5276aacea5d8", + "metadata": {}, + "outputs": [], + "source": [ + "region = [31.30, 30.70, 31.40, 30.80] #xmin, ymin, xmax, ymax\n", + "folder = \"Supplementary_data/WaPOR\" # folder that the data will be sent to\n", + "variable = \"L3-AETI-A\" # level-variable-time, see table above\n", + "period = [\"2018-01-01\", \"2024-02-01\"] # period to load" + ] + }, + { + "cell_type": "markdown", + "id": "f766ff62-23b7-443d-a958-d6c7ddfb44bd", + "metadata": {}, + "source": [ + "This demonstration notebook loads an area of cropland in the Nile Delta, Egypt." + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "id": "f03dad3f-082d-40c2-8b96-661300fd5ba6", + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
Make this Notebook Trusted to load map: File -> Trust Notebook
" + ], + "text/plain": [ + "" + ] + }, + "execution_count": 5, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "display_map(x=(region[0], region[2]), y=(region[1], region[3]))" + ] + }, + { + "cell_type": "markdown", + "id": "7e161380-addb-4dc2-bfc6-4dfcdc2eda60", + "metadata": {}, + "source": [ + "### Download netCDF files\n", + "\n", + "This cell downloads the data specified. The `aeti` object is the file path to the stored netCDF. Once this cell is run, netCDF files will be stored in the directory specified above." + ] + }, + { + "cell_type": "code", + "execution_count": 14, + "id": "ace9923f-c2be-4121-994c-b43d656af500", + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "INFO: Given `region` matches with `ENO` L3 region.\n", + "INFO: Found 6 files for L3-AETI-A.\n", + "INFO: Converting from `.tif` to `.nc`.\n" + ] + }, + { + "ename": "RuntimeError", + "evalue": "Cannot guess driver for Supplementary_data/WaPOR/bb.ENO_L3-AETI-A_NONE_none.nc", + "output_type": "error", + "traceback": [ + "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", + "\u001b[0;31mRuntimeError\u001b[0m Traceback (most recent call last)", + "Cell \u001b[0;32mIn[14], line 1\u001b[0m\n\u001b[0;32m----> 1\u001b[0m aeti \u001b[38;5;241m=\u001b[39m \u001b[43mwapor_map\u001b[49m\u001b[43m(\u001b[49m\u001b[43mregion\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mvariable\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mperiod\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mfolder\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mextension\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43m \u001b[49m\u001b[38;5;124;43m'\u001b[39;49m\u001b[38;5;124;43m.nc\u001b[39;49m\u001b[38;5;124;43m'\u001b[39;49m\u001b[43m)\u001b[49m\n", + "File \u001b[0;32m/opt/venv/lib/python3.10/site-packages/wapordl/main.py:585\u001b[0m, in \u001b[0;36mwapor_map\u001b[0;34m(region, variable, period, folder, unit_conversion, overview, extension)\u001b[0m\n\u001b[1;32m 581\u001b[0m options \u001b[38;5;241m=\u001b[39m gdal\u001b[38;5;241m.\u001b[39mTranslateOptions(\n\u001b[1;32m 582\u001b[0m \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mtoptions\u001b[38;5;241m.\u001b[39mget(extension, {})\n\u001b[1;32m 583\u001b[0m )\n\u001b[1;32m 584\u001b[0m new_fp \u001b[38;5;241m=\u001b[39m fp\u001b[38;5;241m.\u001b[39mreplace(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m.tif\u001b[39m\u001b[38;5;124m\"\u001b[39m, extension)\n\u001b[0;32m--> 585\u001b[0m ds \u001b[38;5;241m=\u001b[39m \u001b[43mgdal\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mTranslate\u001b[49m\u001b[43m(\u001b[49m\u001b[43mnew_fp\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mfp\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43moptions\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43m \u001b[49m\u001b[43moptions\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 586\u001b[0m ds\u001b[38;5;241m.\u001b[39mFlushCache()\n\u001b[1;32m 587\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n", + "File \u001b[0;32m/opt/venv/lib/python3.10/site-packages/osgeo/gdal.py:523\u001b[0m, in \u001b[0;36mTranslate\u001b[0;34m(destName, srcDS, **kwargs)\u001b[0m\n\u001b[1;32m 520\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28misinstance\u001b[39m(srcDS, \u001b[38;5;28mstr\u001b[39m):\n\u001b[1;32m 521\u001b[0m srcDS \u001b[38;5;241m=\u001b[39m Open(srcDS)\n\u001b[0;32m--> 523\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mTranslateInternal\u001b[49m\u001b[43m(\u001b[49m\u001b[43mdestName\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43msrcDS\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mopts\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mcallback\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mcallback_data\u001b[49m\u001b[43m)\u001b[49m\n", + "File \u001b[0;32m/opt/venv/lib/python3.10/site-packages/osgeo/gdal.py:4793\u001b[0m, in \u001b[0;36mTranslateInternal\u001b[0;34m(*args)\u001b[0m\n\u001b[1;32m 4791\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mTranslateInternal\u001b[39m(\u001b[38;5;241m*\u001b[39margs) \u001b[38;5;241m-\u001b[39m\u001b[38;5;241m>\u001b[39m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mGDALDatasetShadow *\u001b[39m\u001b[38;5;124m\"\u001b[39m:\n\u001b[1;32m 4792\u001b[0m \u001b[38;5;250m \u001b[39m\u001b[38;5;124mr\u001b[39m\u001b[38;5;124;03m\"\"\"TranslateInternal(char const * dest, Dataset dataset, GDALTranslateOptions translateOptions, GDALProgressFunc callback=0, void * callback_data=None) -> Dataset\"\"\"\u001b[39;00m\n\u001b[0;32m-> 4793\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43m_gdal\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mTranslateInternal\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43margs\u001b[49m\u001b[43m)\u001b[49m\n", + "\u001b[0;31mRuntimeError\u001b[0m: Cannot guess driver for Supplementary_data/WaPOR/bb.ENO_L3-AETI-A_NONE_none.nc" + ] + } + ], + "source": [ + "aeti = wapor_map(region, variable, period, folder, extension = '.nc')" + ] + }, + { + "cell_type": "markdown", + "id": "80ad7282-df86-4cd5-be88-479c27099624", + "metadata": {}, + "source": [ + "### Convert data to xarray\n", + "\n", + "Below, the netCDFs are brought into the analysis environment as xarray datasets and coordinate labels and attributes are assigned. This means the data is in a common format with other Digital Earth Africa products and services, and makes the data easy to interact with. The `load_wapor_ds()` function brings in the data as xarray." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "9f319486-9cd6-4b3e-bf5f-46b371f28100", + "metadata": {}, + "outputs": [], + "source": [ + "aeti_xr = load_wapor_ds(filename=aeti, variable=variable)" + ] + }, + { + "cell_type": "markdown", + "id": "87ad2f80-f152-41ee-b6da-7a5b941d9665", + "metadata": {}, + "source": [ + "### Inspect the xarray dataset\n", + "\n", + "Now we have x, y, and time as dimensions and our variable (in this case, actual evapotranspiration and interception) as an xarray dataset. This enables us to easily deal with time and space for analysis." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "e805efee-b983-41d7-96dd-57b7145356c1", + "metadata": {}, + "outputs": [], + "source": [ + "aeti_xr" + ] + }, + { + "cell_type": "markdown", + "id": "3e0df299-c1d5-43ba-8e7b-572c860b3967", + "metadata": {}, + "source": [ + "### Plot annual ET\n", + "\n", + "The plots show how AETI varies in space and between years. In the Egypt example, the cropland areas are easily visible as areas with higher AETI.\n", + "\n", + "Note that the scalebar is labelled with information from the WaPOR metadata. This can be accessed by calling `aeti_xr[variable].attrs`, as below, which can be especially useful when checking units for calculation. The `load_wapor_ds()` function takes care of re-scaling when the data is loaded, but it is sensible to check the values are reasonable.\n", + "\n", + "We can also see that the attributes include scale and offset values. These have been incorporated into the `load_wapor_ds()` function so the values returned are in the units shown below, in this case mm/year." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "936dbe3e-8f82-47c2-9bb0-957824c27db0", + "metadata": {}, + "outputs": [], + "source": [ + "aeti_xr[variable].attrs" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "6ab17ba3-1e42-4d0d-b351-a19e8af109a7", + "metadata": {}, + "outputs": [], + "source": [ + "aeti_xr[variable].plot(col='time', col_wrap=3)" + ] + }, + { + "cell_type": "markdown", + "id": "46019aeb-5782-4bf2-868f-af708b31164d", + "metadata": {}, + "source": [ + "## Load dekadal biomass\n", + "\n", + "The cell below loads dekadal actual evapotranspiration using the same procedure as for annual. The only parameter changed is `variable`." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "dadfc36f-ef36-4793-b55e-5b6330568a79", + "metadata": {}, + "outputs": [], + "source": [ + "variable = 'L3-NPP-D'\n", + "period = [\"2024-01-01\", \"2024-03-01\"]\n", + "\n", + "npp_d = wapor_map(region, variable, period, folder, extension = '.nc')\n", + "\n", + "npp_d_xr = load_wapor_ds(filename=npp_d, variable=variable)\n", + "\n", + "npp_d_xr" + ] + }, + { + "cell_type": "markdown", + "id": "4c56bbb5-197f-4e4c-a969-7d860357f221", + "metadata": {}, + "source": [ + "### Plot dekadal net primary productivity\n", + "\n", + "It's interesting to note that in the Egypt example, some areas show very high biomass production > 30t/ha, especially in 2023. This must be considered in the context of several crop cycles occurring within a 12 month period." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "64ba0c94-4bc5-4e85-93c7-0e5c43dcef42", + "metadata": {}, + "outputs": [], + "source": [ + "npp_d_xr[variable].plot(col='time', col_wrap=3)" + ] + }, + { + "cell_type": "markdown", + "id": "ed6fcb77-b261-4a25-a25a-660ddd7b2561", + "metadata": {}, + "source": [ + "## Conclusion\n", + "\n", + "This notebook demonstrated the range of WaPOR variables available and how to load them in the DE Africa Sandbox environment. Subsequent notebooks will dive deeper into analysing WaPOR data alongside DE Africa data." + ] + }, + { + "cell_type": "markdown", + "id": "c6f143f6-587b-47da-9edc-dcac2e26f774", + "metadata": {}, + "source": [ + "***\n", + "\n", + "## Additional information\n", + "\n", + "**License:** The code in this notebook is licensed under the [Apache License, Version 2.0](https://www.apache.org/licenses/LICENSE-2.0). \n", + "Digital Earth Africa data is licensed under the [Creative Commons by Attribution 4.0](https://creativecommons.org/licenses/by/4.0/) license.\n", + "\n", + "**Contact:** If you need assistance, please post a question on the [Open Data Cube Slack channel](http://slack.opendatacube.org/) or on the [GIS Stack Exchange](https://gis.stackexchange.com/questions/ask?tags=open-data-cube) using the `open-data-cube` tag (you can view previously asked questions [here](https://gis.stackexchange.com/questions/tagged/open-data-cube)).\n", + "If you would like to report an issue with this notebook, you can file one on [Github](https://github.com/digitalearthafrica/deafrica-sandbox-notebooks).\n", + "\n", + "**Compatible datacube version:** " + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "df081dc0-a559-4b86-8a71-97906f1733f3", + "metadata": {}, + "outputs": [], + "source": [ + "print(datacube.__version__)" + ] + }, + { + "cell_type": "markdown", + "id": "59ad3d4b-0381-42ff-8365-1239f6308f67", + "metadata": {}, + "source": [ + "**Last Tested:**" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "19804ac3-343f-4e3d-ba62-ffb17c333992", + "metadata": {}, + "outputs": [], + "source": [ + "from datetime import datetime\n", + "datetime.today().strftime('%Y-%m-%d')" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.10.12" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/Tools/deafrica_tools/load_wapor.py b/Tools/deafrica_tools/load_wapor.py new file mode 100644 index 00000000..4fb16ff1 --- /dev/null +++ b/Tools/deafrica_tools/load_wapor.py @@ -0,0 +1,141 @@ +import collections + +import numpy as np +import pandas as pd +import requests +import rioxarray +import xarray as xr + +from deafrica_tools.spatial import add_geobox + +BASE_URL = "https://data.apps.fao.org/gismgr/api/v2/catalog/workspaces/WAPOR-3/mapsets" + + +def get_WaPORv3_info(url: str, info: str | list[str]) -> pd.DataFrame: + """ + Get information on WaPOR v3 data. WaPOR v3 variables are stored in `mapsets`, + which in turn contain `rasters` that contain the data for a particular date or period. + + Parameters + ---------- + url : str + URL to get information from + info : str | list[str] + Attribute of a mapset + + Returns + ------- + pd.DataFrame + A table of the mapset attributes found. + """ + if isinstance(info, str): + info = [info] + elif isinstance(info, list): + info = [str(i) for i in info] + else: + raise TypeError("'info' must be a list or string.") + + data = {"links": [{"rel": "next", "href": url}]} + + output_dict = collections.defaultdict(list) + while "next" in [x["rel"] for x in data["links"]]: + url_ = [x["href"] for x in data["links"] if x["rel"] == "next"][0] + response = requests.get(url_) + response.raise_for_status() + data = response.json()["response"] + for item in data["items"]: + if info == ["all"]: + for key in list(item.keys()): + output_dict[key].append(item[key]) + else: + for key in info: + output_dict[key].append(item[key]) + + output_df = pd.DataFrame(output_dict) + + if "code" in output_df.columns: + output_df.sort_values("code", inplace=True) + output_df.reset_index(drop=True, inplace=True) + return output_df + + +def get_all_WaPORv3_mapsets() -> pd.DataFrame: + """ + List all the available WaPOR v3 mapset codes and descriptions. + + Returns + ------- + pd.DataFrame + A table of all the available WaPOR v3 mapset codes and descriptions. + """ + output_df = get_WaPORv3_info(url=BASE_URL, info=["code", "caption"]) + output_df = output_df.rename(columns={"code": "Mapset Code", "caption": "Mapset Description"}) + + return output_df + + +def load_wapor_ds(filename: str, variable: str) -> xr.Dataset: + """ + Load the netCDF downloaded using `wapordl.wapor_map` as a + xarray Dataset. + Note: Only works for netcdf files, this is because when loading the WAPOR TIFF + files, the start_date and end_date attributes are only loaded for the first band. + + Parameters + ---------- + filename : str + File path of the netCDF downloaded using `wapordl.wapor_map` + variable : str + Name of the WAPOR version 3 mapset downloaded + + Returns + ------- + xr.Dataset + Dataset containing the WAPOR version 3 mapset data downloaded. + """ + # Load the file. + if filename.endswith(".tif"): + raise ValueError("Please set extension='.nc' when downloading data using `wapor_map`") + elif filename.endswith(".nc"): + ds = rioxarray.open_rasterio(filename).squeeze(dim="band").drop_vars("band") + + # Store the crs in the spatial_ref coordinate + if "spatial_ref" in ds.coords: + crs_attrs = ds["spatial_ref"].attrs + ds = add_geobox(ds.drop_vars("spatial_ref"), crs=ds.rio.crs) + elif "crs" in ds.coords: + crs_attrs = ds["crs"].attrs + ds = add_geobox(ds.drop_vars("crs"), crs=ds.rio.crs) + elif "transverse_mercator" in ds.coords: + crs_attrs = ds["transverse_mercator"].attrs + ds = add_geobox(ds.drop_vars("transverse_mercator"), crs=ds.rio.crs) + + ds["spatial_ref"].attrs.update(crs_attrs) + + # Get the bands names. + bands = [var for var in list(ds.variables) if var not in list(ds.coords)] + # Assign the correct time coordinate to each band. + # This is the start date in the attributes. + da_list = [ + ds[band] + .assign_coords(time=np.datetime64(ds[band].attrs["start_date"], "ns")) + .expand_dims(time=1) + .rename(variable) + for band in bands + ] + + # Merge the DataArrays + da = xr.concat(da_list, dim="time") + + # Rescale data + xr.set_options(keep_attrs=True) + if "scale_factor" in da.attrs: + da = da * da.attrs['scale_factor'] + da.attrs['add_offset'] + + # Edit the attributes + if len(da.time) > 1: + attrs_to_drop = ["start_date", "end_date", "number_of_days"] + for attr in attrs_to_drop: + del da.attrs[attr] + + return da.to_dataset(promote_attrs=True).sortby("time", ascending=True)