Merge pull request pypsa-meets-earth#913 from davide-f/add_automatic_…

…WDPA_download

Enable automatic landcover download

Snakefile

@@ -385,7 +385,7 @@ if not config["enable"].get("build_natura_raster", False):
 
     rule copy_defaultnatura_tiff:
         input:
-            "data/natura.tiff",
+            "data/natura/natura.tiff",
         output:
             "resources/" + RDIR + "natura.tiff",
         run:

config.default.yaml

@@ -16,7 +16,7 @@ enable:
   retrieve_databundle: true  #  Recommended 'true', for the first run. Otherwise data might be missing.
   retrieve_cost_data: true  # true: retrieves cost data from technology data and saves in resources/costs.csv, false: uses cost data in data/costs.csv
   download_osm_data: true  # If 'true', OpenStreetMap data will be downloaded for the above given countries
-  build_natura_raster: false  # If True, than an exclusion raster will be build
+  build_natura_raster: false  # If True, then an exclusion raster will be build
   build_cutout: false
   # If "build_cutout" : true, then environmental data is extracted according to `snapshots` date range and `countries`
   # requires cds API key https://cds.climate.copernicus.eu/api-how-to

configs/bundle_config.yaml

@@ -171,7 +171,7 @@ databundles:
     category: common
     destination: "data"
     urls:
-      zenodo: https://sandbox.zenodo.org/records/4114/files/bundle_data_earth.zip?download=1
+      zenodo: https://sandbox.zenodo.org/records/13539/files/bundle_data_earth.zip?download=1
       gdrive: https://drive.google.com/file/d/1zU4VRRBDVIVqoykhK9lnOAr60unVm_x2/view?usp=drive_link
     output:
     - data/eez/eez_v11.gpkg
@@ -188,91 +188,12 @@ databundles:
   bundle_natura_earth:
     countries: [Earth]
     category: natura
-    destination: "data"
+    destination: "data/natura/"
     urls:
-      zenodo: https://sandbox.zenodo.org/record/1183583/files/natura_global.zip?download=1
-      gdrive: https://drive.google.com/file/d/1WmAAwY0TAcHg8YZyAGof0bPewFacm7my/view
+      # zenodo:
+      gdrive: https://drive.google.com/file/d/1H-Y7p8UgnbEzJgo9qPc54BpDXS8loX7Y/view?usp=drive_link
     output:
-    - data/natura.tiff
-
-  # data bundle of the data folder for the African continent
-  bundle_landcover_africa:
-    countries: [Africa]
-    category: landcover
-    destination: "data/landcover"
-    urls:
-      zenodo: https://sandbox.zenodo.org/record/1183583/files/landcover_africa.zip?download=1
-      gdrive: https://drive.google.com/file/d/1qnJ0HAWZ4z_9q3PtDEUrFmZWBZYg-WNW/view
-    output: [data/landcover/*]
-
-  # data bundle of the data folder for the african continent
-  bundle_landcover_asiapacific:
-    countries: [Asia, Oceania]
-    category: landcover
-    destination: "data/landcover"
-    urls:
-      zenodo: https://sandbox.zenodo.org/record/1183583/files/landcover_asiapacific.zip?download=1
-      gdrive: https://drive.google.com/file/d/163Uc9F1DPs2pRgDzkTivnblhXCw4nfd5/view
-    output: [data/landcover/*]
-
-  # data bundle of the data folder for the African continent
-  bundle_landcover_latinamerica_caribbean: # excludes North America
-    countries: [SouthAmerica, NorthAmerica]
-    category: landcover
-    destination: "data/landcover"
-    urls:
-      zenodo: https://sandbox.zenodo.org/record/1183583/files/landcover_latinamerica_caribbean.zip?download=1
-      gdrive: https://drive.google.com/file/d/1d3l9qxPnM9XbXODWkP4kWxK_t5cDcEIZ/view
-    output: [data/landcover/*]
-
-  # data bundle of the data folder for the African continent
-  bundle_landcover_westasia:
-    countries: [WAS]
-    category: landcover
-    destination: "data/landcover"
-    urls:
-      zenodo: https://sandbox.zenodo.org/record/1183583/files/landcover_westasia.zip?download=1
-      gdrive: https://drive.google.com/file/d/1XsZVZDORUFMKyHfAkzqxDVBENiuy4jEM/view
-    output: [data/landcover/*]
-
-  # data bundle of the data folder for Europe
-  bundle_landcover_europe:
-    countries: [Europe]
-    category: landcover
-    destination: "data/landcover"
-    urls:
-      zenodo: https://sandbox.zenodo.org/record/1183583/files/landcover_europe.zip?download=1
-      gdrive: https://drive.google.com/file/d/1MfT3cwHJbNTtTjRLzW2T19f9qBNAxdhw/view
-    output: [data/landcover/*]
-
-  # data bundle of the data folder for the Poles
-  bundle_landcover_northamerica:
-    countries: [NorthAmerica] # Greenland and Antartica
-    category: landcover
-    destination: "data/landcover"
-    urls:
-      zenodo: https://sandbox.zenodo.org/record/1183583/files/landcover_northamerica.zip?download=1
-      gdrive: https://drive.google.com/file/d/1Dc11J2BZQQeJWxwCC6GiiPd_GWMCN0uF/view
-    output: [data/landcover/*]
-
-  # data bundle of the data folder for the Poles
-  bundle_landcover_poles:
-    countries: [GL, ATA] # Greenland and Antartica
-    category: landcover
-    destination: "data/landcover"
-    urls:
-      zenodo: https://sandbox.zenodo.org/record/1183583/files/landcover_polar.zip?download=1
-      gdrive: https://drive.google.com/file/d/18uODyONQZJGoL15P-URPl0lJs7sphQhX/view
-    output: [data/landcover/*]
-
-  # # data bundle of the data folder for the african continent
-  # bundle_landcover_earth:
-  #   countries: [Earth]
-  #   category: landcover
-  #   destination: "data/landcover"
-  #   urls:
-  #     direct: https://d1gam3xoknrgr2.cloudfront.net/current/WDPA_WDOECM_Mar2022_Public_all_shp.zip
-  #   output: [data/landcover/*]
+    - data/natura/natura.tiff
 
   # # data bundle of the data folder for the african continent
   # bundle_landclassification:
@@ -315,7 +236,7 @@ databundles:
     category: cutouts
     destination: "cutouts"
     urls:
-      zenodo: https://sandbox.zenodo.org/records/4114/files/bundle_cutouts_africa.zip?download=1
+      zenodo: https://sandbox.zenodo.org/records/13539/files/bundle_cutouts_africa.zip?download=1
       gdrive: https://drive.google.com/file/d/1WHv5Dm1GtrDZj-AxJZd3T-NMIBXty3eV/view?usp=drive_link
     output: [cutouts/cutout-2013-era5.nc]
     disable_by_opt:
@@ -341,7 +262,7 @@ databundles:
     category: cutouts
     destination: "cutouts"
     urls:
-      # zenodo:
+      zenodo: https://sandbox.zenodo.org/records/13539/files/natura_wpda_CC0.zip?download=1
       gdrive: https://drive.google.com/file/d/1Ew7rQT0VNBqJW1AUrOrOP2IJKSJS7Uoy/view?usp=sharing
     output: [cutouts/cutout-2013-era5.nc]
     disable_by_opt:
@@ -396,3 +317,12 @@ databundles:
     output: [cutouts/cutout-2013-era5.nc]
     disable_by_opt:
       build_cutout: [all]
+
+  # data bundle containing the protected data for the whole world
+  bundle_landcover_earth:
+    countries: [Earth]
+    category: landcover
+    destination: "data/landcover/world_protected_areas"
+    urls:
+      protectedplanet: https://d1gam3xoknrgr2.cloudfront.net/current/WDPA_{month:s}{year:d}_Public_shp.zip
+    output: [data/landcover/world_protected_areas/*]

doc/release_notes.rst

@@ -22,6 +22,8 @@ E.g. if a new rule becomes available describe how to use it `snakemake -j1 run_t
 
 * Add functionality to load shapefiles for hydrobasins directly from the data source directly `PR #919 <https://github.com/pypsa-meets-earth/pypsa-earth/pull/919>`__
 
+* Use `new CC0 v1 dataset <https://doi.org/10.7910/DVN/XIV9BL>`__ for the natura input and automate download of WDPA protected planet data `PR #913 <https://github.com/pypsa-meets-earth/pypsa-earth/pull/913>`__
+
 **Minor Changes and bug-fixing**
 
 * Revise databundles and improve logging in retrieve_databundle `PR #928 <https://github.com/pypsa-meets-earth/pypsa-earth/pull/928>`__

scripts/build_natura_raster.py

@@ -23,33 +23,26 @@
 Inputs
 ------
 
-- ``data/raw/protected_areas/WDPA_WDOECM_Aug2021_Public_AF_shp-points.shp``: `WDPA <https://en.wikipedia.org/wiki/Natura_2000>`_ World Database for Protected Areas.
+- ``data/landcover/world_protected_areas/*.shp``: shapefiles representing the world protected areas, such as the `World Database of Protected Areas (WDPA) <https://www.protectedplanet.net/en/thematic-areas/wdpa?tab=WDPA>`_.
 
     .. image:: /img/natura.png
         :width: 33 %
 
 Outputs
 -------
 
-- ``resources/natura.tiff``: Rasterized version of `Natura 2000 <https://en.wikipedia.org/wiki/Natura_2000>`_ natural protection areas to reduce computation times.
+- ``resources/natura/natura.tiff``: Rasterized version of the world protected areas, such as `WDPA <https://www.protectedplanet.net/en/thematic-areas/wdpa?tab=WDPA>`_ natural protection areas to reduce computation times.
 
     .. image:: /img/natura.png
         :width: 33 %
 
 Description
 -----------
-To operate the script you need all input files. The Snakefile describes what goes in and out. Make sure you didn't skip one of these.
-Maybe not so obvious is the cutout input. An example is this `africa-2013-era5.nc`
-
-Steps to retrieve the protected area data (as apparently no API is given for the WDPA data):
-    - 1. Download the WPDA Dataset: World Database on Protected Areas. UNEP-WCMC and IUCN (2021), Protected Planet: The World Database on Protected Areas (WDPA) and World Database on Other Effective Area-based Conservation Measures (WD-OECM) [Online], August 2021, Cambridge, UK: UNEP-WCMC and IUCN. Available at: www.protectedplanet.net.
-    - 2. Unzipp and rename the folder containing the .shp file to `protected_areas`
-    - 3. Important! Don't delete the other files which come with the .shp file. They are required to build the shape.
-    - 4. Move the file in such a way that the above path is given
-    - 5. Activate the environment of environment-max.yaml
-    - 6. Ready to run the script
-
-Tip: The output file `natura.tiff` contains now the 100x100m rasters of protective areas. This operation can make the filesize of that TIFF quite large and leads to problems when trying to open. QGIS, an open source tool helps exploring the file.
+To operate the script you need all input files.
+
+This script collects all shapefiles available in the folder `data/landcover/*` describing regions of protected areas,
+merges them to one shapefile, and create a rasterized version of the region, that covers the region described by the cutout.
+The output is a raster file with the name `natura.tiff` in the folder `resources/natura/`.
 """
 import logging
 import os
@@ -151,12 +144,7 @@ def unify_protected_shape_areas(inputs, natura_crs, out_logging):
         shp = gpd.read_file(i).to_crs(natura_crs)
         total_files += 1
         try:
-            shp.geometry = shp.apply(
-                lambda row: make_valid(row.geometry)
-                if not row.geometry.is_valid
-                else row.geometry,
-                axis=1,
-            )
+            shp.geometry = shp.geometry.make_valid()
             list_shapes.append(shp)
             read_files += 1
         except: