Merge branch 'pypsa-meets-earth:main' into main

README.md

@@ -333,6 +333,13 @@ The documentation is available here: [documentation](https://pypsa-earth.readthe
         </a>
     </td></tr>
 <tr>
+    <td align="center">
+        <a href="https://github.com/AndreCNF">
+            <img src="https://avatars.githubusercontent.com/u/19359510?v=4" width="100;" alt="AndreCNF"/>
+            <br />
+            <sub><b>André Cristóvão Neves Ferreira</b></sub>
+        </a>
+    </td>
     <td align="center">
         <a href="https://github.com/EmreYorat">
             <img src="https://avatars.githubusercontent.com/u/93644024?v=4" width="100;" alt="EmreYorat"/>
@@ -346,7 +353,8 @@ The documentation is available here: [documentation](https://pypsa-earth.readthe
             <br />
             <sub><b>GridGrapher</b></sub>
         </a>
-    </td>
+    </td></tr>
+<tr>
     <td align="center">
         <a href="https://github.com/HanaElattar">
             <img src="https://avatars.githubusercontent.com/u/87770004?v=4" width="100;" alt="HanaElattar"/>
@@ -389,7 +397,8 @@ The documentation is available here: [documentation](https://pypsa-earth.readthe
             <br />
             <sub><b>Sylvain Quoilin</b></sub>
         </a>
-    </td>
+    </td></tr>
+<tr>
     <td align="center">
         <a href="https://github.com/juli-a-ko">
             <img src="https://avatars.githubusercontent.com/u/126512394?v=4" width="100;" alt="juli-a-ko"/>

Snakefile

@@ -11,9 +11,9 @@ from shutil import copyfile, move
 
 from snakemake.remote.HTTP import RemoteProvider as HTTPRemoteProvider
 
-from scripts._helpers import create_country_list, get_last_commit_message
-from scripts.build_demand_profiles import get_load_paths_gegis
-from scripts.retrieve_databundle_light import datafiles_retrivedatabundle
+from _helpers import create_country_list, get_last_commit_message
+from build_demand_profiles import get_load_paths_gegis
+from retrieve_databundle_light import datafiles_retrivedatabundle
 from pathlib import Path
 
 HTTP = HTTPRemoteProvider()
@@ -28,7 +28,7 @@ if "config" not in globals() or not config:  # skip when used as sub-workflow
 configfile: "configs/bundle_config.yaml"
 
 
-config.update({"git_commit": get_last_commit_message()})
+config.update({"git_commit": get_last_commit_message(".")})
 
 # convert country list according to the desired region
 config["countries"] = create_country_list(config["countries"])
@@ -150,6 +150,7 @@ if config["enable"].get("retrieve_databundle", True):
         params:
             countries=config["countries"],
             tutorial=config["tutorial"],
+            hydrobasins_level=config["renewable"]["hydro"]["hydrobasins_level"],
         output:  #expand(directory('{file}') if isdir('{file}') else '{file}', file=datafiles)
             expand("{file}", file=datafiles_retrivedatabundle(config)),
             directory("data/landcover"),
@@ -1043,7 +1044,7 @@ rule run_scenario:
     resources:
         mem_mb=5000,
     run:
-        from scripts.build_test_configs import create_test_config
+        from build_test_configs import create_test_config
         import yaml
 
         # get base configuration file from diff config

config.default.yaml

@@ -292,9 +292,10 @@ renewable:
     extendable: true
   hydro:
     cutout: cutout-2013-era5
+    hydrobasins_level: 6
     resource:
       method: hydro
-      hydrobasins: data/hydrobasins/hybas_world_lev06_v1c.shp
+      hydrobasins: data/hydrobasins/hybas_world.shp
       flowspeed: 1.0  # m/s
       # weight_with_height: false
       # show_progress: true

config.tutorial.yaml

@@ -290,9 +290,10 @@ renewable:
     extendable: true
   hydro:
     cutout: cutout-2013-era5-tutorial
+    hydrobasins_level: 4
     resource:
       method: hydro
-      hydrobasins: data/hydrobasins/hybas_world_lev04_v1c.shp
+      hydrobasins: data/hydrobasins/hybas_world.shp
       flowspeed: 1.0  # m/s
       # weight_with_height: false
       # show_progress: true

configs/bundle_config.yaml

@@ -36,7 +36,7 @@ databundles:
     category: data
     destination: "data"
     urls:
-      zenodo: https://sandbox.zenodo.org/record/1223907/files/tutorial_data_NGBJ.zip?download=1
+      zenodo: https://sandbox.zenodo.org/records/3853/files/bundle_tutorial_NGBJ.zip?download=1
       gdrive: https://drive.google.com/file/d/1Vb1ISjhy7iwTTZYeezGd6S4nLt-EDGme/view?usp=drive_link
     output:
     - data/gebco/GEBCO_2021_TID.nc
@@ -49,7 +49,7 @@ databundles:
     category: data
     destination: "data"
     urls:
-      zenodo: https://sandbox.zenodo.org/record/1223907/files/tutorial_data_BW.zip?download=1
+      zenodo: https://sandbox.zenodo.org/records/3853/files/bundle_tutorial_BW.zip?download=1
       gdrive: https://drive.google.com/file/d/19IXvTD8gVSzgTInL85ta7QjaNI8ZPCCY/view?usp=drive_link
     output:
     - data/gebco/GEBCO_2021_TID.nc
@@ -62,20 +62,34 @@ databundles:
     category: data
     destination: "data"
     urls:
-      zenodo: https://sandbox.zenodo.org/record/1223907/files/tutorial_data_MA.zip?download=1
+      zenodo: https://sandbox.zenodo.org/records/3853/files/bundle_tutorial_MA.zip?download=1
       gdrive: https://drive.google.com/file/d/1VGzE8ZJHAvAQ9X44QNSX4rH3QF7Yi37D/view?usp=drive_link
     output:
     - data/gebco/GEBCO_2021_TID.nc
     - data/copernicus/PROBAV_LC100_global_v3.0.1_2019-nrt_Discrete-Classification-map_EPSG-4326.tif
 
+  # load tutorial hydrobasins bundle for Africa only
+  bundle_tutorial_hydrobasins:
+    countries: [Africa]
+    tutorial: true
+    category: hydrobasins
+    destination: "data/hydrobasins"
+    urls:
+      hydrobasins:
+        base_url: https://data.hydrosheds.org/file/HydroBASINS/standard/
+        suffixes: ["af"]
+    unzip: true
+    output:
+    - data/hydrobasins/hybas_world.shp
+
   # tutorial bundle specific for Nigeria and Benin only
   bundle_cutouts_tutorial_NGBJ:
     countries: [NG, BJ]
     tutorial: true
     category: cutouts
     destination: "cutouts"
     urls:
-      zenodo: https://sandbox.zenodo.org/record/1223907/files/tutorial_cutouts_NGBJ.zip?download=1
+      zenodo: https://sandbox.zenodo.org/records/3853/files/bundle_cutouts_tutorial_NGBJ.zip?download=1
       gdrive: https://drive.google.com/file/d/1xnomHdXf_c5STrf7jtDiuRlN2zW0FSVC/view?usp=drive_link
     output: [cutouts/cutout-2013-era5-tutorial.nc]
     disable_by_opt:
@@ -88,7 +102,7 @@ databundles:
     category: cutouts
     destination: "cutouts"
     urls:
-      zenodo: https://sandbox.zenodo.org/record/1223907/files/tutorial_cutouts_BW.zip?download=1
+      zenodo: https://sandbox.zenodo.org/records/3853/files/bundle_cutouts_tutorial_BW.zip?download=1
       gdrive: https://drive.google.com/file/d/1DDQAtnIDM0FNC3vCldfHeH__IpTbyIJt/view?usp=drive_link
     output: [cutouts/cutout-2013-era5-tutorial.nc]
     disable_by_opt:
@@ -101,7 +115,7 @@ databundles:
     category: cutouts
     destination: "cutouts"
     urls:
-      zenodo: https://sandbox.zenodo.org/record/1223907/files/tutorial_cutouts_MA.zip?download=1
+      zenodo: https://sandbox.zenodo.org/records/3853/files/bundle_cutouts_tutorial_MA.zip?download=1
       gdrive: https://drive.google.com/file/d/1j5v2f4E756jmDMa707QvdNJq3xM4bYUk/view?usp=drive_link
     output: [cutouts/cutout-2013-era5-tutorial.nc]
     disable_by_opt:
@@ -125,21 +139,30 @@ databundles:
     category: common
     destination: "data"
     urls:
-      zenodo: https://sandbox.zenodo.org/record/1183583/files/tutorial_data_general.zip?download=1
+      zenodo: https://sandbox.zenodo.org/records/3853/files/tutorial_data_general.zip?download=1
       gdrive: https://drive.google.com/file/d/1nRLrs_kP0qVl-IHC4BFLjpoKa3HLk2Py/view
     output:
-    - data/costs.csv
-    - data/hydro_capacities.csv
-    - data/custom_powerplants.csv
     - data/eez/eez_v11.gpkg
     - data/ssp2-2.6/2030/era5_2013/Africa.nc
     - data/ssp2-2.6/2030/era5_2013/Asia.nc
     - data/ssp2-2.6/2030/era5_2013/Europe.nc
     - data/ssp2-2.6/2030/era5_2013/NorthAmerica.nc
     - data/ssp2-2.6/2030/era5_2013/SouthAmerica.nc
     - data/ssp2-2.6/2030/era5_2013/Oceania.nc
-    - data/hydrobasins/hybas_world_lev04_v1c.shp
-    - data/hydrobasins/hybas_world_lev05_v1c.shp
+
+  # global data for hydrobasins
+  bundle_hydrobasins:
+    countries: [Earth]
+    tutorial: false
+    category: hydrobasins
+    destination: "data/hydrobasins"
+    urls:
+      hydrobasins:
+        base_url: https://data.hydrosheds.org/file/HydroBASINS/standard/
+        suffixes: ["af", "ar", "as", "au", "eu", "gr", "na", "sa", "si"]
+    unzip: true
+    output:
+    - data/hydrobasins/hybas_world.shp
 
   # data bundle containing the data of the data folder common to all regions of the world
   bundle_data_earth:

configs/osm_config.yaml

@@ -367,3 +367,12 @@ iso_to_geofk_dict:
   VC: "central-america"  # Saint Vincent e Grenadine
   KN: "central-america"  # Saint Kitts e Nevis
   GD: "central-america"  # Grenada
+  AW: "central-america"  # Aruba
+  AX: "finland"  # Aland
+  BM: "north-america"  # Bermuda
+  CW: "central-america"  # Curaçao
+  KY: "central-america"  # Cayman Islands
+  HK: "china"  # Hong Kong
+  MO: "china"  # Macau
+  SX: "central-america"  # Sint Maarten
+  TC: "central-america"  # Turks and Caicos Islands

doc/release_notes.rst

@@ -18,6 +18,10 @@ E.g. if a new rule becomes available describe how to use it `snakemake -j1 run_t
 
 * Function added in clean_osm_data script to allow the use of custom network data instead or on-top of OSM data. `PR #842 <'https://github.com/pypsa-meets-earth/pypsa-earth/pull/842>`__
 
+* Improve retrieve_databundle to prioritize smallest databundles `PR #911 <https://github.com/pypsa-meets-earth/pypsa-earth/pull/911>`__
+
+* Add functionality to load shapefiles for hydrobasins directly from the data source directly `PR #919 <https://github.com/pypsa-meets-earth/pypsa-earth/pull/919>`__
+
 
 PyPSA-Earth 0.2.3
 =================

envs/environment.yaml

@@ -32,7 +32,7 @@ dependencies:
 - pandas
 - geopandas>=0.11.0
 - fiona!=1.8.22
-- xarray<=2023.8.0
+- xarray
 - netcdf4
 - networkx
 - scipy

scripts/_helpers.py

@@ -386,7 +386,9 @@ def aggregate_costs(n, flatten=False, opts=None, existing_only=False):
     return costs
 
 
-def progress_retrieve(url, file, data=None, disable_progress=False, roundto=1.0):
+def progress_retrieve(
+    url, file, data=None, headers=None, disable_progress=False, roundto=1.0
+):
     """
     Function to download data from a url with a progress bar progress in
     retrieving data.
@@ -418,6 +420,11 @@ def dlProgress(count, blockSize, totalSize, roundto=roundto):
     if data is not None:
         data = urllib.parse.urlencode(data).encode()
 
+    if headers:
+        opener = urllib.request.build_opener()
+        opener.addheaders = headers
+        urllib.request.install_opener(opener)
+
     urllib.request.urlretrieve(url, file, reporthook=dlProgress, data=data)
 
 
@@ -523,7 +530,7 @@ def make_accessable(*ios):
     return snakemake
 
 
-def getContinent(code):
+def getContinent(code, world_iso=read_osm_config("world_iso")):
     """
     Returns continent names that contains list of iso-code countries.
 
@@ -547,7 +554,6 @@ def getContinent(code):
 
     continent_list = []
     code_set = set(code)
-    world_iso = read_osm_config("world_iso")
     for continent in world_iso.keys():
         single_continent_set = set(world_iso[continent])
         if code_set.intersection(single_continent_set):
@@ -787,9 +793,10 @@ def filter_codes(c_list, iso_coding=True):
 
     full_codes_list = []
 
+    world_iso, continent_regions = read_osm_config("world_iso", "continent_regions")
+
     for value1 in input:
         codes_list = []
-        world_iso, continent_regions = read_osm_config("world_iso", "continent_regions")
         # extract countries in world
         if value1 == "Earth":
             for continent in world_iso.keys():
@@ -816,21 +823,29 @@ def filter_codes(c_list, iso_coding=True):
     return full_codes_list
 
 
-def get_last_commit_message():
+def get_last_commit_message(path):
     """
-    Function to get the last Git commit message.
+    Function to get the last PyPSA-Earth Git commit message.
 
     Returns
     -------
     result : string
     """
+    _logger = logging.getLogger(__name__)
+    last_commit_message = None
+    backup_cwd = os.getcwd()
     try:
-        result = subprocess.run(
-            ["git", "log", "-1", "--pretty=format:%H %s"],
-            capture_output=True,
-            text=True,
+        os.chdir(path)
+        last_commit_message = (
+            subprocess.check_output(
+                ["git", "log", "-n", "1", "--pretty=format:%H %s"],
+                stderr=subprocess.STDOUT,
+            )
+            .decode()
+            .strip()
         )
-        return result.stdout.strip()
-    except Exception as e:
-        logger.warning(f"Error getting the last commit message: {e}")
-        return ""
+    except subprocess.CalledProcessError as e:
+        _logger.warning(f"Error executing Git: {e}")
+
+    os.chdir(backup_cwd)
+    return last_commit_message