Merge pull request pypsa-meets-earth#650 from GridGrapher/opt_build_s…

…hapes_gadm

Optimize build shapes add_population_data

Snakefile

@@ -254,7 +254,7 @@ rule build_shapes:
         "benchmarks/" + RDIR + "build_shapes"
     threads: 1
     resources:
-        mem_mb=500,
+        mem_mb=3096,
     script:
         "scripts/build_shapes.py"
 

config.default.yaml

@@ -90,7 +90,6 @@ build_shape_options:
   out_logging: true  # When true, logging is printed to console
   year: 2020  # reference year used to derive shapes, info on population and info on GDP
   nprocesses: 3  # number of processes to be used in build_shapes
-  nchunks: 3  # number of data chuncks for build_shapes. If not specified or smaller than nprocesses, specified as nprocesses
   worldpop_method: "standard"  # "standard" pulls from web 1kmx1km raster, "api" pulls from API 100mx100m raster, false (not "false") no pop addition to shape which is useful when generating only cutout
   gdp_method: "standard"  # "standard" pulls from web 1x1km raster, false (not "false") no gdp addition to shape which useful when generating only cutout
   contended_flag: "set_by_country" # "set_by_country" assigns the contended areas to the countries according to the GADM database, "drop" drops these contended areas from the model

config.tutorial.yaml

@@ -104,7 +104,6 @@ build_shape_options:
   out_logging: true  # When true, logging is printed to console
   year: 2020  # reference year used to derive shapes, info on population and info on GDP
   nprocesses: 2  # number of processes to be used in build_shapes
-  nchunks: 2  # number of data chuncks for build_shapes. If not specified or smaller than nprocesses, specified as nprocesses
   worldpop_method: "standard"  # "standard" pulls from web 1kmx1km raster, "api" pulls from API 100mx100m raster, false (not "false") no pop addition to shape which is useful when generating only cutout
   gdp_method: "standard"  # "standard" pulls from web 1x1km raster, false (not "false") no gdp addition to shape which useful when generating only cutout
   contended_flag: "set_by_country" # "set_by_country" assigns the contended areas to the countries according to the GADM database, "drop" drops these contended areas from the model

doc/configtables/build_shape_options.csv

@@ -4,7 +4,6 @@ update_file, bool, "{True, False}", "True: all input files are downloaded again
 out_logging, bool, "{True, False}", "True: Logging is printed in the console."
 year,, "past year; e.g. YYYY", "Reference year used to derive shapes, info on population and info on GDP."
 nprocesses, int,, "Number of processes to be used in build_shapes."
-nchunks, int,, "Number of data chuncks for build_shapes. If not specified or smaller than nprocesses, specified as nprocesses."
 worldpop_method,, "{""standard"", ""api"", false}", "Specifies how population is added to every shape: ""standard"" pulls from web 1kmx1km raster; ""api"" pulls from API 100mx100m raster; false (not ""false"") no population addition to shape. This is useful when generating only cutout."
 gdp_method,, "{""standard"", false}", "Specifies how GDP is added to every shape: ""standard"" pulls from web 1x1km raster; false (not ""false"") no gdp addition to shape. This is useful when generating only cutout."
 contended_flag,, "{""set_by_country"", ""drop""}", "Specifies what to do with contended countries: ""set_by_country"" assigns the contended areas to the countries according to the GADM database; ""drop"" drops the contended areas from the model."

envs/environment.yaml

@@ -23,7 +23,7 @@ dependencies:
 - xlrd
 - openpyxl
 - seaborn
-- snakemake-minimal
+- snakemake-minimal<8
 - memory_profiler
 - ruamel.yaml<=0.17.26
 - pytables
@@ -43,6 +43,8 @@ dependencies:
 - matplotlib<=3.5.2
 - reverse-geocode
 - country_converter
+- pyogrio
+- numba
 - py7zr
 
   # Keep in conda environment when calling ipython