From 302f45ab8e6b14f6dea4778826841c930191a539 Mon Sep 17 00:00:00 2001 From: SermishaNarayana <156903227+SermishaNarayana@users.noreply.github.com> Date: Sat, 21 Dec 2024 04:46:29 +0530 Subject: [PATCH] Introducing an option for the user to decide on simplifying GADM shapes (#1138) * docs(contributor): contrib-readme-action has updated readme * feature: Introducing configurable option to simplify GADM shapes * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * docs(contributor): contrib-readme-action has updated readme * Adding 'simplify_gadm' description to configtables * doc: Release note update * docs(contributor): contrib-readme-action has updated readme * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * Set simplify_gadm to true by default --- README.md | 687 +++++++++++------------ config.default.yaml | 1 + doc/configtables/build_shape_options.csv | 19 +- doc/release_notes.rst | 4 + scripts/build_shapes.py | 34 +- 5 files changed, 354 insertions(+), 391 deletions(-) diff --git a/README.md b/README.md index 851540909..15556eab2 100644 --- a/README.md +++ b/README.md @@ -187,375 +187,322 @@ The documentation is available here: [documentation](https://pypsa-earth.readthe - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
- - FabianHofmann -
- Fabian Hofmann - - 		- - fneum -
- Fabian Neumann - - 		- - ekatef -
- Ekaterina - - 		- - euronion -
- Euronion - - 		- - Justus-coded -
- Justus Ilemobayo - - 		- - mnm-matin -
- Mnm-matin - -
- - martacki -
- Martha Frysztacki - - 		- - LukasFrankenQ -
- Lukas Franken - - 		- - pz-max -
- Max Parzen - - 		- - davide-f -
- Davide-f - - 		- - koen-vg -
- Koen Van Greevenbroek - - 		- - Eddy-JV -
- Eddy Jalbout - -
- - hazemakhalek -
- Hazem - - 		- - energyLS -
- EnergyLS - - 		- - AnasAlgarei -
- AnasAlgarei - - 		- - yerbol-akhmetov -
- Yerbol Akhmetov - - 		- - GbotemiB -
- Emmanuel Bolarinwa - - 		- - DeniseGiub -
- DeniseGiub - -
- - doneachh -
- Anton Achhammer - - 		- - Tomkourou -
- Thomas Kouroughli - - 		- - finozzifa -
- Finozzifa - - 		- - GridGrapher -
- GridGrapher - - 		- - glenkiely-ieg -
- Null - - 		- - cpschau -
- Cschau - -
- - Emre-Yorat89 -
- Emre_Yorat - - 		- - virio-andreyana -
- Null - - 		- - giacfalk -
- Giacomo Falchetta - - 		- - Ekaterina-Vo -
- Ekaterina-Vo - - 		- - lkstrp -
- Lukas Trippe - - 		- - Tooblippe -
- Tobias - -
- - arizeosalac -
- zeosalac - - 		- - asolavi -
- Null - - 		- - rajesh-ieg -
- Null - - 		- - danielelerede-oet -
- Null - - 		- - carlosfv92 -
- Carlos Fernandez - - 		- - stephenjlee -
- Stephen J Lee - -
- - rsparks3 -
- Ryan - - 		- - ollie-bell -
- Null - - 		- - juli-a-ko -
- Juli-a-ko - - 		- - squoilin -
- Sylvain Quoilin - - 		- - siddharth-krishna -
- Siddharth Krishna - - 		- - pitmonticone -
- Pietro Monticone - -
- - Netotse -
- Null - - 		- - kma33 -
- Katherine M. Antonio - - 		- - jessLryan -
- Jess - - 		- - jarry7 -
- Jarrad Wright - - 		- - HanaElattar -
- HanaElattar - - 		- - EmreYorat -
- EmreYorat - -
- - AndreCNF -
- André Cristóvão Neves Ferreira - - 		- - AlexanderMeisinger -
- Null - -
+ + SermishaNarayana +
+ Null + + 		+ + davide-f +
+ Davide-f + + 		+ + ekatef +
+ Ekaterina + + 		+ + pz-max +
+ Max Parzen + + 		+ + DeniseGiub +
+ DeniseGiub + + 		+ + yerbol-akhmetov +
+ Yerbol Akhmetov + +
+ + GbotemiB +
+ Emmanuel Bolarinwa + + 		+ + mnm-matin +
+ Mnm-matin + + 		+ + hazemakhalek +
+ Hazem + + 		+ + energyLS +
+ EnergyLS + + 		+ + Tomkourou +
+ Thomas Kouroughli + + 		+ + GridGrapher +
+ GridGrapher + +
+ + martacki +
+ Martha Frysztacki + + 		+ + finozzifa +
+ Finozzifa + + 		+ + Emre-Yorat89 +
+ Emre_Yorat + + 		+ + virio-andreyana +
+ Null + + 		+ + giacfalk +
+ Giacomo Falchetta + + 		+ + Ekaterina-Vo +
+ Ekaterina-Vo + +
+ + cpschau +
+ Cschau + + 		+ + euronion +
+ Euronion + + 		+ + AnasAlgarei +
+ AnasAlgarei + + 		+ + LukasFrankenQ +
+ Lukas Franken + + 		+ + Tooblippe +
+ Tobias + + 		+ + doneachh +
+ Anton Achhammer + +
+ + koen-vg +
+ Koen Van Greevenbroek + + 		+ + carlosfv92 +
+ Carlos Fernandez + + 		+ + koen-vg +
+ Koen Van Greevenbroek + + 		+ + asolavi +
+ Null + + 		+ + Netotse +
+ Null + +
+ + pitmonticone +
+ Pietro Monticone + + 		+ + siddharth-krishna +
+ Siddharth Krishna + +
+ + squoilin +
+ Sylvain Quoilin + + 		+ + juli-a-ko +
+ Juli-a-ko + + 		+ + ollie-bell +
+ Null + + 		+ + rsparks3 +
+ Ryan + + 		+ + stephenjlee +
+ Stephen J Lee + + 		+ + kma33 +
+ Katherine M. Antonio + +
+ + jessLryan +
+ Jess + + 		+ + jarry7 +
+ Jarrad Wright + + 		+ + HanaElattar +
+ HanaElattar + + 		+ + FabianHofmann +
+ Fabian Hofmann + + 		+ + EmreYorat +
+ EmreYorat + + 		+ + AndreCNF +
+ André Cristóvão Neves Ferreira + +
+ + AlexanderMeisinger +
+ Null + +
diff --git a/config.default.yaml b/config.default.yaml index c7bfcade6..47053eb25 100644 --- a/config.default.yaml +++ b/config.default.yaml @@ -107,6 +107,7 @@ cluster_options: build_shape_options: gadm_layer_id: 1 # GADM level area used for the gadm_shapes. Codes are country-dependent but roughly: 0: country, 1: region/county-like, 2: municipality-like + simplify_gadm: true # When true, shape polygons are simplified else no update_file: false # When true, all the input files are downloaded again and replace the existing files out_logging: true # When true, logging is printed to console year: 2020 # reference year used to derive shapes, info on population and info on GDP diff --git a/doc/configtables/build_shape_options.csv b/doc/configtables/build_shape_options.csv index 78df78713..16f30ff0c 100644 --- a/doc/configtables/build_shape_options.csv +++ b/doc/configtables/build_shape_options.csv @@ -1,9 +1,10 @@ -,Unit,Values,Description -gadm_layer_id,, "{0, 1, 2}", "GADM level area used for the gadm_shapes. Codes are country-dependent but roughly: 0: country, 1: region/county-like, 2: municipality-like." -update_file, bool, "{True, False}", "True: all input files are downloaded again and replace the existing files." -out_logging, bool, "{True, False}", "True: Logging is printed in the console." -year,, "past year; e.g. YYYY", "Reference year used to derive shapes, info on population and info on GDP." -nprocesses, int,, "Number of processes to be used in build_shapes." -worldpop_method,, "{""standard"", ""api"", false}", "Specifies how population is added to every shape: ""standard"" pulls from web 1kmx1km raster; ""api"" pulls from API 100mx100m raster; false (not ""false"") no population addition to shape. This is useful when generating only cutout." -gdp_method,, "{""standard"", false}", "Specifies how GDP is added to every shape: ""standard"" pulls from web 1x1km raster; false (not ""false"") no gdp addition to shape. This is useful when generating only cutout." -contended_flag,, "{""set_by_country"", ""drop""}", "Specifies what to do with contended countries: ""set_by_country"" assigns the contended areas to the countries according to the GADM database; ""drop"" drops the contended areas from the model." +,Unit,Values,Description,,,, +gadm_layer_id,,"""{0",1,"2}""","""GADM level area used for the gadm_shapes. Codes are country-dependent but roughly: 0: country",1: region/county-like,"2: municipality-like.""" +simplify_gadm,bool," ""{True"," False}""",True: shape polygons are simplified else no,,, +update_file, bool," ""{True"," False}"""," ""True: all input files are downloaded again and replace the existing files.""",,, +out_logging, bool," ""{True"," False}"""," ""True: Logging is printed in the console.""",,, +year,," ""past year; e.g. YYYY"""," ""Reference year used to derive shapes"," info on population and info on GDP.""",,, +nprocesses, int,," ""Number of processes to be used in build_shapes.""",,,, +worldpop_method,," ""{""standard"""," ""api"""," false}""","""Specifies how population is added to every shape: ""standard"" pulls from web 1kmx1km raster; ""api"" pulls from API 100mx100m raster; false (not ""false"") no population addition to shape. This is useful when generating only cutout.""",, +gdp_method,," ""{""standard"""," false}""","""Specifies how GDP is added to every shape: ""standard"" pulls from web 1x1km raster; false (not ""false"") no gdp addition to shape. This is useful when generating only cutout.""",,, +contended_flag,," ""{""set_by_country"""," ""drop""}"""," ""Specifies what to do with contended countries: ""set_by_country"" assigns the contended areas to the countries according to the GADM database; ""drop"" drops the contended areas from the model.""",,, diff --git a/doc/release_notes.rst b/doc/release_notes.rst index fa4f2b346..0156ec70e 100644 --- a/doc/release_notes.rst +++ b/doc/release_notes.rst @@ -81,6 +81,10 @@ PyPSA-Earth 0.5.0 * Drop vrestil dependency `PR #1220 `__ +* Include a configuration option to simplify / not simplify shapefiles based on a boolean value specified under `build_shape_options:simplify_gadm` option in the config file `PR 1138 `_ + +* Fix the mismatch between buses and x, y locations while creating H2 Stores `PR #1134 `_ + * Remove duplicate entries from hydrogen export ports `PR #1233 `__ * Fix the environment placing a version limit to numpoly `PR #1237 `__ diff --git a/scripts/build_shapes.py b/scripts/build_shapes.py index 8d3722de6..3e0e8bd5a 100644 --- a/scripts/build_shapes.py +++ b/scripts/build_shapes.py @@ -363,6 +363,7 @@ def eez( distance=0.01, minarea=0.01, tolerance=0.01, + simplify_gadm=True, ): """ Creates offshore shapes by buffer smooth countryshape (=offset country @@ -387,22 +388,26 @@ def eez( } ).set_index("name") - ret_df = ret_df.geometry.map( - lambda x: _simplify_polys(x, minarea=minarea, tolerance=tolerance) - ) + if simplify_gadm: + ret_df = ret_df.geometry.map( + lambda x: _simplify_polys(x, minarea=minarea, tolerance=tolerance) + ) - ret_df = ret_df.apply(lambda x: make_valid(x)) + ret_df = ret_df.apply(lambda x: make_valid(x)) country_shapes_with_buffer = country_shapes.buffer(distance) ret_df_new = ret_df.difference(country_shapes_with_buffer) - # repeat to simplify after the buffer correction - ret_df_new = ret_df_new.map( - lambda x: ( - x if x is None else _simplify_polys(x, minarea=minarea, tolerance=tolerance) + if simplify_gadm: + # repeat to simplify after the buffer correction + ret_df_new = ret_df_new.map( + lambda x: ( + x + if x is None + else _simplify_polys(x, minarea=minarea, tolerance=tolerance) + ) ) - ) - ret_df_new = ret_df_new.apply(lambda x: x if x is None else make_valid(x)) + ret_df_new = ret_df_new.apply(lambda x: x if x is None else make_valid(x)) # Drops empty geometry ret_df = ret_df_new.dropna() @@ -1253,6 +1258,7 @@ def gadm( out_logging=False, year=2020, nprocesses=None, + simplify_gadm=True, ): if out_logging: logger.info("Stage 3 of 5: Creation GADM GeoDataFrame") @@ -1302,7 +1308,9 @@ def gadm( lambda x: x if x.find(".") == 0 else "." + x ) df_gadm.set_index("GADM_ID", inplace=True) - df_gadm["geometry"] = df_gadm["geometry"].map(_simplify_polys) + + if simplify_gadm: + df_gadm["geometry"] = df_gadm["geometry"].map(_simplify_polys) df_gadm.geometry = df_gadm.geometry.apply( lambda r: make_valid(r) if not r.is_valid else r ) @@ -1335,6 +1343,7 @@ def gadm( contended_flag = snakemake.params.build_shape_options["contended_flag"] worldpop_method = snakemake.params.build_shape_options["worldpop_method"] gdp_method = snakemake.params.build_shape_options["gdp_method"] + simplify_gadm = snakemake.params.build_shape_options["simplify_gadm"] country_shapes = countries( countries_list, @@ -1346,7 +1355,7 @@ def gadm( country_shapes.to_file(snakemake.output.country_shapes) offshore_shapes = eez( - countries_list, geo_crs, country_shapes, EEZ_gpkg, out_logging + countries_list, geo_crs, country_shapes, EEZ_gpkg, out_logging, simplify_gadm ) offshore_shapes.reset_index().to_file(snakemake.output.offshore_shapes) @@ -1368,5 +1377,6 @@ def gadm( out_logging, year, nprocesses=nprocesses, + simplify_gadm=simplify_gadm, ) save_to_geojson(gadm_shapes, out.gadm_shapes)