From a9e89e1400bd8a5667ae7cce16bb4ae6d5de6780 Mon Sep 17 00:00:00 2001 From: Emmanuel Bolarinwa Date: Thu, 7 Nov 2024 13:16:28 +0100 Subject: [PATCH 01/10] revise export data for US --- Snakefile | 118 +++++++++++++++++++++--------------------- scripts/add_export.py | 7 ++- 2 files changed, 64 insertions(+), 61 deletions(-) diff --git a/Snakefile b/Snakefile index 218041c67..17c9d2e1c 100644 --- a/Snakefile +++ b/Snakefile @@ -197,29 +197,29 @@ rule clean_osm_data: "scripts/clean_osm_data.py" -rule build_osm_network: - params: - build_osm_network=config.get("build_osm_network", {}), - countries=config["countries"], - crs=config["crs"], - input: - generators="resources/" + RDIR + "osm/clean/all_clean_generators.geojson", - lines="resources/" + RDIR + "osm/clean/all_clean_lines.geojson", - substations="resources/" + RDIR + "osm/clean/all_clean_substations.geojson", - country_shapes="resources/" + RDIR + "shapes/country_shapes.geojson", - output: - lines="resources/" + RDIR + "base_network/all_lines_build_network.csv", - converters="resources/" + RDIR + "base_network/all_converters_build_network.csv", - transformers="resources/" - + RDIR - + "base_network/all_transformers_build_network.csv", - substations="resources/" + RDIR + "base_network/all_buses_build_network.csv", - log: - "logs/" + RDIR + "build_osm_network.log", - benchmark: - "benchmarks/" + RDIR + "build_osm_network" - script: - "scripts/build_osm_network.py" +# rule build_osm_network: +# params: +# build_osm_network=config.get("build_osm_network", {}), +# countries=config["countries"], +# crs=config["crs"], +# input: +# generators="resources/" + RDIR + "osm/clean/all_clean_generators.geojson", +# lines="resources/" + RDIR + "osm/clean/all_clean_lines.geojson", +# substations="resources/" + RDIR + "osm/clean/all_clean_substations.geojson", +# country_shapes="resources/" + RDIR + "shapes/country_shapes.geojson", +# output: +# lines="resources/" + RDIR + "base_network/all_lines_build_network.csv", +# converters="resources/" + RDIR + "base_network/all_converters_build_network.csv", +# transformers="resources/" +# + RDIR +# + "base_network/all_transformers_build_network.csv", +# substations="resources/" + RDIR + "base_network/all_buses_build_network.csv", +# log: +# "logs/" + RDIR + "build_osm_network.log", +# benchmark: +# "benchmarks/" + RDIR + "build_osm_network" +# script: +# "scripts/build_osm_network.py" rule build_shapes: @@ -447,41 +447,41 @@ rule build_demand_profiles: "scripts/build_demand_profiles.py" -rule build_renewable_profiles: - params: - crs=config["crs"], - renewable=config["renewable"], - countries=config["countries"], - alternative_clustering=config["cluster_options"]["alternative_clustering"], - input: - natura="resources/" + RDIR + "natura.tiff", - copernicus="data/copernicus/PROBAV_LC100_global_v3.0.1_2019-nrt_Discrete-Classification-map_EPSG-4326.tif", - gebco="data/gebco/GEBCO_2021_TID.nc", - country_shapes="resources/" + RDIR + "shapes/country_shapes.geojson", - offshore_shapes="resources/" + RDIR + "shapes/offshore_shapes.geojson", - hydro_capacities="data/hydro_capacities.csv", - eia_hydro_generation="data/eia_hydro_annual_generation.csv", - powerplants="resources/" + RDIR + "powerplants.csv", - regions=lambda w: ( - "resources/" + RDIR + "bus_regions/regions_onshore.geojson" - if w.technology in ("onwind", "solar", "hydro", "csp") - else "resources/" + RDIR + "bus_regions/regions_offshore.geojson" - ), - cutout=lambda w: "cutouts/" - + CDIR - + config["renewable"][w.technology]["cutout"] - + ".nc", - output: - profile="resources/" + RDIR + "renewable_profiles/profile_{technology}.nc", - log: - "logs/" + RDIR + "build_renewable_profile_{technology}.log", - benchmark: - "benchmarks/" + RDIR + "build_renewable_profiles_{technology}" - threads: ATLITE_NPROCESSES - resources: - mem_mb=ATLITE_NPROCESSES * 5000, - script: - "scripts/build_renewable_profiles.py" +# rule build_renewable_profiles: +# params: +# crs=config["crs"], +# renewable=config["renewable"], +# countries=config["countries"], +# alternative_clustering=config["cluster_options"]["alternative_clustering"], +# input: +# natura="resources/" + RDIR + "natura.tiff", +# copernicus="data/copernicus/PROBAV_LC100_global_v3.0.1_2019-nrt_Discrete-Classification-map_EPSG-4326.tif", +# gebco="data/gebco/GEBCO_2021_TID.nc", +# country_shapes="resources/" + RDIR + "shapes/country_shapes.geojson", +# offshore_shapes="resources/" + RDIR + "shapes/offshore_shapes.geojson", +# hydro_capacities="data/hydro_capacities.csv", +# eia_hydro_generation="data/eia_hydro_annual_generation.csv", +# powerplants="resources/" + RDIR + "powerplants.csv", +# regions=lambda w: ( +# "resources/" + RDIR + "bus_regions/regions_onshore.geojson" +# if w.technology in ("onwind", "solar", "hydro", "csp") +# else "resources/" + RDIR + "bus_regions/regions_offshore.geojson" +# ), +# cutout=lambda w: "cutouts/" +# + CDIR +# + config["renewable"][w.technology]["cutout"] +# + ".nc", +# output: +# profile="resources/" + RDIR + "renewable_profiles/profile_{technology}.nc", +# log: +# "logs/" + RDIR + "build_renewable_profile_{technology}.log", +# benchmark: +# "benchmarks/" + RDIR + "build_renewable_profiles_{technology}" +# threads: ATLITE_NPROCESSES +# resources: +# mem_mb=ATLITE_NPROCESSES * 5000, +# script: +# "scripts/build_renewable_profiles.py" rule build_powerplants: @@ -1126,7 +1126,7 @@ rule add_export: costs=config["costs"], input: overrides="data/override_component_attrs", - export_ports="data/export_ports.csv", + export_ports="data/ports.csv", costs=COSTDIR + "costs_{planning_horizons}.csv", ship_profile="resources/ship_profile_{h2export}TWh.csv", network=RESDIR diff --git a/scripts/add_export.py b/scripts/add_export.py index 3a2aab8e8..2d0cb03ea 100644 --- a/scripts/add_export.py +++ b/scripts/add_export.py @@ -33,12 +33,15 @@ def select_ports(n): This function selects the buses where ports are located. """ - ports = pd.read_csv( + raw_ports = pd.read_csv( snakemake.input.export_ports, index_col=None, keep_default_na=False, ).squeeze() + ports = raw_ports[['name', 'country', 'fraction', 'x', 'y']] + ports.loc[:,'fraction'] = ports.fraction.round(1) + ports = ports[ports.country.isin(countries)] if len(ports) < 1: logger.error( @@ -226,4 +229,4 @@ def create_export_profile(): n.export_to_netcdf(snakemake.output[0]) - logger.info("Network successfully exported") + logger.info("Network successfully exported") \ No newline at end of file From 54df2b05d31516fcb640e4217de9c4f836d5dd59 Mon Sep 17 00:00:00 2001 From: Emmanuel Bolarinwa Date: Thu, 7 Nov 2024 13:20:49 +0100 Subject: [PATCH 02/10] revert snakefile --- Snakefile | 116 +++++++++++++++++++++++++++--------------------------- 1 file changed, 58 insertions(+), 58 deletions(-) diff --git a/Snakefile b/Snakefile index 17c9d2e1c..14b4c0ecb 100644 --- a/Snakefile +++ b/Snakefile @@ -197,29 +197,29 @@ rule clean_osm_data: "scripts/clean_osm_data.py" -# rule build_osm_network: -# params: -# build_osm_network=config.get("build_osm_network", {}), -# countries=config["countries"], -# crs=config["crs"], -# input: -# generators="resources/" + RDIR + "osm/clean/all_clean_generators.geojson", -# lines="resources/" + RDIR + "osm/clean/all_clean_lines.geojson", -# substations="resources/" + RDIR + "osm/clean/all_clean_substations.geojson", -# country_shapes="resources/" + RDIR + "shapes/country_shapes.geojson", -# output: -# lines="resources/" + RDIR + "base_network/all_lines_build_network.csv", -# converters="resources/" + RDIR + "base_network/all_converters_build_network.csv", -# transformers="resources/" -# + RDIR -# + "base_network/all_transformers_build_network.csv", -# substations="resources/" + RDIR + "base_network/all_buses_build_network.csv", -# log: -# "logs/" + RDIR + "build_osm_network.log", -# benchmark: -# "benchmarks/" + RDIR + "build_osm_network" -# script: -# "scripts/build_osm_network.py" +rule build_osm_network: + params: + build_osm_network=config.get("build_osm_network", {}), + countries=config["countries"], + crs=config["crs"], + input: + generators="resources/" + RDIR + "osm/clean/all_clean_generators.geojson", + lines="resources/" + RDIR + "osm/clean/all_clean_lines.geojson", + substations="resources/" + RDIR + "osm/clean/all_clean_substations.geojson", + country_shapes="resources/" + RDIR + "shapes/country_shapes.geojson", + output: + lines="resources/" + RDIR + "base_network/all_lines_build_network.csv", + converters="resources/" + RDIR + "base_network/all_converters_build_network.csv", + transformers="resources/" + + RDIR + + "base_network/all_transformers_build_network.csv", + substations="resources/" + RDIR + "base_network/all_buses_build_network.csv", + log: + "logs/" + RDIR + "build_osm_network.log", + benchmark: + "benchmarks/" + RDIR + "build_osm_network" + script: + "scripts/build_osm_network.py" rule build_shapes: @@ -447,41 +447,41 @@ rule build_demand_profiles: "scripts/build_demand_profiles.py" -# rule build_renewable_profiles: -# params: -# crs=config["crs"], -# renewable=config["renewable"], -# countries=config["countries"], -# alternative_clustering=config["cluster_options"]["alternative_clustering"], -# input: -# natura="resources/" + RDIR + "natura.tiff", -# copernicus="data/copernicus/PROBAV_LC100_global_v3.0.1_2019-nrt_Discrete-Classification-map_EPSG-4326.tif", -# gebco="data/gebco/GEBCO_2021_TID.nc", -# country_shapes="resources/" + RDIR + "shapes/country_shapes.geojson", -# offshore_shapes="resources/" + RDIR + "shapes/offshore_shapes.geojson", -# hydro_capacities="data/hydro_capacities.csv", -# eia_hydro_generation="data/eia_hydro_annual_generation.csv", -# powerplants="resources/" + RDIR + "powerplants.csv", -# regions=lambda w: ( -# "resources/" + RDIR + "bus_regions/regions_onshore.geojson" -# if w.technology in ("onwind", "solar", "hydro", "csp") -# else "resources/" + RDIR + "bus_regions/regions_offshore.geojson" -# ), -# cutout=lambda w: "cutouts/" -# + CDIR -# + config["renewable"][w.technology]["cutout"] -# + ".nc", -# output: -# profile="resources/" + RDIR + "renewable_profiles/profile_{technology}.nc", -# log: -# "logs/" + RDIR + "build_renewable_profile_{technology}.log", -# benchmark: -# "benchmarks/" + RDIR + "build_renewable_profiles_{technology}" -# threads: ATLITE_NPROCESSES -# resources: -# mem_mb=ATLITE_NPROCESSES * 5000, -# script: -# "scripts/build_renewable_profiles.py" +rule build_renewable_profiles: + params: + crs=config["crs"], + renewable=config["renewable"], + countries=config["countries"], + alternative_clustering=config["cluster_options"]["alternative_clustering"], + input: + natura="resources/" + RDIR + "natura.tiff", + copernicus="data/copernicus/PROBAV_LC100_global_v3.0.1_2019-nrt_Discrete-Classification-map_EPSG-4326.tif", + gebco="data/gebco/GEBCO_2021_TID.nc", + country_shapes="resources/" + RDIR + "shapes/country_shapes.geojson", + offshore_shapes="resources/" + RDIR + "shapes/offshore_shapes.geojson", + hydro_capacities="data/hydro_capacities.csv", + eia_hydro_generation="data/eia_hydro_annual_generation.csv", + powerplants="resources/" + RDIR + "powerplants.csv", + regions=lambda w: ( + "resources/" + RDIR + "bus_regions/regions_onshore.geojson" + if w.technology in ("onwind", "solar", "hydro", "csp") + else "resources/" + RDIR + "bus_regions/regions_offshore.geojson" + ), + cutout=lambda w: "cutouts/" + + CDIR + + config["renewable"][w.technology]["cutout"] + + ".nc", + output: + profile="resources/" + RDIR + "renewable_profiles/profile_{technology}.nc", + log: + "logs/" + RDIR + "build_renewable_profile_{technology}.log", + benchmark: + "benchmarks/" + RDIR + "build_renewable_profiles_{technology}" + threads: ATLITE_NPROCESSES + resources: + mem_mb=ATLITE_NPROCESSES * 5000, + script: + "scripts/build_renewable_profiles.py" rule build_powerplants: From 8587ec9f8ab3a23308ec5a553c4e224eddd1b56c Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Thu, 7 Nov 2024 12:21:29 +0000 Subject: [PATCH 03/10] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- scripts/add_export.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/scripts/add_export.py b/scripts/add_export.py index 2d0cb03ea..388b7490b 100644 --- a/scripts/add_export.py +++ b/scripts/add_export.py @@ -39,8 +39,8 @@ def select_ports(n): keep_default_na=False, ).squeeze() - ports = raw_ports[['name', 'country', 'fraction', 'x', 'y']] - ports.loc[:,'fraction'] = ports.fraction.round(1) + ports = raw_ports[["name", "country", "fraction", "x", "y"]] + ports.loc[:, "fraction"] = ports.fraction.round(1) ports = ports[ports.country.isin(countries)] if len(ports) < 1: @@ -229,4 +229,4 @@ def create_export_profile(): n.export_to_netcdf(snakemake.output[0]) - logger.info("Network successfully exported") \ No newline at end of file + logger.info("Network successfully exported") From 3018d0fa358a09ab0b73b96d527575a1b01bd699 Mon Sep 17 00:00:00 2001 From: Emmanuel Bolarinwa Date: Fri, 8 Nov 2024 18:10:01 +0100 Subject: [PATCH 04/10] include changes in release note --- doc/release_notes.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/doc/release_notes.rst b/doc/release_notes.rst index 0d1b7c746..02424a88c 100644 --- a/doc/release_notes.rst +++ b/doc/release_notes.rst @@ -42,6 +42,8 @@ E.g. if a new rule becomes available describe how to use it `make test` and in o * Fix bugs in `prepare_sector_network.py` related to links with H2 buses and bug of re-addition of H2 and battery carriers in present `PR #1145 `_ +* Revise ports data for export in `add_export.py` related to sector model `PR #1175 `_ + PyPSA-Earth 0.4.1 ================= From ad826c40720261c9b170795b0ce53b1bf6cb7f8d Mon Sep 17 00:00:00 2001 From: Emmanuel Bolarinwa Date: Wed, 20 Nov 2024 07:04:15 +0100 Subject: [PATCH 05/10] added option to use custom data for ports --- Snakefile | 121 +++++++++++++++-------------- config.default.yaml | 1 + data/{ => custom}/export_ports.csv | 0 scripts/add_export.py | 13 +++- scripts/prepare_ports.py | 33 ++++++++ test/config.test_myopic.yaml | 1 + 6 files changed, 106 insertions(+), 63 deletions(-) rename data/{ => custom}/export_ports.csv (100%) diff --git a/Snakefile b/Snakefile index 2840bd242..caad93326 100644 --- a/Snakefile +++ b/Snakefile @@ -198,29 +198,29 @@ rule clean_osm_data: "scripts/clean_osm_data.py" -rule build_osm_network: - params: - build_osm_network=config.get("build_osm_network", {}), - countries=config["countries"], - crs=config["crs"], - input: - generators="resources/" + RDIR + "osm/clean/all_clean_generators.geojson", - lines="resources/" + RDIR + "osm/clean/all_clean_lines.geojson", - substations="resources/" + RDIR + "osm/clean/all_clean_substations.geojson", - country_shapes="resources/" + RDIR + "shapes/country_shapes.geojson", - output: - lines="resources/" + RDIR + "base_network/all_lines_build_network.csv", - converters="resources/" + RDIR + "base_network/all_converters_build_network.csv", - transformers="resources/" - + RDIR - + "base_network/all_transformers_build_network.csv", - substations="resources/" + RDIR + "base_network/all_buses_build_network.csv", - log: - "logs/" + RDIR + "build_osm_network.log", - benchmark: - "benchmarks/" + RDIR + "build_osm_network" - script: - "scripts/build_osm_network.py" +# rule build_osm_network: +# params: +# build_osm_network=config.get("build_osm_network", {}), +# countries=config["countries"], +# crs=config["crs"], +# input: +# generators="resources/" + RDIR + "osm/clean/all_clean_generators.geojson", +# lines="resources/" + RDIR + "osm/clean/all_clean_lines.geojson", +# substations="resources/" + RDIR + "osm/clean/all_clean_substations.geojson", +# country_shapes="resources/" + RDIR + "shapes/country_shapes.geojson", +# output: +# lines="resources/" + RDIR + "base_network/all_lines_build_network.csv", +# converters="resources/" + RDIR + "base_network/all_converters_build_network.csv", +# transformers="resources/" +# + RDIR +# + "base_network/all_transformers_build_network.csv", +# substations="resources/" + RDIR + "base_network/all_buses_build_network.csv", +# log: +# "logs/" + RDIR + "build_osm_network.log", +# benchmark: +# "benchmarks/" + RDIR + "build_osm_network" +# script: +# "scripts/build_osm_network.py" rule build_shapes: @@ -448,41 +448,41 @@ rule build_demand_profiles: "scripts/build_demand_profiles.py" -rule build_renewable_profiles: - params: - crs=config["crs"], - renewable=config["renewable"], - countries=config["countries"], - alternative_clustering=config["cluster_options"]["alternative_clustering"], - input: - natura="resources/" + RDIR + "natura.tiff", - copernicus="data/copernicus/PROBAV_LC100_global_v3.0.1_2019-nrt_Discrete-Classification-map_EPSG-4326.tif", - gebco="data/gebco/GEBCO_2021_TID.nc", - country_shapes="resources/" + RDIR + "shapes/country_shapes.geojson", - offshore_shapes="resources/" + RDIR + "shapes/offshore_shapes.geojson", - hydro_capacities="data/hydro_capacities.csv", - eia_hydro_generation="data/eia_hydro_annual_generation.csv", - powerplants="resources/" + RDIR + "powerplants.csv", - regions=lambda w: ( - "resources/" + RDIR + "bus_regions/regions_onshore.geojson" - if w.technology in ("onwind", "solar", "hydro", "csp") - else "resources/" + RDIR + "bus_regions/regions_offshore.geojson" - ), - cutout=lambda w: "cutouts/" - + CDIR - + config["renewable"][w.technology]["cutout"] - + ".nc", - output: - profile="resources/" + RDIR + "renewable_profiles/profile_{technology}.nc", - log: - "logs/" + RDIR + "build_renewable_profile_{technology}.log", - benchmark: - "benchmarks/" + RDIR + "build_renewable_profiles_{technology}" - threads: ATLITE_NPROCESSES - resources: - mem_mb=ATLITE_NPROCESSES * 5000, - script: - "scripts/build_renewable_profiles.py" +# rule build_renewable_profiles: +# params: +# crs=config["crs"], +# renewable=config["renewable"], +# countries=config["countries"], +# alternative_clustering=config["cluster_options"]["alternative_clustering"], +# input: +# natura="resources/" + RDIR + "natura.tiff", +# copernicus="data/copernicus/PROBAV_LC100_global_v3.0.1_2019-nrt_Discrete-Classification-map_EPSG-4326.tif", +# gebco="data/gebco/GEBCO_2021_TID.nc", +# country_shapes="resources/" + RDIR + "shapes/country_shapes.geojson", +# offshore_shapes="resources/" + RDIR + "shapes/offshore_shapes.geojson", +# hydro_capacities="data/hydro_capacities.csv", +# eia_hydro_generation="data/eia_hydro_annual_generation.csv", +# powerplants="resources/" + RDIR + "powerplants.csv", +# regions=lambda w: ( +# "resources/" + RDIR + "bus_regions/regions_onshore.geojson" +# if w.technology in ("onwind", "solar", "hydro", "csp") +# else "resources/" + RDIR + "bus_regions/regions_offshore.geojson" +# ), +# cutout=lambda w: "cutouts/" +# + CDIR +# + config["renewable"][w.technology]["cutout"] +# + ".nc", +# output: +# profile="resources/" + RDIR + "renewable_profiles/profile_{technology}.nc", +# log: +# "logs/" + RDIR + "build_renewable_profile_{technology}.log", +# benchmark: +# "benchmarks/" + RDIR + "build_renewable_profiles_{technology}" +# threads: ATLITE_NPROCESSES +# resources: +# mem_mb=ATLITE_NPROCESSES * 5000, +# script: +# "scripts/build_renewable_profiles.py" rule build_powerplants: @@ -1003,6 +1003,7 @@ rule solve_sector_networks: rule prepare_ports: output: ports="data/ports.csv", # TODO move from data to resources + export_ports="resources/" + SECDIR + "export_ports.csv", script: "scripts/prepare_ports.py" @@ -1154,9 +1155,11 @@ rule add_export: export_profile=config["export"]["export_profile"], snapshots=config["snapshots"], costs=config["costs"], + custom_export=config["custom_data"]["export_data"], input: overrides="data/override_component_attrs", - export_ports="data/ports.csv", + custom_export_ports="data/custom/export_ports.csv", + export_ports="resources/" + SECDIR + "export_ports.csv", costs=COSTDIR + "costs_{planning_horizons}.csv", ship_profile="resources/" + SECDIR + "ship_profile_{h2export}TWh.csv", network=RESDIR diff --git a/config.default.yaml b/config.default.yaml index 19f26d96c..0e3116a06 100644 --- a/config.default.yaml +++ b/config.default.yaml @@ -478,6 +478,7 @@ custom_data: add_existing: false custom_sectors: false gas_network: false # If "True" then a custom .csv file must be placed in "resources/custom_data/pipelines.csv" , If "False" the user can choose btw "greenfield" or Model built-in datasets. Please refer to ["sector"] below. + export_data: false # If "True" then a custom .csv file must be placed in "data/custom/export_ports.csv" industry: reference_year: 2015 diff --git a/data/export_ports.csv b/data/custom/export_ports.csv similarity index 100% rename from data/export_ports.csv rename to data/custom/export_ports.csv diff --git a/scripts/add_export.py b/scripts/add_export.py index 388b7490b..cf1e5027d 100644 --- a/scripts/add_export.py +++ b/scripts/add_export.py @@ -33,14 +33,19 @@ def select_ports(n): This function selects the buses where ports are located. """ - raw_ports = pd.read_csv( - snakemake.input.export_ports, + if snakemake.params.custom_export: + input_port_data = snakemake.input.custom_export_ports + else: + input_port_data = snakemake.input.export_ports + + ports = pd.read_csv( + input_port_data, index_col=None, keep_default_na=False, ).squeeze() - ports = raw_ports[["name", "country", "fraction", "x", "y"]] - ports.loc[:, "fraction"] = ports.fraction.round(1) + # ports = raw_ports[["name", "country", "fraction", "x", "y"]] + # ports.loc[:, "fraction"] = ports.fraction.round(1) ports = ports[ports.country.isin(countries)] if len(ports) < 1: diff --git a/scripts/prepare_ports.py b/scripts/prepare_ports.py index c1c0e9716..665653724 100644 --- a/scripts/prepare_ports.py +++ b/scripts/prepare_ports.py @@ -31,6 +31,36 @@ def download_ports(): return wpi_csv +def filter_ports(dataframe): + """ + Filters ports based on their harbor size and returns a DataFrame containing + only the largest port for each country. + """ + # Filter large sized ports + large_ports = dataframe[dataframe["Harbor Size"] == "Large"] + countries_with_large_ports = large_ports["country"].unique() + + # Filter out countries with large ports + remaining_ports = dataframe[~dataframe["country"].isin(countries_with_large_ports)] + + # Filter medium sized ports from remaining ports + medium_ports = remaining_ports[remaining_ports["Harbor Size"] == "Medium"] + countries_with_medium_ports = medium_ports["country"].unique() + + # Filter out countries with medium ports + remaining_ports = remaining_ports[ + ~remaining_ports["country"].isin(countries_with_medium_ports) + ] + + # Filter small sized ports from remaining ports + small_ports = remaining_ports[remaining_ports["Harbor Size"] == "Small"] + + # Combine all filtered ports + filtered_ports = pd.concat([large_ports, medium_ports, small_ports]) + + return filtered_ports + + if __name__ == "__main__": if "snakemake" not in globals(): from _helpers import mock_snakemake @@ -102,3 +132,6 @@ def download_ports(): ports["fraction"] = ports["Harbor_size_nr"] / ports["Total_Harbor_size_nr"] ports.to_csv(snakemake.output[0], sep=",", encoding="utf-8", header="true") + filter_ports(ports).to_csv( + snakemake.output[1], sep=",", encoding="utf-8", header="true" + ) diff --git a/test/config.test_myopic.yaml b/test/config.test_myopic.yaml index dc9dcdecf..cad5f53f5 100644 --- a/test/config.test_myopic.yaml +++ b/test/config.test_myopic.yaml @@ -91,6 +91,7 @@ custom_data: add_existing: false custom_sectors: false gas_network: false # If "True" then a custom .csv file must be placed in "resources/custom_data/pipelines.csv" , If "False" the user can choose btw "greenfield" or Model built-in datasets. Please refer to ["sector"] below. + export_data: false # If "True" then a custom .csv file must be placed in "data/custom/export.csv" costs: # Costs used in PyPSA-Earth-Sec. Year depends on the wildcard planning_horizon in the scenario section From 45fa5f3a9ec8b07bff33e80082a1d4c312647848 Mon Sep 17 00:00:00 2001 From: Emmanuel Bolarinwa Date: Wed, 20 Nov 2024 11:12:48 +0100 Subject: [PATCH 06/10] remove comment from rules --- Snakefile | 116 +++++++++++++++++++++++++++--------------------------- 1 file changed, 58 insertions(+), 58 deletions(-) diff --git a/Snakefile b/Snakefile index caad93326..e398831a9 100644 --- a/Snakefile +++ b/Snakefile @@ -198,29 +198,29 @@ rule clean_osm_data: "scripts/clean_osm_data.py" -# rule build_osm_network: -# params: -# build_osm_network=config.get("build_osm_network", {}), -# countries=config["countries"], -# crs=config["crs"], -# input: -# generators="resources/" + RDIR + "osm/clean/all_clean_generators.geojson", -# lines="resources/" + RDIR + "osm/clean/all_clean_lines.geojson", -# substations="resources/" + RDIR + "osm/clean/all_clean_substations.geojson", -# country_shapes="resources/" + RDIR + "shapes/country_shapes.geojson", -# output: -# lines="resources/" + RDIR + "base_network/all_lines_build_network.csv", -# converters="resources/" + RDIR + "base_network/all_converters_build_network.csv", -# transformers="resources/" -# + RDIR -# + "base_network/all_transformers_build_network.csv", -# substations="resources/" + RDIR + "base_network/all_buses_build_network.csv", -# log: -# "logs/" + RDIR + "build_osm_network.log", -# benchmark: -# "benchmarks/" + RDIR + "build_osm_network" -# script: -# "scripts/build_osm_network.py" +rule build_osm_network: + params: + build_osm_network=config.get("build_osm_network", {}), + countries=config["countries"], + crs=config["crs"], + input: + generators="resources/" + RDIR + "osm/clean/all_clean_generators.geojson", + lines="resources/" + RDIR + "osm/clean/all_clean_lines.geojson", + substations="resources/" + RDIR + "osm/clean/all_clean_substations.geojson", + country_shapes="resources/" + RDIR + "shapes/country_shapes.geojson", + output: + lines="resources/" + RDIR + "base_network/all_lines_build_network.csv", + converters="resources/" + RDIR + "base_network/all_converters_build_network.csv", + transformers="resources/" + + RDIR + + "base_network/all_transformers_build_network.csv", + substations="resources/" + RDIR + "base_network/all_buses_build_network.csv", + log: + "logs/" + RDIR + "build_osm_network.log", + benchmark: + "benchmarks/" + RDIR + "build_osm_network" + script: + "scripts/build_osm_network.py" rule build_shapes: @@ -448,41 +448,41 @@ rule build_demand_profiles: "scripts/build_demand_profiles.py" -# rule build_renewable_profiles: -# params: -# crs=config["crs"], -# renewable=config["renewable"], -# countries=config["countries"], -# alternative_clustering=config["cluster_options"]["alternative_clustering"], -# input: -# natura="resources/" + RDIR + "natura.tiff", -# copernicus="data/copernicus/PROBAV_LC100_global_v3.0.1_2019-nrt_Discrete-Classification-map_EPSG-4326.tif", -# gebco="data/gebco/GEBCO_2021_TID.nc", -# country_shapes="resources/" + RDIR + "shapes/country_shapes.geojson", -# offshore_shapes="resources/" + RDIR + "shapes/offshore_shapes.geojson", -# hydro_capacities="data/hydro_capacities.csv", -# eia_hydro_generation="data/eia_hydro_annual_generation.csv", -# powerplants="resources/" + RDIR + "powerplants.csv", -# regions=lambda w: ( -# "resources/" + RDIR + "bus_regions/regions_onshore.geojson" -# if w.technology in ("onwind", "solar", "hydro", "csp") -# else "resources/" + RDIR + "bus_regions/regions_offshore.geojson" -# ), -# cutout=lambda w: "cutouts/" -# + CDIR -# + config["renewable"][w.technology]["cutout"] -# + ".nc", -# output: -# profile="resources/" + RDIR + "renewable_profiles/profile_{technology}.nc", -# log: -# "logs/" + RDIR + "build_renewable_profile_{technology}.log", -# benchmark: -# "benchmarks/" + RDIR + "build_renewable_profiles_{technology}" -# threads: ATLITE_NPROCESSES -# resources: -# mem_mb=ATLITE_NPROCESSES * 5000, -# script: -# "scripts/build_renewable_profiles.py" +rule build_renewable_profiles: + params: + crs=config["crs"], + renewable=config["renewable"], + countries=config["countries"], + alternative_clustering=config["cluster_options"]["alternative_clustering"], + input: + natura="resources/" + RDIR + "natura.tiff", + copernicus="data/copernicus/PROBAV_LC100_global_v3.0.1_2019-nrt_Discrete-Classification-map_EPSG-4326.tif", + gebco="data/gebco/GEBCO_2021_TID.nc", + country_shapes="resources/" + RDIR + "shapes/country_shapes.geojson", + offshore_shapes="resources/" + RDIR + "shapes/offshore_shapes.geojson", + hydro_capacities="data/hydro_capacities.csv", + eia_hydro_generation="data/eia_hydro_annual_generation.csv", + powerplants="resources/" + RDIR + "powerplants.csv", + regions=lambda w: ( + "resources/" + RDIR + "bus_regions/regions_onshore.geojson" + if w.technology in ("onwind", "solar", "hydro", "csp") + else "resources/" + RDIR + "bus_regions/regions_offshore.geojson" + ), + cutout=lambda w: "cutouts/" + + CDIR + + config["renewable"][w.technology]["cutout"] + + ".nc", + output: + profile="resources/" + RDIR + "renewable_profiles/profile_{technology}.nc", + log: + "logs/" + RDIR + "build_renewable_profile_{technology}.log", + benchmark: + "benchmarks/" + RDIR + "build_renewable_profiles_{technology}" + threads: ATLITE_NPROCESSES + resources: + mem_mb=ATLITE_NPROCESSES * 5000, + script: + "scripts/build_renewable_profiles.py" rule build_powerplants: From a33aa89489cf7281cda988677d34e24c2fdb4fe9 Mon Sep 17 00:00:00 2001 From: Emmanuel Bolarinwa Date: Fri, 22 Nov 2024 14:59:37 +0100 Subject: [PATCH 07/10] rename export variable --- config.default.yaml | 2 +- test/config.test_myopic.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/config.default.yaml b/config.default.yaml index 0e3116a06..8c259c1f3 100644 --- a/config.default.yaml +++ b/config.default.yaml @@ -478,7 +478,7 @@ custom_data: add_existing: false custom_sectors: false gas_network: false # If "True" then a custom .csv file must be placed in "resources/custom_data/pipelines.csv" , If "False" the user can choose btw "greenfield" or Model built-in datasets. Please refer to ["sector"] below. - export_data: false # If "True" then a custom .csv file must be placed in "data/custom/export_ports.csv" + export_ports: false # If "True" then a custom .csv file must be placed in "data/custom/export_ports.csv" industry: reference_year: 2015 diff --git a/test/config.test_myopic.yaml b/test/config.test_myopic.yaml index cad5f53f5..a0e7a2ae3 100644 --- a/test/config.test_myopic.yaml +++ b/test/config.test_myopic.yaml @@ -91,7 +91,7 @@ custom_data: add_existing: false custom_sectors: false gas_network: false # If "True" then a custom .csv file must be placed in "resources/custom_data/pipelines.csv" , If "False" the user can choose btw "greenfield" or Model built-in datasets. Please refer to ["sector"] below. - export_data: false # If "True" then a custom .csv file must be placed in "data/custom/export.csv" + export_ports: false # If "True" then a custom .csv file must be placed in "data/custom/export.csv" costs: # Costs used in PyPSA-Earth-Sec. Year depends on the wildcard planning_horizon in the scenario section From 7a271a8c24e8d3ad6d7407e0113c8b5d7c2c2241 Mon Sep 17 00:00:00 2001 From: Emmanuel Bolarinwa Date: Fri, 22 Nov 2024 15:55:59 +0100 Subject: [PATCH 08/10] move ports data to resources --- Snakefile | 4 ++-- scripts/add_export.py | 10 +++------- scripts/prepare_ports.py | 13 ++++++++++--- 3 files changed, 15 insertions(+), 12 deletions(-) diff --git a/Snakefile b/Snakefile index e398831a9..e26115343 100644 --- a/Snakefile +++ b/Snakefile @@ -1001,6 +1001,8 @@ rule solve_sector_networks: rule prepare_ports: + params: + custom_export=config["custom_data"]["export_data"], output: ports="data/ports.csv", # TODO move from data to resources export_ports="resources/" + SECDIR + "export_ports.csv", @@ -1155,10 +1157,8 @@ rule add_export: export_profile=config["export"]["export_profile"], snapshots=config["snapshots"], costs=config["costs"], - custom_export=config["custom_data"]["export_data"], input: overrides="data/override_component_attrs", - custom_export_ports="data/custom/export_ports.csv", export_ports="resources/" + SECDIR + "export_ports.csv", costs=COSTDIR + "costs_{planning_horizons}.csv", ship_profile="resources/" + SECDIR + "ship_profile_{h2export}TWh.csv", diff --git a/scripts/add_export.py b/scripts/add_export.py index cf1e5027d..bf0c31a77 100644 --- a/scripts/add_export.py +++ b/scripts/add_export.py @@ -33,13 +33,8 @@ def select_ports(n): This function selects the buses where ports are located. """ - if snakemake.params.custom_export: - input_port_data = snakemake.input.custom_export_ports - else: - input_port_data = snakemake.input.export_ports - ports = pd.read_csv( - input_port_data, + snakemake.input.export_ports, index_col=None, keep_default_na=False, ).squeeze() @@ -155,7 +150,8 @@ def create_export_profile(): and resamples it to temp resolution obtained from the wildcard. """ - export_h2 = eval(snakemake.wildcards["h2export"]) * 1e6 # convert TWh to MWh + # convert TWh to MWh + export_h2 = eval(snakemake.wildcards["h2export"]) * 1e6 if snakemake.params.export_profile == "constant": export_profile = export_h2 / 8760 diff --git a/scripts/prepare_ports.py b/scripts/prepare_ports.py index 665653724..b5c86280e 100644 --- a/scripts/prepare_ports.py +++ b/scripts/prepare_ports.py @@ -4,6 +4,7 @@ # SPDX-License-Identifier: AGPL-3.0-or-later import logging import os +import shutil from pathlib import Path import country_converter as coco @@ -67,6 +68,7 @@ def filter_ports(dataframe): snakemake = mock_snakemake("prepare_ports") + config = snakemake.config # configure_logging(snakemake) # run = snakemake.config.get("run", {}) @@ -132,6 +134,11 @@ def filter_ports(dataframe): ports["fraction"] = ports["Harbor_size_nr"] / ports["Total_Harbor_size_nr"] ports.to_csv(snakemake.output[0], sep=",", encoding="utf-8", header="true") - filter_ports(ports).to_csv( - snakemake.output[1], sep=",", encoding="utf-8", header="true" - ) + + if snakemake.params.custom_ports: + custom_export_path = Path.joinpath("data", "custom", "export_ports.csv") + shutil.move(custom_export_path, snakemake.output[1]) + else: + filter_ports(ports).to_csv( + snakemake.output[1], sep=",", encoding="utf-8", header="true" + ) From 2b93191276b5d9a87806d5de0a54c21784f32f5d Mon Sep 17 00:00:00 2001 From: Emmanuel Bolarinwa Date: Fri, 22 Nov 2024 15:59:50 +0100 Subject: [PATCH 09/10] rename ports config --- Snakefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Snakefile b/Snakefile index e26115343..71a18ea83 100644 --- a/Snakefile +++ b/Snakefile @@ -1002,7 +1002,7 @@ rule solve_sector_networks: rule prepare_ports: params: - custom_export=config["custom_data"]["export_data"], + custom_export=config["custom_data"]["export_ports"], output: ports="data/ports.csv", # TODO move from data to resources export_ports="resources/" + SECDIR + "export_ports.csv", From 66d04d687fd91fbc5921732187347ff2e648b021 Mon Sep 17 00:00:00 2001 From: Emmanuel Bolarinwa Date: Fri, 22 Nov 2024 16:10:54 +0100 Subject: [PATCH 10/10] rename config param --- scripts/prepare_ports.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/prepare_ports.py b/scripts/prepare_ports.py index b5c86280e..58ef92f1f 100644 --- a/scripts/prepare_ports.py +++ b/scripts/prepare_ports.py @@ -135,7 +135,7 @@ def filter_ports(dataframe): ports.to_csv(snakemake.output[0], sep=",", encoding="utf-8", header="true") - if snakemake.params.custom_ports: + if snakemake.params.custom_export: custom_export_path = Path.joinpath("data", "custom", "export_ports.csv") shutil.move(custom_export_path, snakemake.output[1]) else: