From 0496d72462770a230d1d9bcbb0697a2bf1a916d3 Mon Sep 17 00:00:00 2001 From: yerbol-akhmetov Date: Sun, 27 Oct 2024 22:23:55 +0500 Subject: [PATCH 01/11] integrate RDIR for sector scripts --- Snakefile | 216 +++++++++++++++++++++++++++--------------------------- 1 file changed, 108 insertions(+), 108 deletions(-) diff --git a/Snakefile b/Snakefile index 218041c67..cfaba8693 100644 --- a/Snakefile +++ b/Snakefile @@ -1024,7 +1024,7 @@ rule prepare_urban_percent: rule prepare_transport_data_input: output: - transport_data_input="resources/transport_data.csv", + transport_data_input="resources/" + RDIR + "transport_data.csv", script: "scripts/prepare_transport_data_input.py" @@ -1049,7 +1049,7 @@ if not config["custom_data"]["gas_network"]: + RDIR + "bus_regions/regions_onshore_elec_s{simpl}_{clusters}.geojson", output: - clustered_gas_network="resources/gas_networks/gas_network_elec_s{simpl}_{clusters}.csv", + clustered_gas_network="resources/" + RDIR + "gas_networks/gas_network_elec_s{simpl}_{clusters}.csv", # TODO: Should be a own snakemake rule # gas_network_fig_1="resources/gas_networks/existing_gas_pipelines_{simpl}_{clusters}.png", # gas_network_fig_2="resources/gas_networks/clustered_gas_pipelines_{simpl}_{clusters}.png", @@ -1065,22 +1065,22 @@ rule prepare_sector_network: + "prenetworks/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}_{sopts}_{planning_horizons}_{discountrate}_{demand}_presec.nc", costs=COSTDIR + "costs_{planning_horizons}.csv", h2_cavern="data/hydrogen_salt_cavern_potentials.csv", - nodal_energy_totals="resources/demand/heat/nodal_energy_heat_totals_{demand}_s{simpl}_{clusters}_{planning_horizons}.csv", - transport="resources/demand/transport_{demand}_s{simpl}_{clusters}_{planning_horizons}.csv", - avail_profile="resources/pattern_profiles/avail_profile_{demand}_s{simpl}_{clusters}_{planning_horizons}.csv", - dsm_profile="resources/pattern_profiles/dsm_profile_{demand}_s{simpl}_{clusters}_{planning_horizons}.csv", - nodal_transport_data="resources/demand/nodal_transport_data_{demand}_s{simpl}_{clusters}_{planning_horizons}.csv", + nodal_energy_totals="resources/" + RDIR + "demand/heat/nodal_energy_heat_totals_{demand}_s{simpl}_{clusters}_{planning_horizons}.csv", + transport="resources/" + RDIR + "demand/transport_{demand}_s{simpl}_{clusters}_{planning_horizons}.csv", + avail_profile="resources/" + RDIR + "pattern_profiles/avail_profile_{demand}_s{simpl}_{clusters}_{planning_horizons}.csv", + dsm_profile="resources/" + RDIR + "pattern_profiles/dsm_profile_{demand}_s{simpl}_{clusters}_{planning_horizons}.csv", + nodal_transport_data="resources/" + RDIR + "demand/nodal_transport_data_{demand}_s{simpl}_{clusters}_{planning_horizons}.csv", overrides="data/override_component_attrs", - clustered_pop_layout="resources/population_shares/pop_layout_elec_s{simpl}_{clusters}_{planning_horizons}.csv", - industrial_demand="resources/demand/industrial_energy_demand_per_node_elec_s{simpl}_{clusters}_{planning_horizons}_{demand}.csv", + clustered_pop_layout="resources/" + RDIR + "population_shares/pop_layout_elec_s{simpl}_{clusters}_{planning_horizons}.csv", + industrial_demand="resources/" + RDIR + "demand/industrial_energy_demand_per_node_elec_s{simpl}_{clusters}_{planning_horizons}_{demand}.csv", energy_totals="data/energy_totals_{demand}_{planning_horizons}.csv", airports="data/airports.csv", ports="data/ports.csv", - heat_demand="resources/demand/heat/heat_demand_{demand}_s{simpl}_{clusters}_{planning_horizons}.csv", - ashp_cop="resources/demand/heat/ashp_cop_{demand}_s{simpl}_{clusters}_{planning_horizons}.csv", - gshp_cop="resources/demand/heat/gshp_cop_{demand}_s{simpl}_{clusters}_{planning_horizons}.csv", - solar_thermal="resources/demand/heat/solar_thermal_{demand}_s{simpl}_{clusters}_{planning_horizons}.csv", - district_heat_share="resources/demand/heat/district_heat_share_{demand}_s{simpl}_{clusters}_{planning_horizons}.csv", + heat_demand="resources/" + RDIR + "demand/heat/heat_demand_{demand}_s{simpl}_{clusters}_{planning_horizons}.csv", + ashp_cop="resources/" + RDIR + "demand/heat/ashp_cop_{demand}_s{simpl}_{clusters}_{planning_horizons}.csv", + gshp_cop="resources/" + RDIR + "demand/heat/gshp_cop_{demand}_s{simpl}_{clusters}_{planning_horizons}.csv", + solar_thermal="resources/" + RDIR + "demand/heat/solar_thermal_{demand}_s{simpl}_{clusters}_{planning_horizons}.csv", + district_heat_share="resources/" + RDIR + "demand/heat/district_heat_share_{demand}_s{simpl}_{clusters}_{planning_horizons}.csv", biomass_transport_costs="data/temp_hard_coded/biomass_transport_costs.csv", shapes_path="resources/" + RDIR @@ -1088,7 +1088,7 @@ rule prepare_sector_network: pipelines=( "data/custom/pipelines.csv" if config["custom_data"]["gas_network"] - else "resources/gas_networks/gas_network_elec_s{simpl}_{clusters}.csv" + else "resources/" + RDIR + "gas_networks/gas_network_elec_s{simpl}_{clusters}.csv" ), output: RESDIR @@ -1110,7 +1110,7 @@ rule build_ship_profile: snapshots=config["snapshots"], ship_opts=config["export"]["ship"], output: - ship_profile="resources/ship_profile_{h2export}TWh.csv", + ship_profile="resources/" + RDIR + "ship_profile_{h2export}TWh.csv", script: "scripts/build_ship_profile.py" @@ -1128,7 +1128,7 @@ rule add_export: overrides="data/override_component_attrs", export_ports="data/export_ports.csv", costs=COSTDIR + "costs_{planning_horizons}.csv", - ship_profile="resources/ship_profile_{h2export}TWh.csv", + ship_profile="resources/" + RDIR + "ship_profile_{h2export}TWh.csv", network=RESDIR + "prenetworks/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}_{sopts}_{planning_horizons}_{discountrate}_{demand}.nc", shapes_path="resources/" @@ -1148,13 +1148,13 @@ rule override_respot: countries=config["countries"], input: **{ - f"custom_res_pot_{tech}_{planning_horizons}_{discountrate}": f"resources/custom_renewables/{tech}_{planning_horizons}_{discountrate}_potential.csv" + f"custom_res_pot_{tech}_{planning_horizons}_{discountrate}": "resources/" + RDIR + f"custom_renewables/{tech}_{planning_horizons}_{discountrate}_potential.csv" for tech in config["custom_data"]["renewables"] for discountrate in config["costs"]["discountrate"] for planning_horizons in config["scenario"]["planning_horizons"] }, **{ - f"custom_res_ins_{tech}_{planning_horizons}_{discountrate}": f"resources/custom_renewables/{tech}_{planning_horizons}_{discountrate}_installable.csv" + f"custom_res_ins_{tech}_{planning_horizons}_{discountrate}": "resources/" + RDIR + f"custom_renewables/{tech}_{planning_horizons}_{discountrate}_installable.csv" for tech in config["custom_data"]["renewables"] for discountrate in config["costs"]["discountrate"] for planning_horizons in config["scenario"]["planning_horizons"] @@ -1175,15 +1175,15 @@ rule prepare_transport_data: energy_totals_name="data/energy_totals_{demand}_{planning_horizons}.csv", traffic_data_KFZ="data/emobility/KFZ__count", traffic_data_Pkw="data/emobility/Pkw__count", - transport_name="resources/transport_data.csv", - clustered_pop_layout="resources/population_shares/pop_layout_elec_s{simpl}_{clusters}_{planning_horizons}.csv", - temp_air_total="resources/temperatures/temp_air_total_elec_s{simpl}_{clusters}_{planning_horizons}.nc", + transport_name="resources/" + RDIR + "transport_data.csv", + clustered_pop_layout="resources/" + RDIR + "population_shares/pop_layout_elec_s{simpl}_{clusters}_{planning_horizons}.csv", + temp_air_total="resources/" + RDIR + "temperatures/temp_air_total_elec_s{simpl}_{clusters}_{planning_horizons}.nc", output: # nodal_energy_totals="resources/nodal_energy_totals_s{simpl}_{clusters}.csv", - transport="resources/demand/transport_{demand}_s{simpl}_{clusters}_{planning_horizons}.csv", - avail_profile="resources/pattern_profiles/avail_profile_{demand}_s{simpl}_{clusters}_{planning_horizons}.csv", - dsm_profile="resources/pattern_profiles/dsm_profile_{demand}_s{simpl}_{clusters}_{planning_horizons}.csv", - nodal_transport_data="resources/demand/nodal_transport_data_{demand}_s{simpl}_{clusters}_{planning_horizons}.csv", + transport="resources/" + RDIR + "demand/transport_{demand}_s{simpl}_{clusters}_{planning_horizons}.csv", + avail_profile="resources/" + RDIR + "pattern_profiles/avail_profile_{demand}_s{simpl}_{clusters}_{planning_horizons}.csv", + dsm_profile="resources/" + RDIR + "pattern_profiles/dsm_profile_{demand}_s{simpl}_{clusters}_{planning_horizons}.csv", + nodal_transport_data="resources/" + RDIR + "demand/nodal_transport_data_{demand}_s{simpl}_{clusters}_{planning_horizons}.csv", script: "scripts/prepare_transport_data.py" @@ -1192,23 +1192,23 @@ rule build_cop_profiles: params: heat_pump_sink_T=config["sector"]["heat_pump_sink_T"], input: - temp_soil_total="resources/temperatures/temp_soil_total_elec_s{simpl}_{clusters}_{planning_horizons}.nc", - temp_soil_rural="resources/temperatures/temp_soil_rural_elec_s{simpl}_{clusters}_{planning_horizons}.nc", - temp_soil_urban="resources/temperatures/temp_soil_urban_elec_s{simpl}_{clusters}_{planning_horizons}.nc", - temp_air_total="resources/temperatures/temp_air_total_elec_s{simpl}_{clusters}_{planning_horizons}.nc", - temp_air_rural="resources/temperatures/temp_air_rural_elec_s{simpl}_{clusters}_{planning_horizons}.nc", - temp_air_urban="resources/temperatures/temp_air_urban_elec_s{simpl}_{clusters}_{planning_horizons}.nc", + temp_soil_total="resources/" + RDIR + "temperatures/temp_soil_total_elec_s{simpl}_{clusters}_{planning_horizons}.nc", + temp_soil_rural="resources/" + RDIR + "temperatures/temp_soil_rural_elec_s{simpl}_{clusters}_{planning_horizons}.nc", + temp_soil_urban="resources/" + RDIR + "temperatures/temp_soil_urban_elec_s{simpl}_{clusters}_{planning_horizons}.nc", + temp_air_total="resources/" + RDIR + "temperatures/temp_air_total_elec_s{simpl}_{clusters}_{planning_horizons}.nc", + temp_air_rural="resources/" + RDIR + "temperatures/temp_air_rural_elec_s{simpl}_{clusters}_{planning_horizons}.nc", + temp_air_urban="resources/" + RDIR + "temperatures/temp_air_urban_elec_s{simpl}_{clusters}_{planning_horizons}.nc", output: - cop_soil_total="resources/cops/cop_soil_total_elec_s{simpl}_{clusters}_{planning_horizons}.nc", - cop_soil_rural="resources/cops/cop_soil_rural_elec_s{simpl}_{clusters}_{planning_horizons}.nc", - cop_soil_urban="resources/cops/cop_soil_urban_elec_s{simpl}_{clusters}_{planning_horizons}.nc", - cop_air_total="resources/cops/cop_air_total_elec_s{simpl}_{clusters}_{planning_horizons}.nc", - cop_air_rural="resources/cops/cop_air_rural_elec_s{simpl}_{clusters}_{planning_horizons}.nc", - cop_air_urban="resources/cops/cop_air_urban_elec_s{simpl}_{clusters}_{planning_horizons}.nc", + cop_soil_total="resources/" + RDIR + "cops/cop_soil_total_elec_s{simpl}_{clusters}_{planning_horizons}.nc", + cop_soil_rural="resources/" + RDIR + "cops/cop_soil_rural_elec_s{simpl}_{clusters}_{planning_horizons}.nc", + cop_soil_urban="resources/" + RDIR + "cops/cop_soil_urban_elec_s{simpl}_{clusters}_{planning_horizons}.nc", + cop_air_total="resources/" + RDIR + "cops/cop_air_total_elec_s{simpl}_{clusters}_{planning_horizons}.nc", + cop_air_rural="resources/" + RDIR + "cops/cop_air_rural_elec_s{simpl}_{clusters}_{planning_horizons}.nc", + cop_air_urban="resources/" + RDIR + "cops/cop_air_urban_elec_s{simpl}_{clusters}_{planning_horizons}.nc", resources: mem_mb=20000, benchmark: - "benchmarks/build_cop_profiles/s{simpl}_{clusters}_{planning_horizons}" + "benchmarks/" + RDIR + "build_cop_profiles/s{simpl}_{clusters}_{planning_horizons}" script: "scripts/build_cop_profiles.py" @@ -1217,20 +1217,20 @@ rule prepare_heat_data: input: network="networks/" + RDIR + "elec_s{simpl}_{clusters}.nc", energy_totals_name="data/energy_totals_{demand}_{planning_horizons}.csv", - clustered_pop_layout="resources/population_shares/pop_layout_elec_s{simpl}_{clusters}_{planning_horizons}.csv", - temp_air_total="resources/temperatures/temp_air_total_elec_s{simpl}_{clusters}_{planning_horizons}.nc", - cop_soil_total="resources/cops/cop_soil_total_elec_s{simpl}_{clusters}_{planning_horizons}.nc", - cop_air_total="resources/cops/cop_air_total_elec_s{simpl}_{clusters}_{planning_horizons}.nc", - solar_thermal_total="resources/demand/heat/solar_thermal_total_elec_s{simpl}_{clusters}_{planning_horizons}.nc", - heat_demand_total="resources/demand/heat/heat_demand_total_elec_s{simpl}_{clusters}_{planning_horizons}.nc", + clustered_pop_layout="resources/" + RDIR + "population_shares/pop_layout_elec_s{simpl}_{clusters}_{planning_horizons}.csv", + temp_air_total="resources/" + RDIR + "temperatures/temp_air_total_elec_s{simpl}_{clusters}_{planning_horizons}.nc", + cop_soil_total="resources/" + RDIR + "cops/cop_soil_total_elec_s{simpl}_{clusters}_{planning_horizons}.nc", + cop_air_total="resources/" + RDIR + "cops/cop_air_total_elec_s{simpl}_{clusters}_{planning_horizons}.nc", + solar_thermal_total="resources/" + RDIR + "demand/heat/solar_thermal_total_elec_s{simpl}_{clusters}_{planning_horizons}.nc", + heat_demand_total="resources/" + RDIR + "demand/heat/heat_demand_total_elec_s{simpl}_{clusters}_{planning_horizons}.nc", heat_profile="data/heat_load_profile_BDEW.csv", output: - nodal_energy_totals="resources/demand/heat/nodal_energy_heat_totals_{demand}_s{simpl}_{clusters}_{planning_horizons}.csv", - heat_demand="resources/demand/heat/heat_demand_{demand}_s{simpl}_{clusters}_{planning_horizons}.csv", - ashp_cop="resources/demand/heat/ashp_cop_{demand}_s{simpl}_{clusters}_{planning_horizons}.csv", - gshp_cop="resources/demand/heat/gshp_cop_{demand}_s{simpl}_{clusters}_{planning_horizons}.csv", - solar_thermal="resources/demand/heat/solar_thermal_{demand}_s{simpl}_{clusters}_{planning_horizons}.csv", - district_heat_share="resources/demand/heat/district_heat_share_{demand}_s{simpl}_{clusters}_{planning_horizons}.csv", + nodal_energy_totals="resources/" + RDIR + "demand/heat/nodal_energy_heat_totals_{demand}_s{simpl}_{clusters}_{planning_horizons}.csv", + heat_demand="resources/" + RDIR + "demand/heat/heat_demand_{demand}_s{simpl}_{clusters}_{planning_horizons}.csv", + ashp_cop="resources/" + RDIR + "demand/heat/ashp_cop_{demand}_s{simpl}_{clusters}_{planning_horizons}.csv", + gshp_cop="resources/" + RDIR + "demand/heat/gshp_cop_{demand}_s{simpl}_{clusters}_{planning_horizons}.csv", + solar_thermal="resources/" + RDIR + "demand/heat/solar_thermal_{demand}_s{simpl}_{clusters}_{planning_horizons}.csv", + district_heat_share="resources/" + RDIR + "demand/heat/district_heat_share_{demand}_s{simpl}_{clusters}_{planning_horizons}.csv", script: "scripts/prepare_heat_data.py" @@ -1272,9 +1272,9 @@ rule build_solar_thermal_profiles: solar_thermal_config=config["solar_thermal"], snapshots=config["snapshots"], input: - pop_layout_total="resources/population_shares/pop_layout_total_{planning_horizons}.nc", - pop_layout_urban="resources/population_shares/pop_layout_urban_{planning_horizons}.nc", - pop_layout_rural="resources/population_shares/pop_layout_rural_{planning_horizons}.nc", + pop_layout_total="resources/" + RDIR + "population_shares/pop_layout_total_{planning_horizons}.nc", + pop_layout_urban="resources/" + RDIR + "population_shares/pop_layout_urban_{planning_horizons}.nc", + pop_layout_rural="resources/" + RDIR + "population_shares/pop_layout_rural_{planning_horizons}.nc", regions_onshore="resources/" + RDIR + "bus_regions/regions_onshore_elec_s{simpl}_{clusters}.geojson", @@ -1284,13 +1284,13 @@ rule build_solar_thermal_profiles: + ".nc", # default to first cutout found output: - solar_thermal_total="resources/demand/heat/solar_thermal_total_elec_s{simpl}_{clusters}_{planning_horizons}.nc", - solar_thermal_urban="resources/demand/heat/solar_thermal_urban_elec_s{simpl}_{clusters}_{planning_horizons}.nc", - solar_thermal_rural="resources/demand/heat/solar_thermal_rural_elec_s{simpl}_{clusters}_{planning_horizons}.nc", + solar_thermal_total="resources/" + RDIR + "demand/heat/solar_thermal_total_elec_s{simpl}_{clusters}_{planning_horizons}.nc", + solar_thermal_urban="resources/" + RDIR + "demand/heat/solar_thermal_urban_elec_s{simpl}_{clusters}_{planning_horizons}.nc", + solar_thermal_rural="resources/" + RDIR + "demand/heat/solar_thermal_rural_elec_s{simpl}_{clusters}_{planning_horizons}.nc", resources: mem_mb=20000, benchmark: - "benchmarks/build_solar_thermal_profiles/s{simpl}_{clusters}_{planning_horizons}" + "benchmarks/" + RDIR + "build_solar_thermal_profiles/s{simpl}_{clusters}_{planning_horizons}" script: "scripts/build_solar_thermal_profiles.py" @@ -1307,14 +1307,14 @@ rule build_population_layouts: + ".nc", # default to first cutout found output: - pop_layout_total="resources/population_shares/pop_layout_total_{planning_horizons}.nc", - pop_layout_urban="resources/population_shares/pop_layout_urban_{planning_horizons}.nc", - pop_layout_rural="resources/population_shares/pop_layout_rural_{planning_horizons}.nc", - gdp_layout="resources/gdp_shares/gdp_layout_{planning_horizons}.nc", + pop_layout_total="resources/" + RDIR + "population_shares/pop_layout_total_{planning_horizons}.nc", + pop_layout_urban="resources/" + RDIR + "population_shares/pop_layout_urban_{planning_horizons}.nc", + pop_layout_rural="resources/" + RDIR + "population_shares/pop_layout_rural_{planning_horizons}.nc", + gdp_layout="resources/" + RDIR + "gdp_shares/gdp_layout_{planning_horizons}.nc", resources: mem_mb=20000, benchmark: - "benchmarks/build_population_layouts_{planning_horizons}" + "benchmarks/" + RDIR + "build_population_layouts_{planning_horizons}" threads: 8 script: "scripts/build_population_layouts.py" @@ -1324,17 +1324,17 @@ rule move_hardcoded_files_temp: input: "data/temp_hard_coded/energy_totals.csv", output: - "resources/energy_totals.csv", + "resources/" + RDIR + "energy_totals.csv", shell: "cp -a data/temp_hard_coded/. resources" rule build_clustered_population_layouts: input: - pop_layout_total="resources/population_shares/pop_layout_total_{planning_horizons}.nc", - pop_layout_urban="resources/population_shares/pop_layout_urban_{planning_horizons}.nc", - pop_layout_rural="resources/population_shares/pop_layout_rural_{planning_horizons}.nc", - gdp_layout="resources/gdp_shares/gdp_layout_{planning_horizons}.nc", + pop_layout_total="resources/" + RDIR + "population_shares/pop_layout_total_{planning_horizons}.nc", + pop_layout_urban="resources/" + RDIR + "population_shares/pop_layout_urban_{planning_horizons}.nc", + pop_layout_rural="resources/" + RDIR + "population_shares/pop_layout_rural_{planning_horizons}.nc", + gdp_layout="resources/" + RDIR + "gdp_shares/gdp_layout_{planning_horizons}.nc", regions_onshore="resources/" + RDIR + "bus_regions/regions_onshore_elec_s{simpl}_{clusters}.geojson", @@ -1344,12 +1344,12 @@ rule build_clustered_population_layouts: + ".nc", # default to first cutout found output: - clustered_pop_layout="resources/population_shares/pop_layout_elec_s{simpl}_{clusters}_{planning_horizons}.csv", - clustered_gdp_layout="resources/gdp_shares/gdp_layout_elec_s{simpl}_{clusters}_{planning_horizons}.csv", + clustered_pop_layout="resources/" + RDIR + "population_shares/pop_layout_elec_s{simpl}_{clusters}_{planning_horizons}.csv", + clustered_gdp_layout="resources/" + RDIR + "gdp_shares/gdp_layout_elec_s{simpl}_{clusters}_{planning_horizons}.csv", resources: mem_mb=10000, benchmark: - "benchmarks/build_clustered_population_layouts/s{simpl}_{clusters}_{planning_horizons}" + "benchmarks/" + RDIR + "build_clustered_population_layouts/s{simpl}_{clusters}_{planning_horizons}" script: "scripts/build_clustered_population_layouts.py" @@ -1358,9 +1358,9 @@ rule build_heat_demand: params: snapshots=config["snapshots"], input: - pop_layout_total="resources/population_shares/pop_layout_total_{planning_horizons}.nc", - pop_layout_urban="resources/population_shares/pop_layout_urban_{planning_horizons}.nc", - pop_layout_rural="resources/population_shares/pop_layout_rural_{planning_horizons}.nc", + pop_layout_total="resources/" + RDIR + "population_shares/pop_layout_total_{planning_horizons}.nc", + pop_layout_urban="resources/" + RDIR + "population_shares/pop_layout_urban_{planning_horizons}.nc", + pop_layout_rural="resources/" + RDIR + "population_shares/pop_layout_rural_{planning_horizons}.nc", regions_onshore="resources/" + RDIR + "bus_regions/regions_onshore_elec_s{simpl}_{clusters}.geojson", @@ -1370,13 +1370,13 @@ rule build_heat_demand: + ".nc", # default to first cutout found output: - heat_demand_urban="resources/demand/heat/heat_demand_urban_elec_s{simpl}_{clusters}_{planning_horizons}.nc", - heat_demand_rural="resources/demand/heat/heat_demand_rural_elec_s{simpl}_{clusters}_{planning_horizons}.nc", - heat_demand_total="resources/demand/heat/heat_demand_total_elec_s{simpl}_{clusters}_{planning_horizons}.nc", + heat_demand_urban="resources/" + RDIR + "demand/heat/heat_demand_urban_elec_s{simpl}_{clusters}_{planning_horizons}.nc", + heat_demand_rural="resources/" + RDIR + "demand/heat/heat_demand_rural_elec_s{simpl}_{clusters}_{planning_horizons}.nc", + heat_demand_total="resources/" + RDIR + "demand/heat/heat_demand_total_elec_s{simpl}_{clusters}_{planning_horizons}.nc", resources: mem_mb=20000, benchmark: - "benchmarks/build_heat_demand/s{simpl}_{clusters}_{planning_horizons}" + "benchmarks/" + RDIR + "build_heat_demand/s{simpl}_{clusters}_{planning_horizons}" script: "scripts/build_heat_demand.py" @@ -1385,9 +1385,9 @@ rule build_temperature_profiles: params: snapshots=config["snapshots"], input: - pop_layout_total="resources/population_shares/pop_layout_total_{planning_horizons}.nc", - pop_layout_urban="resources/population_shares/pop_layout_urban_{planning_horizons}.nc", - pop_layout_rural="resources/population_shares/pop_layout_rural_{planning_horizons}.nc", + pop_layout_total="resources/" + RDIR + "population_shares/pop_layout_total_{planning_horizons}.nc", + pop_layout_urban="resources/" + RDIR + "population_shares/pop_layout_urban_{planning_horizons}.nc", + pop_layout_rural="resources/" + RDIR + "population_shares/pop_layout_rural_{planning_horizons}.nc", regions_onshore="resources/" + RDIR + "bus_regions/regions_onshore_elec_s{simpl}_{clusters}.geojson", @@ -1397,16 +1397,16 @@ rule build_temperature_profiles: + ".nc", # default to first cutout found output: - temp_soil_total="resources/temperatures/temp_soil_total_elec_s{simpl}_{clusters}_{planning_horizons}.nc", - temp_soil_rural="resources/temperatures/temp_soil_rural_elec_s{simpl}_{clusters}_{planning_horizons}.nc", - temp_soil_urban="resources/temperatures/temp_soil_urban_elec_s{simpl}_{clusters}_{planning_horizons}.nc", - temp_air_total="resources/temperatures/temp_air_total_elec_s{simpl}_{clusters}_{planning_horizons}.nc", - temp_air_rural="resources/temperatures/temp_air_rural_elec_s{simpl}_{clusters}_{planning_horizons}.nc", - temp_air_urban="resources/temperatures/temp_air_urban_elec_s{simpl}_{clusters}_{planning_horizons}.nc", + temp_soil_total="resources/" + RDIR + "temperatures/temp_soil_total_elec_s{simpl}_{clusters}_{planning_horizons}.nc", + temp_soil_rural="resources/" + RDIR + "temperatures/temp_soil_rural_elec_s{simpl}_{clusters}_{planning_horizons}.nc", + temp_soil_urban="resources/" + RDIR + "temperatures/temp_soil_urban_elec_s{simpl}_{clusters}_{planning_horizons}.nc", + temp_air_total="resources/" + RDIR + "temperatures/temp_air_total_elec_s{simpl}_{clusters}_{planning_horizons}.nc", + temp_air_rural="resources/" + RDIR + "temperatures/temp_air_rural_elec_s{simpl}_{clusters}_{planning_horizons}.nc", + temp_air_urban="resources/" + RDIR + "temperatures/temp_air_urban_elec_s{simpl}_{clusters}_{planning_horizons}.nc", resources: mem_mb=20000, benchmark: - "benchmarks/build_temperature_profiles/s{simpl}_{clusters}_{planning_horizons}" + "benchmarks/" + RDIR + "build_temperature_profiles/s{simpl}_{clusters}_{planning_horizons}" script: "scripts/build_temperature_profiles.py" @@ -1650,19 +1650,19 @@ rule build_industrial_distribution_key: #default data regions_onshore="resources/" + RDIR + "bus_regions/regions_onshore_elec_s{simpl}_{clusters}.geojson", - clustered_pop_layout="resources/population_shares/pop_layout_elec_s{simpl}_{clusters}_{planning_horizons}.csv", - clustered_gdp_layout="resources/gdp_shares/gdp_layout_elec_s{simpl}_{clusters}_{planning_horizons}.csv", + clustered_pop_layout="resources/" + RDIR + "population_shares/pop_layout_elec_s{simpl}_{clusters}_{planning_horizons}.csv", + clustered_gdp_layout="resources/" + RDIR + "gdp_shares/gdp_layout_elec_s{simpl}_{clusters}_{planning_horizons}.csv", industrial_database="data/industrial_database.csv", shapes_path="resources/" + RDIR + "bus_regions/regions_onshore_elec_s{simpl}_{clusters}.geojson", output: - industrial_distribution_key="resources/demand/industrial_distribution_key_elec_s{simpl}_{clusters}_{planning_horizons}.csv", + industrial_distribution_key="resources/" + RDIR + "demand/industrial_distribution_key_elec_s{simpl}_{clusters}_{planning_horizons}.csv", threads: 1 resources: mem_mb=1000, benchmark: - "benchmarks/build_industrial_distribution_key_elec_s{simpl}_{clusters}_{planning_horizons}" + "benchmarks/" + RDIR + "build_industrial_distribution_key_elec_s{simpl}_{clusters}_{planning_horizons}" script: "scripts/build_industrial_distribution_key.py" @@ -1678,12 +1678,12 @@ rule build_base_industry_totals: #default data energy_totals_base="data/energy_totals_base.csv", transactions_path="data/unsd_transactions.csv", output: - base_industry_totals="resources/demand/base_industry_totals_{planning_horizons}_{demand}.csv", + base_industry_totals="resources/" + RDIR + "demand/base_industry_totals_{planning_horizons}_{demand}.csv", threads: 1 resources: mem_mb=1000, benchmark: - "benchmarks/build_base_industry_totals_{planning_horizons}_{demand}" + "benchmarks/" + RDIR + "build_base_industry_totals_{planning_horizons}_{demand}" script: "scripts/build_base_industry_totals.py" @@ -1696,20 +1696,20 @@ rule build_industry_demand: #default data industry_util_factor=config["sector"]["industry_util_factor"], aluminium_year=config["demand_data"]["aluminium_year"], input: - industrial_distribution_key="resources/demand/industrial_distribution_key_elec_s{simpl}_{clusters}_{planning_horizons}.csv", + industrial_distribution_key="resources/" + RDIR + "demand/industrial_distribution_key_elec_s{simpl}_{clusters}_{planning_horizons}.csv", #industrial_production_per_country_tomorrow="resources/demand/industrial_production_per_country_tomorrow_{planning_horizons}_{demand}.csv", #industrial_production_per_country="data/industrial_production_per_country.csv", - base_industry_totals="resources/demand/base_industry_totals_{planning_horizons}_{demand}.csv", + base_industry_totals="resources/" + RDIR + "demand/base_industry_totals_{planning_horizons}_{demand}.csv", industrial_database="data/industrial_database.csv", costs=COSTDIR + "costs_{planning_horizons}.csv", industry_growth_cagr="data/demand/industry_growth_cagr.csv", output: - industrial_energy_demand_per_node="resources/demand/industrial_energy_demand_per_node_elec_s{simpl}_{clusters}_{planning_horizons}_{demand}.csv", + industrial_energy_demand_per_node="resources/" + RDIR + "demand/industrial_energy_demand_per_node_elec_s{simpl}_{clusters}_{planning_horizons}_{demand}.csv", threads: 1 resources: mem_mb=1000, benchmark: - "benchmarks/industrial_energy_demand_per_node_elec_s{simpl}_{clusters}_{planning_horizons}_{demand}.csv" + "benchmarks/" + RDIR + "industrial_energy_demand_per_node_elec_s{simpl}_{clusters}_{planning_horizons}_{demand}.csv" script: "scripts/build_industry_demand.py" @@ -1721,11 +1721,11 @@ rule build_existing_heating_distribution: existing_capacities=config["existing_capacities"], input: existing_heating="data/existing_infrastructure/existing_heating_raw.csv", - clustered_pop_layout="resources/population_shares/pop_layout_elec_s{simpl}_{clusters}_{planning_horizons}.csv", - clustered_pop_energy_layout="resources/demand/heat/nodal_energy_heat_totals_{demand}_s{simpl}_{clusters}_{planning_horizons}.csv", #"resources/population_shares/pop_weighted_energy_totals_s{simpl}_{clusters}.csv", - district_heat_share="resources/demand/heat/district_heat_share_{demand}_s{simpl}_{clusters}_{planning_horizons}.csv", + clustered_pop_layout="resources/" + RDIR + "population_shares/pop_layout_elec_s{simpl}_{clusters}_{planning_horizons}.csv", + clustered_pop_energy_layout="resources/" + RDIR + "demand/heat/nodal_energy_heat_totals_{demand}_s{simpl}_{clusters}_{planning_horizons}.csv", #"resources/population_shares/pop_weighted_energy_totals_s{simpl}_{clusters}.csv", + district_heat_share="resources/" + RDIR + "demand/heat/district_heat_share_{demand}_s{simpl}_{clusters}_{planning_horizons}.csv", output: - existing_heating_distribution="resources/heating/existing_heating_distribution_{demand}_s{simpl}_{clusters}_{planning_horizons}.csv", + existing_heating_distribution="resources/" + RDIR + "heating/existing_heating_distribution_{demand}_s{simpl}_{clusters}_{planning_horizons}.csv", threads: 1 resources: mem_mb=2000, @@ -1755,12 +1755,12 @@ if config["foresight"] == "myopic": busmap=pypsaearth( "resources/" + RDIR + "bus_regions/busmap_elec_s{simpl}_{clusters}.csv" ), - clustered_pop_layout="resources/population_shares/pop_layout_elec_s{simpl}_{clusters}_{planning_horizons}.csv", + clustered_pop_layout="resources/" + RDIR + "population_shares/pop_layout_elec_s{simpl}_{clusters}_{planning_horizons}.csv", costs=CDIR + "costs_{}.csv".format(config["scenario"]["planning_horizons"][0]), - cop_soil_total="resources/cops/cop_soil_total_elec_s{simpl}_{clusters}_{planning_horizons}.nc", - cop_air_total="resources/cops/cop_air_total_elec_s{simpl}_{clusters}_{planning_horizons}.nc", - existing_heating_distribution="resources/heating/existing_heating_distribution_{demand}_s{simpl}_{clusters}_{planning_horizons}.csv", + cop_soil_total="resources/" + RDIR + "cops/cop_soil_total_elec_s{simpl}_{clusters}_{planning_horizons}.nc", + cop_air_total="resources/" + RDIR + "cops/cop_air_total_elec_s{simpl}_{clusters}_{planning_horizons}.nc", + existing_heating_distribution="resources/" + RDIR + "heating/existing_heating_distribution_{demand}_s{simpl}_{clusters}_{planning_horizons}.csv", output: RESDIR + "prenetworks-brownfield/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sopts}_{planning_horizons}_{discountrate}_{demand}_{h2export}export.nc", @@ -1822,8 +1822,8 @@ if config["foresight"] == "myopic": + "prenetworks/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}_{sopts}_{planning_horizons}_{discountrate}_{demand}_{h2export}export.nc", network_p=solved_previous_horizon, #solved network at previous time step costs=CDIR + "costs_{planning_horizons}.csv", - cop_soil_total="resources/cops/cop_soil_total_elec_s{simpl}_{clusters}_{planning_horizons}.nc", - cop_air_total="resources/cops/cop_air_total_elec_s{simpl}_{clusters}_{planning_horizons}.nc", + cop_soil_total="resources/" + RDIR + "cops/cop_soil_total_elec_s{simpl}_{clusters}_{planning_horizons}.nc", + cop_air_total="resources/" + RDIR + "cops/cop_air_total_elec_s{simpl}_{clusters}_{planning_horizons}.nc", output: RESDIR + "prenetworks-brownfield/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sopts}_{planning_horizons}_{discountrate}_{demand}_{h2export}export.nc", From 31e5a55e16bda98320a590bcd433de067ff042df Mon Sep 17 00:00:00 2001 From: yerbol-akhmetov Date: Sun, 27 Oct 2024 22:31:45 +0500 Subject: [PATCH 02/11] add release notes --- doc/release_notes.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/doc/release_notes.rst b/doc/release_notes.rst index a900e5860..133797f50 100644 --- a/doc/release_notes.rst +++ b/doc/release_notes.rst @@ -30,6 +30,8 @@ E.g. if a new rule becomes available describe how to use it `make test` and in o * Enable configfile specification for mock_snakemake `PR #1135 `_ +* Integrate RDIR into sector rules to store intermediate data in scenario folders `PR #1154 `_ + PyPSA-Earth 0.4.1 ================= From dfe229c886a03ffced9f9fb70f185988432d3d07 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 4 Nov 2024 10:25:51 +0000 Subject: [PATCH 03/11] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- Snakefile | 410 +++++++++++++++++++++++++++++++++++++++++------------- 1 file changed, 311 insertions(+), 99 deletions(-) diff --git a/Snakefile b/Snakefile index cfaba8693..48494e302 100644 --- a/Snakefile +++ b/Snakefile @@ -1049,7 +1049,9 @@ if not config["custom_data"]["gas_network"]: + RDIR + "bus_regions/regions_onshore_elec_s{simpl}_{clusters}.geojson", output: - clustered_gas_network="resources/" + RDIR + "gas_networks/gas_network_elec_s{simpl}_{clusters}.csv", + clustered_gas_network="resources/" + + RDIR + + "gas_networks/gas_network_elec_s{simpl}_{clusters}.csv", # TODO: Should be a own snakemake rule # gas_network_fig_1="resources/gas_networks/existing_gas_pipelines_{simpl}_{clusters}.png", # gas_network_fig_2="resources/gas_networks/clustered_gas_pipelines_{simpl}_{clusters}.png", @@ -1065,22 +1067,46 @@ rule prepare_sector_network: + "prenetworks/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}_{sopts}_{planning_horizons}_{discountrate}_{demand}_presec.nc", costs=COSTDIR + "costs_{planning_horizons}.csv", h2_cavern="data/hydrogen_salt_cavern_potentials.csv", - nodal_energy_totals="resources/" + RDIR + "demand/heat/nodal_energy_heat_totals_{demand}_s{simpl}_{clusters}_{planning_horizons}.csv", - transport="resources/" + RDIR + "demand/transport_{demand}_s{simpl}_{clusters}_{planning_horizons}.csv", - avail_profile="resources/" + RDIR + "pattern_profiles/avail_profile_{demand}_s{simpl}_{clusters}_{planning_horizons}.csv", - dsm_profile="resources/" + RDIR + "pattern_profiles/dsm_profile_{demand}_s{simpl}_{clusters}_{planning_horizons}.csv", - nodal_transport_data="resources/" + RDIR + "demand/nodal_transport_data_{demand}_s{simpl}_{clusters}_{planning_horizons}.csv", + nodal_energy_totals="resources/" + + RDIR + + "demand/heat/nodal_energy_heat_totals_{demand}_s{simpl}_{clusters}_{planning_horizons}.csv", + transport="resources/" + + RDIR + + "demand/transport_{demand}_s{simpl}_{clusters}_{planning_horizons}.csv", + avail_profile="resources/" + + RDIR + + "pattern_profiles/avail_profile_{demand}_s{simpl}_{clusters}_{planning_horizons}.csv", + dsm_profile="resources/" + + RDIR + + "pattern_profiles/dsm_profile_{demand}_s{simpl}_{clusters}_{planning_horizons}.csv", + nodal_transport_data="resources/" + + RDIR + + "demand/nodal_transport_data_{demand}_s{simpl}_{clusters}_{planning_horizons}.csv", overrides="data/override_component_attrs", - clustered_pop_layout="resources/" + RDIR + "population_shares/pop_layout_elec_s{simpl}_{clusters}_{planning_horizons}.csv", - industrial_demand="resources/" + RDIR + "demand/industrial_energy_demand_per_node_elec_s{simpl}_{clusters}_{planning_horizons}_{demand}.csv", + clustered_pop_layout="resources/" + + RDIR + + "population_shares/pop_layout_elec_s{simpl}_{clusters}_{planning_horizons}.csv", + industrial_demand="resources/" + + RDIR + + "demand/industrial_energy_demand_per_node_elec_s{simpl}_{clusters}_{planning_horizons}_{demand}.csv", energy_totals="data/energy_totals_{demand}_{planning_horizons}.csv", airports="data/airports.csv", ports="data/ports.csv", - heat_demand="resources/" + RDIR + "demand/heat/heat_demand_{demand}_s{simpl}_{clusters}_{planning_horizons}.csv", - ashp_cop="resources/" + RDIR + "demand/heat/ashp_cop_{demand}_s{simpl}_{clusters}_{planning_horizons}.csv", - gshp_cop="resources/" + RDIR + "demand/heat/gshp_cop_{demand}_s{simpl}_{clusters}_{planning_horizons}.csv", - solar_thermal="resources/" + RDIR + "demand/heat/solar_thermal_{demand}_s{simpl}_{clusters}_{planning_horizons}.csv", - district_heat_share="resources/" + RDIR + "demand/heat/district_heat_share_{demand}_s{simpl}_{clusters}_{planning_horizons}.csv", + heat_demand="resources/" + + RDIR + + "demand/heat/heat_demand_{demand}_s{simpl}_{clusters}_{planning_horizons}.csv", + ashp_cop="resources/" + + RDIR + + "demand/heat/ashp_cop_{demand}_s{simpl}_{clusters}_{planning_horizons}.csv", + gshp_cop="resources/" + + RDIR + + "demand/heat/gshp_cop_{demand}_s{simpl}_{clusters}_{planning_horizons}.csv", + solar_thermal="resources/" + + RDIR + + "demand/heat/solar_thermal_{demand}_s{simpl}_{clusters}_{planning_horizons}.csv", + district_heat_share="resources/" + + RDIR + + "demand/heat/district_heat_share_{demand}_s{simpl}_{clusters}_{planning_horizons}.csv", biomass_transport_costs="data/temp_hard_coded/biomass_transport_costs.csv", shapes_path="resources/" + RDIR @@ -1088,7 +1114,9 @@ rule prepare_sector_network: pipelines=( "data/custom/pipelines.csv" if config["custom_data"]["gas_network"] - else "resources/" + RDIR + "gas_networks/gas_network_elec_s{simpl}_{clusters}.csv" + else "resources/" + + RDIR + + "gas_networks/gas_network_elec_s{simpl}_{clusters}.csv" ), output: RESDIR @@ -1148,13 +1176,17 @@ rule override_respot: countries=config["countries"], input: **{ - f"custom_res_pot_{tech}_{planning_horizons}_{discountrate}": "resources/" + RDIR + f"custom_renewables/{tech}_{planning_horizons}_{discountrate}_potential.csv" + f"custom_res_pot_{tech}_{planning_horizons}_{discountrate}": "resources/" + + RDIR + + f"custom_renewables/{tech}_{planning_horizons}_{discountrate}_potential.csv" for tech in config["custom_data"]["renewables"] for discountrate in config["costs"]["discountrate"] for planning_horizons in config["scenario"]["planning_horizons"] }, **{ - f"custom_res_ins_{tech}_{planning_horizons}_{discountrate}": "resources/" + RDIR + f"custom_renewables/{tech}_{planning_horizons}_{discountrate}_installable.csv" + f"custom_res_ins_{tech}_{planning_horizons}_{discountrate}": "resources/" + + RDIR + + f"custom_renewables/{tech}_{planning_horizons}_{discountrate}_installable.csv" for tech in config["custom_data"]["renewables"] for discountrate in config["costs"]["discountrate"] for planning_horizons in config["scenario"]["planning_horizons"] @@ -1176,14 +1208,26 @@ rule prepare_transport_data: traffic_data_KFZ="data/emobility/KFZ__count", traffic_data_Pkw="data/emobility/Pkw__count", transport_name="resources/" + RDIR + "transport_data.csv", - clustered_pop_layout="resources/" + RDIR + "population_shares/pop_layout_elec_s{simpl}_{clusters}_{planning_horizons}.csv", - temp_air_total="resources/" + RDIR + "temperatures/temp_air_total_elec_s{simpl}_{clusters}_{planning_horizons}.nc", + clustered_pop_layout="resources/" + + RDIR + + "population_shares/pop_layout_elec_s{simpl}_{clusters}_{planning_horizons}.csv", + temp_air_total="resources/" + + RDIR + + "temperatures/temp_air_total_elec_s{simpl}_{clusters}_{planning_horizons}.nc", output: # nodal_energy_totals="resources/nodal_energy_totals_s{simpl}_{clusters}.csv", - transport="resources/" + RDIR + "demand/transport_{demand}_s{simpl}_{clusters}_{planning_horizons}.csv", - avail_profile="resources/" + RDIR + "pattern_profiles/avail_profile_{demand}_s{simpl}_{clusters}_{planning_horizons}.csv", - dsm_profile="resources/" + RDIR + "pattern_profiles/dsm_profile_{demand}_s{simpl}_{clusters}_{planning_horizons}.csv", - nodal_transport_data="resources/" + RDIR + "demand/nodal_transport_data_{demand}_s{simpl}_{clusters}_{planning_horizons}.csv", + transport="resources/" + + RDIR + + "demand/transport_{demand}_s{simpl}_{clusters}_{planning_horizons}.csv", + avail_profile="resources/" + + RDIR + + "pattern_profiles/avail_profile_{demand}_s{simpl}_{clusters}_{planning_horizons}.csv", + dsm_profile="resources/" + + RDIR + + "pattern_profiles/dsm_profile_{demand}_s{simpl}_{clusters}_{planning_horizons}.csv", + nodal_transport_data="resources/" + + RDIR + + "demand/nodal_transport_data_{demand}_s{simpl}_{clusters}_{planning_horizons}.csv", script: "scripts/prepare_transport_data.py" @@ -1192,23 +1236,51 @@ rule build_cop_profiles: params: heat_pump_sink_T=config["sector"]["heat_pump_sink_T"], input: - temp_soil_total="resources/" + RDIR + "temperatures/temp_soil_total_elec_s{simpl}_{clusters}_{planning_horizons}.nc", - temp_soil_rural="resources/" + RDIR + "temperatures/temp_soil_rural_elec_s{simpl}_{clusters}_{planning_horizons}.nc", - temp_soil_urban="resources/" + RDIR + "temperatures/temp_soil_urban_elec_s{simpl}_{clusters}_{planning_horizons}.nc", - temp_air_total="resources/" + RDIR + "temperatures/temp_air_total_elec_s{simpl}_{clusters}_{planning_horizons}.nc", - temp_air_rural="resources/" + RDIR + "temperatures/temp_air_rural_elec_s{simpl}_{clusters}_{planning_horizons}.nc", - temp_air_urban="resources/" + RDIR + "temperatures/temp_air_urban_elec_s{simpl}_{clusters}_{planning_horizons}.nc", + temp_soil_total="resources/" + + RDIR + + "temperatures/temp_soil_total_elec_s{simpl}_{clusters}_{planning_horizons}.nc", + temp_soil_rural="resources/" + + RDIR + + "temperatures/temp_soil_rural_elec_s{simpl}_{clusters}_{planning_horizons}.nc", + temp_soil_urban="resources/" + + RDIR + + "temperatures/temp_soil_urban_elec_s{simpl}_{clusters}_{planning_horizons}.nc", + temp_air_total="resources/" + + RDIR + + "temperatures/temp_air_total_elec_s{simpl}_{clusters}_{planning_horizons}.nc", + temp_air_rural="resources/" + + RDIR + + "temperatures/temp_air_rural_elec_s{simpl}_{clusters}_{planning_horizons}.nc", + temp_air_urban="resources/" + + RDIR + + "temperatures/temp_air_urban_elec_s{simpl}_{clusters}_{planning_horizons}.nc", output: - cop_soil_total="resources/" + RDIR + "cops/cop_soil_total_elec_s{simpl}_{clusters}_{planning_horizons}.nc", - cop_soil_rural="resources/" + RDIR + "cops/cop_soil_rural_elec_s{simpl}_{clusters}_{planning_horizons}.nc", - cop_soil_urban="resources/" + RDIR + "cops/cop_soil_urban_elec_s{simpl}_{clusters}_{planning_horizons}.nc", - cop_air_total="resources/" + RDIR + "cops/cop_air_total_elec_s{simpl}_{clusters}_{planning_horizons}.nc", - cop_air_rural="resources/" + RDIR + "cops/cop_air_rural_elec_s{simpl}_{clusters}_{planning_horizons}.nc", - cop_air_urban="resources/" + RDIR + "cops/cop_air_urban_elec_s{simpl}_{clusters}_{planning_horizons}.nc", + cop_soil_total="resources/" + + RDIR + + "cops/cop_soil_total_elec_s{simpl}_{clusters}_{planning_horizons}.nc", + cop_soil_rural="resources/" + + RDIR + + "cops/cop_soil_rural_elec_s{simpl}_{clusters}_{planning_horizons}.nc", + cop_soil_urban="resources/" + + RDIR + + "cops/cop_soil_urban_elec_s{simpl}_{clusters}_{planning_horizons}.nc", + cop_air_total="resources/" + + RDIR + + "cops/cop_air_total_elec_s{simpl}_{clusters}_{planning_horizons}.nc", + cop_air_rural="resources/" + + RDIR + + "cops/cop_air_rural_elec_s{simpl}_{clusters}_{planning_horizons}.nc", + cop_air_urban="resources/" + + RDIR + + "cops/cop_air_urban_elec_s{simpl}_{clusters}_{planning_horizons}.nc", resources: mem_mb=20000, benchmark: - "benchmarks/" + RDIR + "build_cop_profiles/s{simpl}_{clusters}_{planning_horizons}" + ( + "benchmarks/" + + RDIR + + "build_cop_profiles/s{simpl}_{clusters}_{planning_horizons}" + ) script: "scripts/build_cop_profiles.py" @@ -1217,20 +1289,44 @@ rule prepare_heat_data: input: network="networks/" + RDIR + "elec_s{simpl}_{clusters}.nc", energy_totals_name="data/energy_totals_{demand}_{planning_horizons}.csv", - clustered_pop_layout="resources/" + RDIR + "population_shares/pop_layout_elec_s{simpl}_{clusters}_{planning_horizons}.csv", - temp_air_total="resources/" + RDIR + "temperatures/temp_air_total_elec_s{simpl}_{clusters}_{planning_horizons}.nc", - cop_soil_total="resources/" + RDIR + "cops/cop_soil_total_elec_s{simpl}_{clusters}_{planning_horizons}.nc", - cop_air_total="resources/" + RDIR + "cops/cop_air_total_elec_s{simpl}_{clusters}_{planning_horizons}.nc", - solar_thermal_total="resources/" + RDIR + "demand/heat/solar_thermal_total_elec_s{simpl}_{clusters}_{planning_horizons}.nc", - heat_demand_total="resources/" + RDIR + "demand/heat/heat_demand_total_elec_s{simpl}_{clusters}_{planning_horizons}.nc", + clustered_pop_layout="resources/" + + RDIR + + "population_shares/pop_layout_elec_s{simpl}_{clusters}_{planning_horizons}.csv", + temp_air_total="resources/" + + RDIR + + "temperatures/temp_air_total_elec_s{simpl}_{clusters}_{planning_horizons}.nc", + cop_soil_total="resources/" + + RDIR + + "cops/cop_soil_total_elec_s{simpl}_{clusters}_{planning_horizons}.nc", + cop_air_total="resources/" + + RDIR + + "cops/cop_air_total_elec_s{simpl}_{clusters}_{planning_horizons}.nc", + solar_thermal_total="resources/" + + RDIR + + "demand/heat/solar_thermal_total_elec_s{simpl}_{clusters}_{planning_horizons}.nc", + heat_demand_total="resources/" + + RDIR + + "demand/heat/heat_demand_total_elec_s{simpl}_{clusters}_{planning_horizons}.nc", heat_profile="data/heat_load_profile_BDEW.csv", output: - nodal_energy_totals="resources/" + RDIR + "demand/heat/nodal_energy_heat_totals_{demand}_s{simpl}_{clusters}_{planning_horizons}.csv", - heat_demand="resources/" + RDIR + "demand/heat/heat_demand_{demand}_s{simpl}_{clusters}_{planning_horizons}.csv", - ashp_cop="resources/" + RDIR + "demand/heat/ashp_cop_{demand}_s{simpl}_{clusters}_{planning_horizons}.csv", - gshp_cop="resources/" + RDIR + "demand/heat/gshp_cop_{demand}_s{simpl}_{clusters}_{planning_horizons}.csv", - solar_thermal="resources/" + RDIR + "demand/heat/solar_thermal_{demand}_s{simpl}_{clusters}_{planning_horizons}.csv", - district_heat_share="resources/" + RDIR + "demand/heat/district_heat_share_{demand}_s{simpl}_{clusters}_{planning_horizons}.csv", + nodal_energy_totals="resources/" + + RDIR + + "demand/heat/nodal_energy_heat_totals_{demand}_s{simpl}_{clusters}_{planning_horizons}.csv", + heat_demand="resources/" + + RDIR + + "demand/heat/heat_demand_{demand}_s{simpl}_{clusters}_{planning_horizons}.csv", + ashp_cop="resources/" + + RDIR + + "demand/heat/ashp_cop_{demand}_s{simpl}_{clusters}_{planning_horizons}.csv", + gshp_cop="resources/" + + RDIR + + "demand/heat/gshp_cop_{demand}_s{simpl}_{clusters}_{planning_horizons}.csv", + solar_thermal="resources/" + + RDIR + + "demand/heat/solar_thermal_{demand}_s{simpl}_{clusters}_{planning_horizons}.csv", + district_heat_share="resources/" + + RDIR + + "demand/heat/district_heat_share_{demand}_s{simpl}_{clusters}_{planning_horizons}.csv", script: "scripts/prepare_heat_data.py" @@ -1272,9 +1368,15 @@ rule build_solar_thermal_profiles: solar_thermal_config=config["solar_thermal"], snapshots=config["snapshots"], input: - pop_layout_total="resources/" + RDIR + "population_shares/pop_layout_total_{planning_horizons}.nc", - pop_layout_urban="resources/" + RDIR + "population_shares/pop_layout_urban_{planning_horizons}.nc", - pop_layout_rural="resources/" + RDIR + "population_shares/pop_layout_rural_{planning_horizons}.nc", + pop_layout_total="resources/" + + RDIR + + "population_shares/pop_layout_total_{planning_horizons}.nc", + pop_layout_urban="resources/" + + RDIR + + "population_shares/pop_layout_urban_{planning_horizons}.nc", + pop_layout_rural="resources/" + + RDIR + + "population_shares/pop_layout_rural_{planning_horizons}.nc", regions_onshore="resources/" + RDIR + "bus_regions/regions_onshore_elec_s{simpl}_{clusters}.geojson", @@ -1284,13 +1386,23 @@ rule build_solar_thermal_profiles: + ".nc", # default to first cutout found output: - solar_thermal_total="resources/" + RDIR + "demand/heat/solar_thermal_total_elec_s{simpl}_{clusters}_{planning_horizons}.nc", - solar_thermal_urban="resources/" + RDIR + "demand/heat/solar_thermal_urban_elec_s{simpl}_{clusters}_{planning_horizons}.nc", - solar_thermal_rural="resources/" + RDIR + "demand/heat/solar_thermal_rural_elec_s{simpl}_{clusters}_{planning_horizons}.nc", + solar_thermal_total="resources/" + + RDIR + + "demand/heat/solar_thermal_total_elec_s{simpl}_{clusters}_{planning_horizons}.nc", + solar_thermal_urban="resources/" + + RDIR + + "demand/heat/solar_thermal_urban_elec_s{simpl}_{clusters}_{planning_horizons}.nc", + solar_thermal_rural="resources/" + + RDIR + + "demand/heat/solar_thermal_rural_elec_s{simpl}_{clusters}_{planning_horizons}.nc", resources: mem_mb=20000, benchmark: - "benchmarks/" + RDIR + "build_solar_thermal_profiles/s{simpl}_{clusters}_{planning_horizons}" + ( + "benchmarks/" + + RDIR + + "build_solar_thermal_profiles/s{simpl}_{clusters}_{planning_horizons}" + ) script: "scripts/build_solar_thermal_profiles.py" @@ -1307,9 +1419,15 @@ rule build_population_layouts: + ".nc", # default to first cutout found output: - pop_layout_total="resources/" + RDIR + "population_shares/pop_layout_total_{planning_horizons}.nc", - pop_layout_urban="resources/" + RDIR + "population_shares/pop_layout_urban_{planning_horizons}.nc", - pop_layout_rural="resources/" + RDIR + "population_shares/pop_layout_rural_{planning_horizons}.nc", + pop_layout_total="resources/" + + RDIR + + "population_shares/pop_layout_total_{planning_horizons}.nc", + pop_layout_urban="resources/" + + RDIR + + "population_shares/pop_layout_urban_{planning_horizons}.nc", + pop_layout_rural="resources/" + + RDIR + + "population_shares/pop_layout_rural_{planning_horizons}.nc", gdp_layout="resources/" + RDIR + "gdp_shares/gdp_layout_{planning_horizons}.nc", resources: mem_mb=20000, @@ -1331,9 +1449,15 @@ rule move_hardcoded_files_temp: rule build_clustered_population_layouts: input: - pop_layout_total="resources/" + RDIR + "population_shares/pop_layout_total_{planning_horizons}.nc", - pop_layout_urban="resources/" + RDIR + "population_shares/pop_layout_urban_{planning_horizons}.nc", - pop_layout_rural="resources/" + RDIR + "population_shares/pop_layout_rural_{planning_horizons}.nc", + pop_layout_total="resources/" + + RDIR + + "population_shares/pop_layout_total_{planning_horizons}.nc", + pop_layout_urban="resources/" + + RDIR + + "population_shares/pop_layout_urban_{planning_horizons}.nc", + pop_layout_rural="resources/" + + RDIR + + "population_shares/pop_layout_rural_{planning_horizons}.nc", gdp_layout="resources/" + RDIR + "gdp_shares/gdp_layout_{planning_horizons}.nc", regions_onshore="resources/" + RDIR @@ -1344,12 +1468,20 @@ rule build_clustered_population_layouts: + ".nc", # default to first cutout found output: - clustered_pop_layout="resources/" + RDIR + "population_shares/pop_layout_elec_s{simpl}_{clusters}_{planning_horizons}.csv", - clustered_gdp_layout="resources/" + RDIR + "gdp_shares/gdp_layout_elec_s{simpl}_{clusters}_{planning_horizons}.csv", + clustered_pop_layout="resources/" + + RDIR + + "population_shares/pop_layout_elec_s{simpl}_{clusters}_{planning_horizons}.csv", + clustered_gdp_layout="resources/" + + RDIR + + "gdp_shares/gdp_layout_elec_s{simpl}_{clusters}_{planning_horizons}.csv", resources: mem_mb=10000, benchmark: - "benchmarks/" + RDIR + "build_clustered_population_layouts/s{simpl}_{clusters}_{planning_horizons}" + ( + "benchmarks/" + + RDIR + + "build_clustered_population_layouts/s{simpl}_{clusters}_{planning_horizons}" + ) script: "scripts/build_clustered_population_layouts.py" @@ -1358,9 +1490,15 @@ rule build_heat_demand: params: snapshots=config["snapshots"], input: - pop_layout_total="resources/" + RDIR + "population_shares/pop_layout_total_{planning_horizons}.nc", - pop_layout_urban="resources/" + RDIR + "population_shares/pop_layout_urban_{planning_horizons}.nc", - pop_layout_rural="resources/" + RDIR + "population_shares/pop_layout_rural_{planning_horizons}.nc", + pop_layout_total="resources/" + + RDIR + + "population_shares/pop_layout_total_{planning_horizons}.nc", + pop_layout_urban="resources/" + + RDIR + + "population_shares/pop_layout_urban_{planning_horizons}.nc", + pop_layout_rural="resources/" + + RDIR + + "population_shares/pop_layout_rural_{planning_horizons}.nc", regions_onshore="resources/" + RDIR + "bus_regions/regions_onshore_elec_s{simpl}_{clusters}.geojson", @@ -1370,13 +1508,23 @@ rule build_heat_demand: + ".nc", # default to first cutout found output: - heat_demand_urban="resources/" + RDIR + "demand/heat/heat_demand_urban_elec_s{simpl}_{clusters}_{planning_horizons}.nc", - heat_demand_rural="resources/" + RDIR + "demand/heat/heat_demand_rural_elec_s{simpl}_{clusters}_{planning_horizons}.nc", - heat_demand_total="resources/" + RDIR + "demand/heat/heat_demand_total_elec_s{simpl}_{clusters}_{planning_horizons}.nc", + heat_demand_urban="resources/" + + RDIR + + "demand/heat/heat_demand_urban_elec_s{simpl}_{clusters}_{planning_horizons}.nc", + heat_demand_rural="resources/" + + RDIR + + "demand/heat/heat_demand_rural_elec_s{simpl}_{clusters}_{planning_horizons}.nc", + heat_demand_total="resources/" + + RDIR + + "demand/heat/heat_demand_total_elec_s{simpl}_{clusters}_{planning_horizons}.nc", resources: mem_mb=20000, benchmark: - "benchmarks/" + RDIR + "build_heat_demand/s{simpl}_{clusters}_{planning_horizons}" + ( + "benchmarks/" + + RDIR + + "build_heat_demand/s{simpl}_{clusters}_{planning_horizons}" + ) script: "scripts/build_heat_demand.py" @@ -1385,9 +1533,15 @@ rule build_temperature_profiles: params: snapshots=config["snapshots"], input: - pop_layout_total="resources/" + RDIR + "population_shares/pop_layout_total_{planning_horizons}.nc", - pop_layout_urban="resources/" + RDIR + "population_shares/pop_layout_urban_{planning_horizons}.nc", - pop_layout_rural="resources/" + RDIR + "population_shares/pop_layout_rural_{planning_horizons}.nc", + pop_layout_total="resources/" + + RDIR + + "population_shares/pop_layout_total_{planning_horizons}.nc", + pop_layout_urban="resources/" + + RDIR + + "population_shares/pop_layout_urban_{planning_horizons}.nc", + pop_layout_rural="resources/" + + RDIR + + "population_shares/pop_layout_rural_{planning_horizons}.nc", regions_onshore="resources/" + RDIR + "bus_regions/regions_onshore_elec_s{simpl}_{clusters}.geojson", @@ -1397,16 +1551,32 @@ rule build_temperature_profiles: + ".nc", # default to first cutout found output: - temp_soil_total="resources/" + RDIR + "temperatures/temp_soil_total_elec_s{simpl}_{clusters}_{planning_horizons}.nc", - temp_soil_rural="resources/" + RDIR + "temperatures/temp_soil_rural_elec_s{simpl}_{clusters}_{planning_horizons}.nc", - temp_soil_urban="resources/" + RDIR + "temperatures/temp_soil_urban_elec_s{simpl}_{clusters}_{planning_horizons}.nc", - temp_air_total="resources/" + RDIR + "temperatures/temp_air_total_elec_s{simpl}_{clusters}_{planning_horizons}.nc", - temp_air_rural="resources/" + RDIR + "temperatures/temp_air_rural_elec_s{simpl}_{clusters}_{planning_horizons}.nc", - temp_air_urban="resources/" + RDIR + "temperatures/temp_air_urban_elec_s{simpl}_{clusters}_{planning_horizons}.nc", + temp_soil_total="resources/" + + RDIR + + "temperatures/temp_soil_total_elec_s{simpl}_{clusters}_{planning_horizons}.nc", + temp_soil_rural="resources/" + + RDIR + + "temperatures/temp_soil_rural_elec_s{simpl}_{clusters}_{planning_horizons}.nc", + temp_soil_urban="resources/" + + RDIR + + "temperatures/temp_soil_urban_elec_s{simpl}_{clusters}_{planning_horizons}.nc", + temp_air_total="resources/" + + RDIR + + "temperatures/temp_air_total_elec_s{simpl}_{clusters}_{planning_horizons}.nc", + temp_air_rural="resources/" + + RDIR + + "temperatures/temp_air_rural_elec_s{simpl}_{clusters}_{planning_horizons}.nc", + temp_air_urban="resources/" + + RDIR + + "temperatures/temp_air_urban_elec_s{simpl}_{clusters}_{planning_horizons}.nc", resources: mem_mb=20000, benchmark: - "benchmarks/" + RDIR + "build_temperature_profiles/s{simpl}_{clusters}_{planning_horizons}" + ( + "benchmarks/" + + RDIR + + "build_temperature_profiles/s{simpl}_{clusters}_{planning_horizons}" + ) script: "scripts/build_temperature_profiles.py" @@ -1650,19 +1820,29 @@ rule build_industrial_distribution_key: #default data regions_onshore="resources/" + RDIR + "bus_regions/regions_onshore_elec_s{simpl}_{clusters}.geojson", - clustered_pop_layout="resources/" + RDIR + "population_shares/pop_layout_elec_s{simpl}_{clusters}_{planning_horizons}.csv", - clustered_gdp_layout="resources/" + RDIR + "gdp_shares/gdp_layout_elec_s{simpl}_{clusters}_{planning_horizons}.csv", + clustered_pop_layout="resources/" + + RDIR + + "population_shares/pop_layout_elec_s{simpl}_{clusters}_{planning_horizons}.csv", + clustered_gdp_layout="resources/" + + RDIR + + "gdp_shares/gdp_layout_elec_s{simpl}_{clusters}_{planning_horizons}.csv", industrial_database="data/industrial_database.csv", shapes_path="resources/" + RDIR + "bus_regions/regions_onshore_elec_s{simpl}_{clusters}.geojson", output: - industrial_distribution_key="resources/" + RDIR + "demand/industrial_distribution_key_elec_s{simpl}_{clusters}_{planning_horizons}.csv", + industrial_distribution_key="resources/" + + RDIR + + "demand/industrial_distribution_key_elec_s{simpl}_{clusters}_{planning_horizons}.csv", threads: 1 resources: mem_mb=1000, benchmark: - "benchmarks/" + RDIR + "build_industrial_distribution_key_elec_s{simpl}_{clusters}_{planning_horizons}" + ( + "benchmarks/" + + RDIR + + "build_industrial_distribution_key_elec_s{simpl}_{clusters}_{planning_horizons}" + ) script: "scripts/build_industrial_distribution_key.py" @@ -1678,7 +1858,9 @@ rule build_base_industry_totals: #default data energy_totals_base="data/energy_totals_base.csv", transactions_path="data/unsd_transactions.csv", output: - base_industry_totals="resources/" + RDIR + "demand/base_industry_totals_{planning_horizons}_{demand}.csv", + base_industry_totals="resources/" + + RDIR + + "demand/base_industry_totals_{planning_horizons}_{demand}.csv", threads: 1 resources: mem_mb=1000, @@ -1696,20 +1878,30 @@ rule build_industry_demand: #default data industry_util_factor=config["sector"]["industry_util_factor"], aluminium_year=config["demand_data"]["aluminium_year"], input: - industrial_distribution_key="resources/" + RDIR + "demand/industrial_distribution_key_elec_s{simpl}_{clusters}_{planning_horizons}.csv", + industrial_distribution_key="resources/" + + RDIR + + "demand/industrial_distribution_key_elec_s{simpl}_{clusters}_{planning_horizons}.csv", #industrial_production_per_country_tomorrow="resources/demand/industrial_production_per_country_tomorrow_{planning_horizons}_{demand}.csv", #industrial_production_per_country="data/industrial_production_per_country.csv", - base_industry_totals="resources/" + RDIR + "demand/base_industry_totals_{planning_horizons}_{demand}.csv", + base_industry_totals="resources/" + + RDIR + + "demand/base_industry_totals_{planning_horizons}_{demand}.csv", industrial_database="data/industrial_database.csv", costs=COSTDIR + "costs_{planning_horizons}.csv", industry_growth_cagr="data/demand/industry_growth_cagr.csv", output: - industrial_energy_demand_per_node="resources/" + RDIR + "demand/industrial_energy_demand_per_node_elec_s{simpl}_{clusters}_{planning_horizons}_{demand}.csv", + industrial_energy_demand_per_node="resources/" + + RDIR + + "demand/industrial_energy_demand_per_node_elec_s{simpl}_{clusters}_{planning_horizons}_{demand}.csv", threads: 1 resources: mem_mb=1000, benchmark: - "benchmarks/" + RDIR + "industrial_energy_demand_per_node_elec_s{simpl}_{clusters}_{planning_horizons}_{demand}.csv" + ( + "benchmarks/" + + RDIR + + "industrial_energy_demand_per_node_elec_s{simpl}_{clusters}_{planning_horizons}_{demand}.csv" + ) script: "scripts/build_industry_demand.py" @@ -1721,11 +1913,19 @@ rule build_existing_heating_distribution: existing_capacities=config["existing_capacities"], input: existing_heating="data/existing_infrastructure/existing_heating_raw.csv", - clustered_pop_layout="resources/" + RDIR + "population_shares/pop_layout_elec_s{simpl}_{clusters}_{planning_horizons}.csv", - clustered_pop_energy_layout="resources/" + RDIR + "demand/heat/nodal_energy_heat_totals_{demand}_s{simpl}_{clusters}_{planning_horizons}.csv", #"resources/population_shares/pop_weighted_energy_totals_s{simpl}_{clusters}.csv", - district_heat_share="resources/" + RDIR + "demand/heat/district_heat_share_{demand}_s{simpl}_{clusters}_{planning_horizons}.csv", + clustered_pop_layout="resources/" + + RDIR + + "population_shares/pop_layout_elec_s{simpl}_{clusters}_{planning_horizons}.csv", + clustered_pop_energy_layout="resources/" + + RDIR + + "demand/heat/nodal_energy_heat_totals_{demand}_s{simpl}_{clusters}_{planning_horizons}.csv", #"resources/population_shares/pop_weighted_energy_totals_s{simpl}_{clusters}.csv", + district_heat_share="resources/" + + RDIR + + "demand/heat/district_heat_share_{demand}_s{simpl}_{clusters}_{planning_horizons}.csv", output: - existing_heating_distribution="resources/" + RDIR + "heating/existing_heating_distribution_{demand}_s{simpl}_{clusters}_{planning_horizons}.csv", + existing_heating_distribution="resources/" + + RDIR + + "heating/existing_heating_distribution_{demand}_s{simpl}_{clusters}_{planning_horizons}.csv", threads: 1 resources: mem_mb=2000, @@ -1755,12 +1955,20 @@ if config["foresight"] == "myopic": busmap=pypsaearth( "resources/" + RDIR + "bus_regions/busmap_elec_s{simpl}_{clusters}.csv" ), - clustered_pop_layout="resources/" + RDIR + "population_shares/pop_layout_elec_s{simpl}_{clusters}_{planning_horizons}.csv", + clustered_pop_layout="resources/" + + RDIR + + "population_shares/pop_layout_elec_s{simpl}_{clusters}_{planning_horizons}.csv", costs=CDIR + "costs_{}.csv".format(config["scenario"]["planning_horizons"][0]), - cop_soil_total="resources/" + RDIR + "cops/cop_soil_total_elec_s{simpl}_{clusters}_{planning_horizons}.nc", - cop_air_total="resources/" + RDIR + "cops/cop_air_total_elec_s{simpl}_{clusters}_{planning_horizons}.nc", - existing_heating_distribution="resources/" + RDIR + "heating/existing_heating_distribution_{demand}_s{simpl}_{clusters}_{planning_horizons}.csv", + cop_soil_total="resources/" + + RDIR + + "cops/cop_soil_total_elec_s{simpl}_{clusters}_{planning_horizons}.nc", + cop_air_total="resources/" + + RDIR + + "cops/cop_air_total_elec_s{simpl}_{clusters}_{planning_horizons}.nc", + existing_heating_distribution="resources/" + + RDIR + + "heating/existing_heating_distribution_{demand}_s{simpl}_{clusters}_{planning_horizons}.csv", output: RESDIR + "prenetworks-brownfield/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sopts}_{planning_horizons}_{discountrate}_{demand}_{h2export}export.nc", @@ -1822,8 +2030,12 @@ if config["foresight"] == "myopic": + "prenetworks/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}_{sopts}_{planning_horizons}_{discountrate}_{demand}_{h2export}export.nc", network_p=solved_previous_horizon, #solved network at previous time step costs=CDIR + "costs_{planning_horizons}.csv", - cop_soil_total="resources/" + RDIR + "cops/cop_soil_total_elec_s{simpl}_{clusters}_{planning_horizons}.nc", - cop_air_total="resources/" + RDIR + "cops/cop_air_total_elec_s{simpl}_{clusters}_{planning_horizons}.nc", + cop_soil_total="resources/" + + RDIR + + "cops/cop_soil_total_elec_s{simpl}_{clusters}_{planning_horizons}.nc", + cop_air_total="resources/" + + RDIR + + "cops/cop_air_total_elec_s{simpl}_{clusters}_{planning_horizons}.nc", output: RESDIR + "prenetworks-brownfield/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sopts}_{planning_horizons}_{discountrate}_{demand}_{h2export}export.nc", From caacb20f4e42c20cb53fef247fa0501e406adecc Mon Sep 17 00:00:00 2001 From: yerbol-akhmetov Date: Wed, 6 Nov 2024 00:35:19 +0500 Subject: [PATCH 04/11] use sector_name to store sector data in separate folders --- Snakefile | 250 +++++++++++++++++++++++++++++++++++++++++--- config.default.yaml | 1 + 2 files changed, 239 insertions(+), 12 deletions(-) diff --git a/Snakefile b/Snakefile index 48494e302..3729761c8 100644 --- a/Snakefile +++ b/Snakefile @@ -50,8 +50,9 @@ config["scenario"]["unc"] = [ run = config.get("run", {}) RDIR = run["name"] + "/" if run.get("name") else "" CDIR = RDIR if not run.get("shared_cutouts") else "" -SDIR = config["summary_dir"].strip("/") + f"/{RDIR}" -RESDIR = config["results_dir"].strip("/") + f"/{RDIR}" +SECDIR = run["sector_name"] + "/" if run.get("sector_name") else "" +SDIR = config["summary_dir"].strip("/") + f"/{RDIR}{SECDIR}" +RESDIR = config["results_dir"].strip("/") + f"/{RDIR}{SECDIR}" COSTDIR = config["costs_dir"] load_data_paths = get_load_paths_gegis("data", config) @@ -1024,7 +1025,11 @@ rule prepare_urban_percent: rule prepare_transport_data_input: output: - transport_data_input="resources/" + RDIR + "transport_data.csv", + transport_data_input="resources/" + + RDIR + + "sector/" + + SECDIR + + "transport_data.csv", script: "scripts/prepare_transport_data_input.py" @@ -1051,6 +1056,8 @@ if not config["custom_data"]["gas_network"]: output: clustered_gas_network="resources/" + RDIR + + "sector/" + + SECDIR + "gas_networks/gas_network_elec_s{simpl}_{clusters}.csv", # TODO: Should be a own snakemake rule # gas_network_fig_1="resources/gas_networks/existing_gas_pipelines_{simpl}_{clusters}.png", @@ -1069,43 +1076,67 @@ rule prepare_sector_network: h2_cavern="data/hydrogen_salt_cavern_potentials.csv", nodal_energy_totals="resources/" + RDIR + + "sector/" + + SECDIR + "demand/heat/nodal_energy_heat_totals_{demand}_s{simpl}_{clusters}_{planning_horizons}.csv", transport="resources/" + RDIR + + "sector/" + + SECDIR + "demand/transport_{demand}_s{simpl}_{clusters}_{planning_horizons}.csv", avail_profile="resources/" + RDIR + + "sector/" + + SECDIR + "pattern_profiles/avail_profile_{demand}_s{simpl}_{clusters}_{planning_horizons}.csv", dsm_profile="resources/" + RDIR + + "sector/" + + SECDIR + "pattern_profiles/dsm_profile_{demand}_s{simpl}_{clusters}_{planning_horizons}.csv", nodal_transport_data="resources/" + RDIR + + "sector/" + + SECDIR + "demand/nodal_transport_data_{demand}_s{simpl}_{clusters}_{planning_horizons}.csv", overrides="data/override_component_attrs", clustered_pop_layout="resources/" + RDIR + + "sector/" + + SECDIR + "population_shares/pop_layout_elec_s{simpl}_{clusters}_{planning_horizons}.csv", industrial_demand="resources/" + RDIR + + "sector/" + + SECDIR + "demand/industrial_energy_demand_per_node_elec_s{simpl}_{clusters}_{planning_horizons}_{demand}.csv", energy_totals="data/energy_totals_{demand}_{planning_horizons}.csv", airports="data/airports.csv", ports="data/ports.csv", heat_demand="resources/" + RDIR + + "sector/" + + SECDIR + "demand/heat/heat_demand_{demand}_s{simpl}_{clusters}_{planning_horizons}.csv", ashp_cop="resources/" + RDIR + + "sector/" + + SECDIR + "demand/heat/ashp_cop_{demand}_s{simpl}_{clusters}_{planning_horizons}.csv", gshp_cop="resources/" + RDIR + + "sector/" + + SECDIR + "demand/heat/gshp_cop_{demand}_s{simpl}_{clusters}_{planning_horizons}.csv", solar_thermal="resources/" + RDIR + + "sector/" + + SECDIR + "demand/heat/solar_thermal_{demand}_s{simpl}_{clusters}_{planning_horizons}.csv", district_heat_share="resources/" + RDIR + + "sector/" + + SECDIR + "demand/heat/district_heat_share_{demand}_s{simpl}_{clusters}_{planning_horizons}.csv", biomass_transport_costs="data/temp_hard_coded/biomass_transport_costs.csv", shapes_path="resources/" @@ -1116,6 +1147,8 @@ rule prepare_sector_network: if config["custom_data"]["gas_network"] else "resources/" + RDIR + + "sector/" + + SECDIR + "gas_networks/gas_network_elec_s{simpl}_{clusters}.csv" ), output: @@ -1138,7 +1171,11 @@ rule build_ship_profile: snapshots=config["snapshots"], ship_opts=config["export"]["ship"], output: - ship_profile="resources/" + RDIR + "ship_profile_{h2export}TWh.csv", + ship_profile="resources/" + + RDIR + + "sector/" + + SECDIR + + "ship_profile_{h2export}TWh.csv", script: "scripts/build_ship_profile.py" @@ -1156,7 +1193,11 @@ rule add_export: overrides="data/override_component_attrs", export_ports="data/export_ports.csv", costs=COSTDIR + "costs_{planning_horizons}.csv", - ship_profile="resources/" + RDIR + "ship_profile_{h2export}TWh.csv", + ship_profile="resources/" + + RDIR + + "sector/" + + SECDIR + + "ship_profile_{h2export}TWh.csv", network=RESDIR + "prenetworks/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}_{sopts}_{planning_horizons}_{discountrate}_{demand}.nc", shapes_path="resources/" @@ -1178,6 +1219,8 @@ rule override_respot: **{ f"custom_res_pot_{tech}_{planning_horizons}_{discountrate}": "resources/" + RDIR + + "sector/" + + SECDIR + f"custom_renewables/{tech}_{planning_horizons}_{discountrate}_potential.csv" for tech in config["custom_data"]["renewables"] for discountrate in config["costs"]["discountrate"] @@ -1186,6 +1229,8 @@ rule override_respot: **{ f"custom_res_ins_{tech}_{planning_horizons}_{discountrate}": "resources/" + RDIR + + "sector/" + + SECDIR + f"custom_renewables/{tech}_{planning_horizons}_{discountrate}_installable.csv" for tech in config["custom_data"]["renewables"] for discountrate in config["costs"]["discountrate"] @@ -1207,26 +1252,38 @@ rule prepare_transport_data: energy_totals_name="data/energy_totals_{demand}_{planning_horizons}.csv", traffic_data_KFZ="data/emobility/KFZ__count", traffic_data_Pkw="data/emobility/Pkw__count", - transport_name="resources/" + RDIR + "transport_data.csv", + transport_name="resources/" + RDIR + "sector/" + SECDIR + "transport_data.csv", clustered_pop_layout="resources/" + RDIR + + "sector/" + + SECDIR + "population_shares/pop_layout_elec_s{simpl}_{clusters}_{planning_horizons}.csv", temp_air_total="resources/" + RDIR + + "sector/" + + SECDIR + "temperatures/temp_air_total_elec_s{simpl}_{clusters}_{planning_horizons}.nc", output: # nodal_energy_totals="resources/nodal_energy_totals_s{simpl}_{clusters}.csv", transport="resources/" + RDIR + + "sector/" + + SECDIR + "demand/transport_{demand}_s{simpl}_{clusters}_{planning_horizons}.csv", avail_profile="resources/" + RDIR + + "sector/" + + SECDIR + "pattern_profiles/avail_profile_{demand}_s{simpl}_{clusters}_{planning_horizons}.csv", dsm_profile="resources/" + RDIR + + "sector/" + + SECDIR + "pattern_profiles/dsm_profile_{demand}_s{simpl}_{clusters}_{planning_horizons}.csv", nodal_transport_data="resources/" + RDIR + + "sector/" + + SECDIR + "demand/nodal_transport_data_{demand}_s{simpl}_{clusters}_{planning_horizons}.csv", script: "scripts/prepare_transport_data.py" @@ -1238,40 +1295,64 @@ rule build_cop_profiles: input: temp_soil_total="resources/" + RDIR + + "sector/" + + SECDIR + "temperatures/temp_soil_total_elec_s{simpl}_{clusters}_{planning_horizons}.nc", temp_soil_rural="resources/" + RDIR + + "sector/" + + SECDIR + "temperatures/temp_soil_rural_elec_s{simpl}_{clusters}_{planning_horizons}.nc", temp_soil_urban="resources/" + RDIR + + "sector/" + + SECDIR + "temperatures/temp_soil_urban_elec_s{simpl}_{clusters}_{planning_horizons}.nc", temp_air_total="resources/" + RDIR + + "sector/" + + SECDIR + "temperatures/temp_air_total_elec_s{simpl}_{clusters}_{planning_horizons}.nc", temp_air_rural="resources/" + RDIR + + "sector/" + + SECDIR + "temperatures/temp_air_rural_elec_s{simpl}_{clusters}_{planning_horizons}.nc", temp_air_urban="resources/" + RDIR + + "sector/" + + SECDIR + "temperatures/temp_air_urban_elec_s{simpl}_{clusters}_{planning_horizons}.nc", output: cop_soil_total="resources/" + RDIR + + "sector/" + + SECDIR + "cops/cop_soil_total_elec_s{simpl}_{clusters}_{planning_horizons}.nc", cop_soil_rural="resources/" + RDIR + + "sector/" + + SECDIR + "cops/cop_soil_rural_elec_s{simpl}_{clusters}_{planning_horizons}.nc", cop_soil_urban="resources/" + RDIR + + "sector/" + + SECDIR + "cops/cop_soil_urban_elec_s{simpl}_{clusters}_{planning_horizons}.nc", cop_air_total="resources/" + RDIR + + "sector/" + + SECDIR + "cops/cop_air_total_elec_s{simpl}_{clusters}_{planning_horizons}.nc", cop_air_rural="resources/" + RDIR + + "sector/" + + SECDIR + "cops/cop_air_rural_elec_s{simpl}_{clusters}_{planning_horizons}.nc", cop_air_urban="resources/" + RDIR + + "sector/" + + SECDIR + "cops/cop_air_urban_elec_s{simpl}_{clusters}_{planning_horizons}.nc", resources: mem_mb=20000, @@ -1279,6 +1360,8 @@ rule build_cop_profiles: ( "benchmarks/" + RDIR + + "sector/" + + SECDIR + "build_cop_profiles/s{simpl}_{clusters}_{planning_horizons}" ) script: @@ -1291,41 +1374,65 @@ rule prepare_heat_data: energy_totals_name="data/energy_totals_{demand}_{planning_horizons}.csv", clustered_pop_layout="resources/" + RDIR + + "sector/" + + SECDIR + "population_shares/pop_layout_elec_s{simpl}_{clusters}_{planning_horizons}.csv", temp_air_total="resources/" + RDIR + + "sector/" + + SECDIR + "temperatures/temp_air_total_elec_s{simpl}_{clusters}_{planning_horizons}.nc", cop_soil_total="resources/" + RDIR + + "sector/" + + SECDIR + "cops/cop_soil_total_elec_s{simpl}_{clusters}_{planning_horizons}.nc", cop_air_total="resources/" + RDIR + + "sector/" + + SECDIR + "cops/cop_air_total_elec_s{simpl}_{clusters}_{planning_horizons}.nc", solar_thermal_total="resources/" + RDIR + + "sector/" + + SECDIR + "demand/heat/solar_thermal_total_elec_s{simpl}_{clusters}_{planning_horizons}.nc", heat_demand_total="resources/" + RDIR + + "sector/" + + SECDIR + "demand/heat/heat_demand_total_elec_s{simpl}_{clusters}_{planning_horizons}.nc", heat_profile="data/heat_load_profile_BDEW.csv", output: nodal_energy_totals="resources/" + RDIR + + "sector/" + + SECDIR + "demand/heat/nodal_energy_heat_totals_{demand}_s{simpl}_{clusters}_{planning_horizons}.csv", heat_demand="resources/" + RDIR + + "sector/" + + SECDIR + "demand/heat/heat_demand_{demand}_s{simpl}_{clusters}_{planning_horizons}.csv", ashp_cop="resources/" + RDIR + + "sector/" + + SECDIR + "demand/heat/ashp_cop_{demand}_s{simpl}_{clusters}_{planning_horizons}.csv", gshp_cop="resources/" + RDIR + + "sector/" + + SECDIR + "demand/heat/gshp_cop_{demand}_s{simpl}_{clusters}_{planning_horizons}.csv", solar_thermal="resources/" + RDIR + + "sector/" + + SECDIR + "demand/heat/solar_thermal_{demand}_s{simpl}_{clusters}_{planning_horizons}.csv", district_heat_share="resources/" + RDIR + + "sector/" + + SECDIR + "demand/heat/district_heat_share_{demand}_s{simpl}_{clusters}_{planning_horizons}.csv", script: "scripts/prepare_heat_data.py" @@ -1370,12 +1477,18 @@ rule build_solar_thermal_profiles: input: pop_layout_total="resources/" + RDIR + + "sector/" + + SECDIR + "population_shares/pop_layout_total_{planning_horizons}.nc", pop_layout_urban="resources/" + RDIR + + "sector/" + + SECDIR + "population_shares/pop_layout_urban_{planning_horizons}.nc", pop_layout_rural="resources/" + RDIR + + "sector/" + + SECDIR + "population_shares/pop_layout_rural_{planning_horizons}.nc", regions_onshore="resources/" + RDIR @@ -1388,12 +1501,18 @@ rule build_solar_thermal_profiles: output: solar_thermal_total="resources/" + RDIR + + "sector/" + + SECDIR + "demand/heat/solar_thermal_total_elec_s{simpl}_{clusters}_{planning_horizons}.nc", solar_thermal_urban="resources/" + RDIR + + "sector/" + + SECDIR + "demand/heat/solar_thermal_urban_elec_s{simpl}_{clusters}_{planning_horizons}.nc", solar_thermal_rural="resources/" + RDIR + + "sector/" + + SECDIR + "demand/heat/solar_thermal_rural_elec_s{simpl}_{clusters}_{planning_horizons}.nc", resources: mem_mb=20000, @@ -1401,6 +1520,8 @@ rule build_solar_thermal_profiles: ( "benchmarks/" + RDIR + + "sector/" + + SECDIR + "build_solar_thermal_profiles/s{simpl}_{clusters}_{planning_horizons}" ) script: @@ -1421,18 +1542,34 @@ rule build_population_layouts: output: pop_layout_total="resources/" + RDIR + + "sector/" + + SECDIR + "population_shares/pop_layout_total_{planning_horizons}.nc", pop_layout_urban="resources/" + RDIR + + "sector/" + + SECDIR + "population_shares/pop_layout_urban_{planning_horizons}.nc", pop_layout_rural="resources/" + RDIR + + "sector/" + + SECDIR + "population_shares/pop_layout_rural_{planning_horizons}.nc", - gdp_layout="resources/" + RDIR + "gdp_shares/gdp_layout_{planning_horizons}.nc", + gdp_layout="resources/" + + RDIR + + "sector/" + + SECDIR + + "gdp_shares/gdp_layout_{planning_horizons}.nc", resources: mem_mb=20000, benchmark: - "benchmarks/" + RDIR + "build_population_layouts_{planning_horizons}" + ( + "benchmarks/" + + RDIR + + "sector/" + + SECDIR + + "build_population_layouts_{planning_horizons}" + ) threads: 8 script: "scripts/build_population_layouts.py" @@ -1442,7 +1579,7 @@ rule move_hardcoded_files_temp: input: "data/temp_hard_coded/energy_totals.csv", output: - "resources/" + RDIR + "energy_totals.csv", + "resources/" + RDIR + "sector/" + SECDIR + "energy_totals.csv", shell: "cp -a data/temp_hard_coded/. resources" @@ -1451,12 +1588,18 @@ rule build_clustered_population_layouts: input: pop_layout_total="resources/" + RDIR + + "sector/" + + SECDIR + "population_shares/pop_layout_total_{planning_horizons}.nc", pop_layout_urban="resources/" + RDIR + + "sector/" + + SECDIR + "population_shares/pop_layout_urban_{planning_horizons}.nc", pop_layout_rural="resources/" + RDIR + + "sector/" + + SECDIR + "population_shares/pop_layout_rural_{planning_horizons}.nc", gdp_layout="resources/" + RDIR + "gdp_shares/gdp_layout_{planning_horizons}.nc", regions_onshore="resources/" @@ -1470,9 +1613,13 @@ rule build_clustered_population_layouts: output: clustered_pop_layout="resources/" + RDIR + + "sector/" + + SECDIR + "population_shares/pop_layout_elec_s{simpl}_{clusters}_{planning_horizons}.csv", clustered_gdp_layout="resources/" + RDIR + + "sector/" + + SECDIR + "gdp_shares/gdp_layout_elec_s{simpl}_{clusters}_{planning_horizons}.csv", resources: mem_mb=10000, @@ -1480,6 +1627,8 @@ rule build_clustered_population_layouts: ( "benchmarks/" + RDIR + + "sector/" + + SECDIR + "build_clustered_population_layouts/s{simpl}_{clusters}_{planning_horizons}" ) script: @@ -1492,12 +1641,18 @@ rule build_heat_demand: input: pop_layout_total="resources/" + RDIR + + "sector/" + + SECDIR + "population_shares/pop_layout_total_{planning_horizons}.nc", pop_layout_urban="resources/" + RDIR + + "sector/" + + SECDIR + "population_shares/pop_layout_urban_{planning_horizons}.nc", pop_layout_rural="resources/" + RDIR + + "sector/" + + SECDIR + "population_shares/pop_layout_rural_{planning_horizons}.nc", regions_onshore="resources/" + RDIR @@ -1510,12 +1665,18 @@ rule build_heat_demand: output: heat_demand_urban="resources/" + RDIR + + "sector/" + + SECDIR + "demand/heat/heat_demand_urban_elec_s{simpl}_{clusters}_{planning_horizons}.nc", heat_demand_rural="resources/" + RDIR + + "sector/" + + SECDIR + "demand/heat/heat_demand_rural_elec_s{simpl}_{clusters}_{planning_horizons}.nc", heat_demand_total="resources/" + RDIR + + "sector/" + + SECDIR + "demand/heat/heat_demand_total_elec_s{simpl}_{clusters}_{planning_horizons}.nc", resources: mem_mb=20000, @@ -1523,6 +1684,8 @@ rule build_heat_demand: ( "benchmarks/" + RDIR + + "sector/" + + SECDIR + "build_heat_demand/s{simpl}_{clusters}_{planning_horizons}" ) script: @@ -1535,12 +1698,18 @@ rule build_temperature_profiles: input: pop_layout_total="resources/" + RDIR + + "sector/" + + SECDIR + "population_shares/pop_layout_total_{planning_horizons}.nc", pop_layout_urban="resources/" + RDIR + + "sector/" + + SECDIR + "population_shares/pop_layout_urban_{planning_horizons}.nc", pop_layout_rural="resources/" + RDIR + + "sector/" + + SECDIR + "population_shares/pop_layout_rural_{planning_horizons}.nc", regions_onshore="resources/" + RDIR @@ -1553,21 +1722,33 @@ rule build_temperature_profiles: output: temp_soil_total="resources/" + RDIR + + "sector/" + + SECDIR + "temperatures/temp_soil_total_elec_s{simpl}_{clusters}_{planning_horizons}.nc", temp_soil_rural="resources/" + RDIR + + "sector/" + + SECDIR + "temperatures/temp_soil_rural_elec_s{simpl}_{clusters}_{planning_horizons}.nc", temp_soil_urban="resources/" + RDIR + + "sector/" + + SECDIR + "temperatures/temp_soil_urban_elec_s{simpl}_{clusters}_{planning_horizons}.nc", temp_air_total="resources/" + RDIR + + "sector/" + + SECDIR + "temperatures/temp_air_total_elec_s{simpl}_{clusters}_{planning_horizons}.nc", temp_air_rural="resources/" + RDIR + + "sector/" + + SECDIR + "temperatures/temp_air_rural_elec_s{simpl}_{clusters}_{planning_horizons}.nc", temp_air_urban="resources/" + RDIR + + "sector/" + + SECDIR + "temperatures/temp_air_urban_elec_s{simpl}_{clusters}_{planning_horizons}.nc", resources: mem_mb=20000, @@ -1575,6 +1756,8 @@ rule build_temperature_profiles: ( "benchmarks/" + RDIR + + "sector/" + + SECDIR + "build_temperature_profiles/s{simpl}_{clusters}_{planning_horizons}" ) script: @@ -1822,9 +2005,13 @@ rule build_industrial_distribution_key: #default data + "bus_regions/regions_onshore_elec_s{simpl}_{clusters}.geojson", clustered_pop_layout="resources/" + RDIR + + "sector/" + + SECDIR + "population_shares/pop_layout_elec_s{simpl}_{clusters}_{planning_horizons}.csv", clustered_gdp_layout="resources/" + RDIR + + "sector/" + + SECDIR + "gdp_shares/gdp_layout_elec_s{simpl}_{clusters}_{planning_horizons}.csv", industrial_database="data/industrial_database.csv", shapes_path="resources/" @@ -1833,6 +2020,8 @@ rule build_industrial_distribution_key: #default data output: industrial_distribution_key="resources/" + RDIR + + "sector/" + + SECDIR + "demand/industrial_distribution_key_elec_s{simpl}_{clusters}_{planning_horizons}.csv", threads: 1 resources: @@ -1860,12 +2049,20 @@ rule build_base_industry_totals: #default data output: base_industry_totals="resources/" + RDIR + + "sector/" + + SECDIR + "demand/base_industry_totals_{planning_horizons}_{demand}.csv", threads: 1 resources: mem_mb=1000, benchmark: - "benchmarks/" + RDIR + "build_base_industry_totals_{planning_horizons}_{demand}" + ( + "benchmarks/" + + RDIR + + "sector/" + + SECDIR + + "build_base_industry_totals_{planning_horizons}_{demand}" + ) script: "scripts/build_base_industry_totals.py" @@ -1880,11 +2077,15 @@ rule build_industry_demand: #default data input: industrial_distribution_key="resources/" + RDIR + + "sector/" + + SECDIR + "demand/industrial_distribution_key_elec_s{simpl}_{clusters}_{planning_horizons}.csv", #industrial_production_per_country_tomorrow="resources/demand/industrial_production_per_country_tomorrow_{planning_horizons}_{demand}.csv", #industrial_production_per_country="data/industrial_production_per_country.csv", base_industry_totals="resources/" + RDIR + + "sector/" + + SECDIR + "demand/base_industry_totals_{planning_horizons}_{demand}.csv", industrial_database="data/industrial_database.csv", costs=COSTDIR + "costs_{planning_horizons}.csv", @@ -1892,6 +2093,8 @@ rule build_industry_demand: #default data output: industrial_energy_demand_per_node="resources/" + RDIR + + "sector/" + + SECDIR + "demand/industrial_energy_demand_per_node_elec_s{simpl}_{clusters}_{planning_horizons}_{demand}.csv", threads: 1 resources: @@ -1900,6 +2103,8 @@ rule build_industry_demand: #default data ( "benchmarks/" + RDIR + + "sector/" + + SECDIR + "industrial_energy_demand_per_node_elec_s{simpl}_{clusters}_{planning_horizons}_{demand}.csv" ) script: @@ -1915,16 +2120,25 @@ rule build_existing_heating_distribution: existing_heating="data/existing_infrastructure/existing_heating_raw.csv", clustered_pop_layout="resources/" + RDIR + + "sector/" + + SECDIR + "population_shares/pop_layout_elec_s{simpl}_{clusters}_{planning_horizons}.csv", clustered_pop_energy_layout="resources/" + RDIR - + "demand/heat/nodal_energy_heat_totals_{demand}_s{simpl}_{clusters}_{planning_horizons}.csv", #"resources/population_shares/pop_weighted_energy_totals_s{simpl}_{clusters}.csv", + + "sector/" + + SECDIR + + "demand/heat/nodal_energy_heat_totals_{demand}_s{simpl}_{clusters}_{planning_horizons}.csv", + #"resources/population_shares/pop_weighted_energy_totals_s{simpl}_{clusters}.csv", district_heat_share="resources/" + RDIR + + "sector/" + + SECDIR + "demand/heat/district_heat_share_{demand}_s{simpl}_{clusters}_{planning_horizons}.csv", output: existing_heating_distribution="resources/" + RDIR + + "sector/" + + SECDIR + "heating/existing_heating_distribution_{demand}_s{simpl}_{clusters}_{planning_horizons}.csv", threads: 1 resources: @@ -1957,17 +2171,25 @@ if config["foresight"] == "myopic": ), clustered_pop_layout="resources/" + RDIR + + "sector/" + + SECDIR + "population_shares/pop_layout_elec_s{simpl}_{clusters}_{planning_horizons}.csv", costs=CDIR + "costs_{}.csv".format(config["scenario"]["planning_horizons"][0]), cop_soil_total="resources/" + RDIR + + "sector/" + + SECDIR + "cops/cop_soil_total_elec_s{simpl}_{clusters}_{planning_horizons}.nc", cop_air_total="resources/" + RDIR + + "sector/" + + SECDIR + "cops/cop_air_total_elec_s{simpl}_{clusters}_{planning_horizons}.nc", existing_heating_distribution="resources/" + RDIR + + "sector/" + + SECDIR + "heating/existing_heating_distribution_{demand}_s{simpl}_{clusters}_{planning_horizons}.csv", output: RESDIR @@ -2004,7 +2226,7 @@ if config["foresight"] == "myopic": planning_horizon_p = str(planning_horizons[i - 1]) return ( - RDIR + RESDIR + "postnetworks/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}_{sopts}_" + planning_horizon_p + "_{discountrate}_{demand}_{h2export}export.nc" @@ -2032,9 +2254,13 @@ if config["foresight"] == "myopic": costs=CDIR + "costs_{planning_horizons}.csv", cop_soil_total="resources/" + RDIR + + "sector/" + + SECDIR + "cops/cop_soil_total_elec_s{simpl}_{clusters}_{planning_horizons}.nc", cop_air_total="resources/" + RDIR + + "sector/" + + SECDIR + "cops/cop_air_total_elec_s{simpl}_{clusters}_{planning_horizons}.nc", output: RESDIR diff --git a/config.default.yaml b/config.default.yaml index 431812b94..aa5135481 100644 --- a/config.default.yaml +++ b/config.default.yaml @@ -37,6 +37,7 @@ custom_rules: [] # Default empty [] or link to custom rule file e.g. ["my_folder run: name: "" # use this to keep track of runs with different settings + sector_name: "" # use this to keep track of sector scenario runs shared_cutouts: true # set to true to share the default cutout(s) across runs # Note: value false requires build_cutout to be enabled allow_scenario_failure: false # If True, the workflow will continue even if a scenario in run_scnenario fails From 051c1511bdc0f49450c882ae7a54544f47420c8f Mon Sep 17 00:00:00 2001 From: Emmanuel Bolarinwa Date: Thu, 7 Nov 2024 10:42:01 +0100 Subject: [PATCH 05/11] remove repeated line (#1171) --- scripts/prepare_sector_network.py | 1 - 1 file changed, 1 deletion(-) diff --git a/scripts/prepare_sector_network.py b/scripts/prepare_sector_network.py index acb6394bb..ac125b0f9 100644 --- a/scripts/prepare_sector_network.py +++ b/scripts/prepare_sector_network.py @@ -2748,7 +2748,6 @@ def add_rail_transport(n, costs): # Fetch wildcards investment_year = int(snakemake.wildcards.planning_horizons[-4:]) demand_sc = snakemake.wildcards.demand # loading the demand scenrario wildcard - pop_layout = pd.read_csv(snakemake.input.clustered_pop_layout, index_col=0) # Prepare the costs dataframe costs = prepare_costs( From 62b6c81ecac37fed42b2817dd9a2c5f33633fb32 Mon Sep 17 00:00:00 2001 From: yerbol-akhmetov Date: Fri, 8 Nov 2024 18:29:50 +0500 Subject: [PATCH 06/11] use SECDIR for sector data only --- Snakefile | 230 ++---------------------------------------------------- 1 file changed, 6 insertions(+), 224 deletions(-) diff --git a/Snakefile b/Snakefile index 3729761c8..2cb7d7d50 100644 --- a/Snakefile +++ b/Snakefile @@ -1025,11 +1025,7 @@ rule prepare_urban_percent: rule prepare_transport_data_input: output: - transport_data_input="resources/" - + RDIR - + "sector/" - + SECDIR - + "transport_data.csv", + transport_data_input="resources/" + SECDIR + "transport_data.csv", script: "scripts/prepare_transport_data_input.py" @@ -1055,8 +1051,6 @@ if not config["custom_data"]["gas_network"]: + "bus_regions/regions_onshore_elec_s{simpl}_{clusters}.geojson", output: clustered_gas_network="resources/" - + RDIR - + "sector/" + SECDIR + "gas_networks/gas_network_elec_s{simpl}_{clusters}.csv", # TODO: Should be a own snakemake rule @@ -1075,67 +1069,43 @@ rule prepare_sector_network: costs=COSTDIR + "costs_{planning_horizons}.csv", h2_cavern="data/hydrogen_salt_cavern_potentials.csv", nodal_energy_totals="resources/" - + RDIR - + "sector/" + SECDIR + "demand/heat/nodal_energy_heat_totals_{demand}_s{simpl}_{clusters}_{planning_horizons}.csv", transport="resources/" - + RDIR - + "sector/" + SECDIR + "demand/transport_{demand}_s{simpl}_{clusters}_{planning_horizons}.csv", avail_profile="resources/" - + RDIR - + "sector/" + SECDIR + "pattern_profiles/avail_profile_{demand}_s{simpl}_{clusters}_{planning_horizons}.csv", dsm_profile="resources/" - + RDIR - + "sector/" + SECDIR + "pattern_profiles/dsm_profile_{demand}_s{simpl}_{clusters}_{planning_horizons}.csv", nodal_transport_data="resources/" - + RDIR - + "sector/" + SECDIR + "demand/nodal_transport_data_{demand}_s{simpl}_{clusters}_{planning_horizons}.csv", overrides="data/override_component_attrs", clustered_pop_layout="resources/" - + RDIR - + "sector/" + SECDIR + "population_shares/pop_layout_elec_s{simpl}_{clusters}_{planning_horizons}.csv", industrial_demand="resources/" - + RDIR - + "sector/" + SECDIR + "demand/industrial_energy_demand_per_node_elec_s{simpl}_{clusters}_{planning_horizons}_{demand}.csv", energy_totals="data/energy_totals_{demand}_{planning_horizons}.csv", airports="data/airports.csv", ports="data/ports.csv", heat_demand="resources/" - + RDIR - + "sector/" + SECDIR + "demand/heat/heat_demand_{demand}_s{simpl}_{clusters}_{planning_horizons}.csv", ashp_cop="resources/" - + RDIR - + "sector/" + SECDIR + "demand/heat/ashp_cop_{demand}_s{simpl}_{clusters}_{planning_horizons}.csv", gshp_cop="resources/" - + RDIR - + "sector/" + SECDIR + "demand/heat/gshp_cop_{demand}_s{simpl}_{clusters}_{planning_horizons}.csv", solar_thermal="resources/" - + RDIR - + "sector/" + SECDIR + "demand/heat/solar_thermal_{demand}_s{simpl}_{clusters}_{planning_horizons}.csv", district_heat_share="resources/" - + RDIR - + "sector/" + SECDIR + "demand/heat/district_heat_share_{demand}_s{simpl}_{clusters}_{planning_horizons}.csv", biomass_transport_costs="data/temp_hard_coded/biomass_transport_costs.csv", @@ -1146,8 +1116,6 @@ rule prepare_sector_network: "data/custom/pipelines.csv" if config["custom_data"]["gas_network"] else "resources/" - + RDIR - + "sector/" + SECDIR + "gas_networks/gas_network_elec_s{simpl}_{clusters}.csv" ), @@ -1171,11 +1139,7 @@ rule build_ship_profile: snapshots=config["snapshots"], ship_opts=config["export"]["ship"], output: - ship_profile="resources/" - + RDIR - + "sector/" - + SECDIR - + "ship_profile_{h2export}TWh.csv", + ship_profile="resources/" + SECDIR + "ship_profile_{h2export}TWh.csv", script: "scripts/build_ship_profile.py" @@ -1193,11 +1157,7 @@ rule add_export: overrides="data/override_component_attrs", export_ports="data/export_ports.csv", costs=COSTDIR + "costs_{planning_horizons}.csv", - ship_profile="resources/" - + RDIR - + "sector/" - + SECDIR - + "ship_profile_{h2export}TWh.csv", + ship_profile="resources/" + SECDIR + "ship_profile_{h2export}TWh.csv", network=RESDIR + "prenetworks/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}_{sopts}_{planning_horizons}_{discountrate}_{demand}.nc", shapes_path="resources/" @@ -1218,8 +1178,6 @@ rule override_respot: input: **{ f"custom_res_pot_{tech}_{planning_horizons}_{discountrate}": "resources/" - + RDIR - + "sector/" + SECDIR + f"custom_renewables/{tech}_{planning_horizons}_{discountrate}_potential.csv" for tech in config["custom_data"]["renewables"] @@ -1228,8 +1186,6 @@ rule override_respot: }, **{ f"custom_res_ins_{tech}_{planning_horizons}_{discountrate}": "resources/" - + RDIR - + "sector/" + SECDIR + f"custom_renewables/{tech}_{planning_horizons}_{discountrate}_installable.csv" for tech in config["custom_data"]["renewables"] @@ -1252,37 +1208,25 @@ rule prepare_transport_data: energy_totals_name="data/energy_totals_{demand}_{planning_horizons}.csv", traffic_data_KFZ="data/emobility/KFZ__count", traffic_data_Pkw="data/emobility/Pkw__count", - transport_name="resources/" + RDIR + "sector/" + SECDIR + "transport_data.csv", + transport_name="resources/" + SECDIR + "transport_data.csv", clustered_pop_layout="resources/" - + RDIR - + "sector/" + SECDIR + "population_shares/pop_layout_elec_s{simpl}_{clusters}_{planning_horizons}.csv", temp_air_total="resources/" - + RDIR - + "sector/" + SECDIR + "temperatures/temp_air_total_elec_s{simpl}_{clusters}_{planning_horizons}.nc", output: # nodal_energy_totals="resources/nodal_energy_totals_s{simpl}_{clusters}.csv", transport="resources/" - + RDIR - + "sector/" + SECDIR + "demand/transport_{demand}_s{simpl}_{clusters}_{planning_horizons}.csv", avail_profile="resources/" - + RDIR - + "sector/" + SECDIR + "pattern_profiles/avail_profile_{demand}_s{simpl}_{clusters}_{planning_horizons}.csv", dsm_profile="resources/" - + RDIR - + "sector/" + SECDIR + "pattern_profiles/dsm_profile_{demand}_s{simpl}_{clusters}_{planning_horizons}.csv", nodal_transport_data="resources/" - + RDIR - + "sector/" + SECDIR + "demand/nodal_transport_data_{demand}_s{simpl}_{clusters}_{planning_horizons}.csv", script: @@ -1294,64 +1238,40 @@ rule build_cop_profiles: heat_pump_sink_T=config["sector"]["heat_pump_sink_T"], input: temp_soil_total="resources/" - + RDIR - + "sector/" + SECDIR + "temperatures/temp_soil_total_elec_s{simpl}_{clusters}_{planning_horizons}.nc", temp_soil_rural="resources/" - + RDIR - + "sector/" + SECDIR + "temperatures/temp_soil_rural_elec_s{simpl}_{clusters}_{planning_horizons}.nc", temp_soil_urban="resources/" - + RDIR - + "sector/" + SECDIR + "temperatures/temp_soil_urban_elec_s{simpl}_{clusters}_{planning_horizons}.nc", temp_air_total="resources/" - + RDIR - + "sector/" + SECDIR + "temperatures/temp_air_total_elec_s{simpl}_{clusters}_{planning_horizons}.nc", temp_air_rural="resources/" - + RDIR - + "sector/" + SECDIR + "temperatures/temp_air_rural_elec_s{simpl}_{clusters}_{planning_horizons}.nc", temp_air_urban="resources/" - + RDIR - + "sector/" + SECDIR + "temperatures/temp_air_urban_elec_s{simpl}_{clusters}_{planning_horizons}.nc", output: cop_soil_total="resources/" - + RDIR - + "sector/" + SECDIR + "cops/cop_soil_total_elec_s{simpl}_{clusters}_{planning_horizons}.nc", cop_soil_rural="resources/" - + RDIR - + "sector/" + SECDIR + "cops/cop_soil_rural_elec_s{simpl}_{clusters}_{planning_horizons}.nc", cop_soil_urban="resources/" - + RDIR - + "sector/" + SECDIR + "cops/cop_soil_urban_elec_s{simpl}_{clusters}_{planning_horizons}.nc", cop_air_total="resources/" - + RDIR - + "sector/" + SECDIR + "cops/cop_air_total_elec_s{simpl}_{clusters}_{planning_horizons}.nc", cop_air_rural="resources/" - + RDIR - + "sector/" + SECDIR + "cops/cop_air_rural_elec_s{simpl}_{clusters}_{planning_horizons}.nc", cop_air_urban="resources/" - + RDIR - + "sector/" + SECDIR + "cops/cop_air_urban_elec_s{simpl}_{clusters}_{planning_horizons}.nc", resources: @@ -1359,8 +1279,6 @@ rule build_cop_profiles: benchmark: ( "benchmarks/" - + RDIR - + "sector/" + SECDIR + "build_cop_profiles/s{simpl}_{clusters}_{planning_horizons}" ) @@ -1373,65 +1291,41 @@ rule prepare_heat_data: network="networks/" + RDIR + "elec_s{simpl}_{clusters}.nc", energy_totals_name="data/energy_totals_{demand}_{planning_horizons}.csv", clustered_pop_layout="resources/" - + RDIR - + "sector/" + SECDIR + "population_shares/pop_layout_elec_s{simpl}_{clusters}_{planning_horizons}.csv", temp_air_total="resources/" - + RDIR - + "sector/" + SECDIR + "temperatures/temp_air_total_elec_s{simpl}_{clusters}_{planning_horizons}.nc", cop_soil_total="resources/" - + RDIR - + "sector/" + SECDIR + "cops/cop_soil_total_elec_s{simpl}_{clusters}_{planning_horizons}.nc", cop_air_total="resources/" - + RDIR - + "sector/" + SECDIR + "cops/cop_air_total_elec_s{simpl}_{clusters}_{planning_horizons}.nc", solar_thermal_total="resources/" - + RDIR - + "sector/" + SECDIR + "demand/heat/solar_thermal_total_elec_s{simpl}_{clusters}_{planning_horizons}.nc", heat_demand_total="resources/" - + RDIR - + "sector/" + SECDIR + "demand/heat/heat_demand_total_elec_s{simpl}_{clusters}_{planning_horizons}.nc", heat_profile="data/heat_load_profile_BDEW.csv", output: nodal_energy_totals="resources/" - + RDIR - + "sector/" + SECDIR + "demand/heat/nodal_energy_heat_totals_{demand}_s{simpl}_{clusters}_{planning_horizons}.csv", heat_demand="resources/" - + RDIR - + "sector/" + SECDIR + "demand/heat/heat_demand_{demand}_s{simpl}_{clusters}_{planning_horizons}.csv", ashp_cop="resources/" - + RDIR - + "sector/" + SECDIR + "demand/heat/ashp_cop_{demand}_s{simpl}_{clusters}_{planning_horizons}.csv", gshp_cop="resources/" - + RDIR - + "sector/" + SECDIR + "demand/heat/gshp_cop_{demand}_s{simpl}_{clusters}_{planning_horizons}.csv", solar_thermal="resources/" - + RDIR - + "sector/" + SECDIR + "demand/heat/solar_thermal_{demand}_s{simpl}_{clusters}_{planning_horizons}.csv", district_heat_share="resources/" - + RDIR - + "sector/" + SECDIR + "demand/heat/district_heat_share_{demand}_s{simpl}_{clusters}_{planning_horizons}.csv", script: @@ -1476,18 +1370,12 @@ rule build_solar_thermal_profiles: snapshots=config["snapshots"], input: pop_layout_total="resources/" - + RDIR - + "sector/" + SECDIR + "population_shares/pop_layout_total_{planning_horizons}.nc", pop_layout_urban="resources/" - + RDIR - + "sector/" + SECDIR + "population_shares/pop_layout_urban_{planning_horizons}.nc", pop_layout_rural="resources/" - + RDIR - + "sector/" + SECDIR + "population_shares/pop_layout_rural_{planning_horizons}.nc", regions_onshore="resources/" @@ -1500,18 +1388,12 @@ rule build_solar_thermal_profiles: # default to first cutout found output: solar_thermal_total="resources/" - + RDIR - + "sector/" + SECDIR + "demand/heat/solar_thermal_total_elec_s{simpl}_{clusters}_{planning_horizons}.nc", solar_thermal_urban="resources/" - + RDIR - + "sector/" + SECDIR + "demand/heat/solar_thermal_urban_elec_s{simpl}_{clusters}_{planning_horizons}.nc", solar_thermal_rural="resources/" - + RDIR - + "sector/" + SECDIR + "demand/heat/solar_thermal_rural_elec_s{simpl}_{clusters}_{planning_horizons}.nc", resources: @@ -1519,8 +1401,6 @@ rule build_solar_thermal_profiles: benchmark: ( "benchmarks/" - + RDIR - + "sector/" + SECDIR + "build_solar_thermal_profiles/s{simpl}_{clusters}_{planning_horizons}" ) @@ -1541,35 +1421,21 @@ rule build_population_layouts: # default to first cutout found output: pop_layout_total="resources/" - + RDIR - + "sector/" + SECDIR + "population_shares/pop_layout_total_{planning_horizons}.nc", pop_layout_urban="resources/" - + RDIR - + "sector/" + SECDIR + "population_shares/pop_layout_urban_{planning_horizons}.nc", pop_layout_rural="resources/" - + RDIR - + "sector/" + SECDIR + "population_shares/pop_layout_rural_{planning_horizons}.nc", gdp_layout="resources/" - + RDIR - + "sector/" + SECDIR + "gdp_shares/gdp_layout_{planning_horizons}.nc", resources: mem_mb=20000, benchmark: - ( - "benchmarks/" - + RDIR - + "sector/" - + SECDIR - + "build_population_layouts_{planning_horizons}" - ) + ("benchmarks/" + SECDIR + "build_population_layouts_{planning_horizons}") threads: 8 script: "scripts/build_population_layouts.py" @@ -1579,7 +1445,7 @@ rule move_hardcoded_files_temp: input: "data/temp_hard_coded/energy_totals.csv", output: - "resources/" + RDIR + "sector/" + SECDIR + "energy_totals.csv", + "resources/" + SECDIR + "energy_totals.csv", shell: "cp -a data/temp_hard_coded/. resources" @@ -1587,18 +1453,12 @@ rule move_hardcoded_files_temp: rule build_clustered_population_layouts: input: pop_layout_total="resources/" - + RDIR - + "sector/" + SECDIR + "population_shares/pop_layout_total_{planning_horizons}.nc", pop_layout_urban="resources/" - + RDIR - + "sector/" + SECDIR + "population_shares/pop_layout_urban_{planning_horizons}.nc", pop_layout_rural="resources/" - + RDIR - + "sector/" + SECDIR + "population_shares/pop_layout_rural_{planning_horizons}.nc", gdp_layout="resources/" + RDIR + "gdp_shares/gdp_layout_{planning_horizons}.nc", @@ -1612,13 +1472,9 @@ rule build_clustered_population_layouts: # default to first cutout found output: clustered_pop_layout="resources/" - + RDIR - + "sector/" + SECDIR + "population_shares/pop_layout_elec_s{simpl}_{clusters}_{planning_horizons}.csv", clustered_gdp_layout="resources/" - + RDIR - + "sector/" + SECDIR + "gdp_shares/gdp_layout_elec_s{simpl}_{clusters}_{planning_horizons}.csv", resources: @@ -1626,8 +1482,6 @@ rule build_clustered_population_layouts: benchmark: ( "benchmarks/" - + RDIR - + "sector/" + SECDIR + "build_clustered_population_layouts/s{simpl}_{clusters}_{planning_horizons}" ) @@ -1640,18 +1494,12 @@ rule build_heat_demand: snapshots=config["snapshots"], input: pop_layout_total="resources/" - + RDIR - + "sector/" + SECDIR + "population_shares/pop_layout_total_{planning_horizons}.nc", pop_layout_urban="resources/" - + RDIR - + "sector/" + SECDIR + "population_shares/pop_layout_urban_{planning_horizons}.nc", pop_layout_rural="resources/" - + RDIR - + "sector/" + SECDIR + "population_shares/pop_layout_rural_{planning_horizons}.nc", regions_onshore="resources/" @@ -1664,18 +1512,12 @@ rule build_heat_demand: # default to first cutout found output: heat_demand_urban="resources/" - + RDIR - + "sector/" + SECDIR + "demand/heat/heat_demand_urban_elec_s{simpl}_{clusters}_{planning_horizons}.nc", heat_demand_rural="resources/" - + RDIR - + "sector/" + SECDIR + "demand/heat/heat_demand_rural_elec_s{simpl}_{clusters}_{planning_horizons}.nc", heat_demand_total="resources/" - + RDIR - + "sector/" + SECDIR + "demand/heat/heat_demand_total_elec_s{simpl}_{clusters}_{planning_horizons}.nc", resources: @@ -1683,8 +1525,6 @@ rule build_heat_demand: benchmark: ( "benchmarks/" - + RDIR - + "sector/" + SECDIR + "build_heat_demand/s{simpl}_{clusters}_{planning_horizons}" ) @@ -1697,18 +1537,12 @@ rule build_temperature_profiles: snapshots=config["snapshots"], input: pop_layout_total="resources/" - + RDIR - + "sector/" + SECDIR + "population_shares/pop_layout_total_{planning_horizons}.nc", pop_layout_urban="resources/" - + RDIR - + "sector/" + SECDIR + "population_shares/pop_layout_urban_{planning_horizons}.nc", pop_layout_rural="resources/" - + RDIR - + "sector/" + SECDIR + "population_shares/pop_layout_rural_{planning_horizons}.nc", regions_onshore="resources/" @@ -1721,33 +1555,21 @@ rule build_temperature_profiles: # default to first cutout found output: temp_soil_total="resources/" - + RDIR - + "sector/" + SECDIR + "temperatures/temp_soil_total_elec_s{simpl}_{clusters}_{planning_horizons}.nc", temp_soil_rural="resources/" - + RDIR - + "sector/" + SECDIR + "temperatures/temp_soil_rural_elec_s{simpl}_{clusters}_{planning_horizons}.nc", temp_soil_urban="resources/" - + RDIR - + "sector/" + SECDIR + "temperatures/temp_soil_urban_elec_s{simpl}_{clusters}_{planning_horizons}.nc", temp_air_total="resources/" - + RDIR - + "sector/" + SECDIR + "temperatures/temp_air_total_elec_s{simpl}_{clusters}_{planning_horizons}.nc", temp_air_rural="resources/" - + RDIR - + "sector/" + SECDIR + "temperatures/temp_air_rural_elec_s{simpl}_{clusters}_{planning_horizons}.nc", temp_air_urban="resources/" - + RDIR - + "sector/" + SECDIR + "temperatures/temp_air_urban_elec_s{simpl}_{clusters}_{planning_horizons}.nc", resources: @@ -1755,8 +1577,6 @@ rule build_temperature_profiles: benchmark: ( "benchmarks/" - + RDIR - + "sector/" + SECDIR + "build_temperature_profiles/s{simpl}_{clusters}_{planning_horizons}" ) @@ -2004,13 +1824,9 @@ rule build_industrial_distribution_key: #default data + RDIR + "bus_regions/regions_onshore_elec_s{simpl}_{clusters}.geojson", clustered_pop_layout="resources/" - + RDIR - + "sector/" + SECDIR + "population_shares/pop_layout_elec_s{simpl}_{clusters}_{planning_horizons}.csv", clustered_gdp_layout="resources/" - + RDIR - + "sector/" + SECDIR + "gdp_shares/gdp_layout_elec_s{simpl}_{clusters}_{planning_horizons}.csv", industrial_database="data/industrial_database.csv", @@ -2019,8 +1835,6 @@ rule build_industrial_distribution_key: #default data + "bus_regions/regions_onshore_elec_s{simpl}_{clusters}.geojson", output: industrial_distribution_key="resources/" - + RDIR - + "sector/" + SECDIR + "demand/industrial_distribution_key_elec_s{simpl}_{clusters}_{planning_horizons}.csv", threads: 1 @@ -2048,8 +1862,6 @@ rule build_base_industry_totals: #default data transactions_path="data/unsd_transactions.csv", output: base_industry_totals="resources/" - + RDIR - + "sector/" + SECDIR + "demand/base_industry_totals_{planning_horizons}_{demand}.csv", threads: 1 @@ -2058,8 +1870,6 @@ rule build_base_industry_totals: #default data benchmark: ( "benchmarks/" - + RDIR - + "sector/" + SECDIR + "build_base_industry_totals_{planning_horizons}_{demand}" ) @@ -2076,15 +1886,11 @@ rule build_industry_demand: #default data aluminium_year=config["demand_data"]["aluminium_year"], input: industrial_distribution_key="resources/" - + RDIR - + "sector/" + SECDIR + "demand/industrial_distribution_key_elec_s{simpl}_{clusters}_{planning_horizons}.csv", #industrial_production_per_country_tomorrow="resources/demand/industrial_production_per_country_tomorrow_{planning_horizons}_{demand}.csv", #industrial_production_per_country="data/industrial_production_per_country.csv", base_industry_totals="resources/" - + RDIR - + "sector/" + SECDIR + "demand/base_industry_totals_{planning_horizons}_{demand}.csv", industrial_database="data/industrial_database.csv", @@ -2092,8 +1898,6 @@ rule build_industry_demand: #default data industry_growth_cagr="data/demand/industry_growth_cagr.csv", output: industrial_energy_demand_per_node="resources/" - + RDIR - + "sector/" + SECDIR + "demand/industrial_energy_demand_per_node_elec_s{simpl}_{clusters}_{planning_horizons}_{demand}.csv", threads: 1 @@ -2102,8 +1906,6 @@ rule build_industry_demand: #default data benchmark: ( "benchmarks/" - + RDIR - + "sector/" + SECDIR + "industrial_energy_demand_per_node_elec_s{simpl}_{clusters}_{planning_horizons}_{demand}.csv" ) @@ -2119,25 +1921,17 @@ rule build_existing_heating_distribution: input: existing_heating="data/existing_infrastructure/existing_heating_raw.csv", clustered_pop_layout="resources/" - + RDIR - + "sector/" + SECDIR + "population_shares/pop_layout_elec_s{simpl}_{clusters}_{planning_horizons}.csv", clustered_pop_energy_layout="resources/" - + RDIR - + "sector/" + SECDIR + "demand/heat/nodal_energy_heat_totals_{demand}_s{simpl}_{clusters}_{planning_horizons}.csv", #"resources/population_shares/pop_weighted_energy_totals_s{simpl}_{clusters}.csv", district_heat_share="resources/" - + RDIR - + "sector/" + SECDIR + "demand/heat/district_heat_share_{demand}_s{simpl}_{clusters}_{planning_horizons}.csv", output: existing_heating_distribution="resources/" - + RDIR - + "sector/" + SECDIR + "heating/existing_heating_distribution_{demand}_s{simpl}_{clusters}_{planning_horizons}.csv", threads: 1 @@ -2170,25 +1964,17 @@ if config["foresight"] == "myopic": "resources/" + RDIR + "bus_regions/busmap_elec_s{simpl}_{clusters}.csv" ), clustered_pop_layout="resources/" - + RDIR - + "sector/" + SECDIR + "population_shares/pop_layout_elec_s{simpl}_{clusters}_{planning_horizons}.csv", costs=CDIR + "costs_{}.csv".format(config["scenario"]["planning_horizons"][0]), cop_soil_total="resources/" - + RDIR - + "sector/" + SECDIR + "cops/cop_soil_total_elec_s{simpl}_{clusters}_{planning_horizons}.nc", cop_air_total="resources/" - + RDIR - + "sector/" + SECDIR + "cops/cop_air_total_elec_s{simpl}_{clusters}_{planning_horizons}.nc", existing_heating_distribution="resources/" - + RDIR - + "sector/" + SECDIR + "heating/existing_heating_distribution_{demand}_s{simpl}_{clusters}_{planning_horizons}.csv", output: @@ -2253,13 +2039,9 @@ if config["foresight"] == "myopic": network_p=solved_previous_horizon, #solved network at previous time step costs=CDIR + "costs_{planning_horizons}.csv", cop_soil_total="resources/" - + RDIR - + "sector/" + SECDIR + "cops/cop_soil_total_elec_s{simpl}_{clusters}_{planning_horizons}.nc", cop_air_total="resources/" - + RDIR - + "sector/" + SECDIR + "cops/cop_air_total_elec_s{simpl}_{clusters}_{planning_horizons}.nc", output: From 434f4e2955943ea73816f3ac9d138ad1ff27aa09 Mon Sep 17 00:00:00 2001 From: yerbol-akhmetov Date: Sun, 10 Nov 2024 20:18:53 +0500 Subject: [PATCH 07/11] use BASE_DIR in sector scripts to enable subworkflow import --- scripts/build_base_energy_totals.py | 10 ++++++---- scripts/build_industry_demand.py | 14 ++++++++++---- scripts/copy_config.py | 12 ++++++++---- scripts/prepare_energy_totals.py | 6 ++++-- scripts/prepare_gas_network.py | 16 ++++++++++++---- scripts/prepare_sector_network.py | 17 ++++++++++++----- scripts/prepare_transport_data_input.py | 6 ++---- 7 files changed, 54 insertions(+), 27 deletions(-) diff --git a/scripts/build_base_energy_totals.py b/scripts/build_base_energy_totals.py index e16f44898..b2d1daedb 100644 --- a/scripts/build_base_energy_totals.py +++ b/scripts/build_base_energy_totals.py @@ -19,7 +19,7 @@ import pandas as pd import py7zr import requests -from _helpers import aggregate_fuels, get_conv_factors +from _helpers import BASE_DIR, aggregate_fuels, get_conv_factors _logger = logging.getLogger(__name__) @@ -375,7 +375,7 @@ def calc_sector(sector): if snakemake.params.update_data: # Delete and existing files to avoid duplication and double counting - files = glob.glob("data/demand/unsd/data/*.txt") + files = glob.glob(os.path.join(BASE_DIR, "data/demand/unsd/data/*.txt")) for f in files: os.remove(f) @@ -387,7 +387,7 @@ def calc_sector(sector): with ZipFile(BytesIO(zipresp.read())) as zfile: zfile.extractall("data/demand/unsd/data") - path = "data/demand/unsd/data" + path = os.path.join(BASE_DIR, "data/demand/unsd/data") # Get the files from the path provided in the OP all_files = list(Path("data/demand/unsd/data").glob("*.txt")) @@ -433,7 +433,9 @@ def calc_sector(sector): df_yr = df_yr[df_yr.country.isin(countries)] # Create an empty dataframe for energy_totals_base - energy_totals_cols = pd.read_csv("data/energy_totals_DF_2030.csv").columns + energy_totals_cols = pd.read_csv( + os.path.join(BASE_DIR, "data/energy_totals_DF_2030.csv") + ).columns energy_totals_base = pd.DataFrame(columns=energy_totals_cols, index=countries) # Lists that combine the different fuels in the dataset to the model's carriers diff --git a/scripts/build_industry_demand.py b/scripts/build_industry_demand.py index 7074e5e89..8617fb466 100644 --- a/scripts/build_industry_demand.py +++ b/scripts/build_industry_demand.py @@ -13,7 +13,7 @@ from itertools import product import pandas as pd -from _helpers import mock_snakemake, read_csv_nafix +from _helpers import BASE_DIR, mock_snakemake, read_csv_nafix _logger = logging.getLogger(__name__) @@ -69,8 +69,12 @@ def country_to_nodal(industrial_production, keys): ) industry_demand = pd.read_csv( - "data/custom/industry_demand_{0}_{1}.csv".format( - snakemake.wildcards["demand"], snakemake.wildcards["planning_horizons"] + os.path.join( + BASE_DIR, + "data/custom/industry_demand_{0}_{1}.csv".format( + snakemake.wildcards["demand"], + snakemake.wildcards["planning_horizons"], + ), ), index_col=[0, 1], ) @@ -204,7 +208,9 @@ def match_technology(df): geo_locs = match_technology(geo_locs).loc[countries_geo] aluminium_year = snakemake.params.aluminium_year - AL = read_csv_nafix("data/AL_production.csv", index_col=0) + AL = read_csv_nafix( + os.path.join(BASE_DIR, "data/AL_production.csv"), index_col=0 + ) # Filter data for the given year and countries AL_prod_tom = AL.query("Year == @aluminium_year and index in @countries_geo")[ "production[ktons/a]" diff --git a/scripts/copy_config.py b/scripts/copy_config.py index 780511d81..b7073c9fa 100644 --- a/scripts/copy_config.py +++ b/scripts/copy_config.py @@ -5,11 +5,15 @@ import os from shutil import copy +from _helpers import BASE_DIR + files_to_copy = { - "./config.yaml": "config.yaml", - "./Snakefile": "Snakefile", - "./scripts/solve_network.py": "solve_network.py", - "./scripts/prepare_sector_network.py": "prepare_sector_network.py", + os.path.join(BASE_DIR, "./config.yaml"): "config.yaml", + os.path.join(BASE_DIR, "./Snakefile"): "Snakefile", + os.path.join(BASE_DIR, "./scripts/solve_network.py"): "solve_network.py", + os.path.join( + BASE_DIR, "./scripts/prepare_sector_network.py" + ): "prepare_sector_network.py", } if __name__ == "__main__": diff --git a/scripts/prepare_energy_totals.py b/scripts/prepare_energy_totals.py index 119083f02..708420b27 100644 --- a/scripts/prepare_energy_totals.py +++ b/scripts/prepare_energy_totals.py @@ -17,7 +17,7 @@ import pandas as pd import py7zr import requests -from _helpers import read_csv_nafix, three_2_two_digits_country +from _helpers import BASE_DIR, read_csv_nafix, three_2_two_digits_country _logger = logging.getLogger(__name__) @@ -53,7 +53,9 @@ def calculate_end_values(df): investment_year = int(snakemake.wildcards.planning_horizons) demand_sc = snakemake.wildcards.demand # loading the demand scenrario wildcard - base_energy_totals = read_csv_nafix("data/energy_totals_base.csv", index_col=0) + base_energy_totals = read_csv_nafix( + os.path.join(BASE_DIR, "data/energy_totals_base.csv"), index_col=0 + ) growth_factors_cagr = read_csv_nafix( snakemake.input.growth_factors_cagr, index_col=0 ) diff --git a/scripts/prepare_gas_network.py b/scripts/prepare_gas_network.py index 59078803e..cbdcd120a 100644 --- a/scripts/prepare_gas_network.py +++ b/scripts/prepare_gas_network.py @@ -19,7 +19,12 @@ import matplotlib.colors as colors import matplotlib.pyplot as plt import pandas as pd -from _helpers import content_retrieve, progress_retrieve, two_2_three_digits_country +from _helpers import ( + BASE_DIR, + content_retrieve, + progress_retrieve, + two_2_three_digits_country, +) from build_shapes import gadm from matplotlib.lines import Line2D from pyproj import CRS @@ -58,8 +63,8 @@ def download_IGGIELGN_gas_network(): url = "https://zenodo.org/record/4767098/files/IGGIELGN.zip" # Save locations - zip_fn = Path("IGGIELGN.zip") - to_fn = Path("data/gas_network/scigrid-gas") + zip_fn = Path(os.path.join(BASE_DIR, "IGGIELGN.zip")) + to_fn = Path(os.path.join(BASE_DIR, "data/gas_network/scigrid-gas")) logger.info(f"Downloading databundle from '{url}'.") progress_retrieve(url, zip_fn) @@ -344,6 +349,7 @@ def download_GADM(country_code, update=False, out_logging=False): GADM_filename = get_GADM_filename(country_code) GADM_inputfile_gpkg = os.path.join( + BASE_DIR, "data", "gadm", GADM_filename, @@ -887,7 +893,9 @@ def check_existence(row): elif snakemake.params.gas_config["network_data"] == "IGGIELGN": download_IGGIELGN_gas_network() - gas_network = "data/gas_network/scigrid-gas/data/IGGIELGN_PipeSegments.geojson" + gas_network = os.path.join( + BASE_DIR, "data/gas_network/scigrid-gas/data/IGGIELGN_PipeSegments.geojson" + ) pipelines = load_IGGIELGN_data(gas_network) pipelines = prepare_IGGIELGN_data(pipelines) diff --git a/scripts/prepare_sector_network.py b/scripts/prepare_sector_network.py index ac125b0f9..9a731fa53 100644 --- a/scripts/prepare_sector_network.py +++ b/scripts/prepare_sector_network.py @@ -16,6 +16,7 @@ import ruamel.yaml import xarray as xr from _helpers import ( + BASE_DIR, create_dummy_data, create_network_topology, cycling_shift, @@ -322,8 +323,11 @@ def add_hydrogen(n, costs): if snakemake.config["sector"]["hydrogen"]["underground_storage"]: if snakemake.config["custom_data"]["h2_underground"]: custom_cavern = pd.read_csv( - "data/custom/h2_underground_{0}_{1}.csv".format( - demand_sc, investment_year + os.path.join( + BASE_DIR, + "data/custom/h2_underground_{0}_{1}.csv".format( + demand_sc, investment_year + ), ) ) # countries = n.buses.country.unique().to_list() @@ -2661,9 +2665,12 @@ def add_residential(n, costs): def add_custom_water_cost(n): for country in countries: water_costs = pd.read_csv( - "resources/custom_data/{}_water_costs.csv".format(country), - sep=",", - index_col=0, + os.path.join( + BASE_DIR, + "resources/custom_data/{}_water_costs.csv".format(country), + sep=",", + index_col=0, + ) ) water_costs = water_costs.filter(like=country, axis=0).loc[spatial.nodes] electrolysis_links = n.links.filter(like=country, axis=0).filter( diff --git a/scripts/prepare_transport_data_input.py b/scripts/prepare_transport_data_input.py index cffa163e2..a15608adc 100644 --- a/scripts/prepare_transport_data_input.py +++ b/scripts/prepare_transport_data_input.py @@ -10,9 +10,7 @@ import country_converter as coco import numpy as np import pandas as pd - -# from _helpers import configure_logging - +from _helpers import BASE_DIR # logger = logging.getLogger(__name__) @@ -127,7 +125,7 @@ def download_CO2_emissions(): if vehicles_csv.empty or CO2_emissions_csv.empty: # In case one of the urls is not working, we can use the hard-coded data - src = os.getcwd() + "/data/temp_hard_coded/transport_data.csv" + src = BASE_DIR + "/data/temp_hard_coded/transport_data.csv" dest = snakemake.output.transport_data_input shutil.copy(src, dest) else: From f5fa5fb421bfa3993ae0563581562fc9d727879b Mon Sep 17 00:00:00 2001 From: yerbol-akhmetov Date: Sun, 10 Nov 2024 20:37:40 +0500 Subject: [PATCH 08/11] add missing BASE_DIR --- scripts/build_base_energy_totals.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/scripts/build_base_energy_totals.py b/scripts/build_base_energy_totals.py index b2d1daedb..4ac1f7889 100644 --- a/scripts/build_base_energy_totals.py +++ b/scripts/build_base_energy_totals.py @@ -385,12 +385,14 @@ def calc_sector(sector): with urlopen(zipurl) as zipresp: with ZipFile(BytesIO(zipresp.read())) as zfile: - zfile.extractall("data/demand/unsd/data") + zfile.extractall(os.path.join(BASE_DIR, "data/demand/unsd/data")) path = os.path.join(BASE_DIR, "data/demand/unsd/data") # Get the files from the path provided in the OP - all_files = list(Path("data/demand/unsd/data").glob("*.txt")) + all_files = list( + Path(os.path.join(BASE_DIR, "data/demand/unsd/data")).glob("*.txt") + ) # Create a dataframe from all downloaded files df = pd.concat( From bae91e22f69b576aab9e5433e6016d91d466975b Mon Sep 17 00:00:00 2001 From: yerbol-akhmetov Date: Sun, 10 Nov 2024 21:35:42 +0500 Subject: [PATCH 09/11] add release notes --- doc/release_notes.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/doc/release_notes.rst b/doc/release_notes.rst index 0d1b7c746..1897326a6 100644 --- a/doc/release_notes.rst +++ b/doc/release_notes.rst @@ -42,6 +42,8 @@ E.g. if a new rule becomes available describe how to use it `make test` and in o * Fix bugs in `prepare_sector_network.py` related to links with H2 buses and bug of re-addition of H2 and battery carriers in present `PR #1145 `_ +* Enable sector rules import in subworkflow `PR #1178 `_ + PyPSA-Earth 0.4.1 ================= From a3167761dd02fcdd6889cab4678531f6c1b7cd59 Mon Sep 17 00:00:00 2001 From: Davide Fioriti <67809479+davide-f@users.noreply.github.com> Date: Mon, 11 Nov 2024 22:41:52 +0100 Subject: [PATCH 10/11] Minor revision release_notes.rst --- doc/release_notes.rst | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/doc/release_notes.rst b/doc/release_notes.rst index a69251d6f..62c21ab2c 100644 --- a/doc/release_notes.rst +++ b/doc/release_notes.rst @@ -14,35 +14,35 @@ E.g. if a new rule becomes available describe how to use it `make test` and in o **New Features and Major Changes** -* The workflow configuration now supports incremental changes to the default configuration in the `config.yaml` and configfiles passed to snakemake via `--configfile myconfig.yaml`. Therefore the user may now only include settings in their `config.yaml` which differ from the default configuration. One can think of the new `config.yaml` as of a list of arguments in a python function that already have a default. So in principle the `config.yaml` could now be empty, and the workflow would still run. `PR #1053 `_ +* The workflow configuration now supports incremental changes to the default configuration in the `config.yaml` and configfiles passed to snakemake via `--configfile myconfig.yaml`. Therefore the user may now only include settings in their `config.yaml` which differ from the default configuration. One can think of the new `config.yaml` as of a list of arguments in a python function that already have a default. So in principle the `config.yaml` could now be empty, and the workflow would still run. `PR #1053 `__ -* Local tests are now run with `make test`. This uses a `Makefile` which runs snakemake calls with different configurations. `PR #1053 `_ +* Local tests are now run with `make test`. This uses a `Makefile` which runs snakemake calls with different configurations. `PR #1053 `__ + +* Integrate RDIR into sector rules to store intermediate data in scenario folders `PR #1154 `__ **Minor Changes and bug-fixing** -* The default configuration for `electricity:estimate_renewable_capacities:year` was updated from 2020 to 2023. `PR #1106 `_ +* The default configuration for `electricity:estimate_renewable_capacities:year` was updated from 2020 to 2023. `PR #1106 `__ -* Include a dedicated cutout for North America in bundle_config.yaml `PR #1121 `_ +* Include a dedicated cutout for North America in bundle_config.yaml `PR #1121 `__ -* Include a dedicated cutout for Europe in bundle_config.yaml `PR #1125 `_ +* Include a dedicated cutout for Europe in bundle_config.yaml `PR #1125 `__ -* Fix the mismatch between buses and x, y locations while creating H2 Stores `PR #1134 `_ +* Fix the mismatch between buses and x, y locations while creating H2 Stores `PR #1134 `__ -* Enable configfile specification for mock_snakemake `PR #1135 `_ +* Enable configfile specification for mock_snakemake `PR #1135 `__ * Fix pre-commit docformatter python issue. `PR #1153 `__ -* Use BASE_DIR in rules and `_helpers.py` script for facilitate module import in subworkflow `PR #1137 `_ - -* Include a dedicated cutout for Oceania in bundle_config.yaml `PR #1157 `_ +* Use BASE_DIR in rules and `_helpers.py` script for facilitate module import in subworkflow `PR #1137 `__ -* Drop duplicate entries in `AL_production.csv` data used in `build_industry_demand` rule `PR #1143 `_ +* Include a dedicated cutout for Oceania in bundle_config.yaml `PR #1157 `__ -* The computation of `hydro_profile.nc` in `build_renewable_profiles.py` is not differentiated whether alternative clustering is applied or not; the indexing of the different power plants in `add_electricity.py` is performed according to the bus either in case alternative clustering is applied or not and a `hydro_inflow_factor` is computed prior to the computation of `inflow_t` to split the inflow according to the capacity of each different unit of each power plant (if more units are present). `PR #1119 `_ +* Drop duplicate entries in `AL_production.csv` data used in `build_industry_demand` rule `PR #1143 `__ -* Fix bugs in `prepare_sector_network.py` related to links with H2 buses and bug of re-addition of H2 and battery carriers in present `PR #1145 `_ +* The computation of `hydro_profile.nc` in `build_renewable_profiles.py` is not differentiated whether alternative clustering is applied or not; the indexing of the different power plants in `add_electricity.py` is performed according to the bus either in case alternative clustering is applied or not and a `hydro_inflow_factor` is computed prior to the computation of `inflow_t` to split the inflow according to the capacity of each different unit of each power plant (if more units are present). `PR #1119 `__ -* Integrate RDIR into sector rules to store intermediate data in scenario folders `PR #1154 `_ +* Fix bugs in `prepare_sector_network.py` related to links with H2 buses and bug of re-addition of H2 and battery carriers in present `PR #1145 `__ PyPSA-Earth 0.4.1 ================= From a59c02d513503a51e46110a47744b8da439ced27 Mon Sep 17 00:00:00 2001 From: Davide Fioriti <67809479+davide-f@users.noreply.github.com> Date: Mon, 11 Nov 2024 22:57:34 +0100 Subject: [PATCH 11/11] Revise location of changes --- doc/release_notes.rst | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/doc/release_notes.rst b/doc/release_notes.rst index db1b7774e..5cc3422c0 100644 --- a/doc/release_notes.rst +++ b/doc/release_notes.rst @@ -20,32 +20,32 @@ E.g. if a new rule becomes available describe how to use it `make test` and in o * Integrate RDIR into sector rules to store intermediate data in scenario folders `PR #1154 `__ -**Minor Changes and bug-fixing** - -* The default configuration for `electricity:estimate_renewable_capacities:year` was updated from 2020 to 2023. `PR #1106 `__ - * Include a dedicated cutout for North America in bundle_config.yaml `PR #1121 `__ * Include a dedicated cutout for Europe in bundle_config.yaml `PR #1125 `__ +* Include a dedicated cutout for Oceania in bundle_config.yaml `PR #1157 `__ + +* Use BASE_DIR in rules and `_helpers.py` script for facilitate module import in subworkflow `PR #1137 `__ + +* Enable sector rules import in subworkflow `PR #1178 `__ + +**Minor Changes and bug-fixing** + +* The default configuration for `electricity:estimate_renewable_capacities:year` was updated from 2020 to 2023. `PR #1106 `__ + * Fix the mismatch between buses and x, y locations while creating H2 Stores `PR #1134 `__ * Enable configfile specification for mock_snakemake `PR #1135 `__ * Fix pre-commit docformatter python issue. `PR #1153 `__ -* Use BASE_DIR in rules and `_helpers.py` script for facilitate module import in subworkflow `PR #1137 `__ - -* Include a dedicated cutout for Oceania in bundle_config.yaml `PR #1157 `__ - * Drop duplicate entries in `AL_production.csv` data used in `build_industry_demand` rule `PR #1143 `__ * The computation of `hydro_profile.nc` in `build_renewable_profiles.py` is not differentiated whether alternative clustering is applied or not; the indexing of the different power plants in `add_electricity.py` is performed according to the bus either in case alternative clustering is applied or not and a `hydro_inflow_factor` is computed prior to the computation of `inflow_t` to split the inflow according to the capacity of each different unit of each power plant (if more units are present). `PR #1119 `__ * Fix bugs in `prepare_sector_network.py` related to links with H2 buses and bug of re-addition of H2 and battery carriers in present `PR #1145 `__ -* Enable sector rules import in subworkflow `PR #1178 `_ - PyPSA-Earth 0.4.1 =================