diff --git a/Snakefile b/Snakefile index 218041c67..2cb7d7d50 100644 --- a/Snakefile +++ b/Snakefile @@ -50,8 +50,9 @@ config["scenario"]["unc"] = [ run = config.get("run", {}) RDIR = run["name"] + "/" if run.get("name") else "" CDIR = RDIR if not run.get("shared_cutouts") else "" -SDIR = config["summary_dir"].strip("/") + f"/{RDIR}" -RESDIR = config["results_dir"].strip("/") + f"/{RDIR}" +SECDIR = run["sector_name"] + "/" if run.get("sector_name") else "" +SDIR = config["summary_dir"].strip("/") + f"/{RDIR}{SECDIR}" +RESDIR = config["results_dir"].strip("/") + f"/{RDIR}{SECDIR}" COSTDIR = config["costs_dir"] load_data_paths = get_load_paths_gegis("data", config) @@ -1024,7 +1025,7 @@ rule prepare_urban_percent: rule prepare_transport_data_input: output: - transport_data_input="resources/transport_data.csv", + transport_data_input="resources/" + SECDIR + "transport_data.csv", script: "scripts/prepare_transport_data_input.py" @@ -1049,7 +1050,9 @@ if not config["custom_data"]["gas_network"]: + RDIR + "bus_regions/regions_onshore_elec_s{simpl}_{clusters}.geojson", output: - clustered_gas_network="resources/gas_networks/gas_network_elec_s{simpl}_{clusters}.csv", + clustered_gas_network="resources/" + + SECDIR + + "gas_networks/gas_network_elec_s{simpl}_{clusters}.csv", # TODO: Should be a own snakemake rule # gas_network_fig_1="resources/gas_networks/existing_gas_pipelines_{simpl}_{clusters}.png", # gas_network_fig_2="resources/gas_networks/clustered_gas_pipelines_{simpl}_{clusters}.png", @@ -1065,22 +1068,46 @@ rule prepare_sector_network: + "prenetworks/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}_{sopts}_{planning_horizons}_{discountrate}_{demand}_presec.nc", costs=COSTDIR + "costs_{planning_horizons}.csv", h2_cavern="data/hydrogen_salt_cavern_potentials.csv", - nodal_energy_totals="resources/demand/heat/nodal_energy_heat_totals_{demand}_s{simpl}_{clusters}_{planning_horizons}.csv", - transport="resources/demand/transport_{demand}_s{simpl}_{clusters}_{planning_horizons}.csv", - avail_profile="resources/pattern_profiles/avail_profile_{demand}_s{simpl}_{clusters}_{planning_horizons}.csv", - dsm_profile="resources/pattern_profiles/dsm_profile_{demand}_s{simpl}_{clusters}_{planning_horizons}.csv", - nodal_transport_data="resources/demand/nodal_transport_data_{demand}_s{simpl}_{clusters}_{planning_horizons}.csv", + nodal_energy_totals="resources/" + + SECDIR + + "demand/heat/nodal_energy_heat_totals_{demand}_s{simpl}_{clusters}_{planning_horizons}.csv", + transport="resources/" + + SECDIR + + "demand/transport_{demand}_s{simpl}_{clusters}_{planning_horizons}.csv", + avail_profile="resources/" + + SECDIR + + "pattern_profiles/avail_profile_{demand}_s{simpl}_{clusters}_{planning_horizons}.csv", + dsm_profile="resources/" + + SECDIR + + "pattern_profiles/dsm_profile_{demand}_s{simpl}_{clusters}_{planning_horizons}.csv", + nodal_transport_data="resources/" + + SECDIR + + "demand/nodal_transport_data_{demand}_s{simpl}_{clusters}_{planning_horizons}.csv", overrides="data/override_component_attrs", - clustered_pop_layout="resources/population_shares/pop_layout_elec_s{simpl}_{clusters}_{planning_horizons}.csv", - industrial_demand="resources/demand/industrial_energy_demand_per_node_elec_s{simpl}_{clusters}_{planning_horizons}_{demand}.csv", + clustered_pop_layout="resources/" + + SECDIR + + "population_shares/pop_layout_elec_s{simpl}_{clusters}_{planning_horizons}.csv", + industrial_demand="resources/" + + SECDIR + + "demand/industrial_energy_demand_per_node_elec_s{simpl}_{clusters}_{planning_horizons}_{demand}.csv", energy_totals="data/energy_totals_{demand}_{planning_horizons}.csv", airports="data/airports.csv", ports="data/ports.csv", - heat_demand="resources/demand/heat/heat_demand_{demand}_s{simpl}_{clusters}_{planning_horizons}.csv", - ashp_cop="resources/demand/heat/ashp_cop_{demand}_s{simpl}_{clusters}_{planning_horizons}.csv", - gshp_cop="resources/demand/heat/gshp_cop_{demand}_s{simpl}_{clusters}_{planning_horizons}.csv", - solar_thermal="resources/demand/heat/solar_thermal_{demand}_s{simpl}_{clusters}_{planning_horizons}.csv", - district_heat_share="resources/demand/heat/district_heat_share_{demand}_s{simpl}_{clusters}_{planning_horizons}.csv", + heat_demand="resources/" + + SECDIR + + "demand/heat/heat_demand_{demand}_s{simpl}_{clusters}_{planning_horizons}.csv", + ashp_cop="resources/" + + SECDIR + + "demand/heat/ashp_cop_{demand}_s{simpl}_{clusters}_{planning_horizons}.csv", + gshp_cop="resources/" + + SECDIR + + "demand/heat/gshp_cop_{demand}_s{simpl}_{clusters}_{planning_horizons}.csv", + solar_thermal="resources/" + + SECDIR + + "demand/heat/solar_thermal_{demand}_s{simpl}_{clusters}_{planning_horizons}.csv", + district_heat_share="resources/" + + SECDIR + + "demand/heat/district_heat_share_{demand}_s{simpl}_{clusters}_{planning_horizons}.csv", biomass_transport_costs="data/temp_hard_coded/biomass_transport_costs.csv", shapes_path="resources/" + RDIR @@ -1088,7 +1115,9 @@ rule prepare_sector_network: pipelines=( "data/custom/pipelines.csv" if config["custom_data"]["gas_network"] - else "resources/gas_networks/gas_network_elec_s{simpl}_{clusters}.csv" + else "resources/" + + SECDIR + + "gas_networks/gas_network_elec_s{simpl}_{clusters}.csv" ), output: RESDIR @@ -1110,7 +1139,7 @@ rule build_ship_profile: snapshots=config["snapshots"], ship_opts=config["export"]["ship"], output: - ship_profile="resources/ship_profile_{h2export}TWh.csv", + ship_profile="resources/" + SECDIR + "ship_profile_{h2export}TWh.csv", script: "scripts/build_ship_profile.py" @@ -1128,7 +1157,7 @@ rule add_export: overrides="data/override_component_attrs", export_ports="data/export_ports.csv", costs=COSTDIR + "costs_{planning_horizons}.csv", - ship_profile="resources/ship_profile_{h2export}TWh.csv", + ship_profile="resources/" + SECDIR + "ship_profile_{h2export}TWh.csv", network=RESDIR + "prenetworks/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}_{sopts}_{planning_horizons}_{discountrate}_{demand}.nc", shapes_path="resources/" @@ -1148,13 +1177,17 @@ rule override_respot: countries=config["countries"], input: **{ - f"custom_res_pot_{tech}_{planning_horizons}_{discountrate}": f"resources/custom_renewables/{tech}_{planning_horizons}_{discountrate}_potential.csv" + f"custom_res_pot_{tech}_{planning_horizons}_{discountrate}": "resources/" + + SECDIR + + f"custom_renewables/{tech}_{planning_horizons}_{discountrate}_potential.csv" for tech in config["custom_data"]["renewables"] for discountrate in config["costs"]["discountrate"] for planning_horizons in config["scenario"]["planning_horizons"] }, **{ - f"custom_res_ins_{tech}_{planning_horizons}_{discountrate}": f"resources/custom_renewables/{tech}_{planning_horizons}_{discountrate}_installable.csv" + f"custom_res_ins_{tech}_{planning_horizons}_{discountrate}": "resources/" + + SECDIR + + f"custom_renewables/{tech}_{planning_horizons}_{discountrate}_installable.csv" for tech in config["custom_data"]["renewables"] for discountrate in config["costs"]["discountrate"] for planning_horizons in config["scenario"]["planning_horizons"] @@ -1175,15 +1208,27 @@ rule prepare_transport_data: energy_totals_name="data/energy_totals_{demand}_{planning_horizons}.csv", traffic_data_KFZ="data/emobility/KFZ__count", traffic_data_Pkw="data/emobility/Pkw__count", - transport_name="resources/transport_data.csv", - clustered_pop_layout="resources/population_shares/pop_layout_elec_s{simpl}_{clusters}_{planning_horizons}.csv", - temp_air_total="resources/temperatures/temp_air_total_elec_s{simpl}_{clusters}_{planning_horizons}.nc", + transport_name="resources/" + SECDIR + "transport_data.csv", + clustered_pop_layout="resources/" + + SECDIR + + "population_shares/pop_layout_elec_s{simpl}_{clusters}_{planning_horizons}.csv", + temp_air_total="resources/" + + SECDIR + + "temperatures/temp_air_total_elec_s{simpl}_{clusters}_{planning_horizons}.nc", output: # nodal_energy_totals="resources/nodal_energy_totals_s{simpl}_{clusters}.csv", - transport="resources/demand/transport_{demand}_s{simpl}_{clusters}_{planning_horizons}.csv", - avail_profile="resources/pattern_profiles/avail_profile_{demand}_s{simpl}_{clusters}_{planning_horizons}.csv", - dsm_profile="resources/pattern_profiles/dsm_profile_{demand}_s{simpl}_{clusters}_{planning_horizons}.csv", - nodal_transport_data="resources/demand/nodal_transport_data_{demand}_s{simpl}_{clusters}_{planning_horizons}.csv", + transport="resources/" + + SECDIR + + "demand/transport_{demand}_s{simpl}_{clusters}_{planning_horizons}.csv", + avail_profile="resources/" + + SECDIR + + "pattern_profiles/avail_profile_{demand}_s{simpl}_{clusters}_{planning_horizons}.csv", + dsm_profile="resources/" + + SECDIR + + "pattern_profiles/dsm_profile_{demand}_s{simpl}_{clusters}_{planning_horizons}.csv", + nodal_transport_data="resources/" + + SECDIR + + "demand/nodal_transport_data_{demand}_s{simpl}_{clusters}_{planning_horizons}.csv", script: "scripts/prepare_transport_data.py" @@ -1192,23 +1237,51 @@ rule build_cop_profiles: params: heat_pump_sink_T=config["sector"]["heat_pump_sink_T"], input: - temp_soil_total="resources/temperatures/temp_soil_total_elec_s{simpl}_{clusters}_{planning_horizons}.nc", - temp_soil_rural="resources/temperatures/temp_soil_rural_elec_s{simpl}_{clusters}_{planning_horizons}.nc", - temp_soil_urban="resources/temperatures/temp_soil_urban_elec_s{simpl}_{clusters}_{planning_horizons}.nc", - temp_air_total="resources/temperatures/temp_air_total_elec_s{simpl}_{clusters}_{planning_horizons}.nc", - temp_air_rural="resources/temperatures/temp_air_rural_elec_s{simpl}_{clusters}_{planning_horizons}.nc", - temp_air_urban="resources/temperatures/temp_air_urban_elec_s{simpl}_{clusters}_{planning_horizons}.nc", + temp_soil_total="resources/" + + SECDIR + + "temperatures/temp_soil_total_elec_s{simpl}_{clusters}_{planning_horizons}.nc", + temp_soil_rural="resources/" + + SECDIR + + "temperatures/temp_soil_rural_elec_s{simpl}_{clusters}_{planning_horizons}.nc", + temp_soil_urban="resources/" + + SECDIR + + "temperatures/temp_soil_urban_elec_s{simpl}_{clusters}_{planning_horizons}.nc", + temp_air_total="resources/" + + SECDIR + + "temperatures/temp_air_total_elec_s{simpl}_{clusters}_{planning_horizons}.nc", + temp_air_rural="resources/" + + SECDIR + + "temperatures/temp_air_rural_elec_s{simpl}_{clusters}_{planning_horizons}.nc", + temp_air_urban="resources/" + + SECDIR + + "temperatures/temp_air_urban_elec_s{simpl}_{clusters}_{planning_horizons}.nc", output: - cop_soil_total="resources/cops/cop_soil_total_elec_s{simpl}_{clusters}_{planning_horizons}.nc", - cop_soil_rural="resources/cops/cop_soil_rural_elec_s{simpl}_{clusters}_{planning_horizons}.nc", - cop_soil_urban="resources/cops/cop_soil_urban_elec_s{simpl}_{clusters}_{planning_horizons}.nc", - cop_air_total="resources/cops/cop_air_total_elec_s{simpl}_{clusters}_{planning_horizons}.nc", - cop_air_rural="resources/cops/cop_air_rural_elec_s{simpl}_{clusters}_{planning_horizons}.nc", - cop_air_urban="resources/cops/cop_air_urban_elec_s{simpl}_{clusters}_{planning_horizons}.nc", + cop_soil_total="resources/" + + SECDIR + + "cops/cop_soil_total_elec_s{simpl}_{clusters}_{planning_horizons}.nc", + cop_soil_rural="resources/" + + SECDIR + + "cops/cop_soil_rural_elec_s{simpl}_{clusters}_{planning_horizons}.nc", + cop_soil_urban="resources/" + + SECDIR + + "cops/cop_soil_urban_elec_s{simpl}_{clusters}_{planning_horizons}.nc", + cop_air_total="resources/" + + SECDIR + + "cops/cop_air_total_elec_s{simpl}_{clusters}_{planning_horizons}.nc", + cop_air_rural="resources/" + + SECDIR + + "cops/cop_air_rural_elec_s{simpl}_{clusters}_{planning_horizons}.nc", + cop_air_urban="resources/" + + SECDIR + + "cops/cop_air_urban_elec_s{simpl}_{clusters}_{planning_horizons}.nc", resources: mem_mb=20000, benchmark: - "benchmarks/build_cop_profiles/s{simpl}_{clusters}_{planning_horizons}" + ( + "benchmarks/" + + SECDIR + + "build_cop_profiles/s{simpl}_{clusters}_{planning_horizons}" + ) script: "scripts/build_cop_profiles.py" @@ -1217,20 +1290,44 @@ rule prepare_heat_data: input: network="networks/" + RDIR + "elec_s{simpl}_{clusters}.nc", energy_totals_name="data/energy_totals_{demand}_{planning_horizons}.csv", - clustered_pop_layout="resources/population_shares/pop_layout_elec_s{simpl}_{clusters}_{planning_horizons}.csv", - temp_air_total="resources/temperatures/temp_air_total_elec_s{simpl}_{clusters}_{planning_horizons}.nc", - cop_soil_total="resources/cops/cop_soil_total_elec_s{simpl}_{clusters}_{planning_horizons}.nc", - cop_air_total="resources/cops/cop_air_total_elec_s{simpl}_{clusters}_{planning_horizons}.nc", - solar_thermal_total="resources/demand/heat/solar_thermal_total_elec_s{simpl}_{clusters}_{planning_horizons}.nc", - heat_demand_total="resources/demand/heat/heat_demand_total_elec_s{simpl}_{clusters}_{planning_horizons}.nc", + clustered_pop_layout="resources/" + + SECDIR + + "population_shares/pop_layout_elec_s{simpl}_{clusters}_{planning_horizons}.csv", + temp_air_total="resources/" + + SECDIR + + "temperatures/temp_air_total_elec_s{simpl}_{clusters}_{planning_horizons}.nc", + cop_soil_total="resources/" + + SECDIR + + "cops/cop_soil_total_elec_s{simpl}_{clusters}_{planning_horizons}.nc", + cop_air_total="resources/" + + SECDIR + + "cops/cop_air_total_elec_s{simpl}_{clusters}_{planning_horizons}.nc", + solar_thermal_total="resources/" + + SECDIR + + "demand/heat/solar_thermal_total_elec_s{simpl}_{clusters}_{planning_horizons}.nc", + heat_demand_total="resources/" + + SECDIR + + "demand/heat/heat_demand_total_elec_s{simpl}_{clusters}_{planning_horizons}.nc", heat_profile="data/heat_load_profile_BDEW.csv", output: - nodal_energy_totals="resources/demand/heat/nodal_energy_heat_totals_{demand}_s{simpl}_{clusters}_{planning_horizons}.csv", - heat_demand="resources/demand/heat/heat_demand_{demand}_s{simpl}_{clusters}_{planning_horizons}.csv", - ashp_cop="resources/demand/heat/ashp_cop_{demand}_s{simpl}_{clusters}_{planning_horizons}.csv", - gshp_cop="resources/demand/heat/gshp_cop_{demand}_s{simpl}_{clusters}_{planning_horizons}.csv", - solar_thermal="resources/demand/heat/solar_thermal_{demand}_s{simpl}_{clusters}_{planning_horizons}.csv", - district_heat_share="resources/demand/heat/district_heat_share_{demand}_s{simpl}_{clusters}_{planning_horizons}.csv", + nodal_energy_totals="resources/" + + SECDIR + + "demand/heat/nodal_energy_heat_totals_{demand}_s{simpl}_{clusters}_{planning_horizons}.csv", + heat_demand="resources/" + + SECDIR + + "demand/heat/heat_demand_{demand}_s{simpl}_{clusters}_{planning_horizons}.csv", + ashp_cop="resources/" + + SECDIR + + "demand/heat/ashp_cop_{demand}_s{simpl}_{clusters}_{planning_horizons}.csv", + gshp_cop="resources/" + + SECDIR + + "demand/heat/gshp_cop_{demand}_s{simpl}_{clusters}_{planning_horizons}.csv", + solar_thermal="resources/" + + SECDIR + + "demand/heat/solar_thermal_{demand}_s{simpl}_{clusters}_{planning_horizons}.csv", + district_heat_share="resources/" + + SECDIR + + "demand/heat/district_heat_share_{demand}_s{simpl}_{clusters}_{planning_horizons}.csv", script: "scripts/prepare_heat_data.py" @@ -1272,9 +1369,15 @@ rule build_solar_thermal_profiles: solar_thermal_config=config["solar_thermal"], snapshots=config["snapshots"], input: - pop_layout_total="resources/population_shares/pop_layout_total_{planning_horizons}.nc", - pop_layout_urban="resources/population_shares/pop_layout_urban_{planning_horizons}.nc", - pop_layout_rural="resources/population_shares/pop_layout_rural_{planning_horizons}.nc", + pop_layout_total="resources/" + + SECDIR + + "population_shares/pop_layout_total_{planning_horizons}.nc", + pop_layout_urban="resources/" + + SECDIR + + "population_shares/pop_layout_urban_{planning_horizons}.nc", + pop_layout_rural="resources/" + + SECDIR + + "population_shares/pop_layout_rural_{planning_horizons}.nc", regions_onshore="resources/" + RDIR + "bus_regions/regions_onshore_elec_s{simpl}_{clusters}.geojson", @@ -1284,13 +1387,23 @@ rule build_solar_thermal_profiles: + ".nc", # default to first cutout found output: - solar_thermal_total="resources/demand/heat/solar_thermal_total_elec_s{simpl}_{clusters}_{planning_horizons}.nc", - solar_thermal_urban="resources/demand/heat/solar_thermal_urban_elec_s{simpl}_{clusters}_{planning_horizons}.nc", - solar_thermal_rural="resources/demand/heat/solar_thermal_rural_elec_s{simpl}_{clusters}_{planning_horizons}.nc", + solar_thermal_total="resources/" + + SECDIR + + "demand/heat/solar_thermal_total_elec_s{simpl}_{clusters}_{planning_horizons}.nc", + solar_thermal_urban="resources/" + + SECDIR + + "demand/heat/solar_thermal_urban_elec_s{simpl}_{clusters}_{planning_horizons}.nc", + solar_thermal_rural="resources/" + + SECDIR + + "demand/heat/solar_thermal_rural_elec_s{simpl}_{clusters}_{planning_horizons}.nc", resources: mem_mb=20000, benchmark: - "benchmarks/build_solar_thermal_profiles/s{simpl}_{clusters}_{planning_horizons}" + ( + "benchmarks/" + + SECDIR + + "build_solar_thermal_profiles/s{simpl}_{clusters}_{planning_horizons}" + ) script: "scripts/build_solar_thermal_profiles.py" @@ -1307,14 +1420,22 @@ rule build_population_layouts: + ".nc", # default to first cutout found output: - pop_layout_total="resources/population_shares/pop_layout_total_{planning_horizons}.nc", - pop_layout_urban="resources/population_shares/pop_layout_urban_{planning_horizons}.nc", - pop_layout_rural="resources/population_shares/pop_layout_rural_{planning_horizons}.nc", - gdp_layout="resources/gdp_shares/gdp_layout_{planning_horizons}.nc", + pop_layout_total="resources/" + + SECDIR + + "population_shares/pop_layout_total_{planning_horizons}.nc", + pop_layout_urban="resources/" + + SECDIR + + "population_shares/pop_layout_urban_{planning_horizons}.nc", + pop_layout_rural="resources/" + + SECDIR + + "population_shares/pop_layout_rural_{planning_horizons}.nc", + gdp_layout="resources/" + + SECDIR + + "gdp_shares/gdp_layout_{planning_horizons}.nc", resources: mem_mb=20000, benchmark: - "benchmarks/build_population_layouts_{planning_horizons}" + ("benchmarks/" + SECDIR + "build_population_layouts_{planning_horizons}") threads: 8 script: "scripts/build_population_layouts.py" @@ -1324,17 +1445,23 @@ rule move_hardcoded_files_temp: input: "data/temp_hard_coded/energy_totals.csv", output: - "resources/energy_totals.csv", + "resources/" + SECDIR + "energy_totals.csv", shell: "cp -a data/temp_hard_coded/. resources" rule build_clustered_population_layouts: input: - pop_layout_total="resources/population_shares/pop_layout_total_{planning_horizons}.nc", - pop_layout_urban="resources/population_shares/pop_layout_urban_{planning_horizons}.nc", - pop_layout_rural="resources/population_shares/pop_layout_rural_{planning_horizons}.nc", - gdp_layout="resources/gdp_shares/gdp_layout_{planning_horizons}.nc", + pop_layout_total="resources/" + + SECDIR + + "population_shares/pop_layout_total_{planning_horizons}.nc", + pop_layout_urban="resources/" + + SECDIR + + "population_shares/pop_layout_urban_{planning_horizons}.nc", + pop_layout_rural="resources/" + + SECDIR + + "population_shares/pop_layout_rural_{planning_horizons}.nc", + gdp_layout="resources/" + RDIR + "gdp_shares/gdp_layout_{planning_horizons}.nc", regions_onshore="resources/" + RDIR + "bus_regions/regions_onshore_elec_s{simpl}_{clusters}.geojson", @@ -1344,12 +1471,20 @@ rule build_clustered_population_layouts: + ".nc", # default to first cutout found output: - clustered_pop_layout="resources/population_shares/pop_layout_elec_s{simpl}_{clusters}_{planning_horizons}.csv", - clustered_gdp_layout="resources/gdp_shares/gdp_layout_elec_s{simpl}_{clusters}_{planning_horizons}.csv", + clustered_pop_layout="resources/" + + SECDIR + + "population_shares/pop_layout_elec_s{simpl}_{clusters}_{planning_horizons}.csv", + clustered_gdp_layout="resources/" + + SECDIR + + "gdp_shares/gdp_layout_elec_s{simpl}_{clusters}_{planning_horizons}.csv", resources: mem_mb=10000, benchmark: - "benchmarks/build_clustered_population_layouts/s{simpl}_{clusters}_{planning_horizons}" + ( + "benchmarks/" + + SECDIR + + "build_clustered_population_layouts/s{simpl}_{clusters}_{planning_horizons}" + ) script: "scripts/build_clustered_population_layouts.py" @@ -1358,9 +1493,15 @@ rule build_heat_demand: params: snapshots=config["snapshots"], input: - pop_layout_total="resources/population_shares/pop_layout_total_{planning_horizons}.nc", - pop_layout_urban="resources/population_shares/pop_layout_urban_{planning_horizons}.nc", - pop_layout_rural="resources/population_shares/pop_layout_rural_{planning_horizons}.nc", + pop_layout_total="resources/" + + SECDIR + + "population_shares/pop_layout_total_{planning_horizons}.nc", + pop_layout_urban="resources/" + + SECDIR + + "population_shares/pop_layout_urban_{planning_horizons}.nc", + pop_layout_rural="resources/" + + SECDIR + + "population_shares/pop_layout_rural_{planning_horizons}.nc", regions_onshore="resources/" + RDIR + "bus_regions/regions_onshore_elec_s{simpl}_{clusters}.geojson", @@ -1370,13 +1511,23 @@ rule build_heat_demand: + ".nc", # default to first cutout found output: - heat_demand_urban="resources/demand/heat/heat_demand_urban_elec_s{simpl}_{clusters}_{planning_horizons}.nc", - heat_demand_rural="resources/demand/heat/heat_demand_rural_elec_s{simpl}_{clusters}_{planning_horizons}.nc", - heat_demand_total="resources/demand/heat/heat_demand_total_elec_s{simpl}_{clusters}_{planning_horizons}.nc", + heat_demand_urban="resources/" + + SECDIR + + "demand/heat/heat_demand_urban_elec_s{simpl}_{clusters}_{planning_horizons}.nc", + heat_demand_rural="resources/" + + SECDIR + + "demand/heat/heat_demand_rural_elec_s{simpl}_{clusters}_{planning_horizons}.nc", + heat_demand_total="resources/" + + SECDIR + + "demand/heat/heat_demand_total_elec_s{simpl}_{clusters}_{planning_horizons}.nc", resources: mem_mb=20000, benchmark: - "benchmarks/build_heat_demand/s{simpl}_{clusters}_{planning_horizons}" + ( + "benchmarks/" + + SECDIR + + "build_heat_demand/s{simpl}_{clusters}_{planning_horizons}" + ) script: "scripts/build_heat_demand.py" @@ -1385,9 +1536,15 @@ rule build_temperature_profiles: params: snapshots=config["snapshots"], input: - pop_layout_total="resources/population_shares/pop_layout_total_{planning_horizons}.nc", - pop_layout_urban="resources/population_shares/pop_layout_urban_{planning_horizons}.nc", - pop_layout_rural="resources/population_shares/pop_layout_rural_{planning_horizons}.nc", + pop_layout_total="resources/" + + SECDIR + + "population_shares/pop_layout_total_{planning_horizons}.nc", + pop_layout_urban="resources/" + + SECDIR + + "population_shares/pop_layout_urban_{planning_horizons}.nc", + pop_layout_rural="resources/" + + SECDIR + + "population_shares/pop_layout_rural_{planning_horizons}.nc", regions_onshore="resources/" + RDIR + "bus_regions/regions_onshore_elec_s{simpl}_{clusters}.geojson", @@ -1397,16 +1554,32 @@ rule build_temperature_profiles: + ".nc", # default to first cutout found output: - temp_soil_total="resources/temperatures/temp_soil_total_elec_s{simpl}_{clusters}_{planning_horizons}.nc", - temp_soil_rural="resources/temperatures/temp_soil_rural_elec_s{simpl}_{clusters}_{planning_horizons}.nc", - temp_soil_urban="resources/temperatures/temp_soil_urban_elec_s{simpl}_{clusters}_{planning_horizons}.nc", - temp_air_total="resources/temperatures/temp_air_total_elec_s{simpl}_{clusters}_{planning_horizons}.nc", - temp_air_rural="resources/temperatures/temp_air_rural_elec_s{simpl}_{clusters}_{planning_horizons}.nc", - temp_air_urban="resources/temperatures/temp_air_urban_elec_s{simpl}_{clusters}_{planning_horizons}.nc", + temp_soil_total="resources/" + + SECDIR + + "temperatures/temp_soil_total_elec_s{simpl}_{clusters}_{planning_horizons}.nc", + temp_soil_rural="resources/" + + SECDIR + + "temperatures/temp_soil_rural_elec_s{simpl}_{clusters}_{planning_horizons}.nc", + temp_soil_urban="resources/" + + SECDIR + + "temperatures/temp_soil_urban_elec_s{simpl}_{clusters}_{planning_horizons}.nc", + temp_air_total="resources/" + + SECDIR + + "temperatures/temp_air_total_elec_s{simpl}_{clusters}_{planning_horizons}.nc", + temp_air_rural="resources/" + + SECDIR + + "temperatures/temp_air_rural_elec_s{simpl}_{clusters}_{planning_horizons}.nc", + temp_air_urban="resources/" + + SECDIR + + "temperatures/temp_air_urban_elec_s{simpl}_{clusters}_{planning_horizons}.nc", resources: mem_mb=20000, benchmark: - "benchmarks/build_temperature_profiles/s{simpl}_{clusters}_{planning_horizons}" + ( + "benchmarks/" + + SECDIR + + "build_temperature_profiles/s{simpl}_{clusters}_{planning_horizons}" + ) script: "scripts/build_temperature_profiles.py" @@ -1650,19 +1823,29 @@ rule build_industrial_distribution_key: #default data regions_onshore="resources/" + RDIR + "bus_regions/regions_onshore_elec_s{simpl}_{clusters}.geojson", - clustered_pop_layout="resources/population_shares/pop_layout_elec_s{simpl}_{clusters}_{planning_horizons}.csv", - clustered_gdp_layout="resources/gdp_shares/gdp_layout_elec_s{simpl}_{clusters}_{planning_horizons}.csv", + clustered_pop_layout="resources/" + + SECDIR + + "population_shares/pop_layout_elec_s{simpl}_{clusters}_{planning_horizons}.csv", + clustered_gdp_layout="resources/" + + SECDIR + + "gdp_shares/gdp_layout_elec_s{simpl}_{clusters}_{planning_horizons}.csv", industrial_database="data/industrial_database.csv", shapes_path="resources/" + RDIR + "bus_regions/regions_onshore_elec_s{simpl}_{clusters}.geojson", output: - industrial_distribution_key="resources/demand/industrial_distribution_key_elec_s{simpl}_{clusters}_{planning_horizons}.csv", + industrial_distribution_key="resources/" + + SECDIR + + "demand/industrial_distribution_key_elec_s{simpl}_{clusters}_{planning_horizons}.csv", threads: 1 resources: mem_mb=1000, benchmark: - "benchmarks/build_industrial_distribution_key_elec_s{simpl}_{clusters}_{planning_horizons}" + ( + "benchmarks/" + + RDIR + + "build_industrial_distribution_key_elec_s{simpl}_{clusters}_{planning_horizons}" + ) script: "scripts/build_industrial_distribution_key.py" @@ -1678,12 +1861,18 @@ rule build_base_industry_totals: #default data energy_totals_base="data/energy_totals_base.csv", transactions_path="data/unsd_transactions.csv", output: - base_industry_totals="resources/demand/base_industry_totals_{planning_horizons}_{demand}.csv", + base_industry_totals="resources/" + + SECDIR + + "demand/base_industry_totals_{planning_horizons}_{demand}.csv", threads: 1 resources: mem_mb=1000, benchmark: - "benchmarks/build_base_industry_totals_{planning_horizons}_{demand}" + ( + "benchmarks/" + + SECDIR + + "build_base_industry_totals_{planning_horizons}_{demand}" + ) script: "scripts/build_base_industry_totals.py" @@ -1696,20 +1885,30 @@ rule build_industry_demand: #default data industry_util_factor=config["sector"]["industry_util_factor"], aluminium_year=config["demand_data"]["aluminium_year"], input: - industrial_distribution_key="resources/demand/industrial_distribution_key_elec_s{simpl}_{clusters}_{planning_horizons}.csv", + industrial_distribution_key="resources/" + + SECDIR + + "demand/industrial_distribution_key_elec_s{simpl}_{clusters}_{planning_horizons}.csv", #industrial_production_per_country_tomorrow="resources/demand/industrial_production_per_country_tomorrow_{planning_horizons}_{demand}.csv", #industrial_production_per_country="data/industrial_production_per_country.csv", - base_industry_totals="resources/demand/base_industry_totals_{planning_horizons}_{demand}.csv", + base_industry_totals="resources/" + + SECDIR + + "demand/base_industry_totals_{planning_horizons}_{demand}.csv", industrial_database="data/industrial_database.csv", costs=COSTDIR + "costs_{planning_horizons}.csv", industry_growth_cagr="data/demand/industry_growth_cagr.csv", output: - industrial_energy_demand_per_node="resources/demand/industrial_energy_demand_per_node_elec_s{simpl}_{clusters}_{planning_horizons}_{demand}.csv", + industrial_energy_demand_per_node="resources/" + + SECDIR + + "demand/industrial_energy_demand_per_node_elec_s{simpl}_{clusters}_{planning_horizons}_{demand}.csv", threads: 1 resources: mem_mb=1000, benchmark: - "benchmarks/industrial_energy_demand_per_node_elec_s{simpl}_{clusters}_{planning_horizons}_{demand}.csv" + ( + "benchmarks/" + + SECDIR + + "industrial_energy_demand_per_node_elec_s{simpl}_{clusters}_{planning_horizons}_{demand}.csv" + ) script: "scripts/build_industry_demand.py" @@ -1721,11 +1920,20 @@ rule build_existing_heating_distribution: existing_capacities=config["existing_capacities"], input: existing_heating="data/existing_infrastructure/existing_heating_raw.csv", - clustered_pop_layout="resources/population_shares/pop_layout_elec_s{simpl}_{clusters}_{planning_horizons}.csv", - clustered_pop_energy_layout="resources/demand/heat/nodal_energy_heat_totals_{demand}_s{simpl}_{clusters}_{planning_horizons}.csv", #"resources/population_shares/pop_weighted_energy_totals_s{simpl}_{clusters}.csv", - district_heat_share="resources/demand/heat/district_heat_share_{demand}_s{simpl}_{clusters}_{planning_horizons}.csv", + clustered_pop_layout="resources/" + + SECDIR + + "population_shares/pop_layout_elec_s{simpl}_{clusters}_{planning_horizons}.csv", + clustered_pop_energy_layout="resources/" + + SECDIR + + "demand/heat/nodal_energy_heat_totals_{demand}_s{simpl}_{clusters}_{planning_horizons}.csv", + #"resources/population_shares/pop_weighted_energy_totals_s{simpl}_{clusters}.csv", + district_heat_share="resources/" + + SECDIR + + "demand/heat/district_heat_share_{demand}_s{simpl}_{clusters}_{planning_horizons}.csv", output: - existing_heating_distribution="resources/heating/existing_heating_distribution_{demand}_s{simpl}_{clusters}_{planning_horizons}.csv", + existing_heating_distribution="resources/" + + SECDIR + + "heating/existing_heating_distribution_{demand}_s{simpl}_{clusters}_{planning_horizons}.csv", threads: 1 resources: mem_mb=2000, @@ -1755,12 +1963,20 @@ if config["foresight"] == "myopic": busmap=pypsaearth( "resources/" + RDIR + "bus_regions/busmap_elec_s{simpl}_{clusters}.csv" ), - clustered_pop_layout="resources/population_shares/pop_layout_elec_s{simpl}_{clusters}_{planning_horizons}.csv", + clustered_pop_layout="resources/" + + SECDIR + + "population_shares/pop_layout_elec_s{simpl}_{clusters}_{planning_horizons}.csv", costs=CDIR + "costs_{}.csv".format(config["scenario"]["planning_horizons"][0]), - cop_soil_total="resources/cops/cop_soil_total_elec_s{simpl}_{clusters}_{planning_horizons}.nc", - cop_air_total="resources/cops/cop_air_total_elec_s{simpl}_{clusters}_{planning_horizons}.nc", - existing_heating_distribution="resources/heating/existing_heating_distribution_{demand}_s{simpl}_{clusters}_{planning_horizons}.csv", + cop_soil_total="resources/" + + SECDIR + + "cops/cop_soil_total_elec_s{simpl}_{clusters}_{planning_horizons}.nc", + cop_air_total="resources/" + + SECDIR + + "cops/cop_air_total_elec_s{simpl}_{clusters}_{planning_horizons}.nc", + existing_heating_distribution="resources/" + + SECDIR + + "heating/existing_heating_distribution_{demand}_s{simpl}_{clusters}_{planning_horizons}.csv", output: RESDIR + "prenetworks-brownfield/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sopts}_{planning_horizons}_{discountrate}_{demand}_{h2export}export.nc", @@ -1796,7 +2012,7 @@ if config["foresight"] == "myopic": planning_horizon_p = str(planning_horizons[i - 1]) return ( - RDIR + RESDIR + "postnetworks/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}_{sopts}_" + planning_horizon_p + "_{discountrate}_{demand}_{h2export}export.nc" @@ -1822,8 +2038,12 @@ if config["foresight"] == "myopic": + "prenetworks/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}_{sopts}_{planning_horizons}_{discountrate}_{demand}_{h2export}export.nc", network_p=solved_previous_horizon, #solved network at previous time step costs=CDIR + "costs_{planning_horizons}.csv", - cop_soil_total="resources/cops/cop_soil_total_elec_s{simpl}_{clusters}_{planning_horizons}.nc", - cop_air_total="resources/cops/cop_air_total_elec_s{simpl}_{clusters}_{planning_horizons}.nc", + cop_soil_total="resources/" + + SECDIR + + "cops/cop_soil_total_elec_s{simpl}_{clusters}_{planning_horizons}.nc", + cop_air_total="resources/" + + SECDIR + + "cops/cop_air_total_elec_s{simpl}_{clusters}_{planning_horizons}.nc", output: RESDIR + "prenetworks-brownfield/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sopts}_{planning_horizons}_{discountrate}_{demand}_{h2export}export.nc", diff --git a/config.default.yaml b/config.default.yaml index 431812b94..aa5135481 100644 --- a/config.default.yaml +++ b/config.default.yaml @@ -37,6 +37,7 @@ custom_rules: [] # Default empty [] or link to custom rule file e.g. ["my_folder run: name: "" # use this to keep track of runs with different settings + sector_name: "" # use this to keep track of sector scenario runs shared_cutouts: true # set to true to share the default cutout(s) across runs # Note: value false requires build_cutout to be enabled allow_scenario_failure: false # If True, the workflow will continue even if a scenario in run_scnenario fails diff --git a/doc/release_notes.rst b/doc/release_notes.rst index 6b3ef19a1..a76740a35 100644 --- a/doc/release_notes.rst +++ b/doc/release_notes.rst @@ -14,35 +14,42 @@ E.g. if a new rule becomes available describe how to use it `make test` and in o **New Features and Major Changes** -* The workflow configuration now supports incremental changes to the default configuration in the `config.yaml` and configfiles passed to snakemake via `--configfile myconfig.yaml`. Therefore the user may now only include settings in their `config.yaml` which differ from the default configuration. One can think of the new `config.yaml` as of a list of arguments in a python function that already have a default. So in principle the `config.yaml` could now be empty, and the workflow would still run. `PR #1053 `_ +* The workflow configuration now supports incremental changes to the default configuration in the `config.yaml` and configfiles passed to snakemake via `--configfile myconfig.yaml`. Therefore the user may now only include settings in their `config.yaml` which differ from the default configuration. One can think of the new `config.yaml` as of a list of arguments in a python function that already have a default. So in principle the `config.yaml` could now be empty, and the workflow would still run. `PR #1053 `__ -* Local tests are now run with `make test`. This uses a `Makefile` which runs snakemake calls with different configurations. `PR #1053 `_ +* Local tests are now run with `make test`. This uses a `Makefile` which runs snakemake calls with different configurations. `PR #1053 `__ -**Minor Changes and bug-fixing** +* Integrate RDIR into sector rules to store intermediate data in scenario folders `PR #1154 `__ -* The default configuration for `electricity:estimate_renewable_capacities:year` was updated from 2020 to 2023. `PR #1106 `_ +* Include a dedicated cutout for North America in bundle_config.yaml `PR #1121 `__ -* Include a dedicated cutout for North America in bundle_config.yaml `PR #1121 `_ +* Include a dedicated cutout for Europe in bundle_config.yaml `PR #1125 `__ -* Include a dedicated cutout for Europe in bundle_config.yaml `PR #1125 `_ +* Include a dedicated cutout for Oceania in bundle_config.yaml `PR #1157 `__ * Removed duplications of devendencies in environment.yaml `PR #1128 `_ * Fix the mismatch between buses and x, y locations while creating H2 Stores `PR #1134 `_ -* Enable configfile specification for mock_snakemake `PR #1135 `_ +* Use BASE_DIR in rules and `_helpers.py` script for facilitate module import in subworkflow `PR #1137 `__ -* Fix pre-commit docformatter python issue. `PR #1153 `__ -* Use BASE_DIR in rules and `_helpers.py` script for facilitate module import in subworkflow `PR #1137 `_ +* Enable sector rules import in subworkflow `PR #1178 `__ + +**Minor Changes and bug-fixing** + +* The default configuration for `electricity:estimate_renewable_capacities:year` was updated from 2020 to 2023. `PR #1106 `__ + +* Fix the mismatch between buses and x, y locations while creating H2 Stores `PR #1134 `__ -* Include a dedicated cutout for Oceania in bundle_config.yaml `PR #1157 `_ +* Enable configfile specification for mock_snakemake `PR #1135 `__ + +* Fix pre-commit docformatter python issue. `PR #1153 `__ -* Drop duplicate entries in `AL_production.csv` data used in `build_industry_demand` rule `PR #1143 `_ +* Drop duplicate entries in `AL_production.csv` data used in `build_industry_demand` rule `PR #1143 `__ -* The computation of `hydro_profile.nc` in `build_renewable_profiles.py` is not differentiated whether alternative clustering is applied or not; the indexing of the different power plants in `add_electricity.py` is performed according to the bus either in case alternative clustering is applied or not and a `hydro_inflow_factor` is computed prior to the computation of `inflow_t` to split the inflow according to the capacity of each different unit of each power plant (if more units are present). `PR #1119 `_ +* The computation of `hydro_profile.nc` in `build_renewable_profiles.py` is not differentiated whether alternative clustering is applied or not; the indexing of the different power plants in `add_electricity.py` is performed according to the bus either in case alternative clustering is applied or not and a `hydro_inflow_factor` is computed prior to the computation of `inflow_t` to split the inflow according to the capacity of each different unit of each power plant (if more units are present). `PR #1119 `__ -* Fix bugs in `prepare_sector_network.py` related to links with H2 buses and bug of re-addition of H2 and battery carriers in present `PR #1145 `_ +* Fix bugs in `prepare_sector_network.py` related to links with H2 buses and bug of re-addition of H2 and battery carriers in present `PR #1145 `__ PyPSA-Earth 0.4.1 ================= diff --git a/scripts/build_base_energy_totals.py b/scripts/build_base_energy_totals.py index e16f44898..4ac1f7889 100644 --- a/scripts/build_base_energy_totals.py +++ b/scripts/build_base_energy_totals.py @@ -19,7 +19,7 @@ import pandas as pd import py7zr import requests -from _helpers import aggregate_fuels, get_conv_factors +from _helpers import BASE_DIR, aggregate_fuels, get_conv_factors _logger = logging.getLogger(__name__) @@ -375,7 +375,7 @@ def calc_sector(sector): if snakemake.params.update_data: # Delete and existing files to avoid duplication and double counting - files = glob.glob("data/demand/unsd/data/*.txt") + files = glob.glob(os.path.join(BASE_DIR, "data/demand/unsd/data/*.txt")) for f in files: os.remove(f) @@ -385,12 +385,14 @@ def calc_sector(sector): with urlopen(zipurl) as zipresp: with ZipFile(BytesIO(zipresp.read())) as zfile: - zfile.extractall("data/demand/unsd/data") + zfile.extractall(os.path.join(BASE_DIR, "data/demand/unsd/data")) - path = "data/demand/unsd/data" + path = os.path.join(BASE_DIR, "data/demand/unsd/data") # Get the files from the path provided in the OP - all_files = list(Path("data/demand/unsd/data").glob("*.txt")) + all_files = list( + Path(os.path.join(BASE_DIR, "data/demand/unsd/data")).glob("*.txt") + ) # Create a dataframe from all downloaded files df = pd.concat( @@ -433,7 +435,9 @@ def calc_sector(sector): df_yr = df_yr[df_yr.country.isin(countries)] # Create an empty dataframe for energy_totals_base - energy_totals_cols = pd.read_csv("data/energy_totals_DF_2030.csv").columns + energy_totals_cols = pd.read_csv( + os.path.join(BASE_DIR, "data/energy_totals_DF_2030.csv") + ).columns energy_totals_base = pd.DataFrame(columns=energy_totals_cols, index=countries) # Lists that combine the different fuels in the dataset to the model's carriers diff --git a/scripts/build_industry_demand.py b/scripts/build_industry_demand.py index 7074e5e89..8617fb466 100644 --- a/scripts/build_industry_demand.py +++ b/scripts/build_industry_demand.py @@ -13,7 +13,7 @@ from itertools import product import pandas as pd -from _helpers import mock_snakemake, read_csv_nafix +from _helpers import BASE_DIR, mock_snakemake, read_csv_nafix _logger = logging.getLogger(__name__) @@ -69,8 +69,12 @@ def country_to_nodal(industrial_production, keys): ) industry_demand = pd.read_csv( - "data/custom/industry_demand_{0}_{1}.csv".format( - snakemake.wildcards["demand"], snakemake.wildcards["planning_horizons"] + os.path.join( + BASE_DIR, + "data/custom/industry_demand_{0}_{1}.csv".format( + snakemake.wildcards["demand"], + snakemake.wildcards["planning_horizons"], + ), ), index_col=[0, 1], ) @@ -204,7 +208,9 @@ def match_technology(df): geo_locs = match_technology(geo_locs).loc[countries_geo] aluminium_year = snakemake.params.aluminium_year - AL = read_csv_nafix("data/AL_production.csv", index_col=0) + AL = read_csv_nafix( + os.path.join(BASE_DIR, "data/AL_production.csv"), index_col=0 + ) # Filter data for the given year and countries AL_prod_tom = AL.query("Year == @aluminium_year and index in @countries_geo")[ "production[ktons/a]" diff --git a/scripts/copy_config.py b/scripts/copy_config.py index 780511d81..b7073c9fa 100644 --- a/scripts/copy_config.py +++ b/scripts/copy_config.py @@ -5,11 +5,15 @@ import os from shutil import copy +from _helpers import BASE_DIR + files_to_copy = { - "./config.yaml": "config.yaml", - "./Snakefile": "Snakefile", - "./scripts/solve_network.py": "solve_network.py", - "./scripts/prepare_sector_network.py": "prepare_sector_network.py", + os.path.join(BASE_DIR, "./config.yaml"): "config.yaml", + os.path.join(BASE_DIR, "./Snakefile"): "Snakefile", + os.path.join(BASE_DIR, "./scripts/solve_network.py"): "solve_network.py", + os.path.join( + BASE_DIR, "./scripts/prepare_sector_network.py" + ): "prepare_sector_network.py", } if __name__ == "__main__": diff --git a/scripts/prepare_energy_totals.py b/scripts/prepare_energy_totals.py index 119083f02..708420b27 100644 --- a/scripts/prepare_energy_totals.py +++ b/scripts/prepare_energy_totals.py @@ -17,7 +17,7 @@ import pandas as pd import py7zr import requests -from _helpers import read_csv_nafix, three_2_two_digits_country +from _helpers import BASE_DIR, read_csv_nafix, three_2_two_digits_country _logger = logging.getLogger(__name__) @@ -53,7 +53,9 @@ def calculate_end_values(df): investment_year = int(snakemake.wildcards.planning_horizons) demand_sc = snakemake.wildcards.demand # loading the demand scenrario wildcard - base_energy_totals = read_csv_nafix("data/energy_totals_base.csv", index_col=0) + base_energy_totals = read_csv_nafix( + os.path.join(BASE_DIR, "data/energy_totals_base.csv"), index_col=0 + ) growth_factors_cagr = read_csv_nafix( snakemake.input.growth_factors_cagr, index_col=0 ) diff --git a/scripts/prepare_gas_network.py b/scripts/prepare_gas_network.py index 59078803e..cbdcd120a 100644 --- a/scripts/prepare_gas_network.py +++ b/scripts/prepare_gas_network.py @@ -19,7 +19,12 @@ import matplotlib.colors as colors import matplotlib.pyplot as plt import pandas as pd -from _helpers import content_retrieve, progress_retrieve, two_2_three_digits_country +from _helpers import ( + BASE_DIR, + content_retrieve, + progress_retrieve, + two_2_three_digits_country, +) from build_shapes import gadm from matplotlib.lines import Line2D from pyproj import CRS @@ -58,8 +63,8 @@ def download_IGGIELGN_gas_network(): url = "https://zenodo.org/record/4767098/files/IGGIELGN.zip" # Save locations - zip_fn = Path("IGGIELGN.zip") - to_fn = Path("data/gas_network/scigrid-gas") + zip_fn = Path(os.path.join(BASE_DIR, "IGGIELGN.zip")) + to_fn = Path(os.path.join(BASE_DIR, "data/gas_network/scigrid-gas")) logger.info(f"Downloading databundle from '{url}'.") progress_retrieve(url, zip_fn) @@ -344,6 +349,7 @@ def download_GADM(country_code, update=False, out_logging=False): GADM_filename = get_GADM_filename(country_code) GADM_inputfile_gpkg = os.path.join( + BASE_DIR, "data", "gadm", GADM_filename, @@ -887,7 +893,9 @@ def check_existence(row): elif snakemake.params.gas_config["network_data"] == "IGGIELGN": download_IGGIELGN_gas_network() - gas_network = "data/gas_network/scigrid-gas/data/IGGIELGN_PipeSegments.geojson" + gas_network = os.path.join( + BASE_DIR, "data/gas_network/scigrid-gas/data/IGGIELGN_PipeSegments.geojson" + ) pipelines = load_IGGIELGN_data(gas_network) pipelines = prepare_IGGIELGN_data(pipelines) diff --git a/scripts/prepare_sector_network.py b/scripts/prepare_sector_network.py index acb6394bb..9a731fa53 100644 --- a/scripts/prepare_sector_network.py +++ b/scripts/prepare_sector_network.py @@ -16,6 +16,7 @@ import ruamel.yaml import xarray as xr from _helpers import ( + BASE_DIR, create_dummy_data, create_network_topology, cycling_shift, @@ -322,8 +323,11 @@ def add_hydrogen(n, costs): if snakemake.config["sector"]["hydrogen"]["underground_storage"]: if snakemake.config["custom_data"]["h2_underground"]: custom_cavern = pd.read_csv( - "data/custom/h2_underground_{0}_{1}.csv".format( - demand_sc, investment_year + os.path.join( + BASE_DIR, + "data/custom/h2_underground_{0}_{1}.csv".format( + demand_sc, investment_year + ), ) ) # countries = n.buses.country.unique().to_list() @@ -2661,9 +2665,12 @@ def add_residential(n, costs): def add_custom_water_cost(n): for country in countries: water_costs = pd.read_csv( - "resources/custom_data/{}_water_costs.csv".format(country), - sep=",", - index_col=0, + os.path.join( + BASE_DIR, + "resources/custom_data/{}_water_costs.csv".format(country), + sep=",", + index_col=0, + ) ) water_costs = water_costs.filter(like=country, axis=0).loc[spatial.nodes] electrolysis_links = n.links.filter(like=country, axis=0).filter( @@ -2748,7 +2755,6 @@ def add_rail_transport(n, costs): # Fetch wildcards investment_year = int(snakemake.wildcards.planning_horizons[-4:]) demand_sc = snakemake.wildcards.demand # loading the demand scenrario wildcard - pop_layout = pd.read_csv(snakemake.input.clustered_pop_layout, index_col=0) # Prepare the costs dataframe costs = prepare_costs( diff --git a/scripts/prepare_transport_data_input.py b/scripts/prepare_transport_data_input.py index cffa163e2..a15608adc 100644 --- a/scripts/prepare_transport_data_input.py +++ b/scripts/prepare_transport_data_input.py @@ -10,9 +10,7 @@ import country_converter as coco import numpy as np import pandas as pd - -# from _helpers import configure_logging - +from _helpers import BASE_DIR # logger = logging.getLogger(__name__) @@ -127,7 +125,7 @@ def download_CO2_emissions(): if vehicles_csv.empty or CO2_emissions_csv.empty: # In case one of the urls is not working, we can use the hard-coded data - src = os.getcwd() + "/data/temp_hard_coded/transport_data.csv" + src = BASE_DIR + "/data/temp_hard_coded/transport_data.csv" dest = snakemake.output.transport_data_input shutil.copy(src, dest) else: