Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Harmonise folder structure for workflow outputs #1236

Merged
merged 7 commits into from
Dec 24, 2024
Merged

Harmonise folder structure for workflow outputs #1236

Show file tree
Hide file tree
Changes from 4 commits
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
7f93dd0
Put country-specific data into resources folder
ekatef Dec 10, 2024
26eafdb
Replace a benchmark path
ekatef Dec 10, 2024
f5a957e
Replace path hardcoding in script
ekatef Dec 11, 2024
8b70c7f
Merge branch 'main' into sec_paths
ekatef Dec 20, 2024
6a90b66
Merge branch 'main' into sec_paths
ekatef Dec 24, 2024
d566e98
Put the path into snakemake inputs
ekatef Dec 24, 2024
09ec93f
Merge branch 'sec_paths' of https://github.com/ekatef/pypsa-earth int…
ekatef Dec 24, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
38 changes: 24 additions & 14 deletions Snakefile
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1004,7 +1004,7 @@ rule prepare_ports:
params:
custom_export=config["custom_data"]["export_ports"],
output:
ports="data/ports.csv", # TODO move from data to resources
ports="resources/" + SECDIR + "ports.csv",
export_ports="resources/" + SECDIR + "export_ports.csv",
script:
"scripts/prepare_ports.py"
Expand All @@ -1015,14 +1015,14 @@ rule prepare_airports:
airport_sizing_factor=config["sector"]["airport_sizing_factor"],
airport_custom_data=config["custom_data"]["airports"],
output:
ports="data/airports.csv", # TODO move from data to resources
ports="resources/" + SECDIR + "airports.csv",
script:
"scripts/prepare_airports.py"


rule prepare_urban_percent:
output:
urban_percent="data/urban_percent.csv", # TODO move from data to resources
urban_percent="resources/" + SECDIR + "urban_percent.csv",
script:
"scripts/prepare_urban_percent.py"

Expand Down Expand Up @@ -1095,9 +1095,11 @@ rule prepare_sector_network:
industrial_demand="resources/"
+ SECDIR
+ "demand/industrial_energy_demand_per_node_elec_s{simpl}_{clusters}_{planning_horizons}_{demand}.csv",
energy_totals="data/energy_totals_{demand}_{planning_horizons}.csv",
airports="data/airports.csv",
ports="data/ports.csv",
energy_totals="resources/"
+ SECDIR
+ "energy_totals_{demand}_{planning_horizons}.csv",
airports="resources/" + SECDIR + "airports.csv",
ports="resources/" + SECDIR + "ports.csv",
heat_demand="resources/"
+ SECDIR
+ "demand/heat/heat_demand_{demand}_s{simpl}_{clusters}_{planning_horizons}.csv",
Expand Down Expand Up @@ -1201,7 +1203,9 @@ rule override_respot:
},
overrides="data/override_component_attrs",
network="networks/" + RDIR + "elec_s{simpl}_{clusters}_ec_l{ll}_{opts}.nc",
energy_totals="data/energy_totals_{demand}_{planning_horizons}.csv",
energy_totals="resources/"
+ SECDIR
+ "energy_totals_{demand}_{planning_horizons}.csv",
output:
RESDIR
+ "prenetworks/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}_{sopts}_{planning_horizons}_{discountrate}_{demand}_presec.nc",
Expand All @@ -1212,7 +1216,9 @@ rule override_respot:
rule prepare_transport_data:
input:
network="networks/" + RDIR + "elec_s{simpl}_{clusters}.nc",
energy_totals_name="data/energy_totals_{demand}_{planning_horizons}.csv",
energy_totals_name="resources/"
+ SECDIR
+ "energy_totals_{demand}_{planning_horizons}.csv",
traffic_data_KFZ="data/emobility/KFZ__count",
traffic_data_Pkw="data/emobility/Pkw__count",
transport_name="resources/" + SECDIR + "transport_data.csv",
Expand Down Expand Up @@ -1296,7 +1302,9 @@ rule build_cop_profiles:
rule prepare_heat_data:
input:
network="networks/" + RDIR + "elec_s{simpl}_{clusters}.nc",
energy_totals_name="data/energy_totals_{demand}_{planning_horizons}.csv",
energy_totals_name="resources/"
+ SECDIR
+ "energy_totals_{demand}_{planning_horizons}.csv",
clustered_pop_layout="resources/"
+ SECDIR
+ "population_shares/pop_layout_elec_s{simpl}_{clusters}_{planning_horizons}.csv",
Expand Down Expand Up @@ -1349,7 +1357,7 @@ rule build_base_energy_totals:
input:
unsd_paths="data/demand/unsd/paths/Energy_Statistics_Database.xlsx",
output:
energy_totals_base="data/energy_totals_base.csv",
energy_totals_base="resources/" + SECDIR + "energy_totals_base.csv",
script:
"scripts/build_base_energy_totals.py"

Expand All @@ -1360,13 +1368,15 @@ rule prepare_energy_totals:
base_year=config["demand_data"]["base_year"],
sector_options=config["sector"],
input:
unsd_paths="data/energy_totals_base.csv",
unsd_paths="resources/" + SECDIR + "energy_totals_base.csv",
efficiency_gains_cagr="data/demand/efficiency_gains_cagr.csv",
growth_factors_cagr="data/demand/growth_factors_cagr.csv",
district_heating="data/demand/district_heating.csv",
fuel_shares="data/demand/fuel_shares.csv",
output:
energy_totals="data/energy_totals_{demand}_{planning_horizons}.csv",
energy_totals="resources/"
+ SECDIR
+ "energy_totals_{demand}_{planning_horizons}.csv",
script:
"scripts/prepare_energy_totals.py"

Expand Down Expand Up @@ -1420,7 +1430,7 @@ rule build_population_layouts:
planning_horizons=config["scenario"]["planning_horizons"][0],
input:
nuts3_shapes="resources/" + RDIR + "shapes/gadm_shapes.geojson",
urban_percent="data/urban_percent.csv",
urban_percent="resources/" + SECDIR + "urban_percent.csv",
cutout="cutouts/"
+ CDIR
+ [c["cutout"] for _, c in config["renewable"].items()][0]
Expand Down Expand Up @@ -1867,7 +1877,7 @@ rule build_base_industry_totals: #default data
input:
#industrial_production_per_country="data/industrial_production_per_country.csv",
#unsd_path="data/demand/unsd/data/",
energy_totals_base="data/energy_totals_base.csv",
energy_totals_base="resources/" + SECDIR + "energy_totals_base.csv",
transactions_path="data/unsd_transactions.csv",
output:
base_industry_totals="resources/"
Expand Down
3 changes: 2 additions & 1 deletion scripts/build_base_industry_totals.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -117,8 +117,9 @@ def create_industry_base_totals(df):
renaming_dit = transaction.set_index("Transaction")["clean_name"].to_dict()
clean_industry_list = list(transaction.clean_name.unique())

# TODO Check with the sector-coupled modelling experts
unsd_path = (
os.path.dirname(snakemake.input["energy_totals_base"]) + "/demand/unsd/data/"
os.path.dirname(snakemake.input["transactions_path"]) + "/demand/unsd/data/"
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Hello!
Apologies, the review of this PR may have been left in drafts.

The whole PR sounds great to me but I'm a bit in doubt about this line.
The rule build_base_energy_totals creates such folder with the target files here loaded.
I believe it would be advisable to add an output to the build base rule and an input here with the directory.
That should avoid the confusion and this inner hidden dependencies.
What do you think?

)

# Get the files from the path provided in the OP
Expand Down
4 changes: 1 addition & 3 deletions scripts/prepare_energy_totals.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -53,9 +53,7 @@ def calculate_end_values(df):
investment_year = int(snakemake.wildcards.planning_horizons)
demand_sc = snakemake.wildcards.demand # loading the demand scenrario wildcard

base_energy_totals = read_csv_nafix(
os.path.join(BASE_DIR, "data/energy_totals_base.csv"), index_col=0
)
base_energy_totals = read_csv_nafix(snakemake.input.unsd_paths, index_col=0)
growth_factors_cagr = read_csv_nafix(
snakemake.input.growth_factors_cagr, index_col=0
)
Expand Down
Loading