From 6e3ec72cbf05a22ea288ca1c625b26c9a412ab37 Mon Sep 17 00:00:00 2001 From: imreddyTeja Date: Fri, 15 Nov 2024 10:10:17 -0800 Subject: [PATCH] Add ERA5 Pressure Levels Monthly Avgs --- README.md | 1 + .../LICENSE.txt | 93 ++++ .../Manifest.toml | 526 ++++++++++++++++++ .../OutputArtifacts.toml | 2 + .../Project.toml | 5 + .../README.md | 104 ++++ .../create_artifact.jl | 139 +++++ .../download_era5.py | 132 +++++ .../requirements.txt | 2 + 9 files changed, 1004 insertions(+) create mode 100644 era5_monthly_averages_pressure_levels_1979_2024/LICENSE.txt create mode 100644 era5_monthly_averages_pressure_levels_1979_2024/Manifest.toml create mode 100644 era5_monthly_averages_pressure_levels_1979_2024/OutputArtifacts.toml create mode 100644 era5_monthly_averages_pressure_levels_1979_2024/Project.toml create mode 100644 era5_monthly_averages_pressure_levels_1979_2024/README.md create mode 100644 era5_monthly_averages_pressure_levels_1979_2024/create_artifact.jl create mode 100644 era5_monthly_averages_pressure_levels_1979_2024/download_era5.py create mode 100644 era5_monthly_averages_pressure_levels_1979_2024/requirements.txt diff --git a/README.md b/README.md index 75e372b..cdf0602 100644 --- a/README.md +++ b/README.md @@ -56,6 +56,7 @@ S. Gupta et al 2022, 2024](https://github.com/CliMA/ClimaArtifacts/tree/main/soi - [ERA5 Monthly Averages 2008](https:////github.com/CliMA/ClimaArtifacts/tree/main/era5_surface_fluxes_2008) - [Initial conditions for summer DYAMOND simulations](https:////github.com/CliMA/ClimaArtifacts/tree/main/atmos_dyamond_summer) - [ERA5 Land Forcing Data 2008](https:////github.com/CliMA/ClimaArtifacts/tree/main/era5_land_forcing_data2008) +- [ERA5 Monthly Averages on Pressure Levels 1979-2024](https:////github.com/CliMA/ClimaArtifacts/tree/main/era5_monthly_averages_pressure_levels_1979_2024) # The ultimate guide to ClimaArtifacts diff --git a/era5_monthly_averages_pressure_levels_1979_2024/LICENSE.txt b/era5_monthly_averages_pressure_levels_1979_2024/LICENSE.txt new file mode 100644 index 0000000..8df6fde --- /dev/null +++ b/era5_monthly_averages_pressure_levels_1979_2024/LICENSE.txt @@ -0,0 +1,93 @@ +I. Licence to Use Copernicus Products + +1. Definitions + +1.1. ‘Licensor’ means the European Union, represented by the European Centre for Medium-Range Weather Forecasts (ECMWF). + +1.2. ‘Licensee’ means all natural or legal persons who agree to the terms of this Licence. + +1.3. ‘Licence’ means this license agreement between the Licensor and the Licensee as amended from time to time. + +1.4. ‘Copernicus Services’ means: + +1.4.1. the Copernicus Atmosphere Monitoring Service (CAMS), which is to provide information on air quality on a local, national, and European scale, and the chemical composition of the atmosphere on a global scale. + +1.4.2. the Copernicus Climate Change Service (C3S), which is to provide information to increase the knowledge base to support policies on adaptation to and mitigation of climate change + +1.5. ‘Copernicus Products’ means all products listed in the C3S or CAMS Service Product Specification or any other items available through an ECMWF Copernicus portal, except those items which are labelled/flagged as being subject to their own separate terms of use. + +1.6. ‘Intellectual Property Rights’ refers to intellectual property rights of all kinds, + +1.6.1. including: all patents; rights to inventions; copyright and related rights; moral rights; trademarks and service marks; trade names and domain names; rights in get-up; rights to goodwill or to sue for passing off or unfair competition; rights in designs; rights in computer software; database rights; rights in confidential information (including know-how and trade secrets); any other rights in the nature of intellectual property rights; + +1.6.2. in each case whether registered or unregistered and including all applications (or rights to apply) for, and renewals or extensions of, such rights and all similar or equivalent rights or forms of protection which subsist or will subsist now or in the future in any part of the world together with all rights of action in relation to the infringement of any of the above. + +1.7. ‘Copernicus Contractor’ refers to providers of Copernicus related goods and services to ECMWF, including information and data, to the Licensor and/or to the users. + +1.8. ‘Copernicus Regulations’ refers to Regulation (EU) No 377/2014 of the European Parliament and of the Council of 3 April 2014 establishing the Copernicus Programme. + +1.9. ‘ECMWF Agreement’ refers to the agreement between the European Commission and ECMWF dated 11 November 2014 on the implementation of CAMS and C3S. + +2. Introduction + +Copernicus is funded under the Copernicus Regulation and operated by ECMWF under the ECMWF Agreement. Access to all Copernicus (previously known as GMES or Global Monitoring for Environment and Security) Information and Data is regulated under Regulation (EU) No 1159/2013 of the European Parliament and of the Council of 12 July 2013 on the European Earth monitoring programme, under the ECMWF Agreement and under the European Commission’s Terms and Conditions. Access to all Copernicus information is regulated under Regulation (EU) No 1159/2013 and under the ECMWF Agreement. + +3. Terms of the Licence + +This Licence sets out the terms for use of Copernicus Products. By agreeing to these terms, the Licensee agrees to abide by all of the terms and conditions in this Licence for the use of Copernicus Products. + +4. Licence Permission + +4.1. This Licence is free of charge, worldwide, non-exclusive, royalty free and perpetual. + +4.2. Access to Copernicus Products is given for any purpose in so far as it is lawful, whereas use may include, but is not limited to: reproduction; distribution; communication to the public; adaptation, modification and combination with other data and information; or any combination of the foregoing. + +5. Attribution + +5.1. All users of Copernicus Products must provide clear and visible attribution to the Copernicus programme. The Licensee will communicate to the public the source of the Copernicus Products by crediting the Copernicus Climate Change and Atmosphere Monitoring Services: + +5.1.1. Where the Licensee communicates or distributes Copernicus Products to the public, the Licensee shall inform the recipients of the source by using the following or any similar notice: +• 'Generated using Copernicus Climate Change Service information [Year]' and/or +• 'Generated using Copernicus Atmosphere Monitoring Service information [Year]'. + +5.1.2. Where the Licensee makes or contributes to a publication or distribution containing adapted or modified Copernicus Products, the Licensee shall provide the following or any similar notice: +• 'Contains modified Copernicus Climate Change Service information [Year]'; and/or +• 'Contains modified Copernicus Atmosphere Monitoring Service information [Year]' + +5.1.3. Any such publication or distribution covered by clauses 5.1.1 and 5.1.2 shall state that neither the European Commission nor ECMWF is responsible for any use that may be made of the Copernicus information or data it contains. + +6. Intellectual Property Rights + +6.1. All Intellectual Property Rights in the Copernicus Products belong, and will continue to belong, to the European Union. + +6.2. All Intellectual Property Rights of new items created as a result of modifying or adapting the Copernicus Products through the applications and workflows accessible on the ECMWF Copernicus portals (e.g. through the CDS Toolbox) will belong to the European Union. + +6.3. All other new Intellectual Property Rights created as a result of modifying or adapting the Copernicus information will be owned by the creator. + +7. Provision of Third Party Information and Data + +This Licence only covers Copernicus Products. Access to third party products, information, and data related to Copernicus information to which the Licensee is directed or which can be directly accessed through any Copernicus portal will be subject to different licence terms. + +8. Disclaimers + +8.1. Neither the Licensor nor ECMWF warrant that Copernicus Products will be free from errors or omissions or that such errors or omissions can or will be rectified, or that the Licensee will have uninterrupted, continuous, or timely access to Copernicus Products. + +8.2. The Licensor, as well as ECMWF, exclude all warranties, conditions, terms, undertakings, obligations whether express or implied by statute including but not limited to the implied warranties of satisfactory quality and fitness for a particular purpose or otherwise to the fullest extent permitted by law. + +9. Liabilities + +Neither the Licensor nor ECMWF will accept liability for any damage, loss whether direct, indirect or consequential resulting from the Licensee’s use of the Copernicus Products. + +10. Termination of and Changes to this Licence + +The Licensor may terminate this licence if the Licensee breaches its obligations under these terms. The Licensor may revise this Licence at any time and will notify the Licensee of any revisions. + +11. Arbitration Clause and Governing Law + +In the event of a dispute arising in connection with this License, the parties shall attempt to settle their differences in an amicable manner. If any dispute cannot be so settled, it shall be settled under the Rules of Arbitration of the International Chamber of Commerce by one arbitrator appointed in accordance with the said rules sitting in London, United Kingdom. The proceedings shall be in the English language. The right of appeal by either party to regular Courts on a question of law arising in the course of any arbitral proceedings or out of an award made in any arbitral proceedings is hereby agreed to be excluded. + +It is the intention of the parties that this License shall comprehensively govern the legal relations between the parties to the Licence, without interference or contradiction by any unspecified law. However, where a matter is not specifically covered by these terms or a provision of the Licence terms is ambiguous or unclear, resolution shall be found by reference to the laws of England and Wales, including any relevant law of the European Union. + +Nothing stated in this License shall be construed as a waiver of any privileges or immunities of the Licensor or of ECMWF. + +Version 1.2 (November 2019) \ No newline at end of file diff --git a/era5_monthly_averages_pressure_levels_1979_2024/Manifest.toml b/era5_monthly_averages_pressure_levels_1979_2024/Manifest.toml new file mode 100644 index 0000000..9c87a33 --- /dev/null +++ b/era5_monthly_averages_pressure_levels_1979_2024/Manifest.toml @@ -0,0 +1,526 @@ +# This file is machine-generated - editing it directly is not advised + +julia_version = "1.11.1" +manifest_format = "2.0" +project_hash = "47774df903631abd3b74628cc1eb8ab16058d1db" + +[[deps.ArgTools]] +uuid = "0dad84c5-d112-42e6-8d28-ef12dabb789f" +version = "1.1.2" + +[[deps.ArtifactUtils]] +deps = ["Downloads", "Git", "HTTP", "Pkg", "ProgressLogging", "SHA", "TOML", "gh_cli_jll"] +git-tree-sha1 = "f7c3ca6c70ea6b035effe5428eb05231ba890c2d" +uuid = "8b73e784-e7d8-4ea5-973d-377fed4e3bce" +version = "0.2.4" + +[[deps.Artifacts]] +uuid = "56f22d72-fd6d-98f1-02f0-08ddc0907c33" +version = "1.11.0" + +[[deps.Base64]] +uuid = "2a0f44e3-6c83-55bd-87e4-b1978d98bd5f" +version = "1.11.0" + +[[deps.BitFlags]] +git-tree-sha1 = "0691e34b3bb8be9307330f88d1a3c3f25466c24d" +uuid = "d1d4a3ce-64b1-5f1a-9ba4-7e7e69966f35" +version = "0.1.9" + +[[deps.Blosc_jll]] +deps = ["Artifacts", "JLLWrappers", "Libdl", "Lz4_jll", "Zlib_jll", "Zstd_jll"] +git-tree-sha1 = "ef12cdd1c7fb7e1dfd6fa8fd60d4db6bc61d2f23" +uuid = "0b7ba130-8d10-5ba8-a3d6-c5182647fed9" +version = "1.21.6+0" + +[[deps.Bzip2_jll]] +deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg"] +git-tree-sha1 = "8873e196c2eb87962a2048b3b8e08946535864a1" +uuid = "6e34b625-4abd-537c-b88f-471c36dfa7a0" +version = "1.0.8+2" + +[[deps.CFTime]] +deps = ["Dates", "Printf"] +git-tree-sha1 = "5afb5c5ba2688ca43a9ad2e5a91cbb93921ccfa1" +uuid = "179af706-886a-5703-950a-314cd64e0468" +version = "0.1.3" + +[[deps.ClimaArtifactsHelper]] +deps = ["ArtifactUtils", "Downloads", "Pkg", "REPL", "SHA"] +path = "/home/treddy/.julia/dev/ClimaArtifacts/ClimaArtifactsHelper.jl" +uuid = "6ffa2572-8378-4377-82eb-ea11db28b255" +version = "0.0.1" + +[[deps.CodecZlib]] +deps = ["TranscodingStreams", "Zlib_jll"] +git-tree-sha1 = "bce6804e5e6044c6daab27bb533d1295e4a2e759" +uuid = "944b1d66-785c-5afd-91f1-9de20f533193" +version = "0.7.6" + +[[deps.CommonDataModel]] +deps = ["CFTime", "DataStructures", "Dates", "Preferences", "Printf", "Statistics"] +git-tree-sha1 = "98d64d5b9e5263884276656a43c45424b3a645c2" +uuid = "1fbeeb36-5f17-413c-809b-666fb144f157" +version = "0.3.7" + +[[deps.Compat]] +deps = ["TOML", "UUIDs"] +git-tree-sha1 = "8ae8d32e09f0dcf42a36b90d4e17f5dd2e4c4215" +uuid = "34da2185-b29b-5c13-b0c7-acf172513d20" +version = "4.16.0" +weakdeps = ["Dates", "LinearAlgebra"] + + [deps.Compat.extensions] + CompatLinearAlgebraExt = "LinearAlgebra" + +[[deps.CompilerSupportLibraries_jll]] +deps = ["Artifacts", "Libdl"] +uuid = "e66e0078-7015-5450-92f7-15fbd957f2ae" +version = "1.1.1+0" + +[[deps.ConcurrentUtilities]] +deps = ["Serialization", "Sockets"] +git-tree-sha1 = "ea32b83ca4fefa1768dc84e504cc0a94fb1ab8d1" +uuid = "f0e56b4a-5159-44fe-b623-3e5288b988bb" +version = "2.4.2" + +[[deps.DataStructures]] +deps = ["Compat", "InteractiveUtils", "OrderedCollections"] +git-tree-sha1 = "1d0a14036acb104d9e89698bd408f63ab58cdc82" +uuid = "864edb3b-99cc-5e75-8d2d-829cb0a9cfe8" +version = "0.18.20" + +[[deps.Dates]] +deps = ["Printf"] +uuid = "ade2ca70-3891-5945-98fb-dc099432e06a" +version = "1.11.0" + +[[deps.DiskArrays]] +deps = ["LRUCache", "OffsetArrays"] +git-tree-sha1 = "e0e89a60637a62d13aa2107f0acd169b9b9b77e7" +uuid = "3c3547ce-8d99-4f5e-a174-61eb10b00ae3" +version = "0.4.6" + +[[deps.Downloads]] +deps = ["ArgTools", "FileWatching", "LibCURL", "NetworkOptions"] +uuid = "f43a241f-c20a-4ad4-852c-f6b1247861c6" +version = "1.6.0" + +[[deps.ExceptionUnwrapping]] +deps = ["Test"] +git-tree-sha1 = "dcb08a0d93ec0b1cdc4af184b26b591e9695423a" +uuid = "460bff9d-24e4-43bc-9d9f-a8973cb893f4" +version = "0.1.10" + +[[deps.Expat_jll]] +deps = ["Artifacts", "JLLWrappers", "Libdl"] +git-tree-sha1 = "1c6317308b9dc757616f0b5cb379db10494443a7" +uuid = "2e619515-83b5-522b-bb60-26c02a35a201" +version = "2.6.2+0" + +[[deps.FileWatching]] +uuid = "7b1f6079-737a-58dc-b8bc-7a2ca5c1b5ee" +version = "1.11.0" + +[[deps.GMP_jll]] +deps = ["Artifacts", "Libdl"] +uuid = "781609d7-10c4-51f6-84f2-b8444358ff6d" +version = "6.3.0+0" + +[[deps.Git]] +deps = ["Git_jll"] +git-tree-sha1 = "04eff47b1354d702c3a85e8ab23d539bb7d5957e" +uuid = "d7ba0133-e1db-5d97-8f8c-041e4b3a1eb2" +version = "1.3.1" + +[[deps.Git_jll]] +deps = ["Artifacts", "Expat_jll", "JLLWrappers", "LibCURL_jll", "Libdl", "Libiconv_jll", "OpenSSL_jll", "PCRE2_jll", "Zlib_jll"] +git-tree-sha1 = "ea372033d09e4552a04fd38361cd019f9003f4f4" +uuid = "f8c6e375-362e-5223-8a59-34ff63f689eb" +version = "2.46.2+0" + +[[deps.GnuTLS_jll]] +deps = ["Artifacts", "GMP_jll", "JLLWrappers", "Libdl", "Nettle_jll", "P11Kit_jll", "Zlib_jll"] +git-tree-sha1 = "383db7d3f900f4c1f47a8a04115b053c095e48d3" +uuid = "0951126a-58fd-58f1-b5b3-b08c7c4a876d" +version = "3.8.4+0" + +[[deps.HDF5_jll]] +deps = ["Artifacts", "CompilerSupportLibraries_jll", "JLLWrappers", "LLVMOpenMP_jll", "LazyArtifacts", "LibCURL_jll", "Libdl", "MPICH_jll", "MPIPreferences", "MPItrampoline_jll", "MicrosoftMPI_jll", "OpenMPI_jll", "OpenSSL_jll", "TOML", "Zlib_jll", "libaec_jll"] +git-tree-sha1 = "38c8874692d48d5440d5752d6c74b0c6b0b60739" +uuid = "0234f1f7-429e-5d53-9886-15a909be8d59" +version = "1.14.2+1" + +[[deps.HTTP]] +deps = ["Base64", "CodecZlib", "ConcurrentUtilities", "Dates", "ExceptionUnwrapping", "Logging", "LoggingExtras", "MbedTLS", "NetworkOptions", "OpenSSL", "Random", "SimpleBufferStream", "Sockets", "URIs", "UUIDs"] +git-tree-sha1 = "1336e07ba2eb75614c99496501a8f4b233e9fafe" +uuid = "cd3eb016-35fb-5094-929b-558a96fad6f3" +version = "1.10.10" + +[[deps.Hwloc_jll]] +deps = ["Artifacts", "JLLWrappers", "Libdl"] +git-tree-sha1 = "50aedf345a709ab75872f80a2779568dc0bb461b" +uuid = "e33a78d0-f292-5ffc-b300-72abe9b543c8" +version = "2.11.2+1" + +[[deps.InteractiveUtils]] +deps = ["Markdown"] +uuid = "b77e0a4c-d291-57a0-90e8-8db25a27a240" +version = "1.11.0" + +[[deps.JLLWrappers]] +deps = ["Artifacts", "Preferences"] +git-tree-sha1 = "be3dc50a92e5a386872a493a10050136d4703f9b" +uuid = "692b3bcd-3c85-4b1f-b108-f13ce0eb3210" +version = "1.6.1" + +[[deps.LLVMOpenMP_jll]] +deps = ["Artifacts", "JLLWrappers", "Libdl"] +git-tree-sha1 = "78211fb6cbc872f77cad3fc0b6cf647d923f4929" +uuid = "1d63c593-3942-5779-bab2-d838dc0a180e" +version = "18.1.7+0" + +[[deps.LRUCache]] +git-tree-sha1 = "b3cc6698599b10e652832c2f23db3cab99d51b59" +uuid = "8ac3fa9e-de4c-5943-b1dc-09c6b5f20637" +version = "1.6.1" +weakdeps = ["Serialization"] + + [deps.LRUCache.extensions] + SerializationExt = ["Serialization"] + +[[deps.LazyArtifacts]] +deps = ["Artifacts", "Pkg"] +uuid = "4af54fe1-eca0-43a8-85a7-787d91b784e3" +version = "1.11.0" + +[[deps.LibCURL]] +deps = ["LibCURL_jll", "MozillaCACerts_jll"] +uuid = "b27032c2-a3e7-50c8-80cd-2d36dbcbfd21" +version = "0.6.4" + +[[deps.LibCURL_jll]] +deps = ["Artifacts", "LibSSH2_jll", "Libdl", "MbedTLS_jll", "Zlib_jll", "nghttp2_jll"] +uuid = "deac9b47-8bc7-5906-a0fe-35ac56dc84c0" +version = "8.6.0+0" + +[[deps.LibGit2]] +deps = ["Base64", "LibGit2_jll", "NetworkOptions", "Printf", "SHA"] +uuid = "76f85450-5226-5b5a-8eaa-529ad045b433" +version = "1.11.0" + +[[deps.LibGit2_jll]] +deps = ["Artifacts", "LibSSH2_jll", "Libdl", "MbedTLS_jll"] +uuid = "e37daf67-58a4-590a-8e99-b0245dd2ffc5" +version = "1.7.2+0" + +[[deps.LibSSH2_jll]] +deps = ["Artifacts", "Libdl", "MbedTLS_jll"] +uuid = "29816b5a-b9ab-546f-933c-edad1886dfa8" +version = "1.11.0+1" + +[[deps.Libdl]] +uuid = "8f399da3-3557-5675-b5ff-fb832c97cbdb" +version = "1.11.0" + +[[deps.Libiconv_jll]] +deps = ["Artifacts", "JLLWrappers", "Libdl"] +git-tree-sha1 = "61dfdba58e585066d8bce214c5a51eaa0539f269" +uuid = "94ce4f54-9a6c-5748-9c1c-f9c7231a4531" +version = "1.17.0+1" + +[[deps.LinearAlgebra]] +deps = ["Libdl", "OpenBLAS_jll", "libblastrampoline_jll"] +uuid = "37e2e46d-f89d-539d-b4ee-838fcccc9c8e" +version = "1.11.0" + +[[deps.Logging]] +uuid = "56ddb016-857b-54e1-b83d-db4d58db5568" +version = "1.11.0" + +[[deps.LoggingExtras]] +deps = ["Dates", "Logging"] +git-tree-sha1 = "f02b56007b064fbfddb4c9cd60161b6dd0f40df3" +uuid = "e6f89c97-d47a-5376-807f-9c37f3926c36" +version = "1.1.0" + +[[deps.Lz4_jll]] +deps = ["Artifacts", "JLLWrappers", "Libdl"] +git-tree-sha1 = "abf88ff67f4fd89839efcae2f4c39cbc4ecd0846" +uuid = "5ced341a-0733-55b8-9ab6-a4889d929147" +version = "1.10.0+1" + +[[deps.MPICH_jll]] +deps = ["Artifacts", "CompilerSupportLibraries_jll", "Hwloc_jll", "JLLWrappers", "LazyArtifacts", "Libdl", "MPIPreferences", "TOML"] +git-tree-sha1 = "7715e65c47ba3941c502bffb7f266a41a7f54423" +uuid = "7cb0a576-ebde-5e09-9194-50597f1243b4" +version = "4.2.3+0" + +[[deps.MPIPreferences]] +deps = ["Libdl", "Preferences"] +git-tree-sha1 = "c105fe467859e7f6e9a852cb15cb4301126fac07" +uuid = "3da0fdf6-3ccc-4f1b-acd9-58baa6c99267" +version = "0.1.11" + +[[deps.MPItrampoline_jll]] +deps = ["Artifacts", "CompilerSupportLibraries_jll", "JLLWrappers", "LazyArtifacts", "Libdl", "MPIPreferences", "TOML"] +git-tree-sha1 = "70e830dab5d0775183c99fc75e4c24c614ed7142" +uuid = "f1f71cc9-e9ae-5b93-9b94-4fe0e1ad3748" +version = "5.5.1+0" + +[[deps.Markdown]] +deps = ["Base64"] +uuid = "d6f4376e-aef5-505a-96c1-9c027394607a" +version = "1.11.0" + +[[deps.MbedTLS]] +deps = ["Dates", "MbedTLS_jll", "MozillaCACerts_jll", "NetworkOptions", "Random", "Sockets"] +git-tree-sha1 = "c067a280ddc25f196b5e7df3877c6b226d390aaf" +uuid = "739be429-bea8-5141-9913-cc70e7f3736d" +version = "1.1.9" + +[[deps.MbedTLS_jll]] +deps = ["Artifacts", "Libdl"] +uuid = "c8ffd9c3-330d-5841-b78e-0817d7145fa1" +version = "2.28.6+0" + +[[deps.MicrosoftMPI_jll]] +deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg"] +git-tree-sha1 = "f12a29c4400ba812841c6ace3f4efbb6dbb3ba01" +uuid = "9237b28f-5490-5468-be7b-bb81f5f5e6cf" +version = "10.1.4+2" + +[[deps.MozillaCACerts_jll]] +uuid = "14a3606d-f60d-562e-9121-12d972cd8159" +version = "2023.12.12" + +[[deps.NCDatasets]] +deps = ["CFTime", "CommonDataModel", "DataStructures", "Dates", "DiskArrays", "NetCDF_jll", "NetworkOptions", "Printf"] +git-tree-sha1 = "2c9dc92001ac06d432f363f37ff5552954d9947c" +uuid = "85f8d34a-cbdd-5861-8df4-14fed0d494ab" +version = "0.14.6" + +[[deps.NetCDF_jll]] +deps = ["Artifacts", "Blosc_jll", "Bzip2_jll", "HDF5_jll", "JLLWrappers", "LibCURL_jll", "Libdl", "OpenMPI_jll", "XML2_jll", "Zlib_jll", "Zstd_jll", "libzip_jll"] +git-tree-sha1 = "a8af1798e4eb9ff768ce7fdefc0e957097793f15" +uuid = "7243133f-43d8-5620-bbf4-c2c921802cf3" +version = "400.902.209+0" + +[[deps.Nettle_jll]] +deps = ["Artifacts", "GMP_jll", "JLLWrappers", "Libdl", "Pkg"] +git-tree-sha1 = "eca63e3847dad608cfa6a3329b95ef674c7160b4" +uuid = "4c82536e-c426-54e4-b420-14f461c4ed8b" +version = "3.7.2+0" + +[[deps.NetworkOptions]] +uuid = "ca575930-c2e3-43a9-ace4-1e988b2c1908" +version = "1.2.0" + +[[deps.OffsetArrays]] +git-tree-sha1 = "1a27764e945a152f7ca7efa04de513d473e9542e" +uuid = "6fe1bfb0-de20-5000-8ca7-80f57d26f881" +version = "1.14.1" + + [deps.OffsetArrays.extensions] + OffsetArraysAdaptExt = "Adapt" + + [deps.OffsetArrays.weakdeps] + Adapt = "79e6a3ab-5dfb-504d-930d-738a2a938a0e" + +[[deps.OpenBLAS_jll]] +deps = ["Artifacts", "CompilerSupportLibraries_jll", "Libdl"] +uuid = "4536629a-c528-5b80-bd46-f80d51c5b363" +version = "0.3.27+1" + +[[deps.OpenMPI_jll]] +deps = ["Artifacts", "CompilerSupportLibraries_jll", "Hwloc_jll", "JLLWrappers", "LazyArtifacts", "Libdl", "MPIPreferences", "TOML", "Zlib_jll"] +git-tree-sha1 = "bfce6d523861a6c562721b262c0d1aaeead2647f" +uuid = "fe0851c0-eecd-5654-98d4-656369965a5c" +version = "5.0.5+0" + +[[deps.OpenSSL]] +deps = ["BitFlags", "Dates", "MozillaCACerts_jll", "OpenSSL_jll", "Sockets"] +git-tree-sha1 = "38cb508d080d21dc1128f7fb04f20387ed4c0af4" +uuid = "4d8831e6-92b7-49fb-bdf8-b643e874388c" +version = "1.4.3" + +[[deps.OpenSSL_jll]] +deps = ["Artifacts", "JLLWrappers", "Libdl"] +git-tree-sha1 = "7493f61f55a6cce7325f197443aa80d32554ba10" +uuid = "458c3c95-2e84-50aa-8efc-19380b2a3a95" +version = "3.0.15+1" + +[[deps.OrderedCollections]] +git-tree-sha1 = "dfdf5519f235516220579f949664f1bf44e741c5" +uuid = "bac558e1-5e72-5ebc-8fee-abe8a469f55d" +version = "1.6.3" + +[[deps.P11Kit_jll]] +deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg"] +git-tree-sha1 = "2cd396108e178f3ae8dedbd8e938a18726ab2fbf" +uuid = "c2071276-7c44-58a7-b746-946036e04d0a" +version = "0.24.1+0" + +[[deps.PCRE2_jll]] +deps = ["Artifacts", "Libdl"] +uuid = "efcefdf7-47ab-520b-bdef-62a2eaa19f15" +version = "10.42.0+1" + +[[deps.Pkg]] +deps = ["Artifacts", "Dates", "Downloads", "FileWatching", "LibGit2", "Libdl", "Logging", "Markdown", "Printf", "Random", "SHA", "TOML", "Tar", "UUIDs", "p7zip_jll"] +uuid = "44cfe95a-1eb2-52ea-b672-e2afdf69b78f" +version = "1.11.0" +weakdeps = ["REPL"] + + [deps.Pkg.extensions] + REPLExt = "REPL" + +[[deps.Preferences]] +deps = ["TOML"] +git-tree-sha1 = "9306f6085165d270f7e3db02af26a400d580f5c6" +uuid = "21216c6a-2e73-6563-6e65-726566657250" +version = "1.4.3" + +[[deps.Printf]] +deps = ["Unicode"] +uuid = "de0858da-6303-5e67-8744-51eddeeeb8d7" +version = "1.11.0" + +[[deps.ProgressLogging]] +deps = ["Logging", "SHA", "UUIDs"] +git-tree-sha1 = "80d919dee55b9c50e8d9e2da5eeafff3fe58b539" +uuid = "33c8b6b6-d38a-422a-b730-caa89a2f386c" +version = "0.1.4" + +[[deps.REPL]] +deps = ["InteractiveUtils", "Markdown", "Sockets", "StyledStrings", "Unicode"] +uuid = "3fa0cd96-eef1-5676-8a61-b3b8758bbffb" +version = "1.11.0" + +[[deps.Random]] +deps = ["SHA"] +uuid = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c" +version = "1.11.0" + +[[deps.SHA]] +uuid = "ea8e919c-243c-51af-8825-aaa63cd721ce" +version = "0.7.0" + +[[deps.Serialization]] +uuid = "9e88b42a-f829-5b0c-bbe9-9e923198166b" +version = "1.11.0" + +[[deps.SimpleBufferStream]] +git-tree-sha1 = "f305871d2f381d21527c770d4788c06c097c9bc1" +uuid = "777ac1f9-54b0-4bf8-805c-2214025038e7" +version = "1.2.0" + +[[deps.Sockets]] +uuid = "6462fe0b-24de-5631-8697-dd941f90decc" +version = "1.11.0" + +[[deps.Statistics]] +deps = ["LinearAlgebra"] +git-tree-sha1 = "ae3bb1eb3bba077cd276bc5cfc337cc65c3075c0" +uuid = "10745b16-79ce-11e8-11f9-7d13ad32a3b2" +version = "1.11.1" + + [deps.Statistics.extensions] + SparseArraysExt = ["SparseArrays"] + + [deps.Statistics.weakdeps] + SparseArrays = "2f01184e-e22b-5df5-ae63-d93ebab69eaf" + +[[deps.StyledStrings]] +uuid = "f489334b-da3d-4c2e-b8f0-e476e12c162b" +version = "1.11.0" + +[[deps.TOML]] +deps = ["Dates"] +uuid = "fa267f1f-6049-4f14-aa54-33bafae1ed76" +version = "1.0.3" + +[[deps.Tar]] +deps = ["ArgTools", "SHA"] +uuid = "a4e569a6-e804-4fa4-b0f3-eef7a1d5b13e" +version = "1.10.0" + +[[deps.Test]] +deps = ["InteractiveUtils", "Logging", "Random", "Serialization"] +uuid = "8dfed614-e22c-5e08-85e1-65c5234f0b40" +version = "1.11.0" + +[[deps.TranscodingStreams]] +git-tree-sha1 = "0c45878dcfdcfa8480052b6ab162cdd138781742" +uuid = "3bb67fe8-82b1-5028-8e26-92a6c54297fa" +version = "0.11.3" + +[[deps.URIs]] +git-tree-sha1 = "67db6cc7b3821e19ebe75791a9dd19c9b1188f2b" +uuid = "5c2747f8-b7ea-4ff2-ba2e-563bfd36b1d4" +version = "1.5.1" + +[[deps.UUIDs]] +deps = ["Random", "SHA"] +uuid = "cf7118a7-6976-5b1a-9a39-7adc72f591a4" +version = "1.11.0" + +[[deps.Unicode]] +uuid = "4ec0a83e-493e-50e2-b9ac-8f72acf5a8f5" +version = "1.11.0" + +[[deps.XML2_jll]] +deps = ["Artifacts", "JLLWrappers", "Libdl", "Libiconv_jll", "Zlib_jll"] +git-tree-sha1 = "6a451c6f33a176150f315726eba8b92fbfdb9ae7" +uuid = "02c8fc9c-b97f-50b9-bbe4-9be30ff0a78a" +version = "2.13.4+0" + +[[deps.XZ_jll]] +deps = ["Artifacts", "JLLWrappers", "Libdl"] +git-tree-sha1 = "15e637a697345f6743674f1322beefbc5dcd5cfc" +uuid = "ffd25f8a-64ca-5728-b0f7-c24cf3aae800" +version = "5.6.3+0" + +[[deps.Zlib_jll]] +deps = ["Libdl"] +uuid = "83775a58-1f1d-513f-b197-d71354ab007a" +version = "1.2.13+1" + +[[deps.Zstd_jll]] +deps = ["Artifacts", "JLLWrappers", "Libdl"] +git-tree-sha1 = "555d1076590a6cc2fdee2ef1469451f872d8b41b" +uuid = "3161d3a3-bdf6-5164-811a-617609db77b4" +version = "1.5.6+1" + +[[deps.gh_cli_jll]] +deps = ["Artifacts", "JLLWrappers", "Libdl"] +git-tree-sha1 = "b54ff233a6a183ad366d29fa0e3ff9bb921692db" +uuid = "5d31d589-30fb-542f-b82d-10325e863e38" +version = "2.35.0+0" + +[[deps.libaec_jll]] +deps = ["Artifacts", "JLLWrappers", "Libdl"] +git-tree-sha1 = "46bf7be2917b59b761247be3f317ddf75e50e997" +uuid = "477f73a3-ac25-53e9-8cc3-50b2fa2566f0" +version = "1.1.2+0" + +[[deps.libblastrampoline_jll]] +deps = ["Artifacts", "Libdl"] +uuid = "8e850b90-86db-534c-a0d3-1478176c7d93" +version = "5.11.0+0" + +[[deps.libzip_jll]] +deps = ["Artifacts", "Bzip2_jll", "GnuTLS_jll", "JLLWrappers", "Libdl", "XZ_jll", "Zlib_jll", "Zstd_jll"] +git-tree-sha1 = "668ac0297e6bd8f4d53dfdcd3ace71f2e00f4a35" +uuid = "337d8026-41b4-5cde-a456-74a10e5b31d1" +version = "1.11.1+0" + +[[deps.nghttp2_jll]] +deps = ["Artifacts", "Libdl"] +uuid = "8e850ede-7688-5339-a07c-302acd2aaf8d" +version = "1.59.0+0" + +[[deps.p7zip_jll]] +deps = ["Artifacts", "Libdl"] +uuid = "3f19e933-33d8-53b3-aaab-bd5110c3b7a0" +version = "17.4.0+2" diff --git a/era5_monthly_averages_pressure_levels_1979_2024/OutputArtifacts.toml b/era5_monthly_averages_pressure_levels_1979_2024/OutputArtifacts.toml new file mode 100644 index 0000000..a2d0d4c --- /dev/null +++ b/era5_monthly_averages_pressure_levels_1979_2024/OutputArtifacts.toml @@ -0,0 +1,2 @@ +[era5_monthly_averages_pressure_levels_1979_2024] +git-tree-sha1 = "dd7ee9500805f590f8fc4c00bfc373eeb7a01587" diff --git a/era5_monthly_averages_pressure_levels_1979_2024/Project.toml b/era5_monthly_averages_pressure_levels_1979_2024/Project.toml new file mode 100644 index 0000000..33e6033 --- /dev/null +++ b/era5_monthly_averages_pressure_levels_1979_2024/Project.toml @@ -0,0 +1,5 @@ +[deps] +ClimaArtifactsHelper = "6ffa2572-8378-4377-82eb-ea11db28b255" +Dates = "ade2ca70-3891-5945-98fb-dc099432e06a" +Downloads = "f43a241f-c20a-4ad4-852c-f6b1247861c6" +NCDatasets = "85f8d34a-cbdd-5861-8df4-14fed0d494ab" diff --git a/era5_monthly_averages_pressure_levels_1979_2024/README.md b/era5_monthly_averages_pressure_levels_1979_2024/README.md new file mode 100644 index 0000000..2983722 --- /dev/null +++ b/era5_monthly_averages_pressure_levels_1979_2024/README.md @@ -0,0 +1,104 @@ +# ERA5 monthly averages on pressure levels + +This folder creates an artifact by proccessing data coming from +[ERA5 monthly averaged reanalysis](https://cds.climate.copernicus.eu/datasets/reanalysis-era5-pressure-levels-monthly-means?tab=overview) + +## Usage + +To create the artifact: + +1. Set up the CDI APS personal access token following the [instruction](https://cds.climate.copernicus.eu/how-to-api#install-the-cds-api-token), +or enter it in when prompted by the script. +2. Create and activate a python virtual environment +3. In the same terminal run `pip install -r requirements.txt` +4. In the same terminal run `julia --project create_artifact.jl` + +## Requirements + +- Python >= 3 +- 37G of free disk space + +## Downloaded data + +The unproccessed downloaded dataset contains the following variables: + +1. `number` - Not included in output. It is introduced by default by the netCDF conversion +and has a single value of 0. +2. `date` +3. `pressure_level` +4. `latitude` +5. `longitude` +6. `expver` +7. `z` +8. `r` +9. `q` +10. `t` +11. `u` +12. `v` +13. `w` + +## Output Dataset + +### Spatial Coverage + +- 1 degree latitude x 1 degree longitude 2d grid +- -90N to 90N and 0E to 359E +- There are 37 pressure levels from 100Pa to 100000Pa. The spacing between each interval increases +unil 10000Pa. After that point, there is a 2500Pa interval between each point until 25000Pa. From there, +each interval is 5000Pa until 75000Pa, where the interval returns to 2500Pa. +- The latitudes and pressure levels are reversed during processing to be in increasing order + +## Temporal Coverage + +These files contain Monthly averaged reanalysis from 1979 to present (October 2024 at time of creation), which is produced by averaging all daily data for each month. This results in 12*(2024-1979 + 10/12) = 550 points on the +time dimension, where each point is the 15th of the month that the point represents. For example, the 6th index of `time` is `1979-06-15T00:00:00`, +which represents the whole month of June in 1979. + +## Variables + +### `expver` + +This variable is only defined on the time dimension, and a value of 5 indicates any data at that point on the time axis is from ERA5T, which is an initial release of ERA5 data. A value of 1 indicates that data for that time is no longer potentially subject to change. No preprocessing is done other than removing residual GRIB attributes. + +### `z` + +This variable is geopotential in units of m^2/s^2, and it is defined on the latitude, longituide, pressure level, and time dimensions. +No processing is done to this variable other than flipping the latitude and pressure level dimensions and removing residual GRIB attributes. + +### `r` + +This variable is relative humidity as a percentage, and it is defined on the latitude, longituide, pressure level, and time dimensions. +No processing is done to this variable other than flipping the latitude and pressure level dimensions and removing residual GRIB attributes. + +### `q` + +This variable is specific humidity as a ratio of kg/kg, and it is defined on the latitude, longituide, pressure level, and time dimensions. +No processing is done to this variable other than flipping the latitude and pressure level dimensions and removing residual GRIB attributes. + +### `t` + +This variable is temperature in units of K, and it is defined on the latitude, longituide, pressure level, and time dimensions. +No processing is done to this variable other than flipping the latitude and pressure level dimensions and removing residual GRIB attributes. + +### `u` + +This variable is the U component of wind in units of m/s, and it is defined on the latitude, longituide, pressure level, and time dimensions. +No processing is done to this variable other than flipping the latitude and pressure level dimensions and removing residual GRIB attributes. + +### `v` + +This variable is the V component of wind in units of m/s, and it is defined on the latitude, longituide, pressure level, and time dimensions. +No processing is done to this variable other than flipping the latitude and pressure level dimensions and removing residual GRIB attributes. + +### `u` + +This variable is vertical velocity in units of Pa/s, and it is defined on the latitude, longituide, pressure level, and time dimensions. +No processing is done to this variable other than flipping the latitude and pressure level dimensions and removing residual GRIB attributes. + +## Citation + +Hersbach, H., Bell, B., Berrisford, P., Biavati, G., Horányi, A., Muñoz Sabater, J., Nicolas, J., Peubey, C., Radu, R., Rozum, I., Schepers, D., Simmons, A., Soci, C., Dee, D., Thépaut, J-N. (2023): ERA5 monthly averaged data on pressure levels from 1940 to present. Copernicus Climate Change Service (C3S) Climate Data Store (CDS), DOI: 10.24381/cds.6860a573 (Accessed on DD-MMM-YYYY) + +## License + +See the [LICENSE](LICENSE.txt) file diff --git a/era5_monthly_averages_pressure_levels_1979_2024/create_artifact.jl b/era5_monthly_averages_pressure_levels_1979_2024/create_artifact.jl new file mode 100644 index 0000000..cf7455d --- /dev/null +++ b/era5_monthly_averages_pressure_levels_1979_2024/create_artifact.jl @@ -0,0 +1,139 @@ +using Downloads +using ClimaArtifactsHelper +using NCDatasets +using Dates + +const DATA_DIR = "/net/sampo/data1/era5/monthly-avg-pressure-levels" +const DOWNLOADED_DATA_PATH = joinpath(DATA_DIR, "single_years/") +const OUTPUT_FILE_NAME = "era5_monthly_averages_pressure_levels_197901-202410.nc" +const OUTPUT_DIR = joinpath(DATA_DIR, basename(@__DIR__) * "_artifact") + +if isdir(OUTPUT_DIR) + @warn "$OUTPUT_DIR already exists. Content will end up in the artifact and may be overwritten." + @warn "Abort this calculation, unless you know what you are doing." +else + mkdir(OUTPUT_DIR) +end + +year_paths = [joinpath(DOWNLOADED_DATA_PATH, string(year) * ".nc") for year = 1979:2024] + +if !all(isfile, year_paths) + @info "Data for 1979-2024 not found in $DOWNLOADED_DATA_PATH, downloading and populatinit (might take a while)" + if isfile(homedir() * "/.cdsapirc") + run(`python download_era5.py -d $DOWNLOADED_DATA_PATH`) + else + println("Enter your CDS Personal Access Token:") + cds_PAT = readline() + println("Downloading data with CDS API using PAT: $cds_PAT") + run(`python download_era5.py -k $cds_PAT -d $DOWNLOADED_DATA_PATH`) + end +end + +@info "Processing data" + +input_ds = NCDataset(year_paths; aggdim = "date", deferopen = false) +output_path = joinpath(OUTPUT_DIR, OUTPUT_FILE_NAME) + +if isfile(output_path) + rm(output_path) + @info "Removed existing file $output_path" +end +output_ds = NCDataset(output_path, "c") +for (attrib_name, attrib_value) in input_ds.attrib + output_ds.attrib[attrib_name] = attrib_value +end + +defDim(output_ds, "longitude", input_ds.dim["longitude"]) +defDim(output_ds, "latitude", input_ds.dim["latitude"]) +defDim(output_ds, "time", input_ds.dim["date"]) +defDim(output_ds, "pressure_level", input_ds.dim["pressure_level"]) + +# The coordinates attribute is incorrect in the original data +# the _FillValue attribute is automatically added by NCDatasets +ignored_attribs = + ["_FillValue", "missing_value", "add_offset", "scale_factor", "coordinates"] + + +deflatelevel = 9 # max compression (lossless) +for (varname, var) in input_ds + if !(varname in ["longitude", "latitude", "date", "pressure_level", "number", "expver"]) + attrib = copy(var.attrib) + for (key, value) in attrib + if key in ignored_attribs || occursin("GRIB", key) || attrib[key] == "unknown" + delete!(attrib, key) + end + end + # store everything as Float32 to save space + # reverse the pressure and latitude dimensions to have them in ascending order + defVar( + output_ds, + varname, + Float32.(reverse(reverse(var[:, :, :, :], dims = 2), dims = 3)), + (dimnames(var)[1:3]..., "time"); + attrib = attrib, + deflatelevel = deflatelevel, + ) + end +end +defVar( + output_ds, + "expver", + input_ds["expver"][:], + ("time",); + attrib = input_ds["expver"].attrib, +) +# reverse the latitude dimension to have it in ascending order +defVar( + output_ds, + "latitude", + reverse(input_ds["latitude"][:]), + ("latitude",); + attrib = delete!(copy(input_ds["latitude"].attrib), "stored_direction"), +) + +defVar( + output_ds, + "longitude", + input_ds["longitude"][:], + ("longitude",); + attrib = input_ds["longitude"].attrib, +) + +pressure_attrib = copy(input_ds["pressure_level"].attrib) +pressure_attrib["stored_direction"] = "increasing" +pressure_attrib["units"] = "Pa" +delete!(pressure_attrib, "positive") +# convert pressure to Pa and reverse the pressure dimension to have it in ascending order +defVar( + output_ds, + "pressure_level", + reverse(input_ds["pressure_level"][:] .* 100), + ("pressure_level",); + attrib = pressure_attrib, +) +# If data is requested as netcdf, and not netcdf_legacy, the data includes a date dimension +# instead of time, where each date is an integer in the format yyyymmdd. +# Here we convert it to a DateTime object, and set the day to the 15th of the month. + +new_times = map(input_ds["date"][:]) do t + d = DateTime(string(t), "yyyymmdd") + d + (Day(15) - Day(d)) +end + +# check that there are no duplicates and that it is sorted +@assert issorted(new_times) +for i = 2:length(new_times) + @assert new_times[i] != new_times[i-1] +end + +new_times_attribs = ["standard_name" => "time", "long_name" => "Time"] + +defVar(output_ds, "time", new_times, ("time",); attrib = new_times_attribs) + +close(output_ds) +close(input_ds) + +create_artifact_guided( + OUTPUT_DIR; + artifact_name = basename(@__DIR__), +) diff --git a/era5_monthly_averages_pressure_levels_1979_2024/download_era5.py b/era5_monthly_averages_pressure_levels_1979_2024/download_era5.py new file mode 100644 index 0000000..0d4b53c --- /dev/null +++ b/era5_monthly_averages_pressure_levels_1979_2024/download_era5.py @@ -0,0 +1,132 @@ +############################################ +# This script was inspired by CDSAPItools, which is deprecated. +# See here: https://github.com/e5k/CDSAPItools/tree/main +############################################ + +import cdsapi +import pandas as pd +import time +import argparse + +parser = argparse.ArgumentParser(description='Use cdsapi to download era5 mean monthly surface fluxes') +parser.add_argument('-k', '--key', help="CDS API KEY", type=str) +parser.add_argument('-r', '--resume', action='store_true') +parser.add_argument('-d', '--dir', help="download target dir", type=str) +args = parser.parse_args() + +dataset = "reanalysis-era5-pressure-levels-monthly-means" + +years = [str(year) for year in range(1979, 2025)] +if args.dir is None: + output_dir = "" +else: + output_dir = args.dir +URL = 'https://cds.climate.copernicus.eu/api' +SLEEP_TIME = 60 +BASE_URL = "https://cds.climate.copernicus.eu/api/retrieve/v1/jobs/" + +def submit_request(year): + print("Submitting Request for data for year: ", year) + if args.key is None: + submit_client = cdsapi.Client(wait_until_complete=False, delete=False) + else: + submit_client = cdsapi.Client(wait_until_complete=False, delete=False, url=URL, key=args.key) + request = { + "product_type": ["monthly_averaged_reanalysis"], + "variable": [ + "geopotential", + "relative_humidity", + "specific_humidity", + "temperature", + "u_component_of_wind", + "v_component_of_wind", + "vertical_velocity" + ], + "pressure_level": [ + "1", "2", "3", + "5", "7", "10", + "20", "30", "50", + "70", "100", "125", + "150", "175", "200", + "225", "250", "300", + "350", "400", "450", + "500", "550", "600", + "650", "700", "750", + "775", "800", "825", + "850", "875", "900", + "925", "950", "975", + "1000" + ], + "year": [year], + "month": [ + "01", "02", "03", + "04", "05", "06", + "07", "08", "09", + "10", "11", "12" + ], + "time": ["00:00"], + "data_format": "netcdf", + "download_format": "unarchived", + "grid" : [1.0, 1.0], + } + r = submit_client.retrieve(dataset, request) + return r.reply["request_id"] + + + +# submit dummy request +if args.key is None: + client = cdsapi.Client(wait_until_complete=False, delete=False) +else: + client = cdsapi.Client(wait_until_complete=False, delete=False, url=URL, key=args.key) +r = client.retrieve('reanalysis-era5-pressure-levels', { + "variable": "temperature", + "pressure_level": "1000", + "product_type": "reanalysis", + "date": "2017-12-01/2017-12-31", + "time": "12:00", + "format": "grib" + }) +r.retry_options["maximum_tries"] = 9999 +if args.resume: + df = pd.read_csv("era5_download_status.csv", index_col=0) +else: + df = pd.DataFrame() + index = 0 + for year in years: + df = pd.concat([df, pd.DataFrame({"year": year, "status": "not started", "request_id": ""}, index=[index])]) + index += 1 + df.to_csv("era5_download_status.csv") +while True: + if all(df["status"] == "done"): + print("All files are downloaded.") + break + not_started = df[df["status"] == "not started"] + started = df[df["status"] == "started"] + if started.shape[0] < 19 and not_started.shape[0] > 0: + add_index = not_started.iloc[0].name + df.iloc[add_index, 2] = submit_request(df.iloc[add_index, 0]) + df.iloc[add_index, 1] = "started" + for index, row in started.iterrows(): + r.url = BASE_URL + row["request_id"] + r.update() + if r.reply["state"] == "completed": + + df.iloc[index, 1] = "done" + print("Downloaded data for year: ", df.iloc[index, 0], end='\r') + try: + r.download(output_dir + str(df.iloc[index, 0]) + ".nc") + except: + print("Error downloading data for year: ", df.iloc[index, 0]) + elif r.reply["state"] == "failed": + df.iloc[index, 1] = "failed" + print("Failed to download data for year: ", df.iloc[index, 0]) + elif r.reply["state"] == "running": + print("Data for year: ", df.iloc[index, 0], "is running.", end='\r') + elif r.reply["state"] == "queued": + print("Data for year: ", df.iloc[index, 0], "is queued.", end='\r') + elif r.reply["state"] == "accepted": + print("Data for year: ", df.iloc[index, 0], "is accepted.", end='\r') + time.sleep(2) + time.sleep(SLEEP_TIME) + df.to_csv("era5_download_status.csv") diff --git a/era5_monthly_averages_pressure_levels_1979_2024/requirements.txt b/era5_monthly_averages_pressure_levels_1979_2024/requirements.txt new file mode 100644 index 0000000..8dada4d --- /dev/null +++ b/era5_monthly_averages_pressure_levels_1979_2024/requirements.txt @@ -0,0 +1,2 @@ +cdsapi==0.7.4 +pandas==2.2.3