-
Notifications
You must be signed in to change notification settings - Fork 13
/
hytest_intake_catalog.yml
248 lines (220 loc) · 11 KB
/
hytest_intake_catalog.yml
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
sources:
conus404-catalog:
args:
path: 'https://raw.githubusercontent.com/hytest-org/hytest/main/dataset_catalog/subcatalogs/conus404-catalog.yml'
description: 'Catalog for CONUS404 datasets'
driver: intake.catalog.local.YAMLFileCatalog
benchmarks-catalog:
args:
path: 'https://raw.githubusercontent.com/hytest-org/hytest/main/dataset_catalog/subcatalogs/benchmarks-catalog.yml'
description: 'Catalog for model benchmarking datasets'
driver: intake.catalog.local.YAMLFileCatalog
conus404-drb-eval-tutorial-catalog:
args:
path: 'https://raw.githubusercontent.com/hytest-org/hytest/main/dataset_catalog/subcatalogs/conus404-drb-eval-tutorial-catalog.yml'
description: 'Catalog for CONUS404 DRB tutorial intermediate datasets'
driver: intake.catalog.local.YAMLFileCatalog
nhm-v1.0-daymet-catalog:
args:
path: 'https://raw.githubusercontent.com/hytest-org/hytest/main/dataset_catalog/subcatalogs/nhm-v1.0-daymet-catalog.yml'
description: 'Catalog for National Hydrologic Model version 1.0 model output variables with Daymet v3 forcings'
driver: intake.catalog.local.YAMLFileCatalog
nhm-v1.1-c404-bc-catalog:
args:
path: 'https://raw.githubusercontent.com/hytest-org/hytest/main/dataset_catalog/subcatalogs/nhm-v1.1-c404-bc-catalog.yml'
description: 'Catalog for National Hydrologic Model version 1.1 model output variables with CONUS404 bias-corrected forcings'
driver: intake.catalog.local.YAMLFileCatalog
nhm-v1.1-gridmet-catalog:
args:
path: 'https://raw.githubusercontent.com/hytest-org/hytest/main/dataset_catalog/subcatalogs/nhm-v1.1-gridmet-catalog.yml'
description: 'Catalog for National Hydrologic Model version 1.1 model output variables with Gridmet forcings'
driver: intake.catalog.local.YAMLFileCatalog
trends-and-drivers-catalog:
args:
path: 'https://raw.githubusercontent.com/hytest-org/hytest/main/dataset_catalog/subcatalogs/trends-and-drivers-catalog.yml'
description: 'Catalog for USGS Trends and Drivers project data. Note: This data is stored in a credentialed bucket; please contact the HyTEST project if you need credentials to read this data.'
driver: intake.catalog.local.YAMLFileCatalog
nhm-prms-v1.1-gridmet-format-testing-catalog:
args:
path: 'https://raw.githubusercontent.com/hytest-org/hytest/main/dataset_catalog/subcatalogs/nhm-prms-v1.1-gridmet-format-testing-catalog.yml'
description: 'Catalog of NHM-PRMS v1.1 with Gridmet Forcings data stored in various file formats, used for testing purposes with the USGS National Modeled Water Atlas Project. Note: This data is stored in a credentialed bucket; please contact the HyTEST project if you need credentials to read this data.'
driver: intake.catalog.local.YAMLFileCatalog
nwis-streamflow-usgs-gages-onprem:
driver: zarr
description: "Streamflow from NWIS, extracted and rechunked into time series (NWM2.1 time period)"
args:
urlpath: '/caldera/projects/usgs/hazards/cmgp/woodshole/rsignell/conus404/zarr/nwis_chanobs.zarr'
consolidated: true
nwis-streamflow-usgs-gages-osn:
driver: zarr
description: "Streamflow from NWIS, extracted and rechunked into time series (NWM2.1 time period)"
args:
urlpath: 's3://hytest/tutorials/evaluation/nwm/nwis_chanobs.zarr'
consolidated: true
storage_options:
anon: true
requester_pays: false
client_kwargs:
endpoint_url: https://usgs.osn.mghpcc.org
nwm21-streamflow-usgs-gages-onprem:
driver: zarr
description: "Streamflow from NWM2.1, extracted and rechunked into time series"
args:
urlpath: '/caldera/projects/usgs/hazards/cmgp/woodshole/rsignell/conus404/zarr/chanobs.zarr'
consolidated: true
nwm21-streamflow-usgs-gages-osn:
driver: zarr
description: "Streamflow from NWM2.1, extracted and rechunked into time series"
args:
urlpath: 's3://hytest/tutorials/evaluation/nwm/chanobs.zarr'
consolidated: true
storage_options:
anon: true
requester_pays: false
client_kwargs:
endpoint_url: https://usgs.osn.mghpcc.org
nwm21-streamflow-cloud:
driver: zarr
description: "National Water Model 2.1 CHRTOUT on AWS"
args:
urlpath: 's3://noaa-nwm-retrospective-2-1-zarr-pds/chrtout.zarr'
consolidated: true
storage_options:
anon: true
geofabric_v1_1-zip-osn:
driver: geoparquet # note - this is the wrong driver, and you cannot open this dataset with intake
description: "GIS Features of the Geospatial Fabric for the National Hydrologic Model, version 1.1 (https://doi.org/10.5066/P971JAGF). This dataset cannot be opened with intake because the GDB driver doesn't exist; however it is still being catalogged here for the purpose of being able to read the urlpath into your workflow to locate the data."
args:
urlpath: 's3://hytest/geofabric_v1_1/GFv1.1.gdb.zip'
storage_options:
anon: true
client_kwargs:
endpoint_url: https://usgs.osn.mghpcc.org
geofabric_v1_1_POIs_v1_1-osn:
driver: geoparquet
description: "POIs_v1_1 layer of the GIS Features of the Geospatial Fabric for the National Hydrologic Model, version 1.1 (https://doi.org/10.5066/P971JAGF) converted to geoparquet format"
args:
urlpath: 's3://hytest/geofabric_v1_1/GFv1.1_POIs_v1_1.geoparquet'
storage_options:
anon: true
client_kwargs:
endpoint_url: https://usgs.osn.mghpcc.org
geofabric_v1_1_TBtoGFv1_POIs-osn:
driver: geoparquet
description: "TBtoGFv1_POIs layer of the GIS Features of the Geospatial Fabric for the National Hydrologic Model, version 1.1 (https://doi.org/10.5066/P971JAGF) converted to geoparquet format"
args:
urlpath: 's3://hytest/geofabric_v1_1/GFv1.1_TBtoGFv1_POIs.geoparquet'
storage_options:
anon: true
client_kwargs:
endpoint_url: https://usgs.osn.mghpcc.org
geofabric_v1_1_nhru_v1_1-osn:
driver: geoparquet
description: "nhru_v1_1 layer of the GIS Features of the Geospatial Fabric for the National Hydrologic Model, version 1.1 (https://doi.org/10.5066/P971JAGF) converted to geoparquet format"
args:
urlpath: 's3://hytest/geofabric_v1_1/GFv1.1_nhru_v1_1.geoparquet'
storage_options:
anon: true
client_kwargs:
endpoint_url: https://usgs.osn.mghpcc.org
geofabric_v1_1_nhru_v1_1_simp-osn:
driver: geoparquet
description: "nhru_v1_1_simp layer of the GIS Features of the Geospatial Fabric for the National Hydrologic Model, version 1.1 (https://doi.org/10.5066/P971JAGF) converted to geoparquet format"
args:
urlpath: 's3://hytest/geofabric_v1_1/GFv1.1_nhru_v1_1_simp.geoparquet'
storage_options:
anon: true
client_kwargs:
endpoint_url: https://usgs.osn.mghpcc.org
geofabric_v1_1_nsegment_v1_1-osn:
driver: geoparquet
description: "nsegment_v1_1 layer of the GIS Features of the Geospatial Fabric for the National Hydrologic Model, version 1.1 (https://doi.org/10.5066/P971JAGF) converted to geoparquet format"
args:
urlpath: 's3://hytest/geofabric_v1_1/GFv1.1_nsegment_v1_1.geoparquet'
storage_options:
anon: true
client_kwargs:
endpoint_url: https://usgs.osn.mghpcc.org
gages2_nndar-osn:
driver: parquet
description: "Estimated streamflow at GAGESII locations (https://doi.org/10.5066/P9XT4WSP) using nearest-neighbor drainage area ratio (NNDAR), converted to geoparquet format"
args:
urlpath: 's3://hytest/nwis_gages2_streamflow_estimates/gages2_nndar.parquet'
storage_options:
anon: true
client_kwargs:
endpoint_url: https://usgs.osn.mghpcc.org
wbd-zip-osn:
driver: geoparquet # note - this is the wrong driver, and you cannot open this dataset with intake
description: "Watershed Boundary Dataset (https://doi.org/10.5066/P9BTKP3T). This dataset cannot be opened with intake because the GDB driver doesn't exist; however it is still being catalogged here for the purpose of being able to read the urlpath into your workflow to locate the data."
args:
urlpath: 's3://hytest/wbd/WBD_National_GDB.zip'
storage_options:
anon: true
client_kwargs:
endpoint_url: https://usgs.osn.mghpcc.org
huc12-geoparquet-osn:
driver: geoparquet
description: "WBDHU12 layer of Watershed Boundary Dataset (https://doi.org/10.5066/P9BTKP3T) converted to geoparquet format"
args:
urlpath: 's3://hytest/wbd/huc12/huc12.geoparquet'
storage_options:
anon: true
client_kwargs:
endpoint_url: https://usgs.osn.mghpcc.org
huc12-gpkg-osn:
driver: geoparquet # This should be geopackage, but this is not currently supported by intake - will be supported in intake v2
description: "WBDHU12 layer of Watershed Boundary Dataset (https://doi.org/10.5066/P9BTKP3T) converted to geopackage format. This file format is not currently supported by intake, so you cannot currently open this dataset with intake. This will be compatible with intake v2 when in is released in early 2024."
args:
urlpath: 's3://hytest/wbd/huc12/huc12.gpkg'
storage_options:
anon: true
client_kwargs:
endpoint_url: https://usgs.osn.mghpcc.org
nwm21-scores:
description: US state information from [CivilServices](https://civil.services/)
driver: csv
args:
urlpath: 'https://raw.githubusercontent.com/nhm-usgs/data-pipeline-helpers/main/hytest/results/nwm_ref_gages_assessment.csv'
lcmap-cloud:
driver: intake_xarray.xzarr.ZarrSource
description: 'LCMAP, all 36 years'
args:
urlpath: "reference://"
consolidated: false
storage_options:
target_options:
requester_pays: true
fo: 's3://nhgf-development/lcmap/lcmap.json'
remote_options:
requester_pays: true
remote_protocol: s3
rechunking-tutorial-osn:
driver: zarr
description: "Sample streamflow and velocity data used in HyTEST rechunking tutorial"
args:
urlpath: 's3://hytest/tutorials/dataset_preprocessing/ReChunking_Tutorial_Data.zarr'
consolidated: true
storage_options:
anon: true
requester_pays: false
client_kwargs:
endpoint_url: https://usgs.osn.mghpcc.org
pointsample-tutorial-sites-osn:
driver: csv
description: "Sample gage/point locations used in CONUS404 point sampling tutorial"
args:
urlpath: 's3://hytest/tutorials/data_access/filtered_temperature_sites.csv'
storage_options:
anon: true
client_kwargs:
endpoint_url: https://usgs.osn.mghpcc.org
pointsample-tutorial-output-osn:
driver: netcdf
description: "Output point sampled CONUS404 data from CONUS404 point sampling tutorial"
args:
urlpath: 's3://hytest/tutorials/data_access/daily_skintemp_at_filtered_temperature_sites.nc'
storage_options:
anon: true
client_kwargs:
endpoint_url: https://usgs.osn.mghpcc.org