Merge pull request #728 from ldecicco-USGS/samplingActivity

Sampling activity

R/AAA.R

@@ -36,7 +36,7 @@ https://doi-usgs.github.io/dataRetrieval/articles/Status.html")
 }
 
 only_legacy <- function(service){
-  legacy <- service %in% c("Activity", "Organization",
+  legacy <- service %in% c("Organization",
                  "ActivityMetric", "SiteSummary",
                  "Project", "ProjectMonitoringLocationWeighting",
                  "ResultDetectionQuantitationLimit", "BiologicalMetric")
@@ -55,7 +55,7 @@ is_legacy <- function(service){
 wqp_message_now <- function(service){
   if(only_legacy(service)){
     return(wqp_message_only_legacy())
-  } else if (service %in% c("Result", "Station")){
+  } else if (service %in% c("Result", "Station", "Activity")){
     return(wqp_message())
   }
 }

R/constructNWISURL.R

@@ -406,7 +406,7 @@ constructWQPURL <- function(siteNumbers,
 
   baseURL <- paste0(baseURL, "&mimeType=csv")
   if(!legacy){
-    baseURL <- paste0(baseURL, "&dataProfile=narrow")
+    baseURL <- paste0(baseURL, "&dataProfile=basicPhysChem")
   }
   return(baseURL)
 }

R/readWQPdata.R

@@ -19,6 +19,7 @@
 #' Full Physical Chemical \tab ResultWQX3 \tab /wqx3/Result/search \tab fullPhysChem \cr
 #' Narrow \tab ResultWQX3 \tab /wqx3/Result/search \tab narrow \cr
 #' Basic Physical Chemical \tab ResultWQX3 \tab /wqx3/Result/search \tab basicPhysChem \cr
+#' Sampling Activity \tab ActivityWQX3 \tab /wqx3/Activity/search \cr
 #' }
 #' 
 #' Legacy:
@@ -103,30 +104,44 @@
 #'                       dataProfile = "narrow")
 #' attr(pHData_narrow, "url") 
 #' ncol(pHData_narrow)
-#'
-#' # vs legacy:
-#' pHData_legacy <- readWQPdata(siteid = "USGS-04024315", 
-#'                       characteristicName = nameToUse,
-#'                       service = "Result",
-#'                       dataProfile = "narrowResult")
-#' attr(pHData_legacy, "url")
 #' 
 #' # Data profiles: "Site Data Only"
 #' site_data <- readWQPdata(
 #'   statecode = "WI",
 #'   countycode = "Dane",
 #'   service = "StationWQX3"
 #' )
-#' 
+#'
 #' # More examples:
 #' # querying by county
 #' DeWitt <- readWQPdata(
 #'   statecode = "Illinois",
 #'   countycode = "DeWitt",
 #'   characteristicName = "Nitrogen"
 #' )
-#'
+#' 
+#' # Data profile: "Sampling Activity"
+#' activity <- readWQPdata(
+#'   siteid = "USGS-04024315",
+#'   service = "ActivityWQX3"
+#' )
+#' 
+#' Dane_activity <- readWQPdata(
+#'   statecode = "Wisconsin",
+#'   countycode = "Dane",
+#'   startDateLo = "2023-01-01",
+#'   startDateHi = "2023-12-31",
+#'   service = "ActivityWQX3"
+#' )
+#' 
+#' ########################################################
 #' # Legacy examples:
+#'
+#' pHData_legacy <- readWQPdata(siteid = "USGS-04024315", 
+#'                       characteristicName = nameToUse,
+#'                       service = "Result",
+#'                       dataProfile = "narrowResult")
+#' attr(pHData_legacy, "url")
 #' 
 #' # Data profiles: "Organization Data" (legacy)
 #' org_data <- readWQPdata(
@@ -213,7 +228,7 @@ readWQPdata <- function(...,
                                   "Project", "ProjectMonitoringLocationWeighting",
                                   "ResultDetectionQuantitationLimit",
                                   "BiologicalMetric", "Organization",
-                                  "ResultWQX3", "StationWQX3"),
+                                  "ResultWQX3", "StationWQX3", "ActivityWQX3"),
                        several.ok = FALSE)
 
   legacy <- is_legacy(service)
@@ -229,7 +244,7 @@ readWQPdata <- function(...,
   wqp_message_now(service)
 
   if(!legacy){
-    if(service != "StationWQX3" & !"dataProfile" %in% names(values)){
+    if(service == "ResultWQX3" & !"dataProfile" %in% names(values)){
       baseURL <- appendDrURL(baseURL, dataProfile = "fullPhysChem")
     }
   } 

R/readWQPdots.R

@@ -25,12 +25,12 @@ readWQPdots <- function(..., legacy = FALSE) {
   if ("dataProfile" %in% names(matchReturn)) {
     profile <- matchReturn$dataProfile
     if (profile == "activityAll") {
-      service <- "Activity"
+      service <- "ActivityWQX3"
       matchReturn$service <- NULL
     } else if (profile %in% c("resultPhysChem","biological","narrowResult")) {
       service <- "Result"
       matchReturn$service <- NULL
-    } else if(profile %in% c("fullPhysChem", "narrow")){
+    } else if(profile %in% c("fullPhysChem", "narrow", "basicPhysChem")){
       service <- "ResultWQX3"
       matchReturn$service <- NULL      
     } 
@@ -41,7 +41,7 @@ readWQPdots <- function(..., legacy = FALSE) {
     "ActivityMetric", "SiteSummary",
     "Project", "ProjectMonitoringLocationWeighting",
     "ResultDetectionQuantitationLimit", "BiologicalMetric",
-    "ResultWQX3", "StationWQX3"
+    "ResultWQX3", "StationWQX3", "ActivityWQX3"
   ))
 
   names(matchReturn)[names(matchReturn) == "bbox"] <- "bBox"

R/setAccess.R

@@ -65,6 +65,7 @@ setAccess <- function(access = "public") {
   pkg.env$Organization <- "https://www.waterqualitydata.us/data/Organization/search"
   pkg.env$ResultWQX3 <- "https://www.waterqualitydata.us/wqx3/Result/search"
   pkg.env$StationWQX3 <- "https://www.waterqualitydata.us/wqx3/Station/search"
+  pkg.env$ActivityWQX3 <- "https://www.waterqualitydata.us/wqx3/Activity/search"
   pkg.env$samplesData <- "https://api.waterdata.usgs.gov/samples-data/summary"
   pkg.env$status <- "https://www.waterqualitydata.us/wqx3/status/"
 

R/whatWQPdata.R

@@ -18,7 +18,7 @@
 #' }
 whatWQPsamples <- function(..., 
                            convertType = TRUE) {
-  values <- readWQPdots(..., legacy = TRUE)
+  values <- readWQPdots(..., legacy = FALSE)
 
   values <- values$values
 
@@ -32,23 +32,16 @@ whatWQPsamples <- function(...,
 
   values <- sapply(values, function(x) utils::URLencode(x, reserved = TRUE))
 
-  baseURL <- drURL("Activity", arg.list = values)
-  wqp_message_now("Activity")
+  baseURL <- drURL("ActivityWQX3", arg.list = values)
+  wqp_message_now("ActivityWQX3")
   baseURL <- appendDrURL(baseURL, mimeType = "csv")
 
-  withCallingHandlers(
-    {
-      retval <- importWQP(baseURL,
-        convertType = convertType
-      )
-    },
-    warning = function(w) {
-      if (any(grepl("Number of rows returned not matched in header", w))) {
-        invokeRestart("muffleWarning")
-      }
-    }
+  retval <- importWQP(baseURL,
+                      convertType = convertType
   )
-
+  attr(retval, "legacy") <- FALSE
+  attr(retval, "wqp-request-id") <- attr(retval, "headerInfo")$`wqp-request-id`
+
   attr(retval, "queryTime") <- Sys.time()
   attr(retval, "url") <- baseURL
 

README.Rmd

@@ -31,6 +31,16 @@ The `dataRetrieval` package was created to simplify the process of loading hydro
 
 :warning: USGS discrete water-quality data availability and format are changing. Beginning March 11, 2024 USGS data obtained from legacy services will not include new USGS data or recent updates to existing data. To view the status of changes in data availability and code functionality, visit: https://doi-usgs.github.io/dataRetrieval/articles/Status.html
 
+:warning: All documentation on <https://doi-usgs.github.io/dataRetrieval/\> assume you have the development version of dataRetrieval installed. To do that:
+
+```{r eval=FALSE}
+library(remotes)
+install_github("DOI-USGS/dataRetrieval",
+               build_vignettes = TRUE, 
+               build_opts = c("--no-resave-data",
+                              "--no-manual"))
+```
+
 If you have additional questions about these changes, email CompTools@usgs.gov.
 
 # What would you like to do?

README.md

@@ -21,6 +21,19 @@ data. To view the status of changes in data availability and code
 functionality, visit:
 <https://doi-usgs.github.io/dataRetrieval/articles/Status.html>
 
+:warning: All documentation on
+<a href="https://doi-usgs.github.io/dataRetrieval/\"
+class="uri">https://doi-usgs.github.io/dataRetrieval/\</a> assume you
+have the development version of dataRetrieval installed. To do that:
+
+``` r
+library(remotes)
+install_github("DOI-USGS/dataRetrieval",
+               build_vignettes = TRUE, 
+               build_opts = c("--no-resave-data",
+                              "--no-manual"))
+```
+
 If you have additional questions about these changes, email
 <CompTools@usgs.gov>.
 
@@ -92,7 +105,7 @@ citation(package = "dataRetrieval")
 #> A BibTeX entry for LaTeX users is
 #> 
 #>   @Manual{,
-#>     author = {Laura DeCicco and Robert Hirsch and David Lorenz and David Watkins and Mike Johnson},
+#>     author = {Laura DeCicco and Robert Hirsch and David Lorenz and Jordan Read and Jordan Walker and Lindsay Platt and David Watkins and David Blodgett and Mike Johnson and Aliesha Krall},
 #>     title = {dataRetrieval: R packages for discovering and retrieving water data available from U.S. federal hydrologic web services},
 #>     publisher = {U.S. Geological Survey},
 #>     address = {Reston, VA},
@@ -122,18 +135,15 @@ NWIScitation <- create_NWIS_bib(dv)
 NWIScitation
 #> U.S. Geological Survey (2024). _National Water Information System data
 #> available on the World Wide Web (USGS Water Data for the Nation)_.
-#> doi:10.5066/F7P55KJN <https://doi.org/10.5066/F7P55KJN>, Accessed Jun
-#> 15, 2024,
+#> doi:10.5066/F7P55KJN <https://doi.org/10.5066/F7P55KJN>, Accessed Aug
+#> 30, 2024,
 #> <https://waterservices.usgs.gov/nwis/dv/?site=09010500&format=waterml,1.1&ParameterCd=00060&StatCd=00003&startDT=1851-01-01>.
-```
-
-``` r
 print(NWIScitation, style = "Bibtex")
 #> @Manual{,
 #>   title = {National Water Information System data available on the World Wide Web (USGS Water Data for the Nation)},
 #>   author = {{U.S. Geological Survey}},
 #>   doi = {10.5066/F7P55KJN},
-#>   note = {Accessed Jun 15, 2024},
+#>   note = {Accessed Aug 30, 2024},
 #>   year = {2024},
 #>   url = {https://waterservices.usgs.gov/nwis/dv/?site=09010500&format=waterml,1.1&ParameterCd=00060&StatCd=00003&startDT=1851-01-01},
 #> }
@@ -157,19 +167,16 @@ WQPcitation <- create_WQP_bib(SC)
 WQPcitation
 #> National Water Quality Monitoring Council (2024). _Water Quality
 #> Portal_. doi:10.5066/P9QRKUVJ <https://doi.org/10.5066/P9QRKUVJ>,
-#> Accessed Jun 15, 2024,
-#> <https://www.waterqualitydata.us/data/Result/search?siteid=USGS-05288705&pCode=00300&mimeType=tsv&zip=yes>.
-```
-
-``` r
+#> Accessed Aug 30, 2024,
+#> <https://www.waterqualitydata.us/wqx3/Result/search?siteid=USGS-05288705&pCode=00300&mimeType=csv&dataProfile=basicPhysChem>.
 print(WQPcitation, style = "Bibtex")
 #> @Manual{,
 #>   title = {Water Quality Portal},
 #>   author = {{National Water Quality Monitoring Council}},
 #>   doi = {10.5066/P9QRKUVJ},
-#>   note = {Accessed Jun 15, 2024},
+#>   note = {Accessed Aug 30, 2024},
 #>   year = {2024},
-#>   url = {https://www.waterqualitydata.us/data/Result/search?siteid=USGS-05288705&pCode=00300&mimeType=tsv&zip=yes},
+#>   url = {https://www.waterqualitydata.us/wqx3/Result/search?siteid=USGS-05288705&pCode=00300&mimeType=csv&dataProfile=basicPhysChem},
 #> }
 ```
 

tests/testthat/tests_userFriendly_fxns.R

@@ -428,7 +428,7 @@ test_that("Construct WQP urls", {
   # nolint start: line_length_linter
   expect_equal(
     url_wqp,
-    "https://www.waterqualitydata.us/wqx3/Result/search?siteid=USGS-01594440&pCode=01075&pCode=00029&pCode=00453&startDateLo=01-01-1985&mimeType=csv&dataProfile=narrow"
+    "https://www.waterqualitydata.us/wqx3/Result/search?siteid=USGS-01594440&pCode=01075&pCode=00029&pCode=00453&startDateLo=01-01-1985&mimeType=csv&dataProfile=basicPhysChem"
   )
 
   # Multiple characteristicNames
@@ -443,7 +443,7 @@ test_that("Construct WQP urls", {
   )
   expect_equal(
     obs_url_orig,
-    "https://www.waterqualitydata.us/wqx3/Result/search?siteid=IIDFG-41WSSPAHS&siteid=USGS-02352560&characteristicName=Temperature&characteristicName=Temperature%2C%20sample&characteristicName=Temperature%2C%20water&characteristicName=Temperature%2C%20water%2C%20deg%20F&mimeType=csv&dataProfile=narrow"
+    "https://www.waterqualitydata.us/wqx3/Result/search?siteid=IIDFG-41WSSPAHS&siteid=USGS-02352560&characteristicName=Temperature&characteristicName=Temperature%2C%20sample&characteristicName=Temperature%2C%20water&characteristicName=Temperature%2C%20water%2C%20deg%20F&mimeType=csv&dataProfile=basicPhysChem"
   )
 
   # nolint end
@@ -463,7 +463,7 @@ test_that("Construct WQP urls", {
   # nolint start: line_length_linter
   expect_equal(
     url_wqp,
-    "https://www.waterqualitydata.us/wqx3/Result/search?siteid=USGS-01594440&pCode=01075&pCode=00029&pCode=00453&startDateLo=01-01-1985&mimeType=csv&dataProfile=narrow"
+    "https://www.waterqualitydata.us/wqx3/Result/search?siteid=USGS-01594440&pCode=01075&pCode=00029&pCode=00453&startDateLo=01-01-1985&mimeType=csv&dataProfile=basicPhysChem"
   )
 
   # nolint end