Move chunk_size to argument and retry_on_failure = TRUE

.Rbuildignore

@@ -67,5 +67,7 @@ vignettes/changes_slides.Rmd
 vignettes/daily_data_statistics.Rmd
 vignettes/continuous_pr.Rmd
 vignettes/quick_slides.Rmd
+vignettes/Reference_Lists.Rmd
 ^[.]?air[.]toml$
 ^\.vscode$
+environment.yml

.github/workflows/pkgdown.yaml

@@ -46,11 +46,9 @@ jobs:
             any::pkgdown
             any::rcmdcheck
             any::DT
-            any::data.table
             any::dplyr
             any::tidyr
             any::ggplot2
-            any::zoo
             any::sf
             any::patchwork
             any::maps
@@ -59,6 +57,12 @@ jobs:
             any::gridExtra
             local::.
           needs: website
+      - name: Setup Micromamba
+        uses: mamba-org/setup-micromamba@v1
+        with:
+          environment-file: environment.yml
+          cache-environment: false
+          cache-downloads: false
       - name: Create public directory
         run: |
           mkdir public

.gitignore

@@ -12,10 +12,8 @@ docs
 /doc/
 /Meta/
 /Temp/
+/public/
 vignettes/*.html
 vignettes/*.R
-
-
 /.quarto/
-
 **/*.quarto_ipynb

.gitlab-ci.yml

@@ -42,14 +42,13 @@ build-image:
     alias: docker
   rules:
     - changes:
+      - .gitlab-ci.yml
       - docker/Dockerfile
+      - environment.yml
   script:
     - echo ${CI_REGISTRY_PASSWORD} | docker login -u ${CI_REGISTRY_USER} --password-stdin $CI_REGISTRY
     - docker pull ${CI_REGISTRY_IMAGE}:latest || true
-    - cd docker
-    - docker build
-        -t ${CI_REGISTRY_IMAGE}:latest
-        .
+    - docker build --cache-from ${CI_REGISTRY_IMAGE}:latest -t ${CI_REGISTRY_IMAGE}:latest -t ${CI_REGISTRY_IMAGE}:BUILD_${CI_COMMIT_SHORT_SHA} -f docker/Dockerfile .
     - docker push --all-tags ${CI_REGISTRY_IMAGE}
 
 buildcheck:
@@ -58,8 +57,15 @@ buildcheck:
   dependencies:
     - build-image
   script:
-    - Rscript -e 'devtools::install(quick = TRUE, upgrade = "never")'
-    - Rscript -e 'devtools::check(document = FALSE, args = "--no-tests", check_dir = Sys.getenv("BUILD_LOGS_DIR"), vignettes = FALSE)'
+    - |
+      Rscript -e "
+        rcmdcheck::rcmdcheck(
+          args = c('--no-manual', '--no-tests'),
+          build_args = c('--no-manual', '--no-resave-data'),
+          check_dir = '.',
+          error_on = 'warning'
+        )
+      "
 
 unittests:
   stage: test
@@ -68,8 +74,16 @@ unittests:
     - build-image
     - buildcheck
   script:
-    - R -e 'library(testthat); options(testthat.output_file = file.path(Sys.getenv("CI_PROJECT_DIR"), "test-out.xml")); devtools::test(reporter = "junit")'
-    - R -e 'x <- covr::package_coverage(); covr::to_cobertura(x); x; '
+    - |
+      Rscript -e '
+        library(testthat)
+        options(testthat.output_file = file.path(Sys.getenv("CI_PROJECT_DIR"), "test-out.xml"))
+        test_local(reporter = "junit")'
+    - |
+      Rscript -e '
+        x <- covr::package_coverage()
+        covr::to_cobertura(x)
+        x'
   artifacts:
     when: always
     expire_in: 1 week
@@ -100,11 +114,9 @@ pages:
     - build-image
     - buildcheck
   script:
-    - Rscript -e 'devtools::install(quick = TRUE, upgrade = "never")'
     - Rscript -e 'pkgdown::build_site(override = list(destination = "public"))'
     - Rscript -e 'file.copy(from = "./public/articles/logo.png", to = "./public/reference/logo.png")'
     - quarto render
-
   artifacts:
     paths:
       - $PAGES_OUTDIR

DESCRIPTION

@@ -1,7 +1,7 @@
 Package: dataRetrieval
 Type: Package
 Title: Retrieval Functions for USGS and EPA Hydrology and Water Quality Data
-Version: 2.7.23.9001
+Version: 2.7.24.9001
 Authors@R: c(
     person("Laura", "DeCicco", role = c("aut","cre"),
             email = "ldecicco@usgs.gov",

NAMESPACE

@@ -55,6 +55,7 @@ export(read_waterdata_latest_daily)
 export(read_waterdata_metadata)
 export(read_waterdata_monitoring_location)
 export(read_waterdata_parameter_codes)
+export(read_waterdata_ratings)
 export(read_waterdata_samples)
 export(read_waterdata_stats_daterange)
 export(read_waterdata_stats_por)

NEWS

@@ -1,6 +1,24 @@
+dataRetrieval 2.7.25
+===================
+* Added read_waterdata_ratings to access USGS rating curves with
+new modern endpoint.
+* Increase flexibility of chunking by monitoring_location_id by 
+including it as an argument in each relevant function.
+* Clean up deprecated code.
+* Updated retry strategy to include retry_on_failure = TRUE.
+* Added countries, methods, method-categories, method-citations, and
+citations to possible values in read_waterdata_metadata.
+* Added field_measurements_series_id argument to read_waterdata_field_measurement
+* Removed NWIS tests
+
+
+
 dataRetrieval 2.7.24
 ===================
 * Let dataRetrieval take care of chunking up requests by monitoring_location_id.
+* Fixed bug causing some character columns in importWQP to be numeric
+* Fixed bug causing open ended date requests in waterdata funcitons to 
+return full period of record.
 
 dataRetrieval 2.7.23
 ===================

R/AAA.R

@@ -7,6 +7,11 @@ pkg.env <- new.env()
   options("dataRetrieval.api_version" = "v0")
   options("dataRetrieval.api_version_stat" = "v0")
   options("dataRetrieval.attach_request" = TRUE)
+  options("dataRetrieval.convertType" = TRUE)
+  options("dataRetrieval.no_paging" = FALSE)
+  options("dataRetrieval.site_chunk_size_meta" = 250)
+  options("dataRetrieval.site_chunk_size_data" = 10)
+  options("dataRetrieval.limit" = 50000)
 
   services <- c(
     "server",
@@ -32,7 +37,12 @@ pkg.env <- new.env()
     "coordinate-method-codes",
     "medium-codes",
     "counties",
+    "countries",
     "hydrologic-unit-codes",
+    "methods",
+    "method-categories",
+    "method-citations",
+    "citations",
     "states",
     "national-aquifer-codes",
     "reliability-codes",
@@ -52,7 +62,12 @@ pkg.env <- new.env()
     "coordinate_method_code",
     "medium_code",
     "county",
+    "country",
     "hydrologic_unit_code",
+    "methods",
+    "method_categories",
+    "method_citations",
+    "citations",
     "state",
     "national_aquifer_code",
     "reliability_code",

R/construct_api_requests.R

@@ -5,48 +5,67 @@
 #'
 #' @export
 #' @param service Which service available on <https://api.waterdata.usgs.gov/ogcapi/v0/>.
+#' @param output_id Name of id column to return
 #' @param ... Extra parameters from the specific services.
 #' @param bbox Only features that have a geometry that intersects the bounding
 #' box are selected.The bounding box is provided as four or six numbers, depending
 #' on whether the coordinate reference system includes a vertical axis (height or
 #' depth).
-#' @param properties The properties that should be included for each feature. The
-#' parameter value is a comma-separated list of property names which depend on the
-#' service being called.
-#' @param skipGeometry This option can be used to skip response geometries for
-#' each feature. The returning object will be a data frame with no spatial
-#' information.
 #' @keywords internal
+#'
+#' @inheritParams check_arguments_api
+#' @inheritParams check_arguments_non_api
+#'
 #' @examples
 #' site <- "USGS-02238500"
 #' pcode <- "00060"
 #' req_dv <- construct_api_requests("daily",
+#'                                  output_id = "daily_id",
 #'                                  monitoring_location_id = site,
 #'                                  parameter_code = "00060")
 #'
 #' req_dv <- construct_api_requests("daily",
+#'                                  output_id = "daily_id",
 #'                                  monitoring_location_id = site,
 #'                                  parameter_code = c("00060", "00065"))
 #'
 #' sites <- c("USGS-01491000", "USGS-01645000")
 #' start_date <- "2018-01-01"
 #' end_date <- "2022-01-01"
 #' req_dv <- construct_api_requests("daily",
-#'                                 monitoring_location_id = sites,
-#'                                 parameter_code = c("00060", "00065"),
-#'                                 datetime = c(start_date, end_date))
+#'                                  output_id = "daily_id",
+#'                                  monitoring_location_id = sites,
+#'                                  parameter_code = c("00060", "00065"),
+#'                                  datetime = c(start_date, end_date))
 #'
 construct_api_requests <- function(
   service,
-  properties = NA_character_,
+  output_id,
+  ...,
   bbox = NA,
-  skipGeometry = FALSE,
-  no_paging = FALSE,
-  ...
+  convertType = getOption("dataRetrieval.convertType"),
+  no_paging = getOption("dataRetrieval.no_paging"),
+  chunk_size = getOption("dataRetrieval.site_chunk_size_data"),
+  limit = getOption("dataRetrieval.limit"),
+  attach_request = getOption("dataRetrieval.attach_request")
 ) {
   POST <- FALSE
 
   full_list <- list(...)
+  full_list[["limit"]] <- limit
+
+  check_arguments_non_api(
+    convertType = convertType,
+    no_paging = no_paging,
+    limit = full_list[["limit"]],
+    attach_request = attach_request,
+    chunk_size = chunk_size
+  )
+
+  check_arguments_api(
+    bbox = full_list[["bbox"]],
+    skipGeometry = full_list[["skipGeometry"]]
+  )
 
   time_periods <- c(
     "last_modified",
@@ -58,6 +77,20 @@ construct_api_requests <- function(
     "end_utc"
   )
 
+  full_list <- switch_arg_id(
+    full_list,
+    id_name = output_id,
+    service = service
+  )
+
+  # Clean out non-API arguments:
+  properties <- switch_properties_id(
+    properties = full_list[["properties"]],
+    id = output_id
+  )
+
+  full_list[["properties"]] <- NULL
+
   if (any(time_periods %in% names(full_list))) {
     for (i in time_periods[time_periods %in% names(full_list)]) {
       dates <- FALSE
@@ -74,9 +107,10 @@ construct_api_requests <- function(
     "begin",
     "end",
     "time",
-    "limit",
     "begin_utc",
-    "end_utc"
+    "end_utc",
+    "limit",
+    "skipGeometry"
   )
 
   comma_params <- c(
@@ -85,7 +119,8 @@ construct_api_requests <- function(
     "statistic_id",
     "time_series_id",
     "computation_period_identifier",
-    "computation_identifier"
+    "computation_identifier",
+    "data_type"
   )
 
   if (
@@ -110,6 +145,7 @@ construct_api_requests <- function(
     Negate(anyNA),
     lapply(full_list[comma_params], function(x) x[!is.na(x)])
   )
+
   comma_params_filtered <- comma_params_filtered[
     !sapply(comma_params_filtered, is.null)
   ]
@@ -129,8 +165,6 @@ construct_api_requests <- function(
     get_list <- c(single_params_filtered, comma_params_filtered)
   }
 
-  get_list[["skipGeometry"]] <- skipGeometry
-
   get_list <- get_list[!is.na(get_list)]
 
   format_type <- ifelse(isTRUE(no_paging), "csv", "json")
@@ -207,15 +241,6 @@ construct_api_requests <- function(
   return(baseURL)
 }
 
-check_limits <- function(args) {
-  current_api_limit <- 50000
-
-  if (is.na(args[["limit"]])) {
-    args[["limit"]] <- current_api_limit
-  }
-
-  return(args)
-}
 
 #' Setup the request for the OGC API requests
 #'
@@ -595,6 +620,7 @@ basic_request <- function(url_base, format = "json") {
     httr2::req_headers(`Accept-Encoding` = c("compress", "gzip")) |>
     httr2::req_url_query(f = format, lang = "en-US") |>
     httr2::req_error(body = error_body) |>
+    httr2::req_retry(max_tries = 3, retry_on_failure = TRUE) |>
     httr2::req_timeout(seconds = 180)
 
   req <- add_api_token(req)

R/dataRetrieval-package.R

@@ -211,7 +211,7 @@ NULL
 #
 # property_list <- list()
 # for(service in services){
-#   property_list[[service]] <- get_properties_for_docs(service)
+#   property_list[[service]] <- dataRetrieval:::get_properties_for_docs(service)
 # }
 #
 # num_cols <- c("value", "contributing_drainage_area", "drainage_area",

R/getWebServiceData.R

@@ -30,7 +30,11 @@ getWebServiceData <- function(obs_url, ...) {
 
   obs_url <- httr2::req_user_agent(obs_url, default_ua())
   obs_url <- httr2::req_throttle(obs_url, rate = 30 / 60)
-  obs_url <- httr2::req_retry(obs_url, max_tries = 3, max_seconds = 180)
+  obs_url <- httr2::req_retry(
+    obs_url,
+    max_tries = 3,
+    retry_on_failure = TRUE
+  )
   obs_url <- httr2::req_headers(
     obs_url,
     `Accept-Encoding` = c("compress", "gzip")