Initial paginate function

docs/paginators.md

@@ -0,0 +1,119 @@
+# Paginators
+
+> Some AWS operations return results that are incomplete and require subsequent requests in order to attain the entire result set. The process of sending subsequent requests to continue where a previous request left off is called pagination. For example, the list_objects operation of Amazon S3 returns up to 1000 objects at a time, and you must send subsequent requests with the appropriate Marker in order to retrieve the next page of results.
+(https://boto3.amazonaws.com/v1/documentation/api/latest/guide/paginators.html#paginators)
+
+
+As of `paws v0.4.0+` paginators are supported within `paws`.
+
+## Basic Usage
+
+A paginator can be applied to a `paws` operation. `paws` support 3 different methods of paginator (`paginate`, `paginate_lapply`, `paginate_sapply`). 
+
+### `paginate`:
+Return all response from the `paws` operation.
+
+```r
+library(paws)
+
+svc <- s3(region = "us-west-2")
+
+results <- paginate(svc$list_objects(Bucket = "my-bucket"))
+```
+
+### `paginate_lapply`:
+Allows you to apply a function on each returning response.
+```r
+library(paws)
+
+svc <- s3(region = "us-west-2")
+
+results <- paginate_lapply(svc$list_objects(Bucket = "my-bucket"), \(resp) resp$Contents)
+```
+
+### `paginate_sapply`:
+Allows you to apply a function on each returning response, however the final result is simplified similar to `base::sapply`.
+```r
+library(paws)
+
+svc <- s3(region = "us-west-2")
+
+results <- paginate_sapply(
+    svc$list_objects(Bucket = "my-bucket"),
+    \(resp) resp$Contents,
+    simplify = T
+)
+```
+
+## Customizing page Iterators
+
+You can modify the operation by
+
+* `MaxItems:` 
+    Limits the maximum number of total returned items returned while paginating.
+* `StartingToken:`
+    Can be used to modify the starting marker or token of a paginator. This argument if useful for resuming pagination from a previous token or starting pagination at a known position.
+* `PageSize:`
+    Controls the number of items returned per page of each result.
+
+
+### `paginate`
+```r
+library(paws)
+
+svc <- s3(region = "us-west-2")
+
+results <- paginate(svc$list_objects(Bucket = "my-bucket"), MaxItems = 10)
+```
+
+#### `paginate_lapply`
+```r
+library(paws)
+
+svc <- s3(region = "us-west-2")
+
+results <- paginate_lapply(svc$list_objects(Bucket = "my-bucket"), \(page) page$Contents)
+```
+
+#### `paginate_sapply`
+```r
+library(paws)
+
+svc <- s3(region = "us-west-2")
+
+results <- paginate_lapply(svc$list_objects(Bucket = "my-bucket"), \(page) page$Contents)
+```
+
+## Piping:
+
+`paws` paginator support R native piping `|>`. However we currently don't support magrittr piping `%>%`.
+
+```r
+library(paws)
+library(magrittr)
+
+svc <- s3(region = "us-west-2")
+
+# Will Work
+results <- svc$list_objects(Bucket = "my-bucket") |> paginate(MaxItems = 10)
+
+# Will error:
+results <- svc$list_objects(Bucket = "my-bucket") %>% paginate(MaxItems = 10)
+```
+
+
+## Filtering results:
+
+You can filter the paginator results by limiting the response for the paws operation. For example `list_objects` accepts `Prefix` parameter to filter page server-side before returning to `R`.
+
+```r
+library(paws)
+
+svc <- s3(region = "us-west-2")
+
+kwargs <- list(
+    Bucket='my-bucket',
+    Prefix='foo/baz'
+)
+result <- do.call(svc$list_objects, kwargs) |> paginate_lapply(\(page) page$Contents)
+```

make.paws/R/cran_category.R

@@ -1,6 +1,8 @@
 #' @include package.R service.R
 NULL
 
+.paws.common.import.version <- "paws.common (>= 0.5.9)"
+
 # Make all category-level packages.
 make_categories <- function(sdk_dir, out_dir, categories, service_names) {
   for (category in categories) {
@@ -14,7 +16,7 @@ make_category <- function(category, service_names, sdk_dir, out_dir) {
   services <- category$services
   title <- category$title
   description <- category$description
-  imports <- "paws.common (>= 0.5.4)"
+  imports <- .paws.common.import.version
   version <- get_version(sdk_dir)
 
   if (is.null(name) || is.null(title) || is.null(description)) {
@@ -34,6 +36,7 @@ make_category <- function(category, service_names, sdk_dir, out_dir) {
   for (service in services) {
     copy_files(service_names[[service]], from = sdk_dir, to = package_dir)
   }
+  copy_files("reexports", from = sdk_dir, to = package_dir)
   write_documentation(package_dir)
 }
 

make.paws/R/cran_collection.R

@@ -62,6 +62,7 @@ write_source_collection <- function(sdk_dir,
     }
   }
   write_list(clients, file.path(out_dir, "R", "paws.R"))
+  write_list(make_reexports(), file.path(out_dir, "R", "reexports_paws.common.R"))
 }
 
 # Add the category packages to the DESCRIPTION file's Imports.
@@ -70,6 +71,7 @@ write_source_collection <- function(sdk_dir,
 # generate the package.
 write_imports_collection <- function(path, version, imports) {
   packages <- sprintf("%s (>= %s)", imports, version)
+  packages <- c(packages, .paws.common.import.version)
   desc::desc_set(
     Imports = paste0(packages, collapse = ","),
     file = file.path(path, "DESCRIPTION"),

make.paws/R/operations.R

@@ -1,4 +1,5 @@
 #' @include templates.R
+#' @include utils.R
 NULL
 
 operation_file_template <- template(
@@ -36,7 +37,7 @@ operation_template <- template(
       name = ${operation_name},
       http_method = ${http_method},
       http_path = ${http_path},
-      paginator = list()
+      paginator = ${paginator}
     )
     input <- .${service}$${operation_input}
     output <- .${service}$${operation_output}
@@ -62,10 +63,32 @@ make_operation <- function(operation, api, doc_maker) {
     operation_input = get_operation_input(operation, api),
     operation_output = get_operation_output(operation),
     http_method = quoted(operation$http$method),
-    http_path = quoted(operation$http$requestUri)
+    http_path = quoted(operation$http$requestUri),
+    paginator = set_paginator(operation$paginators)
   )
 }
 
+
+set_paginator <- function(paginator) {
+  if (!is.null(paginator)) {
+    output_token <- paginator$output_token
+    if (!is.null(output_token)) {
+      for (i in seq_along(output_token)) {
+        # output_token[[i]] <- strsplit(output_token[[i]], " ")[[1]][[1]]
+        output_token[i] <- list(trimws(strsplit(output_token[[i]], split = "||", fixed = T)[[1]]))
+      }
+      paginator$output_token <- unlist(output_token, use.names = FALSE)
+      paginator$input_token <- rep_len(
+        paginator$input_token,
+        length(paginator$output_token)
+      )
+    }
+    paste(trimws(deparse(paginator)), collapse = " ")
+  } else {
+    "list()"
+  }
+}
+
 # Override operation name from extdata/operation_name_override.yml
 operation_name_override <- function(operation_name) {
   path <- system_file(

make.paws/R/process_api.R

@@ -27,6 +27,7 @@ make_code_files <- function(api) {
   result$interfaces <- make_interfaces_files(api)
   result$service <- make_service_files(api)
   result$custom <- make_custom_operations_files(api)
+  result$reexports <- make_reexports()
   return(result)
 }
 
@@ -78,11 +79,23 @@ make_custom_operations_files <- function(api) {
   return(result)
 }
 
+make_reexports <- function() {
+  result <- list()
+  from <- system_file("templates/reexports_paws.common.R", package = methods::getPackageName())
+  filename <- "reexports_paws.common.R"
+  if (from != "" && file.exists(from)) {
+    contents <- readLines(from)
+    result[[file.path(CODE_DIR, filename)]] <- paste(contents, collapse = "\n")
+  }
+  return(result)
+}
+
 make_docs_files <- function(api) {
   result <- list()
   result$operations <- make_operations_files(api, doc_maker = make_docs_long)
   result$service <- make_service_files(api)
   result$custom <- make_custom_operations_files(api)
+  result$reexports <- make_reexports()
   return(result)
 }
 

make.paws/R/read_api.R

@@ -14,6 +14,10 @@ read_api <- function(api_name, path) {
     examples <- jsonlite::read_json(files$examples)
     api <- merge_examples(api, examples$examples)
   }
+  if (!is.null(files$paginators)) {
+    paginators <- jsonlite::read_json(files$paginators)
+    api <- merge_paginators(api, paginators$pagination)
+  }
   region_config <- jsonlite::read_json(region_config_path)
   api <- merge_region_config(api, region_config)
   api <- fix_region_config(api)
@@ -48,6 +52,16 @@ merge_examples <- function(api, examples) {
   return(api)
 }
 
+# Returns an API object with paginators merged into the corresponding operations.
+merge_paginators <- function(api, paginators) {
+  for (name in names(paginators)) {
+    operation <- api$operations[[name]]
+    operation[["paginators"]] <- paginators[[name]]
+    api$operations[[name]] <- operation
+  }
+  return(api)
+}
+
 # Returns an API object with region config info attached. Region config info
 # lists endpoints for each service and region, if different from the default.
 merge_region_config <- function(api, region_config) {

make.paws/R/utils.R

@@ -13,7 +13,7 @@ parse_operations <- function(text) {
   ids <- rep(NA, length(text))
   id <- 1
   for (i in seq_along(text)) {
-    if (i > 1 && startsWith(text[i], "#'") && !startsWith(text[i-1], "#'")) {
+    if (i > 1 && startsWith(text[i], "#'") && !startsWith(text[i - 1], "#'")) {
       id <- id + 1
     }
     ids[i] <- id
@@ -35,9 +35,11 @@ parse_operation <- function(text) {
   comment_lines <- startsWith(text, "#'")
   comment <- text[comment_lines]
   code <- text[!comment_lines]
-  if (length(code) == 0 || all(code == "") || code[1] == "NULL") return(NULL)
+  if (length(code) == 0 || all(code == "") || code[1] == "NULL") {
+    return(NULL)
+  }
   func <- strsplit(code[1], " ")[[1]][1]
-  name <- substring(func, regexpr("_", func)+1)
+  name <- substring(func, regexpr("_", func) + 1)
   operation <- list(
     name = name,
     http = list(),
@@ -65,14 +67,18 @@ system_file <- function(..., package = "base") {
     pkg_path <- find.package(package)
     subfolder <- list(...)
     if (length(subfolder) > 0) {
-      if (subfolder[[1]] == "src")
+      if (subfolder[[1]] == "src") {
         subfolder[[1]] <- "R"
-      else
+      } else {
         subfolder <- c("inst", subfolder)
+      }
     }
     path <- do.call(file.path, c(pkg_path, subfolder))
-    if (file.exists(path)) return(path)
-    else return("")
+    if (file.exists(path)) {
+      return(path)
+    } else {
+      return("")
+    }
   }
 }
 
@@ -125,3 +131,6 @@ get_url <- function(url, tries = 3) {
     return(NULL)
   })
 }
+
+# helper function to make it easy to replace NULLs with default value
+`%||%` <- function(x, y) if (is.null(x)) y else x

make.paws/inst/templates/reexports_paws.common.R

@@ -0,0 +1,23 @@
+#' @importFrom paws.common paginate
+#' @export
+paws.common::paginate
+
+#' @importFrom paws.common paginate_lapply
+#' @export
+paws.common::paginate_lapply
+
+#' @importFrom paws.common paginate_sapply
+#' @export
+paws.common::paginate_sapply
+
+#' @importFrom paws.common config
+#' @export
+paws.common::config
+
+#' @importFrom paws.common credentials
+#' @export
+paws.common::credentials
+
+#' @importFrom paws.common creds
+#' @export
+paws.common::creds

paws.common/DESCRIPTION

@@ -38,6 +38,7 @@ Roxygen: list(markdown = TRUE, roclets = c("rd", "namespace", "collate"))
 RoxygenNote: 7.2.3
 Collate:
     'RcppExports.R'
+    'util.R'
     'struct.R'
     'handlers.R'
     'logging.R'
@@ -55,7 +56,6 @@ Collate:
     'service.R'
     'custom_dynamodb.R'
     'custom_rds.R'
-    'util.R'
     'xmlutil.R'
     'stream.R'
     'custom_s3.R'
@@ -70,6 +70,7 @@ Collate:
     'idempotency.R'
     'jsonutil.R'
     'onLoad.R'
+    'paginate.R'
     'populate.R'
     'populateutil.R'
     'tags.R'

paws.common/NAMESPACE

@@ -24,6 +24,9 @@ export(new_handlers)
 export(new_operation)
 export(new_request)
 export(new_service)
+export(paginate)
+export(paginate_lapply)
+export(paginate_sapply)
 export(paws_config_log)
 export(populate)
 export(send_request)

paws.common/NEWS.md

@@ -1,6 +1,7 @@
 # paws.common 0.5.9
 * add expiration parameter to creds
 * add signature_version to config
+* add the ability to paginate paws methods (#30)
 
 # paws.common 0.5.8
 * fix mismatch apparent method as.list.struct (#634)