Record requests in responses and errors

NAMESPACE

@@ -114,9 +114,10 @@ export(resp_url_queries)
 export(resp_url_query)
 export(response)
 export(response_json)
-export(resps_combine)
-export(resps_errors)
-export(resps_responses)
+export(resps_data)
+export(resps_failures)
+export(resps_requests)
+export(resps_successes)
 export(secret_decrypt)
 export(secret_decrypt_file)
 export(secret_encrypt)

NEWS.md

@@ -1,5 +1,13 @@
 # httr2 (development version)
 
+* A new family of functions `resps_successes()`, `resps_failures()`, 
+  `resps_requests()` and `resps_data()` make it easier to work with lists of
+  responses as returned by `req_perform_parallel()` and 
+  `req_perform_iteratively()` (#357).
+
+* The request is now stored in the response object (and errors that httr2
+  throws), making it easier to debug when things go wrong (#357).
+
 * New `oauth_token_cached()` that allows you to get an OAuth token while still
   taking advantage of httr2's caching and auto-renewal features. For expert
   use only (#328).

R/iterate-responses.R

@@ -1,31 +1,63 @@
 #' Tools for working with lists of responses
 #'
-#' * `resps_combine()` combines the data from each response into a single
-#'   vector.
-#' * `resps_response()` returns all successful responses.
-#' * `resps_error()` returns all errors.
+#' @description
+#' These function provide a basic toolkit for operating with lists of
+#' responses as returned [req_perform_parallel()] and
+#' [req_perform_iteratively()].
+#'
+#' * `resps_successes()` returns a list successful responses.
+#' * `resps_failures()` returns a list failed responses (i.e. errors).
+#' * `resps_requests()` returns the list of requests that corresponds to
+#'   each request.
+#' * `resps_data()` returns all the data in a single vector or data frame.
+#'   It requires the vctrs package to be installed.
 #'
 #' @export
 #' @param resps A list of responses (possibly including errors).
 #' @param resp_data A function that takes a response (`resp`) and
 #'   returns the data foind inside that response as a vector or data frame.
-resps_combine <- function(resps, resp_data) {
-  check_installed("vctrs")
+#' @examples
+#' reqs <- list(
+#'   request(example_url()) |> req_url_path("/ip"),
+#'   request(example_url()) |> req_url_path("/user-agent"),
+#'   request(example_url()) |> req_template("/status/:status", status = 404),
+#'   request("INVALID")
+#' )
+#' resps <- req_perform_parallel(reqs)
+#'
+#' # find successful responses
+#' resps |> resps_successes()
+#'
+#' # collect all their data
+#' resps |> resps_successes() |> resps_data(\(resp) resp_body_json(resp))
+#'
+#' # find requests corresponding to failure responses
+#' resps |> resps_failures() |> resps_requests()
+resps_successes <- function(resps) {
+  resps[resps_ok(resps)]
+}
 
-  check_function2(resp_data, "resp")
-  vctrs::list_unchop(lapply(resps, resp_data))
+#' @export
+#' @rdname resps_successes
+resps_failures <- function(resps) {
+  resps[!resps_ok(resps)]
 }
-resps_is_resp <- function(resps) {
+
+resps_ok <- function(resps) {
   vapply(resps, is_response, logical(1))
 }
 
 #' @export
-#' @rdname resps_combine
-resps_responses <- function(resps) {
-  resps[resps_is_resp(resps)]
+#' @rdname resps_successes
+resps_requests <- function(resps) {
+  lapply(resps, function(x) x$request)
 }
+
 #' @export
-#' @rdname resps_combine
-resps_errors <- function(resps) {
-  resps[!resps_is_resp(resps)]
+#' @rdname resps_successes
+resps_data <- function(resps, resp_data) {
+  check_installed("vctrs")
+  check_function2(resp_data, "resp")
+
+  vctrs::list_unchop(lapply(resps, resp_data))
 }

R/iterate.R

@@ -7,7 +7,7 @@
 #' using a callback function, `next_req`, to define the next request based on
 #' the current request and response. You will probably want to it pair with an
 #' [iteration helper][iterate_with_offset] and use a
-#' [multi-response handler][resps_combine] to process the result.
+#' [multi-response handler][resps_successes] to process the result.
 #'
 #' @inheritParams req_perform
 #' @param next_req A function that takes the previous response (`resp`) and
@@ -32,7 +32,7 @@
 #'
 #' resps <- req_perform_iteratively(req, iterate_with_offset("page_index"))
 #'
-#' resps |> resps_combine(function(resp) {
+#' resps |> resps_data(function(resp) {
 #'   data <- resp_body_json(resp)$data
 #'   data.frame(
 #'     Sepal.Length = sapply(data, `[[`, "Sepal.Length"),

R/multi-req.R

@@ -53,9 +53,12 @@
 #' )
 #' # req_perform_parallel() will always succeed
 #' resps <- req_perform_parallel(reqs)
-#' # you'll need to inspect the results to figure out which requests fails
-#' fail <- vapply(resps, inherits, "error", FUN.VALUE = logical(1))
-#' resps[fail]
+#'
+#' # Inspect the successful responses
+#' resps |> resps_successes()
+#'
+#' # And the failed responses
+#' resps |> resps_failures() |> resps_requests()
 req_perform_parallel <- function(reqs, paths = NULL, pool = NULL, cancel_on_error = FALSE) {
   if (!is.null(paths)) {
     if (length(reqs) != length(paths)) {
@@ -161,7 +164,8 @@ Performance <- R6Class("Performance", public = list(
       url = res$url,
       status_code = res$status_code,
       headers = as_headers(res$headers),
-      body = body
+      body = body,
+      request = self$req
     )
     resp <- cache_post_fetch(self$reqs, resp, path = self$paths)
 
@@ -172,7 +176,7 @@ Performance <- R6Class("Performance", public = list(
   },
 
   fail = function(msg) {
-    self$resp <- error_cnd("httr2_failure", message = msg)
+    self$resp <- error_cnd("httr2_failure", message = msg, request = self$req)
     signal("", class = "httr2:::failed")
   },
 

R/req-perform.R

@@ -112,6 +112,7 @@ req_perform <- function(
           message = "Failed to perform HTTP request.",
           class = "httr2_failure",
           parent = err,
+          request = req,
           call = error_call,
           trace = trace_back()
         )
@@ -146,7 +147,7 @@ handle_resp <- function(req, resp, error_call = caller_env()) {
     cnd_signal(resp)
   } else if (error_is_error(req, resp)) {
     body <- error_body(req, resp, error_call)
-    resp_abort(resp, body, call = error_call)
+    resp_abort(resp, req, body, call = error_call)
   } else {
     resp
   }
@@ -173,7 +174,8 @@ req_perform1 <- function(req, path = NULL, handle = NULL) {
     url = res$url,
     status_code = res$status_code,
     headers = as_headers(res$headers),
-    body = body
+    body = body,
+    request = req
   )
   the$last_response <- resp
   resp

R/req.R

@@ -56,15 +56,24 @@ is_request <- function(x) {
   inherits(x, "httr2_request")
 }
 
-check_request <- function(req, arg = caller_arg(req), call = caller_env()) {
-  if (!missing(req) && is_request(req)) {
-    return(invisible(NULL))
+check_request <- function(req,
+                          arg = caller_arg(req),
+                          call = caller_env(),
+                          allow_null = FALSE) {
+  if (!missing(req)) {
+    if (is_request(req)) {
+      return(invisible(NULL))
+    }
+
+    if (allow_null && is.null(req)) {
+      return(invisible(NULL))
+    }
   }
 
   stop_input_type(
     req,
     "an HTTP request object",
-    allow_null = FALSE,
+    allow_null = allow_null,
     arg = arg,
     call = call
   )

R/resp-body.R

@@ -78,13 +78,14 @@ resp_body_string <- function(resp, encoding = NULL) {
 #' @rdname resp_body_raw
 #' @export
 resp_body_json <- function(resp, check_type = TRUE, simplifyVector = FALSE, ...) {
+  check_response(resp)
+  check_installed("jsonlite")
+
   key <- body_cache_key("json", simplifyVector = simplifyVector, ...)
   if (env_has(resp$cache, key)) {
     return(resp$cache[[key]])
   }
 
-  check_response(resp)
-  check_installed("jsonlite")
   resp_check_content_type(
     resp,
     valid_types = "application/json",
@@ -114,14 +115,14 @@ resp_body_html <- function(resp, check_type = TRUE, ...) {
 #' @rdname resp_body_raw
 #' @export
 resp_body_xml <- function(resp, check_type = TRUE, ...) {
+  check_response(resp)
+  check_installed("xml2")
+
   key <- body_cache_key("xml", ...)
   if (env_has(resp$cache, key)) {
     return(resp$cache[[key]])
   }
 
-
-  check_response(resp)
-  check_installed("xml2")
   resp_check_content_type(
     resp,
     valid_types = c("application/xml", "text/xml"),

R/resp-status.R

@@ -66,11 +66,11 @@ resp_check_status <- function(resp, info = NULL, error_call = caller_env()) {
   if (!resp_is_error(resp)) {
     invisible(resp)
   } else {
-    resp_abort(resp, info, call = error_call)
+    resp_abort(resp, resp$request, info, call = error_call)
   }
 }
 
-resp_abort <- function(resp, info = NULL, call = caller_env()) {
+resp_abort <- function(resp, req, info = NULL, call = caller_env()) {
   status <- resp_status(resp)
   desc <- resp_status_desc(resp)
   message <- glue("HTTP {status} {desc}.")
@@ -80,6 +80,7 @@ resp_abort <- function(resp, info = NULL, call = caller_env()) {
     status = status,
     resp = resp,
     class = c(glue("httr2_http_{status}"), "httr2_http"),
+    request = req,
     call = call
   )
 }

R/resp.R

@@ -73,10 +73,12 @@ new_response <- function(method,
                          status_code,
                          headers,
                          body,
+                         request = NULL,
                          error_call = caller_env()) {
   check_string(method, call = error_call)
   check_string(url, call = error_call)
   check_number_whole(status_code, call = error_call)
+  check_request(request, allow_null = TRUE)
 
   headers <- as_headers(headers, error_call = error_call)
   # ensure we always have a date field
@@ -91,6 +93,7 @@ new_response <- function(method,
       status_code = status_code,
       headers = headers,
       body = body,
+      request = request,
       cache = new_environment()
     ),
     class = "httr2_response"

_pkgdown.yml

@@ -41,9 +41,10 @@ reference:
   - starts_with("req_auth")
   - starts_with("req_oauth")
 
-- title: Perform one or more requests
+- title: Perform a request
   contents:
-  - starts_with("req_perform")
+  - req_perform
+  - req_perform_stream
 
 - subtitle: Control the process
   desc: >
@@ -55,9 +56,10 @@ reference:
   - req_throttle
   - req_retry
 
-- title: Iteration
+- title: Perform multiple requests
   contents:
   - req_perform_iteratively
+  - req_perform_parallel
   - starts_with("iterate_")
   - starts_with("resps_")