Fix Pending Request causing timeout (#636)

Don't wait for requests that have been not intercepted (`intercepting` is not set) and are not loaded asynchronously (`asyncLoading` is not set) in awaitPageResources() when page is done. Occasionally, it seems some pending requests that only get added via `Network.requestWillBeSent` but never receive a finished/failed message may persist in the pending request list, and will now be discarded.
(Large requests that have a streaming response body will have either `intercepting or `asyncLoading` set and will not be affected)

src/util/recorder.ts

@@ -328,18 +328,19 @@ export class Recorder {
   }
 
   handleRequestWillBeSent(params: Protocol.Network.RequestWillBeSentEvent) {
-    // only handling redirect here, committing last response in redirect chain
-    // request data stored from requestPaused
     const { redirectResponse, requestId, request, type } = params;
 
     const { headers, method, url } = request;
 
     logNetwork("Network.requestWillBeSent", {
       requestId,
+      url,
       redirectResponse,
       ...this.logDetails,
     });
 
+    // handling redirect here, committing last response in redirect chain
+    // request data stored from requestPaused
     if (redirectResponse) {
       this.handleRedirectResponse(params);
     } else {
@@ -808,8 +809,17 @@ export class Recorder {
       if (reqresp.payload) {
         this.removeReqResp(requestId);
         await this.serializeToWARC(reqresp);
-        // no url, likely invalid
-      } else if (!reqresp.url) {
+        // if no url, and not fetch intercept or async loading,
+        // drop this request, as it was not being loaded
+      } else if (
+        !reqresp.url ||
+        (!reqresp.intercepting && !reqresp.asyncLoading)
+      ) {
+        logger.debug(
+          "Removing pending request that was never fetched",
+          { requestId, url: reqresp.url, ...this.logDetails },
+          "recorder",
+        );
         this.removeReqResp(requestId);
       }
     }
@@ -825,13 +835,17 @@ export class Recorder {
           url: string;
           expectedSize?: number;
           readSize?: number;
+          resourceType?: string;
         } = { requestId, url };
         if (reqresp.expectedSize) {
           entry.expectedSize = reqresp.expectedSize;
         }
         if (reqresp.readSize) {
           entry.readSize = reqresp.readSize;
         }
+        if (reqresp.resourceType) {
+          entry.resourceType = reqresp.resourceType;
+        }
         pending.push(entry);
       }
 

src/util/reqresp.ts

@@ -53,8 +53,6 @@ export class RequestResponseInfo {
 
   frameId?: string;
 
-  fetch = false;
-
   resourceType?: string;
 
   // TODO: Fix this the next time the file is edited.
@@ -65,6 +63,9 @@ export class RequestResponseInfo {
   readSize: number = 0;
   expectedSize: number = 0;
 
+  // set to true to indicate request intercepted via Fetch.requestPaused
+  intercepting = false;
+
   // set to true to indicate async loading in progress
   asyncLoading: boolean = false;
 
@@ -83,7 +84,7 @@ export class RequestResponseInfo {
 
     this.responseHeadersList = params.responseHeaders;
 
-    this.fetch = true;
+    this.intercepting = true;
 
     this.frameId = params.frameId;
   }