feat(client): add http cache and load failures

Author: blakebyrnes

client/lib/ResourceResponse.ts

@@ -2,6 +2,7 @@ import initializeConstantsAndProperties from 'awaited-dom/base/initializeConstan
 import StateMachine from 'awaited-dom/base/StateMachine';
 import IResourceHeaders from '@secret-agent/core-interfaces/IResourceHeaders';
 import IResourceResponse from '@secret-agent/core-interfaces/IResourceResponse';
+import IHttpResourceLoadDetails from '@secret-agent/core-interfaces/IHttpResourceLoadDetails';
 import CoreSession from './CoreTab';
 
 const { getState, setState } = StateMachine<ResourceResponse, IState>();
@@ -18,6 +19,8 @@ const propertyKeys: (keyof ResourceResponse)[] = [
   'remoteAddress',
   'statusCode',
   'statusMessage',
+  'browserLoadFailure',
+  'browserServedFromCache',
   'data',
 ];
 
@@ -26,6 +29,16 @@ export default class ResourceResponse {
     initializeConstantsAndProperties(this, [], propertyKeys);
   }
 
+  public get browserServedFromCache(): Promise<
+    null | IHttpResourceLoadDetails['browserServedFromCache']
+  > {
+    return getResponseProperty(this, 'browserServedFromCache');
+  }
+
+  public get browserLoadFailure(): Promise<string> {
+    return getResponseProperty(this, 'browserLoadFailure');
+  }
+
   public get headers(): Promise<IResourceHeaders> {
     return getResponseProperty(this, 'headers');
   }

core-interfaces/IHttpResourceLoadDetails.ts

@@ -39,7 +39,7 @@ export default interface IHttpResourceLoadDetails {
   responseTrailers?: IResourceHeaders;
   resourceType?: ResourceType;
   browserRequestId?: string;
-  browserServedFromCache?: 'service-worker' | 'disk' | 'prefetch' | 'unspecified';
+  browserServedFromCache?: 'service-worker' | 'disk' | 'prefetch' | 'memory';
   browserLoadFailure?: string;
   browserBlockedReason?: string;
   browserCanceled?: boolean;

core-interfaces/IResourceResponse.ts

@@ -1,10 +1,13 @@
 import IResourceHeaders from './IResourceHeaders';
+import IHttpResourceLoadDetails from './IHttpResourceLoadDetails';
 
 export default interface IResourceResponse {
   url: string;
   timestamp: string;
   headers: IResourceHeaders;
   trailers?: IResourceHeaders;
+  browserServedFromCache?: IHttpResourceLoadDetails['browserServedFromCache'];
+  browserLoadFailure?: string;
   remoteAddress: string;
   statusCode: number;
   statusMessage?: string;

core/lib/SessionState.ts

@@ -220,6 +220,9 @@ export default class SessionState {
     const resource = this.resourceEventToMeta(tabId, resourceEvent);
     this.db.resources.insert(tabId, resource, null, resourceEvent, error);
 
+    if (!this.resources.some(x => x.id === resourceEvent.id)) {
+      this.resources.push(resource);
+    }
     const navigations = this.navigationsByTabId[tabId];
     if (!navigations) return;
 
@@ -301,7 +304,7 @@ export default class SessionState {
       },
     } as IResourceMeta;
 
-    if (response?.statusCode) {
+    if (response?.statusCode || response?.browserServedFromCache || response?.browserLoadFailure) {
       resource.response = response;
       if (response.url) resource.url = response.url;
       else resource.response.url = request.url;

core/models/ResourcesTable.ts

@@ -63,8 +63,6 @@ export default class ResourcesTable extends SqliteTable<IResourcesRecord> {
       didBlockResource: boolean;
       browserRequestId?: string;
       isHttp2Push: boolean;
-      browserServedFromCache?: 'service-worker' | 'disk' | 'prefetch' | 'unspecified';
-      browserLoadFailure?: string;
       browserBlockedReason?: string;
       browserCanceled?: boolean;
     },
@@ -81,6 +79,12 @@ export default class ResourcesTable extends SqliteTable<IResourcesRecord> {
           ...error,
         });
     }
+    let contentEncoding: string;
+    if (meta.response && meta.response.headers) {
+      contentEncoding = <string>(
+        (meta.response.headers['Content-Encoding'] ?? meta.response.headers['content-encoding'])
+      );
+    }
     return this.queuePendingInsert([
       meta.id,
       extras.browserRequestId,
@@ -101,7 +105,7 @@ export default class ResourcesTable extends SqliteTable<IResourcesRecord> {
       meta.response ? JSON.stringify(meta.response.headers ?? {}) : undefined,
       meta.response ? JSON.stringify(meta.response.trailers ?? {}) : undefined,
       meta.response?.timestamp,
-      meta.response?.headers['Content-Encoding'] ?? meta.response?.headers['content-encoding'],
+      contentEncoding,
       meta.response ? body : undefined,
       extras.socketId,
       extras.clientAlpn,
@@ -111,8 +115,8 @@ export default class ResourcesTable extends SqliteTable<IResourcesRecord> {
       extras.didBlockResource ? 1 : 0,
       JSON.stringify(extras.originalHeaders ?? {}),
       errorString,
-      extras.browserServedFromCache,
-      extras.browserLoadFailure,
+      meta.response?.browserServedFromCache,
+      meta.response?.browserLoadFailure,
       extras.browserBlockedReason,
       extras.browserCanceled ? 1 : 0,
     ]);

mitm/handlers/RequestSession.ts

@@ -412,7 +412,7 @@ export interface IRequestSessionResponseEvent extends IRequestSessionRequestEven
   body: Buffer;
   redirectedToUrl?: string;
   executionMillis: number;
-  browserServedFromCache?: 'service-worker' | 'disk' | 'prefetch' | 'unspecified';
+  browserServedFromCache?: IHttpResourceLoadDetails['browserServedFromCache'];
   browserLoadFailure?: string;
   browserBlockedReason?: string;
   browserCanceled?: boolean;

mitm/lib/MitmRequestContext.ts

@@ -159,6 +159,8 @@ export default class MitmRequestContext {
       headers: ctx.responseHeaders,
       trailers: ctx.responseTrailers,
       timestamp: ctx.responseTime?.toISOString(),
+      browserServedFromCache: ctx.browserServedFromCache,
+      browserLoadFailure: ctx.browserLoadFailure,
       remoteAddress: ctx.remoteAddress,
     } as IResourceResponse;
 
@@ -181,8 +183,6 @@ export default class MitmRequestContext {
       executionMillis: (ctx.responseTime ?? new Date()).getTime() - ctx.requestTime.getTime(),
       isHttp2Push: ctx.isHttp2Push,
       browserBlockedReason: ctx.browserBlockedReason,
-      browserServedFromCache: ctx.browserServedFromCache,
-      browserLoadFailure: ctx.browserLoadFailure,
       browserCanceled: ctx.browserCanceled,
     };
   }

puppet-chrome/lib/NetworkManager.ts

@@ -273,7 +273,7 @@ export class NetworkManager extends TypedEventEmitter<IPuppetNetworkEvents> {
     const { requestId } = event;
     const resource = this.requestsById.get(requestId);
     if (resource) {
-      resource.browserServedFromCache = 'unspecified';
+      resource.browserServedFromCache = 'memory';
       this.emitLoaded(requestId);
     }
   }

website/docs/Advanced/ResourceResponse.md

@@ -6,6 +6,18 @@ Instances of this class are associated with a resource returned from `window.wai
 
 ## Properties
 
+### browserServedFromCache
+
+Will have a value if http response was served from a browser cache.
+
+#### **Returns** `Promise<null | 'service-worker' | 'disk' | 'prefetch' | 'memory'>`
+
+### browserLoadFailure
+
+Will have a value if an http error occurred loading this request.
+
+#### **Returns** `Promise<string | null>`
+
 ### headers
 
 Retrieve the actual headers returned to the client (order and casing is preserved)
@@ -34,6 +46,8 @@ IPv4/6 and port of remote socket. `192.168.172.2:5001`
 
 Http response status code.
 
+NOTE: this value might be null if no HTTP response occurred, or an error occurred.
+
 #### **Returns** `Promise<number>`
 
 ### statusText