feat(session-state): record mitm resource states

Author: blakebyrnes

commons/interfaces/IHttpResourceLoadDetails.ts

@@ -20,7 +20,7 @@ export default interface IHttpResourceLoadDetails {
   firstRedirectingUrl?: string; // track back to first redirection
   redirectedToUrl?: string;
   clientAlpn: string;
-
+  dnsResolvedIp?: string;
   url: URL;
   method: string;
   requestTime: Date;

core/lib/Session.ts

@@ -9,6 +9,7 @@ import RequestSession, {
   IRequestSessionHttpErrorEvent,
   IRequestSessionRequestEvent,
   IRequestSessionResponseEvent,
+  IResourceStateChangeEvent,
 } from '@secret-agent/mitm/handlers/RequestSession';
 import * as Os from 'os';
 import IPuppetContext, {
@@ -105,7 +106,8 @@ export default class Session {
     const requestSession = this.mitmRequestSession;
     requestSession.on('request', this.onMitmRequest.bind(this));
     requestSession.on('response', this.onMitmResponse.bind(this));
-    requestSession.on('httpError', this.onMitmError.bind(this));
+    requestSession.on('http-error', this.onMitmError.bind(this));
+    requestSession.on('resource-state', this.onResourceStates.bind(this));
   }
 
   public async createTab() {
@@ -174,6 +176,10 @@ export default class Session {
     this.sessionState.captureResourceError(tabId, event.request, event.error);
   }
 
+  private onResourceStates(event: IResourceStateChangeEvent) {
+    this.sessionState.captureResourceState(event.context.id, event.context.stateChanges);
+  }
+
   private async onNewTab(
     parentTab: Tab,
     page: IPuppetPage,

injected-scripts/scripts/jsPath.ts

@@ -252,10 +252,10 @@ class ObjectAtPath {
     if (!element) return false;
     const { top, right, left, bottom } = element.getBoundingClientRect();
     // adjust coordinates to get more accurate results
-    if (document.elementFromPoint(left + 1, top + 1) !== element) return true;
-    if (document.elementFromPoint(left + 1, bottom - 1) !== element) return true;
-    if (document.elementFromPoint(right - 1, top + 1) !== element) return true;
-    if (document.elementFromPoint(right - 1, bottom - 1) !== element) return true;
+    if (!element.contains(document.elementFromPoint(left + 1, top + 1))) return true;
+    if (!element.contains(document.elementFromPoint(left + 1, bottom - 1))) return true;
+    if (!element.contains(document.elementFromPoint(right - 1, top + 1))) return true;
+    if (!element.contains(document.elementFromPoint(right - 1, bottom - 1))) return true;
     return false;
   }
 

mitm/handlers/BaseHttpHandler.ts

@@ -5,6 +5,7 @@ import HeadersHandler from './HeadersHandler';
 import CookieHandler from './CookieHandler';
 import MitmRequestContext from '../lib/MitmRequestContext';
 import HttpResponseCache from '../lib/HttpResponseCache';
+import ResourceState from '../interfaces/ResourceState';
 
 const { log } = Log(module);
 
@@ -39,9 +40,10 @@ export default abstract class BaseHttpHandler {
         method: context.method,
         ws: context.isUpgrade,
       });
-      if (session.isClosing) return;
+      if (session.isClosing) return context.setState(ResourceState.SessionClosed);
 
       if (BlockHandler.shouldBlockRequest(context)) {
+        context.setState(ResourceState.Blocked);
         log.info(`Http.RequestBlocked`, {
           sessionId: session.sessionId,
           url: context.url.href,
@@ -58,7 +60,7 @@ export default abstract class BaseHttpHandler {
       context.cacheHandler.onRequest();
 
       // do one more check on the session before doing a connect
-      if (session.isClosing) return;
+      if (session.isClosing) return context.setState(ResourceState.SessionClosed);
 
       this.context.proxyToServerRequest = await session.requestAgent.request(context);
       this.context.proxyToServerRequest.on(

mitm/handlers/BlockHandler.ts

@@ -1,11 +1,14 @@
-import IMitmRequestContext from '../interfaces/IMitmRequestContext';
+import IMitmRequestContext from "../interfaces/IMitmRequestContext";
+import ResourceState from "../interfaces/ResourceState";
 
 export default class BlockHandler {
   public static shouldBlockRequest(ctx: IMitmRequestContext) {
+    ctx.setState(ResourceState.BlockHandler);
     const session = ctx.requestSession;
     if (!session) return false;
     if (session.isClosing) return true;
 
+
     const shouldBlock =
       (ctx.resourceType && session.blockedResources?.types?.includes(ctx.resourceType)) ||
       session.shouldBlockRequest(ctx.url.href);

mitm/handlers/CacheHandler.ts

@@ -1,5 +1,6 @@
 import HttpResponseCache from '../lib/HttpResponseCache';
 import IMitmRequestContext from '../interfaces/IMitmRequestContext';
+import ResourceState from "../interfaces/ResourceState";
 
 export default class CacheHandler {
   public didProposeCachedResource = false;
@@ -19,6 +20,7 @@ export default class CacheHandler {
 
   public onRequest() {
     const ctx = this.ctx;
+    ctx.setState(ResourceState.CheckCacheOnRequest);
     // only cache get (don't do preflight, post, etc)
     if (ctx.method === 'GET' && !ctx.requestLowerHeaders['if-none-match']) {
       const cache = this.responseCache?.get(ctx.url.href);
@@ -32,6 +34,7 @@ export default class CacheHandler {
   }
 
   public onHttp2PushStream() {
+    this.ctx.setState(ResourceState.CheckCacheOnRequest);
     if (this.ctx.method === 'GET') {
       const cached = this.responseCache?.get(this.ctx.url.href);
       if (cached) {
@@ -60,6 +63,7 @@ export default class CacheHandler {
 
   public onResponseEnd() {
     const ctx = this.ctx;
+    ctx.setState(ResourceState.CheckCacheOnResponseEnd);
     if (
       ctx.method === 'GET' &&
       !this.didProposeCachedResource &&

mitm/handlers/CookieHandler.ts

@@ -1,11 +1,13 @@
 import Log from '@secret-agent/commons/Logger';
 import IResourceHeaders from '@secret-agent/core-interfaces/IResourceHeaders';
 import IMitmRequestContext from '../interfaces/IMitmRequestContext';
+import ResourceState from "../interfaces/ResourceState";
 
 const { log } = Log(module);
 
 export default class CookieHandler {
   public static async setProxyToServerCookies(ctx: IMitmRequestContext) {
+    ctx.setState(ResourceState.SetCookieHeader);
     const session = ctx.requestSession;
     if (!session || !session.delegate?.getCookieHeader) return;
     // never send cookies to preflight requests
@@ -16,6 +18,7 @@ export default class CookieHandler {
   }
 
   public static async readServerResponseCookies(ctx: IMitmRequestContext) {
+    ctx.setState(ResourceState.ReadResponseCookies);
     const session = ctx.requestSession;
     if (!session || !session.delegate?.setCookie) return;
 

mitm/handlers/HeadersHandler.ts

@@ -6,10 +6,12 @@ import * as http from 'http';
 import http2 from 'http2';
 import { parseRawHeaders } from '../lib/Utils';
 import IMitmRequestContext from '../interfaces/IMitmRequestContext';
+import ResourceState from '../interfaces/ResourceState';
 
 const { log } = Log(module);
 export default class HeadersHandler {
   public static async waitForBrowserRequest(ctx: IMitmRequestContext) {
+    ctx.setState(ResourceState.WaitForBrowserRequest);
     const session = ctx.requestSession;
 
     const { method, requestHeaders } = ctx;
@@ -46,6 +48,7 @@ export default class HeadersHandler {
   }
 
   public static modifyHeaders(ctx: IMitmRequestContext) {
+    ctx.setState(ResourceState.ModifyHeaders);
     const session = ctx.requestSession;
     if (session.delegate?.modifyHeadersBeforeSend) {
       const updatedHeaders = session.delegate.modifyHeadersBeforeSend({

mitm/handlers/HttpRequestHandler.ts

@@ -9,6 +9,7 @@ import MitmRequestContext from '../lib/MitmRequestContext';
 import { parseRawHeaders } from '../lib/Utils';
 import BaseHttpHandler from './BaseHttpHandler';
 import HttpResponseCache from '../lib/HttpResponseCache';
+import ResourceState from '../interfaces/ResourceState';
 
 const { log } = Log(module);
 const redirectCodes = new Set([300, 301, 302, 303, 305, 307, 308]);
@@ -23,23 +24,25 @@ export default class HttpRequestHandler extends BaseHttpHandler {
     >,
   ) {
     super(request, false, HttpRequestHandler.responseCache);
+    this.context.setState(ResourceState.ClientToProxyRequest);
+
+    // register error listeners first
+    const { clientToProxyRequest, proxyToClientResponse } = request;
+    clientToProxyRequest.on('error', this.onError.bind(this, 'ClientToProxy.RequestError'));
+    if (clientToProxyRequest instanceof Http2ServerRequest) {
+      const http2Session = clientToProxyRequest.stream.session;
+      if (!http2Session.listenerCount('error')) {
+        http2Session.on('error', this.onError.bind(this, 'ClientToProxy.Http2SessionError'));
+      }
+    }
+    proxyToClientResponse.on('error', this.onError.bind(this, 'ProxyToClient.ResponseError'));
   }
 
   public async onRequest() {
-    const { clientToProxyRequest, proxyToClientResponse } = this.context;
+    const { clientToProxyRequest } = this.context;
 
     try {
       clientToProxyRequest.pause();
-      clientToProxyRequest.on('error', this.onError.bind(this, 'ClientToProxy.RequestError'));
-      if (clientToProxyRequest instanceof Http2ServerRequest) {
-        if (clientToProxyRequest.stream.session.listenerCount('error') === 0) {
-          clientToProxyRequest.stream.session.on(
-            'error',
-            this.onError.bind(this, 'ClientToProxy.SessionError'),
-          );
-        }
-      }
-      proxyToClientResponse.on('error', this.onError.bind(this, 'ProxyToClient.ResponseError'));
 
       const proxyToServerRequest = await this.createProxyToServerRequest();
       if (!proxyToServerRequest) return;
@@ -61,6 +64,8 @@ export default class HttpRequestHandler extends BaseHttpHandler {
     rawHeaders?: string[],
   ) {
     const context = this.context;
+    context.setState(ResourceState.ServerToProxyOnResponse);
+
     if (response instanceof http.IncomingMessage) {
       MitmRequestContext.readHttp1Response(context, response);
     } else {
@@ -91,6 +96,8 @@ export default class HttpRequestHandler extends BaseHttpHandler {
 
     await this.writeResponse();
 
+    context.setState(ResourceState.End);
+
     process.nextTick(agent => agent.freeSocket(context), context.requestSession.requestAgent);
   }
 
@@ -99,7 +106,8 @@ export default class HttpRequestHandler extends BaseHttpHandler {
     const { method, requestSession, proxyToClientResponse } = this.context;
     const sessionId = requestSession.sessionId;
 
-    requestSession.emit('httpError', {
+    this.context.setState(ResourceState.Error);
+    requestSession.emit('http-error', {
       request: MitmRequestContext.toEmittedResource(this.context),
       error,
     });
@@ -120,6 +128,7 @@ export default class HttpRequestHandler extends BaseHttpHandler {
   }
 
   private async writeRequest() {
+    this.context.setState(ResourceState.WriteProxyToServerRequestBody);
     const { proxyToServerRequest, clientToProxyRequest } = this.context;
 
     const onWriteError = error => {
@@ -142,6 +151,7 @@ export default class HttpRequestHandler extends BaseHttpHandler {
       log.warn('Error.NoProxyToClientResponse', {
         sessionId: context.requestSession.sessionId,
       });
+      context.setState(ResourceState.PrematurelyClosed);
       return;
     }
 
@@ -157,6 +167,7 @@ export default class HttpRequestHandler extends BaseHttpHandler {
       context.responseTrailers = headers;
     });
 
+    context.setState(ResourceState.WriteProxyToClientResponseBody);
     for await (const chunk of serverToProxyResponse) {
       const data = context.cacheHandler.onResponseData(chunk as Buffer);
       if (data && !(proxyToClientResponse as Http2ServerResponse).stream?.destroyed) {

mitm/handlers/HttpUpgradeHandler.ts

@@ -5,6 +5,7 @@ import MitmRequestContext from '../lib/MitmRequestContext';
 import CookieHandler from './CookieHandler';
 import BaseHttpHandler from './BaseHttpHandler';
 import IMitmRequestContext from '../interfaces/IMitmRequestContext';
+import ResourceState from '../interfaces/ResourceState';
 
 const { log } = Log(module);
 
@@ -15,12 +16,12 @@ export default class HttpUpgradeHandler extends BaseHttpHandler {
     readonly clientHead: Buffer,
   ) {
     super(request, true, null);
+    this.context.setState(ResourceState.ClientToProxyRequest);
+    this.clientSocket.on('error', this.onError.bind(this, 'ClientToProxy.UpgradeSocketError'));
   }
 
   public async onUpgrade() {
     try {
-      this.clientSocket.on('error', this.onError.bind(this, 'ClientToProxy.UpgradeSocketError'));
-
       const proxyToServerRequest = await this.createProxyToServerRequest();
       if (!proxyToServerRequest) return;
 
@@ -37,7 +38,9 @@ export default class HttpUpgradeHandler extends BaseHttpHandler {
     const url = context.url.href;
     const session = context.requestSession;
     const sessionId = session.sessionId;
-    session.emit('httpError', { request: MitmRequestContext.toEmittedResource(context), error });
+    context.setState(ResourceState.Error);
+
+    session.emit('http-error', { request: MitmRequestContext.toEmittedResource(context), error });
 
     if (!(error as any)?.isLogged) {
       log.error(`MitmWebSocket.${errorType}`, {
@@ -54,6 +57,7 @@ export default class HttpUpgradeHandler extends BaseHttpHandler {
     serverSocket: net.Socket,
     serverHead: Buffer,
   ) {
+    this.context.setState(ResourceState.ServerToProxyOnResponse);
     serverSocket.pause();
     MitmRequestContext.readHttp1Response(this.context, serverResponse);
     this.context.serverToProxyResponse = serverResponse;
@@ -65,6 +69,7 @@ export default class HttpUpgradeHandler extends BaseHttpHandler {
     clientSocket.on('end', () => proxyToServerMitmSocket.close());
     serverSocket.on('end', () => proxyToServerMitmSocket.close());
     proxyToServerMitmSocket.on('close', () => {
+      this.context.setState(ResourceState.End);
       // don't try to write again
       clientSocket.destroy();
       serverSocket.destroy();
@@ -77,11 +82,13 @@ export default class HttpUpgradeHandler extends BaseHttpHandler {
     }
     await CookieHandler.readServerResponseCookies(this.context);
 
+    this.context.setState(ResourceState.WriteProxyToClientResponseBody);
     clientSocket.write(`${responseMessage}\r\n`, error => {
       if (error) this.onError('ProxyToClient.UpgradeWriteError', error);
     });
 
     if (!serverSocket.readable || !serverSocket.writable) {
+      this.context.setState(ResourceState.PrematurelyClosed);
       return serverSocket.destroy();
     }
     serverSocket.on('error', this.onError.bind(this, 'ServerToProxy.UpgradeSocketError'));
@@ -104,6 +111,7 @@ export default class HttpUpgradeHandler extends BaseHttpHandler {
 
     const formattedResponse = MitmRequestContext.toEmittedResource(this.context);
     this.context.requestSession.emit('response', formattedResponse);
+    // don't log close since this stays open...
   }
 
   public static async onUpgrade(