feat(core): add screenshot capability

Author: blakebyrnes

client/lib/Agent.ts

@@ -19,6 +19,7 @@ import { IElementIsolate } from 'awaited-dom/base/interfaces/isolate';
 import CSSStyleDeclaration from 'awaited-dom/impl/official-klasses/CSSStyleDeclaration';
 import { CanceledPromiseError } from '@secret-agent/commons/interfaces/IPendingWaitEvent';
 import IAgentMeta from '@secret-agent/core-interfaces/IAgentMeta';
+import IScreenshotOptions from '@secret-agent/core-interfaces/IScreenshotOptions';
 import WebsocketResource from './WebsocketResource';
 import IWaitForResourceFilter from '../interfaces/IWaitForResourceFilter';
 import Resource from './Resource';
@@ -256,6 +257,10 @@ export default class Agent extends AwaitedEventTarget<{ close: void }> {
     return this.activeTab.isElementVisible(element);
   }
 
+  public takeScreenshot(options?: IScreenshotOptions): Promise<Buffer> {
+    return this.activeTab.takeScreenshot(options);
+  }
+
   public waitForPaintingStable(options?: IWaitForOptions): Promise<void> {
     return this.activeTab.waitForPaintingStable(options);
   }

client/lib/CoreTab.ts

@@ -13,6 +13,7 @@ import IAttachedState from 'awaited-dom/base/IAttachedState';
 import ISetCookieOptions from '@secret-agent/core-interfaces/ISetCookieOptions';
 import IConfigureSessionOptions from '@secret-agent/core-interfaces/IConfigureSessionOptions';
 import IWaitForOptions from '@secret-agent/core-interfaces/IWaitForOptions';
+import IScreenshotOptions from '@secret-agent/core-interfaces/IScreenshotOptions';
 import CoreCommandQueue from './CoreCommandQueue';
 import CoreEventHeap from './CoreEventHeap';
 import IWaitForResourceFilter from '../interfaces/IWaitForResourceFilter';
@@ -117,6 +118,10 @@ export default class CoreTab implements IJsPathEventTarget {
     return await this.commandQueue.run('removeCookie', name);
   }
 
+  public async takeScreenshot(options: IScreenshotOptions): Promise<Buffer> {
+    return await this.commandQueue.run('takeScreenshot', options);
+  }
+
   public async isElementVisible(jsPath: IJsPath): Promise<boolean> {
     return await this.commandQueue.run('isElementVisible', jsPath);
   }

client/lib/Tab.ts

@@ -19,6 +19,7 @@ import IWaitForElementOptions from '@secret-agent/core-interfaces/IWaitForElemen
 import Response from 'awaited-dom/impl/official-klasses/Response';
 import IWaitForOptions from '@secret-agent/core-interfaces/IWaitForOptions';
 import { IElementIsolate } from 'awaited-dom/base/interfaces/isolate';
+import IScreenshotOptions from '@secret-agent/core-interfaces/IScreenshotOptions';
 import CoreTab from './CoreTab';
 import Resource, { createResource } from './Resource';
 import IWaitForResourceFilter from '../interfaces/IWaitForResourceFilter';
@@ -161,6 +162,11 @@ export default class Tab extends AwaitedEventTarget<IEventType> {
     return coreTab.isElementVisible(awaitedPath.toJSON());
   }
 
+  public async takeScreenshot(options?: IScreenshotOptions): Promise<Buffer> {
+    const coreTab = await getCoreTab(this);
+    return coreTab.takeScreenshot(options);
+  }
+
   public async waitForPaintingStable(options?: IWaitForOptions): Promise<void> {
     const coreTab = await getCoreTab(this);
     await coreTab.waitForLoad(LocationStatus.PaintingStable, options);

core-interfaces/IScreenshotOptions.ts

@@ -0,0 +1,7 @@
+import IRect from './IRect';
+
+export default interface IScreenshotOptions {
+  format?: 'jpeg' | 'png';
+  rectangle?: IRect & { scale: number };
+  jpegQuality?: number;
+}

core/injected-scripts/pageEventsRecorder.ts

@@ -168,7 +168,7 @@ class PageEventsRecorder {
     const time = new Date().toISOString();
     const event = [this.commandId, eventType, nodeId, relatedNodeId, time] as IFocusEvent;
     this.focusEvents.push(event);
-    this.checkForPropertyChanges(time);
+    this.getPropertyChanges(time, this.domChanges);
   }
 
   public trackMouse(eventType: MouseEventType, mouseEvent: MouseEvent) {
@@ -206,59 +206,8 @@ class PageEventsRecorder {
     this.uploadChanges();
   }
 
-  public checkForLocationChange(changeTime?: string) {
-    const timestamp = changeTime || new Date().toISOString();
-    const currentLocation = window.self.location.href;
-    if (this.location !== currentLocation) {
-      this.location = currentLocation;
-      this.domChanges.push([
-        this.commandId,
-        'location',
-        { id: -1, textContent: currentLocation },
-        timestamp,
-        idx(),
-      ]);
-    }
-  }
-
-  public checkForStylesheetChanges(changeTime?: string) {
-    const timestamp = changeTime || new Date().toISOString();
-    for (const [style, current] of this.stylesheets) {
-      if (!style.sheet || !style.isConnected) continue;
-      const sheet = style.sheet as CSSStyleSheet;
-      const newPropValue = [...sheet.cssRules].map(x => x.cssText);
-      if (newPropValue.toString() !== current.toString()) {
-        const nodeId = nodeTracker.getId(style);
-        this.domChanges.push([
-          this.commandId,
-          'property',
-          { id: nodeId, properties: { 'sheet.cssRules': newPropValue } },
-          timestamp,
-          idx(),
-        ]);
-        this.stylesheets.set(style, newPropValue);
-      }
-    }
-  }
-
-  public checkForPropertyChanges(changeTime?: string) {
-    const timestamp = changeTime || new Date().toISOString();
-    for (const [input, propertyMap] of this.propertyTrackingElements) {
-      for (const [propertyName, value] of propertyMap) {
-        const newPropValue = input[propertyName];
-        if (newPropValue !== value) {
-          const nodeId = nodeTracker.getId(input);
-          this.domChanges.push([
-            this.commandId,
-            'property',
-            { id: nodeId, properties: { [propertyName]: newPropValue } },
-            timestamp,
-            idx(),
-          ]);
-          propertyMap.set(propertyName, newPropValue);
-        }
-      }
-    }
+  public checkForAllPropertyChanges() {
+    this.getPropertyChanges(new Date().toISOString(), this.domChanges);
   }
 
   public get pageResultset(): PageRecorderResultSet {
@@ -291,15 +240,50 @@ class PageEventsRecorder {
     }
   }
 
-  private trackStylesheet(element: HTMLLinkElement | HTMLStyleElement) {
+  private getLocationChange(changeTime: string, changes: IDomChangeEvent[]) {
+    const timestamp = changeTime || new Date().toISOString();
+    const currentLocation = window.self.location.href;
+    if (this.location !== currentLocation) {
+      this.location = currentLocation;
+      changes.push([
+        this.commandId,
+        'location',
+        { id: -1, textContent: currentLocation },
+        timestamp,
+        idx(),
+      ]);
+    }
+  }
+
+  private getPropertyChanges(changeTime: string, changes: IDomChangeEvent[]) {
+    const timestamp = changeTime;
+    for (const [input, propertyMap] of this.propertyTrackingElements) {
+      for (const [propertyName, value] of propertyMap) {
+        const newPropValue = input[propertyName];
+        if (newPropValue !== value) {
+          const nodeId = nodeTracker.getId(input);
+          changes.push([
+            this.commandId,
+            'property',
+            { id: nodeId, properties: { [propertyName]: newPropValue } },
+            timestamp,
+            idx(),
+          ]);
+          propertyMap.set(propertyName, newPropValue);
+        }
+      }
+    }
+  }
+
+  private trackStylesheet(element: HTMLStyleElement) {
     if (!element || this.stylesheets.has(element)) return;
     if (!element.sheet) return;
 
-    const shouldRecordInitialStyle = element.textContent || element instanceof HTMLStyleElement;
+    const shouldStoreCurrentStyleState = !!element.textContent;
     if (element.sheet instanceof CSSStyleSheet) {
       try {
         // if there's style text, record the current state
-        const startingStyle = shouldRecordInitialStyle
+        const startingStyle = shouldStoreCurrentStyleState
           ? [...element.sheet.cssRules].map(x => x.cssText)
           : [];
         this.stylesheets.set(element, startingStyle);
@@ -309,6 +293,26 @@ class PageEventsRecorder {
     }
   }
 
+  private checkForStylesheetChanges(changeTime: string, changes: IDomChangeEvent[]) {
+    const timestamp = changeTime || new Date().toISOString();
+    for (const [style, current] of this.stylesheets) {
+      if (!style.sheet || !style.isConnected) continue;
+      const sheet = style.sheet as CSSStyleSheet;
+      const newPropValue = [...sheet.cssRules].map(x => x.cssText);
+      if (newPropValue.toString() !== current.toString()) {
+        const nodeId = nodeTracker.getId(style);
+        changes.push([
+          this.commandId,
+          'property',
+          { id: nodeId, properties: { 'sheet.cssRules': newPropValue } },
+          timestamp,
+          idx(),
+        ]);
+        this.stylesheets.set(style, newPropValue);
+      }
+    }
+  }
+
   private onMutation(mutations: MutationRecord[]) {
     const changes = this.convertMutationsToChanges(mutations);
     this.domChanges.push(...changes);
@@ -319,8 +323,8 @@ class PageEventsRecorder {
     const currentCommandId = this.commandId;
     const stamp = new Date().toISOString();
 
-    this.checkForLocationChange(stamp);
-    this.checkForPropertyChanges(stamp);
+    this.getLocationChange(stamp, changes);
+    this.getPropertyChanges(stamp, changes);
 
     const addedNodeMap = new Map<Node, INodeData>();
     const removedNodes = new Set<Node>();
@@ -411,7 +415,7 @@ class PageEventsRecorder {
       }
     }
 
-    this.checkForStylesheetChanges(stamp);
+    this.checkForStylesheetChanges(stamp, changes);
 
     return changes;
   }
@@ -451,10 +455,6 @@ class PageEventsRecorder {
     }
     changes.push([currentCommandId, 'added', serial, stamp, idx()]);
     addedNodeMap.set(node, serial);
-    const childRecords = this.serializeChildren(node, addedNodeMap);
-    for (const childData of childRecords) {
-      changes.push([currentCommandId, 'added', childData, stamp, idx()]);
-    }
     return serial;
   }
 
@@ -603,17 +603,17 @@ const perfObserver = new PerformanceObserver(() => {
 });
 perfObserver.observe({ type: 'largest-contentful-paint', buffered: true });
 
-document.addEventListener('input', () => recorder.checkForPropertyChanges(), {
+document.addEventListener('input', () => recorder.checkForAllPropertyChanges(), {
   capture: true,
   passive: true,
 });
 
-document.addEventListener('keydown', () => recorder.checkForPropertyChanges(), {
+document.addEventListener('keydown', () => recorder.checkForAllPropertyChanges(), {
   capture: true,
   passive: true,
 });
 
-document.addEventListener('change', () => recorder.checkForPropertyChanges(), {
+document.addEventListener('change', () => recorder.checkForAllPropertyChanges(), {
   capture: true,
   passive: true,
 });

core/lib/CommandFormatter.ts

@@ -9,7 +9,7 @@ export default class CommandFormatter {
     if (!command.args) {
       return `${command.name}()`;
     }
-    const args = JSON.parse(command.args);
+    const args = JSON.parse(command.args).filter(x => x !== null);
     if (command.name === 'execJsPath') {
       return formatJsPath(args[0]);
     }
@@ -64,7 +64,7 @@ export default class CommandFormatter {
       return `waitForElement( ${formatJsPath(args[0])} )`;
     }
 
-    return `${command.name}(${args.map(JSON.stringify)})`;
+    return `${command.name}(${args.map(JSON.stringify).join(', ')})`;
   }
 
   public static parseResult(meta: ICommandMeta) {
@@ -106,6 +106,14 @@ export default class CommandFormatter {
         }
       }
 
+      if (result?.type === 'Buffer' && meta.name === 'takeScreenshot') {
+        const imageType = command.label.includes('jpeg') ? 'jpeg' : 'png';
+        command.result = `data:image/${imageType}; base64,${Buffer.from(result.data).toString(
+          'base64',
+        )}`;
+        command.resultType = 'image';
+      }
+
       if (meta.resultType.toLowerCase().includes('error')) {
         command.isError = true;
         command.result = result.message;

core/lib/SessionState.ts

@@ -220,6 +220,7 @@ export default class SessionState {
     this.db.resources.insert(tabId, resource, null, resourceEvent, error);
 
     const navigations = this.navigationsByTabId[tabId];
+    if (!navigations) return;
 
     const isNavigationToCurrentUrl =
       resource.url === navigations.currentUrl && resourceEvent.request.method !== 'OPTIONS';
@@ -353,7 +354,7 @@ export default class SessionState {
   }
 
   public captureError(tabId: string, frameId: string, source: string, error: Error): void {
-    this.logger.error('Window.error', { source, error });
+    this.logger.info('Window.error', { source, error });
     this.db.pageLogs.insert(tabId, frameId, source, error.stack || String(error), new Date());
   }
 
@@ -364,11 +365,7 @@ export default class SessionState {
     message: string,
     location?: string,
   ): void {
-    if (message.match(/error/gi)) {
-      this.logger.error('Window.error', { message });
-    } else {
-      this.logger.info('Window.console', { message });
-    }
+    this.logger.info('Window.console', { message });
     this.db.pageLogs.insert(tabId, frameId, consoleType, message, new Date(), location);
   }
 

core/lib/Tab.ts

@@ -29,6 +29,7 @@ import { IBoundLog } from '@secret-agent/core-interfaces/ILog';
 import IWebsocketResourceMessage from '@secret-agent/core-interfaces/IWebsocketResourceMessage';
 import IAttachedState from 'awaited-dom/base/IAttachedState';
 import IWaitForOptions from '@secret-agent/core-interfaces/IWaitForOptions';
+import IScreenshotOptions from '@secret-agent/core-interfaces/IScreenshotOptions';
 import TabNavigations from './TabNavigations';
 import SessionState from './SessionState';
 import TabNavigationObserver from './TabNavigationsObserver';
@@ -136,6 +137,7 @@ export default class Tab extends TypedEventEmitter<ITabEventParams> {
       this.isElementVisible,
       this.removeCookie,
       this.setCookie,
+      this.takeScreenshot,
       this.waitForMillis,
       this.waitForElement,
       this.waitForLoad,
@@ -383,6 +385,10 @@ export default class Tab extends TypedEventEmitter<ITabEventParams> {
     return isVisible.value;
   }
 
+  public takeScreenshot(options: IScreenshotOptions = {}): Promise<Buffer> {
+    return this.puppetPage.screenshot(options.format, options.rectangle, options.jpegQuality);
+  }
+
   public async waitForNewTab(options: IWaitForOptions = {}): Promise<Tab> {
     // last command is the one running right now
     const startCommandId = options?.sinceCommandId ?? this.lastCommandId - 1;

core/server/ConnectionToReplay.ts

@@ -249,7 +249,7 @@ export default class ConnectionToReplay {
     }
 
     const json = JSON.stringify({ event, data }, (_, value) => {
-      if (value !== null) return value;
+      if (value !== undefined) return value;
     });
 
     const sendPromise = this.sendMessage(json).catch(err => err);