fix(replay): fix command overlays

feat(replay): add keyboard control to playbar for fine grained navigation

fix(replay): fix unresponsive window dialog

Author: blakebyrnes

replay-app/backend/Application.ts

@@ -188,6 +188,11 @@ export default class Application {
       this.overlayManager.show(name, browserWindow, rect, ...args);
     });
 
+    ipcMain.on('message-overlay:hide', (e, webContentsId, messageId) => {
+      this.overlayManager.getByWebContentsId(webContentsId).hide();
+      Window.current.hideMessageOverlay(messageId);
+    });
+
     ipcMain.on('overlay:hide', (e, webContentsId) => {
       this.overlayManager.getByWebContentsId(webContentsId).hide();
     });

replay-app/backend/api/ReplayTabState.ts

@@ -71,6 +71,11 @@ export default class ReplayTabState extends EventEmitter {
     return this.ticks.find(x => x.commandId === pageToLoad.commandId)?.playbarOffsetPercent ?? 0;
   }
 
+  public gotoPreviousTick() {
+    const prevTickIdx = this.currentTickIdx > 0 ? this.currentTickIdx - 1 : this.currentTickIdx;
+    return this.loadTick(prevTickIdx);
+  }
+
   public gotoNextTick() {
     const result = this.loadTick(this.currentTickIdx + 1);
     if (
@@ -147,7 +152,7 @@ export default class ReplayTabState extends EventEmitter {
     }
 
     console.log(
-      'Paint load. Current=%s. New: %s->%s (%s, back? %s)',
+      'Paint load. Current=%s. New: %s->%s (paints: %s, back? %s)',
       this.paintEventsLoadedIdx,
       startIndex,
       newTick.paintEventIdx,
@@ -318,15 +323,16 @@ export default class ReplayTabState extends EventEmitter {
           if (command.resultNodeIds) {
             lastSelectedNodeIds = command.resultNodeIds;
           }
-          if (lastFocusEventIdx !== null) {
-            const focusEvent = this.focusEvents[lastFocusEventIdx];
-            if (focusEvent.event === 0 && !lastSelectedNodeIds) {
-              lastSelectedNodeIds = [focusEvent.targetNodeId];
-            }
-          }
           break;
         case 'focus':
           lastFocusEventIdx = tick.eventTypeIdx;
+          const focusEvent = this.focusEvents[lastFocusEventIdx];
+          if (focusEvent.event === 0 && focusEvent.targetNodeId) {
+            lastSelectedNodeIds = [focusEvent.targetNodeId];
+          } else if (focusEvent.event === 1) {
+            lastSelectedNodeIds = null;
+          }
+
           break;
         case 'paint':
           lastPaintEventIdx = tick.eventTypeIdx;
@@ -336,6 +342,12 @@ export default class ReplayTabState extends EventEmitter {
           break;
         case 'mouse':
           lastMouseEventIdx = tick.eventTypeIdx;
+          const mouseEvent = this.mouseEvents[lastMouseEventIdx];
+          if (mouseEvent.event === 1 && mouseEvent.targetNodeId) {
+            lastSelectedNodeIds = [mouseEvent.targetNodeId];
+          } else if (mouseEvent.event === 2) {
+            lastSelectedNodeIds = null;
+          }
           break;
       }
 
@@ -353,6 +365,7 @@ export default class ReplayTabState extends EventEmitter {
         tick.paintEventIdx = -1;
       }
     }
+    this.checkBroadcast();
   }
 
   public checkBroadcast() {

replay-app/backend/api/index.ts

@@ -18,7 +18,9 @@ export default class ReplayApi {
   public readonly saSession: ISaSession;
   public tabs: ReplayTabState[] = [];
   public apiHost: string;
-  public unresponsiveSeconds = 0;
+  public lastActivityDate: Date;
+  public lastCommandName: string;
+  public showUnresponsiveMessage = true;
 
   public onNewTab?: (tab: ReplayTabState) => any;
 
@@ -131,7 +133,8 @@ export default class ReplayApi {
       console.log('ScriptState', data);
       const closeDate = data.closeDate ? new Date(data.closeDate) : null;
       this.replayTime.update(closeDate);
-      this.unresponsiveSeconds = data.unresponsiveSeconds;
+      this.lastActivityDate = data.lastActivityDate ? new Date(data.lastActivityDate) : null;
+      this.lastCommandName = data.lastCommandName;
       for (const tab of this.tabs) tabsWithChanges.add(tab);
     } else {
       for (const event of data) {

replay-app/backend/managers/OverlayManager.ts

@@ -1,11 +1,11 @@
-import { BrowserWindow } from "electron";
-import BaseOverlay from "../overlays/BaseOverlay";
-import MainMenu from "../overlays/MainMenu";
-import LocationsMenu from "../overlays/LocationsMenu";
-import IRectangle from "~shared/interfaces/IRectangle";
-import CommandOverlay from "../overlays/CommandOverlay";
-import MessageOverlay from "../overlays/MessageOverlay";
-import ListMenu from "~backend/overlays/ListMenu";
+import { BrowserWindow } from 'electron';
+import BaseOverlay from '../overlays/BaseOverlay';
+import MainMenu from '../overlays/MainMenu';
+import LocationsMenu from '../overlays/LocationsMenu';
+import IRectangle from '~shared/interfaces/IRectangle';
+import CommandOverlay from '../overlays/CommandOverlay';
+import MessageOverlay from '../overlays/MessageOverlay';
+import ListMenu from '~backend/overlays/ListMenu';
 
 export default class OverlayManager {
   private overlays: BaseOverlay[] = [];

replay-app/backend/menus/generateAppMenu.ts

@@ -1,6 +1,6 @@
-import { app, BrowserWindow, ipcMain, Menu, MenuItem, webContents } from "electron";
-import { saveAs, viewSource } from "./CommonActions";
-import Window from "../models/Window";
+import { app, BrowserWindow, ipcMain, Menu, MenuItem, webContents } from 'electron';
+import { saveAs, viewSource } from './CommonActions';
+import Window from '../models/Window';
 
 const isMac = process.platform === 'darwin';
 
@@ -126,6 +126,25 @@ export default function generateAppMenu() {
         ),
       ],
     },
+    {
+      label: 'Replay',
+      submenu: [
+        {
+          label: 'View History',
+          click: () => {
+            return Window.current.openAppLocation('History');
+          },
+        },
+        ...createMenuItem(['Left'], () => {
+          if (!Window.current) return;
+          return Window.current.replayView?.gotoPreviousTick();
+        }),
+        ...createMenuItem(['Right'], () => {
+          if (!Window.current) return;
+          return Window.current.replayView?.gotoNextTick();
+        }),
+      ],
+    },
     {
       label: 'Tools',
       submenu: [
@@ -177,23 +196,6 @@ export default function generateAppMenu() {
     },
   ];
 
-  // Ctrl+1 - Ctrl+8
-  template[0].submenu = template[0].submenu.concat(
-    createMenuItem(
-      Array.from({ length: 8 }, (v, k) => k + 1).map(i => `CmdOrCtrl+${i}`),
-      (window, menuItem, i) => {
-        Window.current.webContents.send('select-tab-index', i);
-      },
-    ),
-  );
-
-  // Ctrl+9
-  template[0].submenu = template[0].submenu.concat(
-    createMenuItem(['CmdOrCtrl+9'], () => {
-      Window.current.webContents.send('select-last-tab');
-    }),
-  );
-
   return Menu.buildFromTemplate(template);
 }
 

replay-app/backend/menus/generateContextMenu.ts

@@ -1,5 +1,6 @@
 import { clipboard, Menu, nativeImage } from 'electron';
 import { printPage, saveAs, viewSource } from './CommonActions';
+import Window from '../models/Window';
 
 export default function generateContextMenu(
   params: Electron.ContextMenuParams,
@@ -109,17 +110,17 @@ export default function generateContextMenu(
       {
         label: 'Go back',
         accelerator: 'Alt+Left',
-        enabled: webContents.canGoBack(),
+        enabled: Window.current.hasBack(),
         click: () => {
-          webContents.goBack();
+          return Window.current.goBack();
         },
       },
       {
         label: 'Go forward',
         accelerator: 'Alt+Right',
-        enabled: webContents.canGoForward(),
+        enabled: Window.current.hasNext(),
         click: () => {
-          webContents.goForward();
+          return Window.current.goForward();
         },
       },
       {

replay-app/backend/models/PlaybarView.ts

@@ -4,6 +4,7 @@ import IRectangle from '~shared/interfaces/IRectangle';
 import ReplayTabState from '~backend/api/ReplayTabState';
 import ViewBackend from '~backend/models/ViewBackend';
 
+
 export default class PlaybarView extends ViewBackend {
   private readonly isReady: Promise<void>;
   private tabState: ReplayTabState;
@@ -55,7 +56,12 @@ export default class PlaybarView extends ViewBackend {
     }
   }
 
-  public async changeTickOffset(offset: number) {
+  public pause() {
+    this.playOnLoaded = false;
+    this.browserView.webContents.send('pause');
+  }
+
+  public changeTickOffset(offset: number) {
     this.browserView.webContents.send('ticks:change-offset', offset);
   }
 
@@ -64,7 +70,8 @@ export default class PlaybarView extends ViewBackend {
     const tick = this.tabState.ticks.find(x => x.playbarOffsetPercent === tickValue);
     if (!tick) return;
 
-    rect.y += this.bounds.y;
+    const bounds = this.browserView.getBounds();
+    rect.y += bounds.y - bounds.height;
     const commandLabel = tick.label;
     const commandResult =
       tick.eventType === 'command'

replay-app/backend/models/ReplayView.ts

@@ -13,10 +13,13 @@ import { IMouseEvent, IScrollRecord } from '~shared/interfaces/ISaSession';
 const domReplayerScript = require.resolve('../../injected-scripts/domReplayerSubscribe');
 
 export default class ReplayView extends ViewBackend {
+  public static MESSAGE_HANG_ID = 'script-hang';
   public replayApi: ReplayApi;
   public tabState: ReplayTabState;
   public readonly playbarView: PlaybarView;
 
+  private lastInactivityMillis = 0;
+
   public constructor(window: Window) {
     super(window, {
       preload: domReplayerScript,
@@ -25,6 +28,7 @@ export default class ReplayView extends ViewBackend {
       contextIsolation: true,
       javascript: false,
     });
+
     this.interceptHttpRequests();
     this.playbarView = new PlaybarView(window);
     this.checkResponsive = this.checkResponsive.bind(this);
@@ -102,10 +106,22 @@ export default class ReplayView extends ViewBackend {
     await this.publishTickChanges(events);
   }
 
+  public async gotoNextTick() {
+    const offset = await this.nextTick();
+    this.playbarView.changeTickOffset(offset);
+  }
+
+  public async gotoPreviousTick() {
+    const state = this.tabState.gotoPreviousTick();
+    await this.publishTickChanges(state);
+    this.playbarView.changeTickOffset(this.tabState.currentPlaybarOffsetPct);
+  }
+
   public async nextTick() {
     if (!this.isAttached || !this.tabState) return 0;
     const events = this.tabState.gotoNextTick();
     await this.publishTickChanges(events);
+    setImmediate(() => this.checkResponsive());
 
     return this.tabState.currentPlaybarOffsetPct;
   }
@@ -123,9 +139,13 @@ export default class ReplayView extends ViewBackend {
     const [domChanges] = events;
     if (domChanges?.length) {
       if (domChanges[0].action === 'newDocument') {
-        const nav = domChanges.shift();
-        console.log('Re-navigating', nav.textContent);
-        await this.webContents.loadURL(nav.textContent);
+        const url = new URL(domChanges[0].textContent);
+        const currentUrl = new URL(this.webContents.getURL());
+        if (url.origin !== currentUrl.origin) {
+          console.log('Re-navigating', url.href);
+          domChanges.shift();
+          await this.webContents.loadURL(url.href);
+        }
       }
     }
     this.webContents.send('dom:apply', ...events);
@@ -142,14 +162,30 @@ export default class ReplayView extends ViewBackend {
   }
 
   private checkResponsive() {
-    if (this.replayApi.unresponsiveSeconds >= 5) {
+    const lastActivityMillis =
+      new Date().getTime() - (this.replayApi.lastActivityDate ?? new Date()).getTime();
+
+    if (lastActivityMillis < this.lastInactivityMillis) {
+      this.lastInactivityMillis = lastActivityMillis;
+      return;
+    }
+    if (lastActivityMillis - this.lastInactivityMillis < 500) return;
+    this.lastInactivityMillis = lastActivityMillis;
+
+    if (
+      lastActivityMillis >= 5e3 &&
+      this.replayApi.lastCommandName !== 'waitForMillis' &&
+      this.replayApi.showUnresponsiveMessage
+    ) {
+      const lastActivitySecs = Math.floor(lastActivityMillis / 1e3);
       Application.instance.overlayManager.show(
         'message-overlay',
         this.window.browserWindow,
         this.window.browserWindow.getContentBounds(),
         {
           title: 'Did your script hang?',
-          message: `The last update was ${this.replayApi.unresponsiveSeconds} seconds ago.`,
+          message: `The last update was ${lastActivitySecs} seconds ago.`,
+          id: ReplayView.MESSAGE_HANG_ID,
         },
       );
     } else {

replay-app/backend/models/Window.ts

@@ -99,6 +99,14 @@ export default class Window {
     }
   }
 
+  public hasBack() {
+    return this.navCursor > 0;
+  }
+
+  public hasNext() {
+    return this.navCursor + 1 < this.navHistory.length;
+  }
+
   public async openAppLocation(location: IWindowLocation, navigateToHistoryIdx?: number) {
     console.log('Navigating to %s', location);
 
@@ -173,6 +181,12 @@ export default class Window {
     }
   }
 
+  public hideMessageOverlay(messageId: string) {
+    if (messageId === ReplayView.MESSAGE_HANG_ID) {
+      this.replayView.replayApi.showUnresponsiveMessage = false;
+    }
+  }
+
   protected async load(state: { replayApi?: ReplayApi; location?: IWindowLocation }) {
     const { replayApi, location } = state || {};
     await this.browserWindow.loadURL(Application.instance.getPageUrl('header'));
@@ -203,22 +217,14 @@ export default class Window {
     return this.openReplayApi(api, index);
   }
 
-  private hasBack() {
-    return this.navCursor > 0;
-  }
-
-  private hasNext() {
-    return this.navCursor + 1 < this.navHistory.length;
-  }
-
   private logHistory(
     history: { replayMeta?: IReplayMeta; location?: IWindowLocation },
     historyIdx?: number,
   ) {
     if (historyIdx !== undefined) {
       this.navCursor = historyIdx;
     } else {
-      this.navHistory.length = this.navCursor+1;
+      this.navHistory.length = this.navCursor + 1;
       this.navCursor = this.navHistory.length;
       this.navHistory.push(history);
     }