UI testing with Galata

.github/workflows/build.yml

@@ -85,6 +85,31 @@ jobs:
     - name: JavaScript prettyfier
       run: yarn run prettier:check
 
+    - name: Install Galata
+      run: yarn install
+      working-directory: ui-tests
+
+    - name: Launch JupyterLab
+      run: yarn run start-jlab:detached
+      working-directory: ui-tests
+
+    - name: Wait for JupyterLab
+      uses: ifaxity/wait-on-action@v1
+      with:
+        resource: http-get://localhost:8888/api
+        timeout: 20000
+
+    - name: Run UI Tests
+      run: yarn run test
+      working-directory: ui-tests
+
+    - name: Upload UI Test artifacts
+      if: always()
+      uses: actions/upload-artifact@v2
+      with:
+        name: ui-test-output
+        path: ui-tests/test-output
+
     # - name: JavaScript tests
     #   run: yarn run test
     #   working-directory: js
@@ -106,58 +131,6 @@ jobs:
         name: dist ${{ github.run_number }}
         path: ./dist
 
-# visual-regression-tests:
-#   runs-on: ubuntu-latest
-#   needs: [build]
-
-#   steps:
-#     - name: Checkout
-#       uses: actions/checkout@v2
-
-#     - name: Setup conda
-#       uses: conda-incubator/setup-miniconda@v2
-#       with:
-#         activate-environment: bqplot-test
-#         environment-file: test-environment.yml
-#         python-version: 3.9
-#         mamba-version: "*"
-#         auto-activate-base: false
-#         channels: conda-forge
-
-#     - uses: actions/download-artifact@v2
-#       with:
-#         name: dist ${{ github.run_number }}
-#         path: ./dist
-
-#     - name: Install the package
-#       run: pip install -vv bqplot*.whl
-#       working-directory: dist
-
-#     - name: Install Galata
-#       run: yarn install
-#       working-directory: ui-tests
-
-#     - name: Launch JupyterLab
-#       run: yarn run start-jlab:detached
-#       working-directory: ui-tests
-
-#     - name: Wait for JupyterLab
-#       uses: ifaxity/wait-on-action@v1
-#       with:
-#         resource: http-get://localhost:8888/api
-#         timeout: 20000
-
-#     - name: Run UI Tests
-#       run: yarn run test
-#       working-directory: ui-tests
-
-#     - name: Upload UI Test artifacts
-#       if: always()
-#       uses: actions/upload-artifact@v2
-#       with:
-#         name: ui-test-output
-#         path: ui-tests/test-output
-
   install:
     runs-on: ${{ matrix.os }}-latest
     needs: [build]

.gitignore

@@ -154,3 +154,5 @@ bqplot_gl/labextension/*.tgz
 
 # Packed lab extensions
 bqplot_gl/labextension
+
+ui-tests/test-output/*

babel.config.js

@@ -9,5 +9,6 @@ module.exports = {
         },
       },
     ],
+    '@babel/preset-typescript',
   ],
 };

bqplot_gl/_version.py

@@ -4,5 +4,5 @@
 # Copyright (c) The bqplot Development Team.
 # Distributed under the terms of the Modified BSD License.
 
-version_info = (0, 1, 0, 'dev')
+version_info = (0, 1, 0)
 __version__ = ".".join(map(str, version_info))

package.json

@@ -61,6 +61,7 @@
   "devDependencies": {
     "@babel/core": "^7.5.0",
     "@babel/preset-env": "^7.5.0",
+    "@babel/preset-typescript": "^7.15.0",
     "@jupyterlab/builder": "^3.0.0",
     "@phosphor/application": "^1.6.0",
     "@phosphor/widgets": "^1.6.0",

src/ScatterGLView.ts

@@ -13,6 +13,7 @@
  * limitations under the License.
  */
 
+import { ColorScale } from 'bqscales';
 import { Mark, symbol as bqSymbol, deepCopy, Scale } from 'bqplot';
 import * as d3 from 'd3';
 import * as _ from 'underscore';
@@ -56,9 +57,9 @@ const color_to_array_rgba = function (color: any, default_color?: any) {
   }
 };
 
-const create_colormap = function (scale: any) {
+const create_colormap = function (scale: ColorScale) {
   // convert the d3 color scale to a texture
-  const colors = scale ? scale.model.color_range : ['#ff0000', '#ff0000'];
+  const colors = scale ? scale.model.colorRange : [0, 1];
   const color_scale = d3
     .scaleLinear()
     .range(colors)

src/plugin.ts

@@ -7,7 +7,7 @@ import { Widget } from '@phosphor/widgets';
 
 import { IJupyterWidgetRegistry } from '@jupyter-widgets/base';
 
-import * as widgetExports from './figure';
+import * as widgetExports from './index';
 
 import { MODULE_NAME, MODULE_VERSION } from './version';
 

tsconfig.json

@@ -1,7 +1,7 @@
 {
   "compilerOptions": {
     "declaration": true,
-    "esModuleInterop":true,
+    "esModuleInterop": true,
     "lib": ["es2015", "dom"],
     "module": "commonjs",
     "moduleResolution": "node",

ui-tests/README.md

@@ -0,0 +1,36 @@
+# Visual regression tests using Galata
+
+This directory contains visual regression tests for bqplot, using Galata.
+
+In order to run them, you need to install dependencies:
+
+```bash
+const install -c conda-forge yarn jupyterlab=3.0.11
+
+yarn install
+```
+
+Then start JupyterLab in one terminal (you need to check that it properly starts on port 8888):
+```bash
+yarn run start-jlab
+```
+
+Finally, run the galata tests:
+```bash
+yarn run test
+```
+
+If bqplot visuals change, you can re-generate reference images by running:
+```bash
+yarn run update-references
+```
+
+## Notebooks directory
+
+The `tests/notebooks` directory contains the test notebooks. For most notebooks (*e.g.* `bars.ipynb`, `scatter.ipynb`) Galata will run them cell by cell and take a screenshot of each output, comparing with the reference images.
+
+When running notebooks named `*_update.ipynb`, Galata will always take the first cell output as reference which must contain the plot, later cells will only be used to update the plot, those notebooks are checking that bqplot is properly taking updates into account on already-created plots.
+
+## Add a new test
+
+You can add a new test by simply adding a new notebook to the `tests/notebooks` directory and updating the references. If you want to test updating plots, create notebook named `*_update.ipynb`, create a plot in your first cell then update the plot in later cells.

ui-tests/galata-config.json

@@ -0,0 +1,3 @@
+{
+  "testId": "test"
+}

ui-tests/jupyter_server_config.py

@@ -0,0 +1,7 @@
+c.ServerApp.port = 8888
+c.ServerApp.token = ""
+c.ServerApp.password = ""
+c.ServerApp.disable_check_xsrf = True
+c.ServerApp.open_browser = False
+c.LabApp.open_browser = False
+c.LabApp.expose_app_in_browser = True

ui-tests/package.json

@@ -0,0 +1,20 @@
+{
+  "name": "bqplot-gl-ui-tests",
+  "version": "1.0.0",
+  "description": "bqplot-gl UI Tests",
+  "private": true,
+  "scripts": {
+    "start-jlab": "jupyter lab --config ./jupyter_server_config.py",
+    "start-jlab:detached": "yarn run start-jlab&",
+    "clean": "rimraf tests/notebooks/.ipynb_checkpoints && rimraf test-output",
+    "test": "yarn run clean && galata --image-match-threshold 0.33",
+    "update-references": "galata --update-references"
+  },
+  "author": "bqplot",
+  "license": "Apache-2.0",
+  "dependencies": {
+    "@jupyterlab/galata": "3.0.11-2",
+    "klaw-sync": "^6.0.0",
+    "rimraf": "^3.0.2"
+  }
+}

ui-tests/tests/bqplot.test.ts

@@ -0,0 +1,171 @@
+// Copyright (c) Jupyter Development Team.
+// Distributed under the terms of the Modified BSD License.
+
+import { galata, describe, test } from '@jupyterlab/galata';
+import * as path from 'path';
+import * as klaw from 'klaw-sync';
+
+jest.setTimeout(600000);
+
+const filterUpdateNotebooks = item => {
+  const basename = path.basename(item.path);
+  return basename.includes('_update');
+}
+
+const testCellOutputs = async (theme: 'JupyterLab Light' | 'JupyterLab Dark') => {
+  const paths = klaw('tests/notebooks', {filter: item => !filterUpdateNotebooks(item), nodir: true});
+  const notebooks = paths.map(item => path.basename(item.path));
+
+  let results = [];
+
+  const contextPrefix = theme == 'JupyterLab Light' ? 'light_' : 'dark_';
+  galata.theme.setTheme(theme);
+
+  for (const notebook of notebooks) {
+    galata.context.capturePrefix = contextPrefix + notebook;
+
+    await galata.notebook.open(notebook);
+    expect(await galata.notebook.isOpen(notebook)).toBeTruthy();
+    await galata.notebook.activate(notebook);
+    expect(await galata.notebook.isActive(notebook)).toBeTruthy();
+
+    let numCellImages = 0;
+
+    const getCaptureImageName = (id: number): string => {
+      return `cell-${id}`;
+    };
+
+    await galata.notebook.runCellByCell({
+      onAfterCellRun: async (cellIndex: number) => {
+        const cell = await galata.notebook.getCellOutput(cellIndex);
+        if (cell) {
+          if (
+            await galata.capture.screenshot(
+              getCaptureImageName(numCellImages),
+              cell
+            )
+          ) {
+            numCellImages++;
+          }
+        }
+      }
+    });
+
+    for (let c = 0; c < numCellImages; ++c) {
+      results.push(await galata.capture.compareScreenshot(getCaptureImageName(c)));
+    }
+
+    await galata.notebook.close(true);
+  }
+
+  for (const result of results) {
+    expect(result).toBe('same');
+  }
+}
+
+const testPlotUpdates = async (theme: 'JupyterLab Light' | 'JupyterLab Dark') => {
+  const paths = klaw('tests/notebooks', {filter: item => filterUpdateNotebooks(item), nodir: true});
+  const notebooks = paths.map(item => path.basename(item.path));
+
+  let results = [];
+
+  const contextPrefix = theme == 'JupyterLab Light' ? 'light_' : 'dark_';
+  galata.theme.setTheme(theme);
+
+  for (const notebook of notebooks) {
+    galata.context.capturePrefix = contextPrefix + notebook;
+
+    await galata.notebook.open(notebook);
+    expect(await galata.notebook.isOpen(notebook)).toBeTruthy();
+    await galata.notebook.activate(notebook);
+    expect(await galata.notebook.isActive(notebook)).toBeTruthy();
+
+    let numCellImages = 0;
+
+    const getCaptureImageName = (id: number): string => {
+      return `cell-${id}`;
+    };
+
+    await galata.notebook.runCellByCell({
+      onAfterCellRun: async (cellIndex: number) => {
+        // Always get first cell output which must contain the plot
+        const cell = await galata.notebook.getCellOutput(0);
+        if (cell) {
+          if (
+            await galata.capture.screenshot(
+              getCaptureImageName(numCellImages),
+              cell
+            )
+          ) {
+            numCellImages++;
+          }
+        }
+      }
+    });
+
+    for (let c = 0; c < numCellImages; ++c) {
+      results.push(await galata.capture.compareScreenshot(getCaptureImageName(c)));
+    }
+
+    await galata.notebook.close(true);
+  }
+
+  for (const result of results) {
+    expect(result).toBe('same');
+  }
+};
+
+describe('bqplot Visual Regression', () => {
+  beforeAll(async () => {
+    await galata.resetUI();
+  });
+
+  afterAll(async () => {
+    galata.context.capturePrefix = '';
+  });
+
+  test('Upload files to JupyterLab', async () => {
+    await galata.contents.moveDirectoryToServer(
+      path.resolve(__dirname, `./notebooks`),
+      'uploaded'
+    );
+    expect(
+      await galata.contents.fileExists('uploaded/scatter.ipynb')
+    ).toBeTruthy();
+  });
+
+  test('Refresh File Browser', async () => {
+    await galata.filebrowser.refresh();
+  });
+
+  test('Open directory uploaded', async () => {
+    await galata.filebrowser.openDirectory('uploaded');
+    expect(
+      await galata.filebrowser.isFileListedInBrowser('scatter.ipynb')
+    ).toBeTruthy();
+  });
+
+  test('Light theme: Check bqplot first renders', async () => {
+    await testCellOutputs('JupyterLab Light');
+  });
+
+  test('Dark theme: Check bqplot first renders', async () => {
+    await testCellOutputs('JupyterLab Dark');
+  });
+
+  test('Light theme: Check bqplot update plot properties', async () => {
+    await testPlotUpdates('JupyterLab Light');
+  });
+
+  test('Dark theme: Check bqplot update plot properties', async () => {
+    await testPlotUpdates('JupyterLab Dark');
+  });
+
+  test('Open home directory', async () => {
+    await galata.filebrowser.openHomeDirectory();
+  });
+
+  test('Delete uploaded directory', async () => {
+    await galata.contents.deleteDirectory('uploaded');
+  });
+});