Let test suites define completion criterias

runtime-nodejs/example-project/src/custom-globals-examples.spec.js

@@ -0,0 +1,23 @@
+describe("Custom globals", () => {
+
+  describe("promFetch", () => {
+
+    it("Prepends to resource, the root URL to the relevant Prometheus instance", async () => {
+      const flags = await promFetch('/api/v1/status/flags').then(res => res.json());
+      expect(flags).toHaveProperty('status','success');
+      expect(flags.data['web.listen-address']).toEqual('0.0.0.0:9090');
+    });
+
+  });
+
+  describe("promValue", () => {
+
+    it("Returns a metric value as Number via Promise", async () => {
+      const pod = process.env.POD_NAME;
+      const completions = await promValue(`assert_completions_remaining{pod="${pod}"}`);
+      expect(completions).toBeGreaterThanOrEqual(0);
+    });
+
+  });
+
+});

runtime-nodejs/example-project/src/metrics.spec.js

@@ -1,11 +1,15 @@
-const fetch = require('node-fetch');
 
 describe("A sub-module", () => {
 
-  it("Can use the runtime's default dependencies", () => {
+  it("Can use the runtime's global helpers", () => {
     expect(fetch).toBeDefined();
   });
 
+  it("Can use the runtime's default dependencies", () => {
+    expect(require('node-fetch')).toBeDefined();
+    expect(require('@kubernetes/client-node')).toBeDefined();
+  });
+
   it("Gets the metrics reporter despite having its own package.json", async () => {
     const metrics = await fetch('http://localhost:9090/metrics').then(res => res.text());
     expect(metrics).toMatch(/\bassertions_failed /);

runtime-nodejs/jest.config.js

@@ -1,6 +1,10 @@
 module.exports = {
+  testEnvironment: 'node',
   reporters: [
     "default",
     ["<rootDir>/jest.kubernetes-assertions-reporter.js", {}]
+  ],
+  setupFilesAfterEnv: [
+    "<rootDir>/jest.setup-global-fetch.js"
   ]
 };

runtime-nodejs/jest.kubernetes-assertions-reporter.js

@@ -37,6 +37,21 @@ const test_suites_run_total = new client.Counter({
   help: 'inc\'d for every onRunComplete numTotalTestSuites'
 });
 
+const assert_files_seen = new client.Gauge({
+  name: 'assert_files_seen',
+  help: 'Unique spec file paths that have been seen in onTestResult'
+});
+
+const assert_completions_passed = new client.Gauge({
+  name: 'assert_completions_passed',
+  help: 'Gets a value if a spec names assert-completion.spec.js has run'
+});
+
+const assert_completions_remaining = new client.Gauge({
+  name: 'assert_completions_remaining',
+  help: 'The number of failures for assert-completion.spec.js (pending and todo are ignored)'
+});
+
 class MetricsServer {
 
   constructor({ port, getMetrics }) {
@@ -75,6 +90,7 @@ class MetricsReporter {
   constructor(globalConfig, options) {
     this._globalConfig = globalConfig;
     this._options = options;
+    this._pathsSeen = {};
   }
 
   onRunStart() {
@@ -91,16 +107,35 @@ class MetricsReporter {
     test_suites_run_total.inc(results.numTotalTestSuites);
     tests_run.set(results.numTotalTests);
     tests_run_total.inc(results.numTotalTests);
-  }
-
-  onTestResult(test, testResult, aggregatedResult) {
-    assertions_failed.set(testResult.numFailingTests);
-    assertions_failed_total.inc(testResult.numFailingTests);
-    //console.log('onTestResult', testResult, aggregatedResult);
+    assertions_failed.set(results.numFailedTests);
+    assertions_failed_total.inc(results.numFailedTests);
     if (!this._globalConfig.watch && !this._globalConfig.watchAll) {
       //console.log('Not a watch run. Exiting');
       server.stop();
     }
+    // onRunComplete seems to happen before each onTestResult so this placement isn't great
+    assert_files_seen.set(Object.keys(this._pathsSeen).length);
+  }
+
+  onTestResult(test, testResult, aggregatedResult) {
+    //console.log('onTestResult', testResult);
+    const path = testResult.testFilePath;
+    if (!this._pathsSeen[path]) {
+      this._pathsSeen[path] = {};
+    }
+    if (this.isAssertCompletion(path)) {
+      this.onAssertCompletion(testResult);
+    }
+  }
+
+  isAssertCompletion(testFilePath) {
+    const match = /.*\/assert-completion\.spec\.js$/.test(testFilePath);
+    return match;
+  }
+
+  onAssertCompletion(testResult) {
+    assert_completions_passed.set(testResult.numPassingTests);
+    assert_completions_remaining.set(testResult.numFailingTests);
   }
 
 }

runtime-nodejs/jest.setup-global-fetch.js

@@ -0,0 +1,23 @@
+global.fetch = require('node-fetch');
+
+global.prometheusHost = process.env.PROMETHEUS_HOST 
+  || 'http://prometheus-now.monitoring:9090';
+
+global.promFetch = (resource, init) => fetch(global.prometheusHost + resource, init);
+
+// Good enough errors for test failure outputs
+const err = (message, data) => {
+  return "promValue failed: " + message + " " + JSON.stringify(data);
+}
+
+global.promValue = async query => {
+  const req = promFetch(`/api/v1/query?query=${query}`);
+  const json = await req.then(res => res.json());
+  if (json.status != "success") return err("Expected status=success", json);
+  const result = json.data.result;
+  if (!result.length) return err("Empty result", json);
+  if (result.length > 1) return err("Too many results", json);
+  const value = result[0].value[1];
+  if (isNaN(value)) return err("Not a number", value);
+  return Number(value);
+}

runtime-nodejs/specs/assert-completion.spec.js

@@ -0,0 +1,23 @@
+
+// The recommended approach for assert_completions is ...
+// - Check test health indirectly through Prometheus
+// - For example existence of some key metric of the system under test
+// - Actual system specs are done in properly names .spec.js files
+
+describe("Assert completion", () => {
+
+  describe("Suite size", () => {
+    // This is quite imporant to keep up-to-date while we're learning non-interactive jest watch
+    const ASSERT_FILES_MIN = 2;
+
+    it(`Has seen at least ${ASSERT_FILES_MIN} spec files`, async () => {
+      //expect(await promValue(`assert_files_seen{pod="${process.env.POD_NAME}"}`))
+      //  .toBeGreaterThanOrEqual(ASSERT_FILES_MIN);
+    });
+  });
+
+  it("The runtime has produced metrics", () => {
+
+  });
+
+});

runtime-nodejs/specs/custom-globals.spec.js

@@ -0,0 +1,27 @@
+describe("Custom globals", () => {
+
+  describe("fetch", () => {
+
+    it("Makes node-fetch global like in browsers", () => {
+      expect(fetch).toBeInstanceOf(Function);
+    });
+
+  });
+
+  describe("promFetch", () => {
+
+    it("Wraps fetch", () => {
+      expect(promFetch).toBeInstanceOf(Function);
+    });
+
+  });
+
+  describe("promValue", () => {
+
+    it("Is a function", () => {
+      expect(promValue).toBeInstanceOf(Function);
+    });
+
+  });
+
+});