diff --git a/runtime-nodejs/example-project/src/custom-globals-examples.spec.js b/runtime-nodejs/example-project/src/custom-globals-examples.spec.js new file mode 100644 index 0000000..c19f5a8 --- /dev/null +++ b/runtime-nodejs/example-project/src/custom-globals-examples.spec.js @@ -0,0 +1,23 @@ +describe("Custom globals", () => { + + describe("promFetch", () => { + + it("Prepends to resource, the root URL to the relevant Prometheus instance", async () => { + const flags = await promFetch('/api/v1/status/flags').then(res => res.json()); + expect(flags).toHaveProperty('status','success'); + expect(flags.data['web.listen-address']).toEqual('0.0.0.0:9090'); + }); + + }); + + describe("promValue", () => { + + it("Returns a metric value as Number via Promise", async () => { + const pod = process.env.POD_NAME; + const completions = await promValue(`assert_completions_remaining{pod="${pod}"}`); + expect(completions).toBeGreaterThanOrEqual(0); + }); + + }); + +}); diff --git a/runtime-nodejs/example-project/src/metrics.spec.js b/runtime-nodejs/example-project/src/metrics.spec.js index 517733d..e5b2e24 100644 --- a/runtime-nodejs/example-project/src/metrics.spec.js +++ b/runtime-nodejs/example-project/src/metrics.spec.js @@ -1,11 +1,15 @@ -const fetch = require('node-fetch'); describe("A sub-module", () => { - it("Can use the runtime's default dependencies", () => { + it("Can use the runtime's global helpers", () => { expect(fetch).toBeDefined(); }); + it("Can use the runtime's default dependencies", () => { + expect(require('node-fetch')).toBeDefined(); + expect(require('@kubernetes/client-node')).toBeDefined(); + }); + it("Gets the metrics reporter despite having its own package.json", async () => { const metrics = await fetch('http://localhost:9090/metrics').then(res => res.text()); expect(metrics).toMatch(/\bassertions_failed /); diff --git a/runtime-nodejs/jest.config.js b/runtime-nodejs/jest.config.js index 197f4c3..9c7254b 100644 --- a/runtime-nodejs/jest.config.js +++ b/runtime-nodejs/jest.config.js @@ -1,6 +1,10 @@ module.exports = { + testEnvironment: 'node', reporters: [ "default", ["/jest.kubernetes-assertions-reporter.js", {}] + ], + setupFilesAfterEnv: [ + "/jest.setup-global-fetch.js" ] }; diff --git a/runtime-nodejs/jest.kubernetes-assertions-reporter.js b/runtime-nodejs/jest.kubernetes-assertions-reporter.js index ade01a0..8151351 100644 --- a/runtime-nodejs/jest.kubernetes-assertions-reporter.js +++ b/runtime-nodejs/jest.kubernetes-assertions-reporter.js @@ -37,6 +37,21 @@ const test_suites_run_total = new client.Counter({ help: 'inc\'d for every onRunComplete numTotalTestSuites' }); +const assert_files_seen = new client.Gauge({ + name: 'assert_files_seen', + help: 'Unique spec file paths that have been seen in onTestResult' +}); + +const assert_completions_passed = new client.Gauge({ + name: 'assert_completions_passed', + help: 'Gets a value if a spec names assert-completion.spec.js has run' +}); + +const assert_completions_remaining = new client.Gauge({ + name: 'assert_completions_remaining', + help: 'The number of failures for assert-completion.spec.js (pending and todo are ignored)' +}); + class MetricsServer { constructor({ port, getMetrics }) { @@ -75,6 +90,7 @@ class MetricsReporter { constructor(globalConfig, options) { this._globalConfig = globalConfig; this._options = options; + this._pathsSeen = {}; } onRunStart() { @@ -91,16 +107,35 @@ class MetricsReporter { test_suites_run_total.inc(results.numTotalTestSuites); tests_run.set(results.numTotalTests); tests_run_total.inc(results.numTotalTests); - } - - onTestResult(test, testResult, aggregatedResult) { - assertions_failed.set(testResult.numFailingTests); - assertions_failed_total.inc(testResult.numFailingTests); - //console.log('onTestResult', testResult, aggregatedResult); + assertions_failed.set(results.numFailedTests); + assertions_failed_total.inc(results.numFailedTests); if (!this._globalConfig.watch && !this._globalConfig.watchAll) { //console.log('Not a watch run. Exiting'); server.stop(); } + // onRunComplete seems to happen before each onTestResult so this placement isn't great + assert_files_seen.set(Object.keys(this._pathsSeen).length); + } + + onTestResult(test, testResult, aggregatedResult) { + //console.log('onTestResult', testResult); + const path = testResult.testFilePath; + if (!this._pathsSeen[path]) { + this._pathsSeen[path] = {}; + } + if (this.isAssertCompletion(path)) { + this.onAssertCompletion(testResult); + } + } + + isAssertCompletion(testFilePath) { + const match = /.*\/assert-completion\.spec\.js$/.test(testFilePath); + return match; + } + + onAssertCompletion(testResult) { + assert_completions_passed.set(testResult.numPassingTests); + assert_completions_remaining.set(testResult.numFailingTests); } } diff --git a/runtime-nodejs/jest.setup-global-fetch.js b/runtime-nodejs/jest.setup-global-fetch.js new file mode 100644 index 0000000..097fb5c --- /dev/null +++ b/runtime-nodejs/jest.setup-global-fetch.js @@ -0,0 +1,23 @@ +global.fetch = require('node-fetch'); + +global.prometheusHost = process.env.PROMETHEUS_HOST + || 'http://prometheus-now.monitoring:9090'; + +global.promFetch = (resource, init) => fetch(global.prometheusHost + resource, init); + +// Good enough errors for test failure outputs +const err = (message, data) => { + return "promValue failed: " + message + " " + JSON.stringify(data); +} + +global.promValue = async query => { + const req = promFetch(`/api/v1/query?query=${query}`); + const json = await req.then(res => res.json()); + if (json.status != "success") return err("Expected status=success", json); + const result = json.data.result; + if (!result.length) return err("Empty result", json); + if (result.length > 1) return err("Too many results", json); + const value = result[0].value[1]; + if (isNaN(value)) return err("Not a number", value); + return Number(value); +} diff --git a/runtime-nodejs/specs/assert-completion.spec.js b/runtime-nodejs/specs/assert-completion.spec.js new file mode 100644 index 0000000..832c4cb --- /dev/null +++ b/runtime-nodejs/specs/assert-completion.spec.js @@ -0,0 +1,23 @@ + +// The recommended approach for assert_completions is ... +// - Check test health indirectly through Prometheus +// - For example existence of some key metric of the system under test +// - Actual system specs are done in properly names .spec.js files + +describe("Assert completion", () => { + + describe("Suite size", () => { + // This is quite imporant to keep up-to-date while we're learning non-interactive jest watch + const ASSERT_FILES_MIN = 2; + + it(`Has seen at least ${ASSERT_FILES_MIN} spec files`, async () => { + //expect(await promValue(`assert_files_seen{pod="${process.env.POD_NAME}"}`)) + // .toBeGreaterThanOrEqual(ASSERT_FILES_MIN); + }); + }); + + it("The runtime has produced metrics", () => { + + }); + +}); diff --git a/runtime-nodejs/specs/custom-globals.spec.js b/runtime-nodejs/specs/custom-globals.spec.js new file mode 100644 index 0000000..19de2cd --- /dev/null +++ b/runtime-nodejs/specs/custom-globals.spec.js @@ -0,0 +1,27 @@ +describe("Custom globals", () => { + + describe("fetch", () => { + + it("Makes node-fetch global like in browsers", () => { + expect(fetch).toBeInstanceOf(Function); + }); + + }); + + describe("promFetch", () => { + + it("Wraps fetch", () => { + expect(promFetch).toBeInstanceOf(Function); + }); + + }); + + describe("promValue", () => { + + it("Is a function", () => { + expect(promValue).toBeInstanceOf(Function); + }); + + }); + +});