From 1a5f65d293419e87fabd6186e58469692f2454e9 Mon Sep 17 00:00:00 2001 From: Staffan Olsson Date: Fri, 5 Jun 2020 09:18:34 +0200 Subject: [PATCH 1/6] Avoids onTestResult for metrics because while aggregatedResults has the right metrics the callback is per test, not per run, thus much more frequent in large test suites. --- .../jest.kubernetes-assertions-reporter.js | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/runtime-nodejs/jest.kubernetes-assertions-reporter.js b/runtime-nodejs/jest.kubernetes-assertions-reporter.js index ade01a0..7d0a6b4 100644 --- a/runtime-nodejs/jest.kubernetes-assertions-reporter.js +++ b/runtime-nodejs/jest.kubernetes-assertions-reporter.js @@ -91,18 +91,18 @@ class MetricsReporter { test_suites_run_total.inc(results.numTotalTestSuites); tests_run.set(results.numTotalTests); tests_run_total.inc(results.numTotalTests); - } - - onTestResult(test, testResult, aggregatedResult) { - assertions_failed.set(testResult.numFailingTests); - assertions_failed_total.inc(testResult.numFailingTests); - //console.log('onTestResult', testResult, aggregatedResult); + assertions_failed.set(results.numFailedTests); + assertions_failed_total.inc(results.numFailedTests); if (!this._globalConfig.watch && !this._globalConfig.watchAll) { //console.log('Not a watch run. Exiting'); server.stop(); } } + onTestResult(test, testResult, aggregatedResult) { + //console.log('onTestResult', testResult, aggregatedResult); + } + } module.exports = MetricsReporter; From 6c352fedbd447f3e323c10424fc5bfe6b658caba Mon Sep 17 00:00:00 2001 From: Staffan Olsson Date: Fri, 5 Jun 2020 10:33:47 +0200 Subject: [PATCH 2/6] Begins a concept for defining when a test run has completed --- .../jest.kubernetes-assertions-reporter.js | 38 ++++++++++++++++++- .../specs/assert-completion.spec.js | 24 ++++++++++++ 2 files changed, 61 insertions(+), 1 deletion(-) create mode 100644 runtime-nodejs/specs/assert-completion.spec.js diff --git a/runtime-nodejs/jest.kubernetes-assertions-reporter.js b/runtime-nodejs/jest.kubernetes-assertions-reporter.js index 7d0a6b4..b8b57f4 100644 --- a/runtime-nodejs/jest.kubernetes-assertions-reporter.js +++ b/runtime-nodejs/jest.kubernetes-assertions-reporter.js @@ -37,6 +37,21 @@ const test_suites_run_total = new client.Counter({ help: 'inc\'d for every onRunComplete numTotalTestSuites' }); +const assert_files_seen = new client.Gauge({ + name: 'assert_files_seen', + help: 'Unique spec file paths that have been seen in onTestResult' +}); + +const assert_completions_passed = new client.Gauge({ + name: 'assert_completions_passed', + help: 'Gets a value if a spec names assert-completion.spec.js has run' +}); + +const assert_completions_remaining = new client.Gauge({ + name: 'assert_completions_remaining', + help: 'The number of failures for assert-completion.spec.js (pending and todo are ignored)' +}); + class MetricsServer { constructor({ port, getMetrics }) { @@ -75,6 +90,7 @@ class MetricsReporter { constructor(globalConfig, options) { this._globalConfig = globalConfig; this._options = options; + this._pathsSeen = {}; } onRunStart() { @@ -100,7 +116,27 @@ class MetricsReporter { } onTestResult(test, testResult, aggregatedResult) { - //console.log('onTestResult', testResult, aggregatedResult); + console.log('onTestResult', testResult); + const path = testResult.testFilePath; + if (!this._pathsSeen[path]) { + console.log('Saw new spec path', path); + this._pathsSeen[path] = {}; + assert_files_seen.inc(1); + } + if (this.isAssertCompletion(path)) { + this.onAssertCompletion(testResult); + } + } + + isAssertCompletion(testFilePath) { + const match = /.*\/assert-completion\.spec\.js$/.test(testFilePath); + console.log('Completion?', testFilePath, match); + return match; + } + + onAssertCompletion(testResult) { + assert_completions_passed.set(testResult.numPassingTests); + assert_completions_remaining.set(testResult.numFailingTests); } } diff --git a/runtime-nodejs/specs/assert-completion.spec.js b/runtime-nodejs/specs/assert-completion.spec.js new file mode 100644 index 0000000..ef96708 --- /dev/null +++ b/runtime-nodejs/specs/assert-completion.spec.js @@ -0,0 +1,24 @@ + +// TODO +const promFetch = require('node-fetch'); +const promValue = require('node-fetch'); + +// The recommended approach for assert_completions is ... +// - Check test health indirectly through Prometheus +// - For example existence of some key metric of the system under test +// - Actual system specs are done in properly names .spec.js files + +describe("Assert completion", () => { + + // This is quite imporant to keep up-to-date while we're learning non-interactive jest watch + const ASSERT_FILES_MIN = 2; + + it(`Has seen at least ${ASSERT_FILES_MIN} spec files`, () => { + // TODOOO + }); + + it("The runtime has produced metrics", () => { + + }); + +}); From a37053500670cc936b3b9fb05362bea5005ee20c Mon Sep 17 00:00:00 2001 From: Staffan Olsson Date: Fri, 5 Jun 2020 20:32:03 +0200 Subject: [PATCH 3/6] Adds prom helpers, primarily for assert-completion.spec.js --- .../src/custom-globals-examples.spec.js | 23 ++++++++++++++++ runtime-nodejs/jest.config.js | 4 +++ .../jest.kubernetes-assertions-reporter.js | 4 +-- runtime-nodejs/jest.setup-global-fetch.js | 23 ++++++++++++++++ runtime-nodejs/specs/custom-globals.spec.js | 27 +++++++++++++++++++ 5 files changed, 78 insertions(+), 3 deletions(-) create mode 100644 runtime-nodejs/example-project/src/custom-globals-examples.spec.js create mode 100644 runtime-nodejs/jest.setup-global-fetch.js create mode 100644 runtime-nodejs/specs/custom-globals.spec.js diff --git a/runtime-nodejs/example-project/src/custom-globals-examples.spec.js b/runtime-nodejs/example-project/src/custom-globals-examples.spec.js new file mode 100644 index 0000000..c19f5a8 --- /dev/null +++ b/runtime-nodejs/example-project/src/custom-globals-examples.spec.js @@ -0,0 +1,23 @@ +describe("Custom globals", () => { + + describe("promFetch", () => { + + it("Prepends to resource, the root URL to the relevant Prometheus instance", async () => { + const flags = await promFetch('/api/v1/status/flags').then(res => res.json()); + expect(flags).toHaveProperty('status','success'); + expect(flags.data['web.listen-address']).toEqual('0.0.0.0:9090'); + }); + + }); + + describe("promValue", () => { + + it("Returns a metric value as Number via Promise", async () => { + const pod = process.env.POD_NAME; + const completions = await promValue(`assert_completions_remaining{pod="${pod}"}`); + expect(completions).toBeGreaterThanOrEqual(0); + }); + + }); + +}); diff --git a/runtime-nodejs/jest.config.js b/runtime-nodejs/jest.config.js index 197f4c3..9c7254b 100644 --- a/runtime-nodejs/jest.config.js +++ b/runtime-nodejs/jest.config.js @@ -1,6 +1,10 @@ module.exports = { + testEnvironment: 'node', reporters: [ "default", ["/jest.kubernetes-assertions-reporter.js", {}] + ], + setupFilesAfterEnv: [ + "/jest.setup-global-fetch.js" ] }; diff --git a/runtime-nodejs/jest.kubernetes-assertions-reporter.js b/runtime-nodejs/jest.kubernetes-assertions-reporter.js index b8b57f4..e1fd8ec 100644 --- a/runtime-nodejs/jest.kubernetes-assertions-reporter.js +++ b/runtime-nodejs/jest.kubernetes-assertions-reporter.js @@ -116,10 +116,9 @@ class MetricsReporter { } onTestResult(test, testResult, aggregatedResult) { - console.log('onTestResult', testResult); + //console.log('onTestResult', testResult); const path = testResult.testFilePath; if (!this._pathsSeen[path]) { - console.log('Saw new spec path', path); this._pathsSeen[path] = {}; assert_files_seen.inc(1); } @@ -130,7 +129,6 @@ class MetricsReporter { isAssertCompletion(testFilePath) { const match = /.*\/assert-completion\.spec\.js$/.test(testFilePath); - console.log('Completion?', testFilePath, match); return match; } diff --git a/runtime-nodejs/jest.setup-global-fetch.js b/runtime-nodejs/jest.setup-global-fetch.js new file mode 100644 index 0000000..097fb5c --- /dev/null +++ b/runtime-nodejs/jest.setup-global-fetch.js @@ -0,0 +1,23 @@ +global.fetch = require('node-fetch'); + +global.prometheusHost = process.env.PROMETHEUS_HOST + || 'http://prometheus-now.monitoring:9090'; + +global.promFetch = (resource, init) => fetch(global.prometheusHost + resource, init); + +// Good enough errors for test failure outputs +const err = (message, data) => { + return "promValue failed: " + message + " " + JSON.stringify(data); +} + +global.promValue = async query => { + const req = promFetch(`/api/v1/query?query=${query}`); + const json = await req.then(res => res.json()); + if (json.status != "success") return err("Expected status=success", json); + const result = json.data.result; + if (!result.length) return err("Empty result", json); + if (result.length > 1) return err("Too many results", json); + const value = result[0].value[1]; + if (isNaN(value)) return err("Not a number", value); + return Number(value); +} diff --git a/runtime-nodejs/specs/custom-globals.spec.js b/runtime-nodejs/specs/custom-globals.spec.js new file mode 100644 index 0000000..19de2cd --- /dev/null +++ b/runtime-nodejs/specs/custom-globals.spec.js @@ -0,0 +1,27 @@ +describe("Custom globals", () => { + + describe("fetch", () => { + + it("Makes node-fetch global like in browsers", () => { + expect(fetch).toBeInstanceOf(Function); + }); + + }); + + describe("promFetch", () => { + + it("Wraps fetch", () => { + expect(promFetch).toBeInstanceOf(Function); + }); + + }); + + describe("promValue", () => { + + it("Is a function", () => { + expect(promValue).toBeInstanceOf(Function); + }); + + }); + +}); From 594d9886eca5175b90dcd96584578b8a42d5fc26 Mon Sep 17 00:00:00 2001 From: Staffan Olsson Date: Fri, 5 Jun 2020 21:10:32 +0200 Subject: [PATCH 4/6] Fixes a flaw with exporting specs file count --- .../jest.kubernetes-assertions-reporter.js | 3 ++- runtime-nodejs/specs/assert-completion.spec.js | 13 ++++++++----- 2 files changed, 10 insertions(+), 6 deletions(-) diff --git a/runtime-nodejs/jest.kubernetes-assertions-reporter.js b/runtime-nodejs/jest.kubernetes-assertions-reporter.js index e1fd8ec..8151351 100644 --- a/runtime-nodejs/jest.kubernetes-assertions-reporter.js +++ b/runtime-nodejs/jest.kubernetes-assertions-reporter.js @@ -113,6 +113,8 @@ class MetricsReporter { //console.log('Not a watch run. Exiting'); server.stop(); } + // onRunComplete seems to happen before each onTestResult so this placement isn't great + assert_files_seen.set(Object.keys(this._pathsSeen).length); } onTestResult(test, testResult, aggregatedResult) { @@ -120,7 +122,6 @@ class MetricsReporter { const path = testResult.testFilePath; if (!this._pathsSeen[path]) { this._pathsSeen[path] = {}; - assert_files_seen.inc(1); } if (this.isAssertCompletion(path)) { this.onAssertCompletion(testResult); diff --git a/runtime-nodejs/specs/assert-completion.spec.js b/runtime-nodejs/specs/assert-completion.spec.js index ef96708..f6155f3 100644 --- a/runtime-nodejs/specs/assert-completion.spec.js +++ b/runtime-nodejs/specs/assert-completion.spec.js @@ -10,11 +10,14 @@ const promValue = require('node-fetch'); describe("Assert completion", () => { - // This is quite imporant to keep up-to-date while we're learning non-interactive jest watch - const ASSERT_FILES_MIN = 2; - - it(`Has seen at least ${ASSERT_FILES_MIN} spec files`, () => { - // TODOOO + describe("Suite size", () => { + // This is quite imporant to keep up-to-date while we're learning non-interactive jest watch + const ASSERT_FILES_MIN = 2; + + it(`Has seen at least ${ASSERT_FILES_MIN} spec files`, async () => { + //expect(await promValue(`assert_files_seen{pod="${process.env.POD_NAME}"}`)) + // .toBeGreaterThanOrEqual(ASSERT_FILES_MIN); + }); }); it("The runtime has produced metrics", () => { From 0d9b066b71bb68164ce9fb4c14992998528a12e9 Mon Sep 17 00:00:00 2001 From: Staffan Olsson Date: Fri, 5 Jun 2020 21:13:27 +0200 Subject: [PATCH 5/6] Done --- runtime-nodejs/specs/assert-completion.spec.js | 4 ---- 1 file changed, 4 deletions(-) diff --git a/runtime-nodejs/specs/assert-completion.spec.js b/runtime-nodejs/specs/assert-completion.spec.js index f6155f3..832c4cb 100644 --- a/runtime-nodejs/specs/assert-completion.spec.js +++ b/runtime-nodejs/specs/assert-completion.spec.js @@ -1,8 +1,4 @@ -// TODO -const promFetch = require('node-fetch'); -const promValue = require('node-fetch'); - // The recommended approach for assert_completions is ... // - Check test health indirectly through Prometheus // - For example existence of some key metric of the system under test From 5655407824fab85b32afa4884306a428776acb79 Mon Sep 17 00:00:00 2001 From: Staffan Olsson Date: Fri, 5 Jun 2020 21:17:47 +0200 Subject: [PATCH 6/6] Adapts to new globals --- runtime-nodejs/example-project/src/metrics.spec.js | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/runtime-nodejs/example-project/src/metrics.spec.js b/runtime-nodejs/example-project/src/metrics.spec.js index 517733d..e5b2e24 100644 --- a/runtime-nodejs/example-project/src/metrics.spec.js +++ b/runtime-nodejs/example-project/src/metrics.spec.js @@ -1,11 +1,15 @@ -const fetch = require('node-fetch'); describe("A sub-module", () => { - it("Can use the runtime's default dependencies", () => { + it("Can use the runtime's global helpers", () => { expect(fetch).toBeDefined(); }); + it("Can use the runtime's default dependencies", () => { + expect(require('node-fetch')).toBeDefined(); + expect(require('@kubernetes/client-node')).toBeDefined(); + }); + it("Gets the metrics reporter despite having its own package.json", async () => { const metrics = await fetch('http://localhost:9090/metrics').then(res => res.text()); expect(metrics).toMatch(/\bassertions_failed /);