From f0e22cb5df4983a6e9849a5cc3683efea18adf8a Mon Sep 17 00:00:00 2001 From: Andrew Date: Fri, 15 Dec 2023 10:09:20 -0500 Subject: [PATCH 1/8] adds xmlwriter dependency for writing junit files --- package-lock.json | 90 ++++++++++++--------------------- packages/bruno-cli/package.json | 3 +- 2 files changed, 35 insertions(+), 58 deletions(-) diff --git a/package-lock.json b/package-lock.json index f09a7bcdb..156cb2555 100644 --- a/package-lock.json +++ b/package-lock.json @@ -14315,6 +14315,7 @@ }, "node_modules/react-is": { "version": "18.2.0", + "dev": true, "license": "MIT" }, "node_modules/react-pdf": { @@ -17310,7 +17311,6 @@ }, "node_modules/xmlbuilder": { "version": "15.1.1", - "devOptional": true, "license": "MIT", "engines": { "node": ">=8.0" @@ -17600,7 +17600,7 @@ }, "packages/bruno-cli": { "name": "@usebruno/cli", - "version": "1.2.1", + "version": "1.2.2", "license": "MIT", "dependencies": { "@usebruno/js": "0.9.4", @@ -17619,6 +17619,7 @@ "mustache": "^4.2.0", "qs": "^6.11.0", "socks-proxy-agent": "^8.0.2", + "xmlbuilder": "^15.1.1", "yargs": "^17.6.2" }, "bin": { @@ -21435,8 +21436,7 @@ } }, "@tabler/icons": { - "version": "1.119.0", - "requires": {} + "version": "1.119.0" }, "@tippyjs/react": { "version": "4.2.6", @@ -21840,6 +21840,7 @@ "mustache": "^4.2.0", "qs": "^6.11.0", "socks-proxy-agent": "^8.0.2", + "xmlbuilder": "^15.1.1", "yargs": "^17.6.2" }, "dependencies": { @@ -21989,8 +21990,7 @@ } }, "@usebruno/schema": { - "version": "file:packages/bruno-schema", - "requires": {} + "version": "file:packages/bruno-schema" }, "@usebruno/testbench": { "version": "file:packages/bruno-testbench", @@ -22134,8 +22134,7 @@ }, "@webpack-cli/configtest": { "version": "1.2.0", - "dev": true, - "requires": {} + "dev": true }, "@webpack-cli/info": { "version": "1.5.0", @@ -22146,8 +22145,7 @@ }, "@webpack-cli/serve": { "version": "1.7.0", - "dev": true, - "requires": {} + "dev": true }, "@xtuc/ieee754": { "version": "1.2.0", @@ -22229,8 +22227,7 @@ }, "ajv-keywords": { "version": "3.5.2", - "dev": true, - "requires": {} + "dev": true }, "amdefine": { "version": "0.0.8" @@ -23179,8 +23176,7 @@ "chai-string": { "version": "1.5.0", "resolved": "https://registry.npmjs.org/chai-string/-/chai-string-1.5.0.tgz", - "integrity": "sha512-sydDC3S3pNAQMYwJrs6dQX0oBQ6KfIPuOZ78n7rocW0eJJlsHPh2t3kwW7xfwYA/1Bf6/arGtSUo16rxR2JFlw==", - "requires": {} + "integrity": "sha512-sydDC3S3pNAQMYwJrs6dQX0oBQ6KfIPuOZ78n7rocW0eJJlsHPh2t3kwW7xfwYA/1Bf6/arGtSUo16rxR2JFlw==" }, "chalk": { "version": "4.1.2", @@ -23615,8 +23611,7 @@ }, "css-declaration-sorter": { "version": "6.3.1", - "dev": true, - "requires": {} + "dev": true }, "css-loader": { "version": "6.7.3", @@ -23735,8 +23730,7 @@ }, "cssnano-utils": { "version": "3.1.0", - "dev": true, - "requires": {} + "dev": true }, "csso": { "version": "4.2.0", @@ -24963,8 +24957,7 @@ } }, "goober": { - "version": "2.1.11", - "requires": {} + "version": "2.1.11" }, "gopd": { "version": "1.0.1", @@ -25358,8 +25351,7 @@ }, "icss-utils": { "version": "5.1.0", - "dev": true, - "requires": {} + "dev": true }, "idb": { "version": "7.1.1" @@ -25959,8 +25951,7 @@ }, "jest-pnp-resolver": { "version": "1.2.3", - "dev": true, - "requires": {} + "dev": true }, "jest-regex-util": { "version": "29.2.0", @@ -26679,8 +26670,7 @@ "merge-refs": { "version": "1.2.2", "resolved": "https://registry.npmjs.org/merge-refs/-/merge-refs-1.2.2.tgz", - "integrity": "sha512-RwcT7GsQR3KbuLw1rRuodq4Nt547BKEBkliZ0qqsrpyNne9bGTFtsFIsIpx82huWhcl3kOlOlH4H0xkPk/DqVw==", - "requires": {} + "integrity": "sha512-RwcT7GsQR3KbuLw1rRuodq4Nt547BKEBkliZ0qqsrpyNne9bGTFtsFIsIpx82huWhcl3kOlOlH4H0xkPk/DqVw==" }, "merge-stream": { "version": "2.0.0", @@ -26690,8 +26680,7 @@ "version": "1.4.1" }, "meros": { - "version": "1.2.1", - "requires": {} + "version": "1.2.1" }, "methods": { "version": "1.1.2" @@ -27568,23 +27557,19 @@ }, "postcss-discard-comments": { "version": "5.1.2", - "dev": true, - "requires": {} + "dev": true }, "postcss-discard-duplicates": { "version": "5.1.0", - "dev": true, - "requires": {} + "dev": true }, "postcss-discard-empty": { "version": "5.1.1", - "dev": true, - "requires": {} + "dev": true }, "postcss-discard-overridden": { "version": "5.1.0", - "dev": true, - "requires": {} + "dev": true }, "postcss-js": { "version": "3.0.3", @@ -27666,8 +27651,7 @@ }, "postcss-modules-extract-imports": { "version": "3.0.0", - "dev": true, - "requires": {} + "dev": true }, "postcss-modules-local-by-default": { "version": "4.0.0", @@ -27700,8 +27684,7 @@ }, "postcss-normalize-charset": { "version": "5.1.0", - "dev": true, - "requires": {} + "dev": true }, "postcss-normalize-display-values": { "version": "5.1.0", @@ -28186,11 +28169,11 @@ } }, "react-inspector": { - "version": "6.0.2", - "requires": {} + "version": "6.0.2" }, "react-is": { - "version": "18.2.0" + "version": "18.2.0", + "dev": true }, "react-pdf": { "version": "7.5.1", @@ -28352,8 +28335,7 @@ } }, "redux-thunk": { - "version": "2.4.2", - "requires": {} + "version": "2.4.2" }, "regenerate": { "version": "1.4.2", @@ -28591,8 +28573,7 @@ }, "rollup-plugin-peer-deps-external": { "version": "2.2.4", - "dev": true, - "requires": {} + "dev": true }, "rollup-plugin-postcss": { "version": "4.0.2", @@ -29152,8 +29133,7 @@ }, "style-loader": { "version": "3.3.1", - "dev": true, - "requires": {} + "dev": true }, "styled-components": { "version": "5.3.6", @@ -29182,8 +29162,7 @@ } }, "styled-jsx": { - "version": "5.0.7", - "requires": {} + "version": "5.0.7" }, "stylehacks": { "version": "5.1.1", @@ -29856,8 +29835,7 @@ } }, "use-sync-external-store": { - "version": "1.2.0", - "requires": {} + "version": "1.2.0" }, "utf8-byte-length": { "version": "1.0.4", @@ -30018,8 +29996,7 @@ }, "acorn-import-assertions": { "version": "1.8.0", - "dev": true, - "requires": {} + "dev": true }, "schema-utils": { "version": "3.1.1", @@ -30168,8 +30145,7 @@ } }, "xmlbuilder": { - "version": "15.1.1", - "devOptional": true + "version": "15.1.1" }, "xtend": { "version": "4.0.2" diff --git a/packages/bruno-cli/package.json b/packages/bruno-cli/package.json index 41b586a5e..c19c8c8ae 100644 --- a/packages/bruno-cli/package.json +++ b/packages/bruno-cli/package.json @@ -1,6 +1,6 @@ { "name": "@usebruno/cli", - "version": "1.2.1", + "version": "1.2.2", "license": "MIT", "main": "src/index.js", "bin": { @@ -40,6 +40,7 @@ "mustache": "^4.2.0", "qs": "^6.11.0", "socks-proxy-agent": "^8.0.2", + "xmlbuilder": "^15.1.1", "yargs": "^17.6.2" } } From 2103ab20bf5b624234679632821f81e23b9a98f3 Mon Sep 17 00:00:00 2001 From: Andrew Date: Fri, 15 Dec 2023 10:09:20 -0500 Subject: [PATCH 2/8] implements a reporter flag w/ junit reporter type --- packages/bruno-cli/src/commands/run.js | 108 +++++++++++++++++++++++-- 1 file changed, 102 insertions(+), 6 deletions(-) diff --git a/packages/bruno-cli/src/commands/run.js b/packages/bruno-cli/src/commands/run.js index d1eea59a8..6fee148fe 100644 --- a/packages/bruno-cli/src/commands/run.js +++ b/packages/bruno-cli/src/commands/run.js @@ -1,7 +1,9 @@ const fs = require('fs'); const chalk = require('chalk'); +const os = require('os'); const path = require('path'); -const { forOwn } = require('lodash'); +const xmlbuilder = require('xmlbuilder'); +const { forOwn, result } = require('lodash'); const { exists, isFile, isDirectory } = require('../utils/filesystem'); const { runSingleRequest } = require('../runner/run-single-request'); const { bruToEnvJson, getEnvVars } = require('../utils/bru'); @@ -165,6 +167,73 @@ const getCollectionRoot = (dir) => { return collectionBruToJson(content); }; +const makeJunitOutput = async (results, outputPath) => { + const output = { + testsuites: { + testsuite: [] + } + }; + + results.forEach((result, idx) => { + const assertionTestCount = result.assertionResults ? result.assertionResults.length : 0; + const testCount = result.testResults ? result.testResults.length : 0; + const totalTests = assertionTestCount + testCount; + + const suite = { + '@name': result.suitename, + '@errors': 0, + '@failures': 0, + '@skipped': 0, + '@tests': totalTests, + '@timestamp': new Date().toISOString().split('Z')[0], + '@hostname': os.hostname(), + '@time': result.runtime.toFixed(3), + testcase: [] + }; + + result.assertionResults && + result.assertionResults.forEach((assertion) => { + const testcase = { + '@name': `${assertion.lhsExpr} ${assertion.rhsExpr}`, + '@status': assertion.status, + '@classname': result.request.url, + '@time': (result.runtime / totalTests).toFixed(3) + }; + + if (assertion.status === 'fail') { + suite['@failures']++; + + testcase.failure = [{ '@type': 'failure', '@message': assertion.error }]; + } + + suite.testcase.push(testcase); + }); + + result.testResults && + result.testResults.forEach((test) => { + const testcase = { + '@type': 'testcase', + '@name': test.description, + '@status': test.status, + '@classname': result.request.url, + '@time': (result.runtime / totalTests).toFixed(3) + }; + + if (test.status === 'fail') { + suite['@failures']++; + + testcase.failure = [{ '@type': 'failure', '@message': test.error }]; + } + + suite.testcase.push(testcase); + }); + + output.testsuites.testsuite.push(suite); + }); + + fs.writeFileSync(outputPath, xmlbuilder.create(output).end({ pretty: true })); +}; + const builder = async (yargs) => { yargs .option('r', { @@ -186,7 +255,13 @@ const builder = async (yargs) => { }) .option('output', { alias: 'o', - describe: 'Path to write JSON results to', + describe: 'Path to write file results to', + type: 'string' + }) + .option('format', { + alias: 'f', + describe: 'Format for the file results', + default: 'json', type: 'string' }) .option('insecure', { @@ -204,12 +279,16 @@ const builder = async (yargs) => { .example( '$0 run request.bru --output results.json', 'Run a request and write the results to results.json in the current directory' + ) + .example( + '$0 run request.bru --output results.xml --format junit', + 'Run a request and write the results to results.xml in junit format in the current directory' ); }; const handler = async function (argv) { try { - let { filename, cacert, env, envVar, insecure, r: recursive, output: outputPath } = argv; + let { filename, cacert, env, envVar, insecure, r: recursive, output: outputPath, format } = argv; const collectionPath = process.cwd(); // todo @@ -297,6 +376,11 @@ const handler = async function (argv) { } } + if (['json', 'junit'].indexOf(format) === -1) { + console.error(chalk.red(`Format must be one of "json" or "junit"`)); + return; + } + // load .env file at root of collection if it exists const dotEnvPath = path.join(collectionPath, '.env'); const dotEnvExists = await exists(dotEnvPath); @@ -360,6 +444,8 @@ const handler = async function (argv) { while (currentRequestIndex < bruJsons.length) { const iter = bruJsons[currentRequestIndex]; const { bruFilepath, bruJson } = iter; + + const start = process.hrtime(); const result = await runSingleRequest( bruFilepath, bruJson, @@ -371,7 +457,11 @@ const handler = async function (argv) { collectionRoot ); - results.push(result); + results.push({ + ...result, + runtime: process.hrtime(start)[0] + process.hrtime(start)[1] / 1e9, + suitename: bruFilepath.replace('.bru', '') + }); // determine next request const nextRequestName = result?.nextRequestName; @@ -413,7 +503,12 @@ const handler = async function (argv) { results }; - fs.writeFileSync(outputPath, JSON.stringify(outputJson, null, 2)); + if (format === 'json') { + fs.writeFileSync(outputPath, JSON.stringify(outputJson, null, 2)); + } else if (format === 'junit') { + makeJunitOutput(results, outputPath); + } + console.log(chalk.dim(chalk.grey(`Wrote results to ${outputPath}`))); } @@ -432,5 +527,6 @@ module.exports = { desc, builder, handler, - printRunSummary + printRunSummary, + makeJunitOutput }; From 174f99f9fbfa8688ff53f961bb447a2e866b56f4 Mon Sep 17 00:00:00 2001 From: Andrew Date: Fri, 15 Dec 2023 10:09:20 -0500 Subject: [PATCH 3/8] test coverage for junit reporter --- packages/bruno-cli/tests/commands/run.spec.js | 223 +++++++++++++++++- 1 file changed, 222 insertions(+), 1 deletion(-) diff --git a/packages/bruno-cli/tests/commands/run.spec.js b/packages/bruno-cli/tests/commands/run.spec.js index 10cdf42b4..0c34bd373 100644 --- a/packages/bruno-cli/tests/commands/run.spec.js +++ b/packages/bruno-cli/tests/commands/run.spec.js @@ -1,6 +1,8 @@ const { describe, it, expect } = require('@jest/globals'); +const xmlbuilder = require('xmlbuilder'); +const fs = require('fs'); -const { printRunSummary } = require('../../src/commands/run'); +const { printRunSummary, makeJunitOutput } = require('../../src/commands/run'); describe('printRunSummary', () => { // Suppress console.log output @@ -65,3 +67,222 @@ describe('printRunSummary', () => { expect(summary.failedTests).toBe(2); }); }); + +describe('printRunSummary', () => { + // Suppress console.log output + jest.spyOn(console, 'log').mockImplementation(() => {}); + + it('should produce the correct summary for a successful run', () => { + const results = [ + { + testResults: [{ status: 'pass' }, { status: 'pass' }, { status: 'pass' }], + assertionResults: [{ status: 'pass' }, { status: 'pass' }], + error: null + }, + { + testResults: [{ status: 'pass' }, { status: 'pass' }], + assertionResults: [{ status: 'pass' }, { status: 'pass' }, { status: 'pass' }], + error: null + } + ]; + + const summary = printRunSummary(results); + + expect(summary.totalRequests).toBe(2); + expect(summary.passedRequests).toBe(2); + expect(summary.failedRequests).toBe(0); + expect(summary.totalAssertions).toBe(5); + expect(summary.passedAssertions).toBe(5); + expect(summary.failedAssertions).toBe(0); + expect(summary.totalTests).toBe(5); + expect(summary.passedTests).toBe(5); + expect(summary.failedTests).toBe(0); + }); + + it('should produce the correct summary for a failed run', () => { + const results = [ + { + testResults: [{ status: 'fail' }, { status: 'pass' }, { status: 'pass' }], + assertionResults: [{ status: 'pass' }, { status: 'fail' }], + error: null + }, + { + testResults: [{ status: 'pass' }, { status: 'fail' }], + assertionResults: [{ status: 'pass' }, { status: 'fail' }, { status: 'fail' }], + error: null + }, + { + testResults: [], + assertionResults: [], + error: new Error('Request failed') + } + ]; + + const summary = printRunSummary(results); + + expect(summary.totalRequests).toBe(3); + expect(summary.passedRequests).toBe(2); + expect(summary.failedRequests).toBe(1); + expect(summary.totalAssertions).toBe(5); + expect(summary.passedAssertions).toBe(2); + expect(summary.failedAssertions).toBe(3); + expect(summary.totalTests).toBe(5); + expect(summary.passedTests).toBe(3); + expect(summary.failedTests).toBe(2); + }); +}); + +describe('printRunSummary', () => { + // Suppress console.log output + jest.spyOn(console, 'log').mockImplementation(() => {}); + + it('should produce the correct summary for a successful run', () => { + const results = [ + { + testResults: [{ status: 'pass' }, { status: 'pass' }, { status: 'pass' }], + assertionResults: [{ status: 'pass' }, { status: 'pass' }], + error: null + }, + { + testResults: [{ status: 'pass' }, { status: 'pass' }], + assertionResults: [{ status: 'pass' }, { status: 'pass' }, { status: 'pass' }], + error: null + } + ]; + + const summary = printRunSummary(results); + + expect(summary.totalRequests).toBe(2); + expect(summary.passedRequests).toBe(2); + expect(summary.failedRequests).toBe(0); + expect(summary.totalAssertions).toBe(5); + expect(summary.passedAssertions).toBe(5); + expect(summary.failedAssertions).toBe(0); + expect(summary.totalTests).toBe(5); + expect(summary.passedTests).toBe(5); + expect(summary.failedTests).toBe(0); + }); + + it('should produce the correct summary for a failed run', () => { + const results = [ + { + testResults: [{ status: 'fail' }, { status: 'pass' }, { status: 'pass' }], + assertionResults: [{ status: 'pass' }, { status: 'fail' }], + error: null + }, + { + testResults: [{ status: 'pass' }, { status: 'fail' }], + assertionResults: [{ status: 'pass' }, { status: 'fail' }, { status: 'fail' }], + error: null + }, + { + testResults: [], + assertionResults: [], + error: new Error('Request failed') + } + ]; + + const summary = printRunSummary(results); + + expect(summary.totalRequests).toBe(3); + expect(summary.passedRequests).toBe(2); + expect(summary.failedRequests).toBe(1); + expect(summary.totalAssertions).toBe(5); + expect(summary.passedAssertions).toBe(2); + expect(summary.failedAssertions).toBe(3); + expect(summary.totalTests).toBe(5); + expect(summary.passedTests).toBe(3); + expect(summary.failedTests).toBe(2); + }); +}); + +describe('makeJUnitOutput', () => { + let createStub = jest.fn(); + + beforeEach(() => { + jest.spyOn(xmlbuilder, 'create').mockImplementation(() => { + return { end: createStub }; + }); + jest.spyOn(fs, 'writeFileSync').mockImplementation(() => {}); + }); + + afterEach(() => { + jest.restoreAllMocks(); + }); + + it('should produce a junit spec object for serialization', () => { + const results = [ + { + description: 'description provided', + suitename: 'Tests/Suite A', + request: { + method: 'GET', + url: 'https://ima.test' + }, + assertionResults: [ + { + lhsExpr: 'res.status', + rhsExpr: 'eq 200', + status: 'pass' + }, + { + lhsExpr: 'res.status', + rhsExpr: 'neq 200', + status: 'fail', + error: 'expected 200 to not equal 200' + } + ], + runtime: 1.2345678 + }, + { + request: { + method: 'GET', + url: 'https://imanother.test' + }, + suitename: 'Tests/Suite B', + testResults: [ + { + lhsExpr: 'res.status', + rhsExpr: 'eq 200', + description: 'A test that passes', + status: 'pass' + }, + { + description: 'A test that fails', + status: 'fail', + error: 'expected 200 to not equal 200', + status: 'fail' + } + ], + runtime: 2.3456789 + } + ]; + + makeJunitOutput(results, '/tmp/testfile.xml'); + expect(createStub).toBeCalled; + + const junit = xmlbuilder.create.mock.calls[0][0]; + + expect(junit.testsuites).toBeDefined; + expect(junit.testsuites.testsuite.length).toBe(2); + expect(junit.testsuites.testsuite[0].testcase.length).toBe(2); + expect(junit.testsuites.testsuite[1].testcase.length).toBe(2); + + expect(junit.testsuites.testsuite[0]['@name']).toBe('Tests/Suite A'); + expect(junit.testsuites.testsuite[1]['@name']).toBe('Tests/Suite B'); + + expect(junit.testsuites.testsuite[0]['@tests']).toBe(2); + expect(junit.testsuites.testsuite[1]['@tests']).toBe(2); + + const testcase = junit.testsuites.testsuite[0].testcase[0]; + + expect(testcase['@name']).toBe('res.status eq 200'); + expect(testcase['@status']).toBe('pass'); + + const failcase = junit.testsuites.testsuite[0].testcase[1]; + + expect(failcase['@name']).toBe('res.status neq 200'); + expect(failcase.failure).toBeDefined; + expect(failcase.failure[0]['@type']).toBe('failure'); + }); +}); From 882341c35be7f30388f5fa9e063044dc79cbd179 Mon Sep 17 00:00:00 2001 From: Andrew Date: Fri, 15 Dec 2023 10:09:20 -0500 Subject: [PATCH 4/8] remove duplicated test suite --- packages/bruno-cli/tests/commands/run.spec.js | 128 ------------------ 1 file changed, 128 deletions(-) diff --git a/packages/bruno-cli/tests/commands/run.spec.js b/packages/bruno-cli/tests/commands/run.spec.js index 0c34bd373..7b69da7f8 100644 --- a/packages/bruno-cli/tests/commands/run.spec.js +++ b/packages/bruno-cli/tests/commands/run.spec.js @@ -68,134 +68,6 @@ describe('printRunSummary', () => { }); }); -describe('printRunSummary', () => { - // Suppress console.log output - jest.spyOn(console, 'log').mockImplementation(() => {}); - - it('should produce the correct summary for a successful run', () => { - const results = [ - { - testResults: [{ status: 'pass' }, { status: 'pass' }, { status: 'pass' }], - assertionResults: [{ status: 'pass' }, { status: 'pass' }], - error: null - }, - { - testResults: [{ status: 'pass' }, { status: 'pass' }], - assertionResults: [{ status: 'pass' }, { status: 'pass' }, { status: 'pass' }], - error: null - } - ]; - - const summary = printRunSummary(results); - - expect(summary.totalRequests).toBe(2); - expect(summary.passedRequests).toBe(2); - expect(summary.failedRequests).toBe(0); - expect(summary.totalAssertions).toBe(5); - expect(summary.passedAssertions).toBe(5); - expect(summary.failedAssertions).toBe(0); - expect(summary.totalTests).toBe(5); - expect(summary.passedTests).toBe(5); - expect(summary.failedTests).toBe(0); - }); - - it('should produce the correct summary for a failed run', () => { - const results = [ - { - testResults: [{ status: 'fail' }, { status: 'pass' }, { status: 'pass' }], - assertionResults: [{ status: 'pass' }, { status: 'fail' }], - error: null - }, - { - testResults: [{ status: 'pass' }, { status: 'fail' }], - assertionResults: [{ status: 'pass' }, { status: 'fail' }, { status: 'fail' }], - error: null - }, - { - testResults: [], - assertionResults: [], - error: new Error('Request failed') - } - ]; - - const summary = printRunSummary(results); - - expect(summary.totalRequests).toBe(3); - expect(summary.passedRequests).toBe(2); - expect(summary.failedRequests).toBe(1); - expect(summary.totalAssertions).toBe(5); - expect(summary.passedAssertions).toBe(2); - expect(summary.failedAssertions).toBe(3); - expect(summary.totalTests).toBe(5); - expect(summary.passedTests).toBe(3); - expect(summary.failedTests).toBe(2); - }); -}); - -describe('printRunSummary', () => { - // Suppress console.log output - jest.spyOn(console, 'log').mockImplementation(() => {}); - - it('should produce the correct summary for a successful run', () => { - const results = [ - { - testResults: [{ status: 'pass' }, { status: 'pass' }, { status: 'pass' }], - assertionResults: [{ status: 'pass' }, { status: 'pass' }], - error: null - }, - { - testResults: [{ status: 'pass' }, { status: 'pass' }], - assertionResults: [{ status: 'pass' }, { status: 'pass' }, { status: 'pass' }], - error: null - } - ]; - - const summary = printRunSummary(results); - - expect(summary.totalRequests).toBe(2); - expect(summary.passedRequests).toBe(2); - expect(summary.failedRequests).toBe(0); - expect(summary.totalAssertions).toBe(5); - expect(summary.passedAssertions).toBe(5); - expect(summary.failedAssertions).toBe(0); - expect(summary.totalTests).toBe(5); - expect(summary.passedTests).toBe(5); - expect(summary.failedTests).toBe(0); - }); - - it('should produce the correct summary for a failed run', () => { - const results = [ - { - testResults: [{ status: 'fail' }, { status: 'pass' }, { status: 'pass' }], - assertionResults: [{ status: 'pass' }, { status: 'fail' }], - error: null - }, - { - testResults: [{ status: 'pass' }, { status: 'fail' }], - assertionResults: [{ status: 'pass' }, { status: 'fail' }, { status: 'fail' }], - error: null - }, - { - testResults: [], - assertionResults: [], - error: new Error('Request failed') - } - ]; - - const summary = printRunSummary(results); - - expect(summary.totalRequests).toBe(3); - expect(summary.passedRequests).toBe(2); - expect(summary.failedRequests).toBe(1); - expect(summary.totalAssertions).toBe(5); - expect(summary.passedAssertions).toBe(2); - expect(summary.failedAssertions).toBe(3); - expect(summary.totalTests).toBe(5); - expect(summary.passedTests).toBe(3); - expect(summary.failedTests).toBe(2); - }); -}); - describe('makeJUnitOutput', () => { let createStub = jest.fn(); From de530a889c97c6ed235ecb16d3a77cf2c456f639 Mon Sep 17 00:00:00 2001 From: Andrew Date: Fri, 15 Dec 2023 10:09:20 -0500 Subject: [PATCH 5/8] adding request-level error reporting --- packages/bruno-cli/src/commands/run.js | 17 +++++++- packages/bruno-cli/tests/commands/run.spec.js | 39 +++++++++++++++++++ 2 files changed, 54 insertions(+), 2 deletions(-) diff --git a/packages/bruno-cli/src/commands/run.js b/packages/bruno-cli/src/commands/run.js index 6fee148fe..42b45b637 100644 --- a/packages/bruno-cli/src/commands/run.js +++ b/packages/bruno-cli/src/commands/run.js @@ -174,7 +174,7 @@ const makeJunitOutput = async (results, outputPath) => { } }; - results.forEach((result, idx) => { + results.forEach((result) => { const assertionTestCount = result.assertionResults ? result.assertionResults.length : 0; const testCount = result.testResults ? result.testResults.length : 0; const totalTests = assertionTestCount + testCount; @@ -212,7 +212,6 @@ const makeJunitOutput = async (results, outputPath) => { result.testResults && result.testResults.forEach((test) => { const testcase = { - '@type': 'testcase', '@name': test.description, '@status': test.status, '@classname': result.request.url, @@ -228,6 +227,20 @@ const makeJunitOutput = async (results, outputPath) => { suite.testcase.push(testcase); }); + if (result.error) { + suite['@errors'] = 1; + suite['@tests'] = 1; + suite.testcase = [ + { + '@name': 'Test suite has no errors', + '@status': 'fail', + '@classname': result.request.url, + '@time': result.runtime.toFixed(3), + error: [{ '@type': 'error', '@message': result.error }] + } + ]; + } + output.testsuites.testsuite.push(suite); }); diff --git a/packages/bruno-cli/tests/commands/run.spec.js b/packages/bruno-cli/tests/commands/run.spec.js index 7b69da7f8..375620885 100644 --- a/packages/bruno-cli/tests/commands/run.spec.js +++ b/packages/bruno-cli/tests/commands/run.spec.js @@ -157,4 +157,43 @@ describe('makeJUnitOutput', () => { expect(failcase.failure).toBeDefined; expect(failcase.failure[0]['@type']).toBe('failure'); }); + + it('should handle request errors', () => { + const results = [ + { + description: 'description provided', + suitename: 'Tests/Suite A', + request: { + method: 'GET', + url: 'https://ima.test' + }, + assertionResults: [ + { + lhsExpr: 'res.status', + rhsExpr: 'eq 200', + status: 'fail' + } + ], + runtime: 1.2345678, + error: 'timeout of 2000ms exceeded' + } + ]; + + makeJunitOutput(results, '/tmp/testfile.xml'); + + const junit = xmlbuilder.create.mock.calls[0][0]; + + expect(createStub).toBeCalled; + + expect(junit.testsuites).toBeDefined; + expect(junit.testsuites.testsuite.length).toBe(1); + expect(junit.testsuites.testsuite[0].testcase.length).toBe(1); + + const failcase = junit.testsuites.testsuite[0].testcase[0]; + + expect(failcase['@name']).toBe('Test suite has no errors'); + expect(failcase.error).toBeDefined; + expect(failcase.error[0]['@type']).toBe('error'); + expect(failcase.error[0]['@message']).toBe('timeout of 2000ms exceeded'); + }); }); From ab37e533460b90a7c45791e2fd6704ca64e00dba Mon Sep 17 00:00:00 2001 From: Andrew Winder Date: Fri, 15 Dec 2023 12:04:41 -0500 Subject: [PATCH 6/8] refactor for reporters directory --- packages/bruno-cli/src/commands/run.js | 88 +---------------------- packages/bruno-cli/src/reporters/junit.js | 85 ++++++++++++++++++++++ 2 files changed, 88 insertions(+), 85 deletions(-) create mode 100644 packages/bruno-cli/src/reporters/junit.js diff --git a/packages/bruno-cli/src/commands/run.js b/packages/bruno-cli/src/commands/run.js index 42b45b637..3e67dba1b 100644 --- a/packages/bruno-cli/src/commands/run.js +++ b/packages/bruno-cli/src/commands/run.js @@ -1,12 +1,11 @@ const fs = require('fs'); const chalk = require('chalk'); -const os = require('os'); const path = require('path'); -const xmlbuilder = require('xmlbuilder'); const { forOwn, result } = require('lodash'); const { exists, isFile, isDirectory } = require('../utils/filesystem'); const { runSingleRequest } = require('../runner/run-single-request'); const { bruToEnvJson, getEnvVars } = require('../utils/bru'); +const makeJUnitOutput = require('../reporters/junit'); const { rpad } = require('../utils/common'); const { bruToJson, getOptions, collectionBruToJson } = require('../utils/bru'); const { dotenvToJson } = require('@usebruno/lang'); @@ -167,86 +166,6 @@ const getCollectionRoot = (dir) => { return collectionBruToJson(content); }; -const makeJunitOutput = async (results, outputPath) => { - const output = { - testsuites: { - testsuite: [] - } - }; - - results.forEach((result) => { - const assertionTestCount = result.assertionResults ? result.assertionResults.length : 0; - const testCount = result.testResults ? result.testResults.length : 0; - const totalTests = assertionTestCount + testCount; - - const suite = { - '@name': result.suitename, - '@errors': 0, - '@failures': 0, - '@skipped': 0, - '@tests': totalTests, - '@timestamp': new Date().toISOString().split('Z')[0], - '@hostname': os.hostname(), - '@time': result.runtime.toFixed(3), - testcase: [] - }; - - result.assertionResults && - result.assertionResults.forEach((assertion) => { - const testcase = { - '@name': `${assertion.lhsExpr} ${assertion.rhsExpr}`, - '@status': assertion.status, - '@classname': result.request.url, - '@time': (result.runtime / totalTests).toFixed(3) - }; - - if (assertion.status === 'fail') { - suite['@failures']++; - - testcase.failure = [{ '@type': 'failure', '@message': assertion.error }]; - } - - suite.testcase.push(testcase); - }); - - result.testResults && - result.testResults.forEach((test) => { - const testcase = { - '@name': test.description, - '@status': test.status, - '@classname': result.request.url, - '@time': (result.runtime / totalTests).toFixed(3) - }; - - if (test.status === 'fail') { - suite['@failures']++; - - testcase.failure = [{ '@type': 'failure', '@message': test.error }]; - } - - suite.testcase.push(testcase); - }); - - if (result.error) { - suite['@errors'] = 1; - suite['@tests'] = 1; - suite.testcase = [ - { - '@name': 'Test suite has no errors', - '@status': 'fail', - '@classname': result.request.url, - '@time': result.runtime.toFixed(3), - error: [{ '@type': 'error', '@message': result.error }] - } - ]; - } - - output.testsuites.testsuite.push(suite); - }); - - fs.writeFileSync(outputPath, xmlbuilder.create(output).end({ pretty: true })); -}; - const builder = async (yargs) => { yargs .option('r', { @@ -519,7 +438,7 @@ const handler = async function (argv) { if (format === 'json') { fs.writeFileSync(outputPath, JSON.stringify(outputJson, null, 2)); } else if (format === 'junit') { - makeJunitOutput(results, outputPath); + makeJUnitOutput(results, outputPath); } console.log(chalk.dim(chalk.grey(`Wrote results to ${outputPath}`))); @@ -540,6 +459,5 @@ module.exports = { desc, builder, handler, - printRunSummary, - makeJunitOutput + printRunSummary }; diff --git a/packages/bruno-cli/src/reporters/junit.js b/packages/bruno-cli/src/reporters/junit.js new file mode 100644 index 000000000..30fb51939 --- /dev/null +++ b/packages/bruno-cli/src/reporters/junit.js @@ -0,0 +1,85 @@ +const os = require('os'); +const fs = require('fs'); +const xmlbuilder = require('xmlbuilder'); + +const makeJUnitOutput = async (results, outputPath) => { + const output = { + testsuites: { + testsuite: [] + } + }; + + results.forEach((result) => { + const assertionTestCount = result.assertionResults ? result.assertionResults.length : 0; + const testCount = result.testResults ? result.testResults.length : 0; + const totalTests = assertionTestCount + testCount; + + const suite = { + '@name': result.suitename, + '@errors': 0, + '@failures': 0, + '@skipped': 0, + '@tests': totalTests, + '@timestamp': new Date().toISOString().split('Z')[0], + '@hostname': os.hostname(), + '@time': result.runtime.toFixed(3), + testcase: [] + }; + + result.assertionResults && + result.assertionResults.forEach((assertion) => { + const testcase = { + '@name': `${assertion.lhsExpr} ${assertion.rhsExpr}`, + '@status': assertion.status, + '@classname': result.request.url, + '@time': (result.runtime / totalTests).toFixed(3) + }; + + if (assertion.status === 'fail') { + suite['@failures']++; + + testcase.failure = [{ '@type': 'failure', '@message': assertion.error }]; + } + + suite.testcase.push(testcase); + }); + + result.testResults && + result.testResults.forEach((test) => { + const testcase = { + '@name': test.description, + '@status': test.status, + '@classname': result.request.url, + '@time': (result.runtime / totalTests).toFixed(3) + }; + + if (test.status === 'fail') { + suite['@failures']++; + + testcase.failure = [{ '@type': 'failure', '@message': test.error }]; + } + + suite.testcase.push(testcase); + }); + + if (result.error) { + suite['@errors'] = 1; + suite['@tests'] = 1; + suite.testcase = [ + { + '@name': 'Test suite has no errors', + '@status': 'fail', + '@classname': result.request.url, + '@time': result.runtime.toFixed(3), + error: [{ '@type': 'error', '@message': result.error }] + } + ]; + } + + output.testsuites.testsuite.push(suite); + }); + + fs.writeFileSync(outputPath, xmlbuilder.create(output).end({ pretty: true })); +}; + +module.exports = makeJUnitOutput; From 99c2dd9030c339c05b67a44ac11e43fd00a2e52a Mon Sep 17 00:00:00 2001 From: Andrew Winder Date: Fri, 15 Dec 2023 12:04:52 -0500 Subject: [PATCH 7/8] test updates --- packages/bruno-cli/tests/commands/run.spec.js | 134 +---------------- .../bruno-cli/tests/reporters/junit.spec.js | 135 ++++++++++++++++++ 2 files changed, 136 insertions(+), 133 deletions(-) create mode 100644 packages/bruno-cli/tests/reporters/junit.spec.js diff --git a/packages/bruno-cli/tests/commands/run.spec.js b/packages/bruno-cli/tests/commands/run.spec.js index 375620885..10cdf42b4 100644 --- a/packages/bruno-cli/tests/commands/run.spec.js +++ b/packages/bruno-cli/tests/commands/run.spec.js @@ -1,8 +1,6 @@ const { describe, it, expect } = require('@jest/globals'); -const xmlbuilder = require('xmlbuilder'); -const fs = require('fs'); -const { printRunSummary, makeJunitOutput } = require('../../src/commands/run'); +const { printRunSummary } = require('../../src/commands/run'); describe('printRunSummary', () => { // Suppress console.log output @@ -67,133 +65,3 @@ describe('printRunSummary', () => { expect(summary.failedTests).toBe(2); }); }); - -describe('makeJUnitOutput', () => { - let createStub = jest.fn(); - - beforeEach(() => { - jest.spyOn(xmlbuilder, 'create').mockImplementation(() => { - return { end: createStub }; - }); - jest.spyOn(fs, 'writeFileSync').mockImplementation(() => {}); - }); - - afterEach(() => { - jest.restoreAllMocks(); - }); - - it('should produce a junit spec object for serialization', () => { - const results = [ - { - description: 'description provided', - suitename: 'Tests/Suite A', - request: { - method: 'GET', - url: 'https://ima.test' - }, - assertionResults: [ - { - lhsExpr: 'res.status', - rhsExpr: 'eq 200', - status: 'pass' - }, - { - lhsExpr: 'res.status', - rhsExpr: 'neq 200', - status: 'fail', - error: 'expected 200 to not equal 200' - } - ], - runtime: 1.2345678 - }, - { - request: { - method: 'GET', - url: 'https://imanother.test' - }, - suitename: 'Tests/Suite B', - testResults: [ - { - lhsExpr: 'res.status', - rhsExpr: 'eq 200', - description: 'A test that passes', - status: 'pass' - }, - { - description: 'A test that fails', - status: 'fail', - error: 'expected 200 to not equal 200', - status: 'fail' - } - ], - runtime: 2.3456789 - } - ]; - - makeJunitOutput(results, '/tmp/testfile.xml'); - expect(createStub).toBeCalled; - - const junit = xmlbuilder.create.mock.calls[0][0]; - - expect(junit.testsuites).toBeDefined; - expect(junit.testsuites.testsuite.length).toBe(2); - expect(junit.testsuites.testsuite[0].testcase.length).toBe(2); - expect(junit.testsuites.testsuite[1].testcase.length).toBe(2); - - expect(junit.testsuites.testsuite[0]['@name']).toBe('Tests/Suite A'); - expect(junit.testsuites.testsuite[1]['@name']).toBe('Tests/Suite B'); - - expect(junit.testsuites.testsuite[0]['@tests']).toBe(2); - expect(junit.testsuites.testsuite[1]['@tests']).toBe(2); - - const testcase = junit.testsuites.testsuite[0].testcase[0]; - - expect(testcase['@name']).toBe('res.status eq 200'); - expect(testcase['@status']).toBe('pass'); - - const failcase = junit.testsuites.testsuite[0].testcase[1]; - - expect(failcase['@name']).toBe('res.status neq 200'); - expect(failcase.failure).toBeDefined; - expect(failcase.failure[0]['@type']).toBe('failure'); - }); - - it('should handle request errors', () => { - const results = [ - { - description: 'description provided', - suitename: 'Tests/Suite A', - request: { - method: 'GET', - url: 'https://ima.test' - }, - assertionResults: [ - { - lhsExpr: 'res.status', - rhsExpr: 'eq 200', - status: 'fail' - } - ], - runtime: 1.2345678, - error: 'timeout of 2000ms exceeded' - } - ]; - - makeJunitOutput(results, '/tmp/testfile.xml'); - - const junit = xmlbuilder.create.mock.calls[0][0]; - - expect(createStub).toBeCalled; - - expect(junit.testsuites).toBeDefined; - expect(junit.testsuites.testsuite.length).toBe(1); - expect(junit.testsuites.testsuite[0].testcase.length).toBe(1); - - const failcase = junit.testsuites.testsuite[0].testcase[0]; - - expect(failcase['@name']).toBe('Test suite has no errors'); - expect(failcase.error).toBeDefined; - expect(failcase.error[0]['@type']).toBe('error'); - expect(failcase.error[0]['@message']).toBe('timeout of 2000ms exceeded'); - }); -}); diff --git a/packages/bruno-cli/tests/reporters/junit.spec.js b/packages/bruno-cli/tests/reporters/junit.spec.js new file mode 100644 index 000000000..5d2154a88 --- /dev/null +++ b/packages/bruno-cli/tests/reporters/junit.spec.js @@ -0,0 +1,135 @@ +const { describe, it, expect } = require('@jest/globals'); +const xmlbuilder = require('xmlbuilder'); +const fs = require('fs'); + +const makeJUnitOutput = require('../../src/reporters/junit'); + +describe('makeJUnitOutput', () => { + let createStub = jest.fn(); + + beforeEach(() => { + jest.spyOn(xmlbuilder, 'create').mockImplementation(() => { + return { end: createStub }; + }); + jest.spyOn(fs, 'writeFileSync').mockImplementation(() => {}); + }); + + afterEach(() => { + jest.restoreAllMocks(); + }); + + it('should produce a junit spec object for serialization', () => { + const results = [ + { + description: 'description provided', + suitename: 'Tests/Suite A', + request: { + method: 'GET', + url: 'https://ima.test' + }, + assertionResults: [ + { + lhsExpr: 'res.status', + rhsExpr: 'eq 200', + status: 'pass' + }, + { + lhsExpr: 'res.status', + rhsExpr: 'neq 200', + status: 'fail', + error: 'expected 200 to not equal 200' + } + ], + runtime: 1.2345678 + }, + { + request: { + method: 'GET', + url: 'https://imanother.test' + }, + suitename: 'Tests/Suite B', + testResults: [ + { + lhsExpr: 'res.status', + rhsExpr: 'eq 200', + description: 'A test that passes', + status: 'pass' + }, + { + description: 'A test that fails', + status: 'fail', + error: 'expected 200 to not equal 200', + status: 'fail' + } + ], + runtime: 2.3456789 + } + ]; + + makeJUnitOutput(results, '/tmp/testfile.xml'); + expect(createStub).toBeCalled; + + const junit = xmlbuilder.create.mock.calls[0][0]; + + expect(junit.testsuites).toBeDefined; + expect(junit.testsuites.testsuite.length).toBe(2); + expect(junit.testsuites.testsuite[0].testcase.length).toBe(2); + expect(junit.testsuites.testsuite[1].testcase.length).toBe(2); + + expect(junit.testsuites.testsuite[0]['@name']).toBe('Tests/Suite A'); + expect(junit.testsuites.testsuite[1]['@name']).toBe('Tests/Suite B'); + + expect(junit.testsuites.testsuite[0]['@tests']).toBe(2); + expect(junit.testsuites.testsuite[1]['@tests']).toBe(2); + + const testcase = junit.testsuites.testsuite[0].testcase[0]; + + expect(testcase['@name']).toBe('res.status eq 200'); + expect(testcase['@status']).toBe('pass'); + + const failcase = junit.testsuites.testsuite[0].testcase[1]; + + expect(failcase['@name']).toBe('res.status neq 200'); + expect(failcase.failure).toBeDefined; + expect(failcase.failure[0]['@type']).toBe('failure'); + }); + + it('should handle request errors', () => { + const results = [ + { + description: 'description provided', + suitename: 'Tests/Suite A', + request: { + method: 'GET', + url: 'https://ima.test' + }, + assertionResults: [ + { + lhsExpr: 'res.status', + rhsExpr: 'eq 200', + status: 'fail' + } + ], + runtime: 1.2345678, + error: 'timeout of 2000ms exceeded' + } + ]; + + makeJUnitOutput(results, '/tmp/testfile.xml'); + + const junit = xmlbuilder.create.mock.calls[0][0]; + + expect(createStub).toBeCalled; + + expect(junit.testsuites).toBeDefined; + expect(junit.testsuites.testsuite.length).toBe(1); + expect(junit.testsuites.testsuite[0].testcase.length).toBe(1); + + const failcase = junit.testsuites.testsuite[0].testcase[0]; + + expect(failcase['@name']).toBe('Test suite has no errors'); + expect(failcase.error).toBeDefined; + expect(failcase.error[0]['@type']).toBe('error'); + expect(failcase.error[0]['@message']).toBe('timeout of 2000ms exceeded'); + }); +}); From 647a819051f3c06212e279a10a8cbaeb350fa8ac Mon Sep 17 00:00:00 2001 From: Andrew Winder Date: Fri, 15 Dec 2023 12:06:36 -0500 Subject: [PATCH 8/8] remove extraneous import --- packages/bruno-cli/src/commands/run.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/bruno-cli/src/commands/run.js b/packages/bruno-cli/src/commands/run.js index 3e67dba1b..2151ff56b 100644 --- a/packages/bruno-cli/src/commands/run.js +++ b/packages/bruno-cli/src/commands/run.js @@ -1,7 +1,7 @@ const fs = require('fs'); const chalk = require('chalk'); const path = require('path'); -const { forOwn, result } = require('lodash'); +const { forOwn } = require('lodash'); const { exists, isFile, isDirectory } = require('../utils/filesystem'); const { runSingleRequest } = require('../runner/run-single-request'); const { bruToEnvJson, getEnvVars } = require('../utils/bru');