fix: flaky test count (#46)

package.json

@@ -25,8 +25,8 @@
     "test:all": "run-s test:unit test:integration",
     "test:unit": "mocha test/unit/**/*.test.js",
     "test:integration": "run-s test:integration:mocha test:integration:playwright",
-    "test:integration:mocha": "mocha --config test/integration/configs/mocha.cjs",
-    "test:integration:playwright": "playwright test --config test/integration/configs/playwright.js",
+    "test:integration:mocha": "mocha --config test/integration/configs/mocha.cjs || exit 0",
+    "test:integration:playwright": "playwright test --config test/integration/configs/playwright.js || exit 0",
     "test:coverage": "c8 npm run test:all"
   },
   "engines": {

src/reporters/mocha.cjs

@@ -89,7 +89,7 @@ class TestReportingMochaReporter extends Spec {
 
 		values.started = values.started ?? (new Date()).toISOString();
 		values.location = values.location ?? makeLocation(test.file);
-		values.retries = values.retries === undefined ? 0 : values.retries + 1;
+		values.retries = values.retries ?? 0;
 		values.totalDuration = values.totalDuration ?? 0;
 
 		this._tests.set(name, values);
@@ -100,9 +100,9 @@ class TestReportingMochaReporter extends Spec {
 		const values = this._tests.get(name);
 
 		values.totalDuration += test.duration;
+		values.retries += 1;
 
 		this._tests.set(name, values);
-		this._testsFlaky.add(name);
 	}
 
 	_onTestEnd(test) {
@@ -114,6 +114,10 @@ class TestReportingMochaReporter extends Spec {
 		values.totalDuration += values.duration;
 
 		this._tests.set(name, values);
+
+		if (values.status === 'passed' && values.retries !== 0) {
+			this._testsFlaky.add(name);
+		}
 	}
 
 	_onRunEnd(stats) {

src/reporters/playwright.js

@@ -117,16 +117,16 @@ export default class Reporter {
 
 			if (values.status === 'passed') {
 				countPassed++;
+
+				if (values.retries !== 0) {
+					countFlaky++;
+				}
 			} else if (values.status === 'failed') {
 				countFailed++;
 			} else if (values.status === 'skipped') {
 				countSkipped++;
 			}
 
-			if (values.retries !== 0) {
-				countFlaky++;
-			}
-
 			return { ...values };
 		});
 

test/integration/mocha-1.test.js

@@ -5,11 +5,11 @@ const delay = () => {
 describe('reporter tests 1', () => {
 	let count = 0;
 
-	it('test', () => {});
+	it('test', () => { });
 
-	it.skip('skipped test', () => {});
+	it.skip('skipped test', () => { });
 
-	it('flaky test', async() => {
+	it('flaky test that succeeds eventually', async() => {
 		if (count < 2) {
 			await delay();
 
@@ -18,4 +18,6 @@ describe('reporter tests 1', () => {
 			throw new Error('flaky test failure');
 		}
 	});
+
+	it('failed test', () => { throw new Error('fail'); });
 });

test/integration/mocha-2.test.js

@@ -5,9 +5,9 @@ const delay = () => {
 describe('reporter tests 2', () => {
 	let count = 0;
 
-	it('test', () => {});
+	it('test', () => { });
 
-	it.skip('skipped test', () => {});
+	it.skip('skipped test', () => { });
 
 	it('flaky test', async() => {
 		if (count < 2) {
@@ -18,4 +18,6 @@ describe('reporter tests 2', () => {
 			throw new Error('flaky test failure');
 		}
 	});
+
+	it('failed test', () => { throw new Error('fail'); });
 });

test/integration/playwright-1.test.js

@@ -17,4 +17,6 @@ test.describe('reporter tests 1', () => {
 			throw new Error('flaky test failure');
 		}
 	});
+
+	test('failed test', () => { throw new Error('fail'); });
 });

test/integration/playwright-2.test.js

@@ -17,4 +17,6 @@ test.describe('reporter tests 2', () => {
 			throw new Error('flaky test failure');
 		}
 	});
+
+	test('failed test', () => { throw new Error('fail'); });
 });