Error handling in custom benchmark

examples/benchmarks/tablebench/package.json

@@ -43,6 +43,7 @@
 		"test": "npm run test:mocha",
 		"test:benchmark:report": "mocha --exit --perfMode --parentProcess --fgrep @Benchmark --reporter @fluid-tools/benchmark/dist/MochaReporter.js --timeout 60000",
 		"test:customBenchmarks": "mocha --config ./.mocharc.customBenchmarks.cjs",
+		"test:customBenchmarks:verbose": "cross-env FLUID_TEST_VERBOSE=1 npm run test:customBenchmarks",
 		"test:mocha": "npm run test:mocha:esm",
 		"test:mocha:esm": "mocha --exit",
 		"test:mocha:verbose": "cross-env FLUID_TEST_VERBOSE=1 npm run test:mocha",

packages/dds/tree/package.json

@@ -87,6 +87,7 @@
 		"test:benchmark:report": "mocha --exit --perfMode --parentProcess --fgrep @Benchmark --reporter @fluid-tools/benchmark/dist/MochaReporter.js --timeout 60000",
 		"test:coverage": "c8 npm test",
 		"test:customBenchmarks": "mocha --config ./.mocharc.customBenchmarks.cjs",
+		"test:customBenchmarks:verbose": "cross-env FLUID_TEST_VERBOSE=1 npm run test:customBenchmarks",
 		"test:memory": "mocha --config ./src/test/memory/.mocharc.cjs",
 		"test:memory-profiling:report": "mocha --config ./src/test/memory/.mocharc.cjs",
 		"test:mocha": "npm run test:mocha:esm && echo skipping cjs to avoid overhead - npm run test:mocha:cjs",

tools/benchmark/src/mocha/customOutputRunner.ts

@@ -6,7 +6,7 @@
 import { Test } from "mocha";
 
 import type { BenchmarkDescription, MochaExclusiveOptions, Titled } from "../Configuration";
-import type { BenchmarkData, CustomData } from "../ResultTypes";
+import type { BenchmarkData, BenchmarkError, CustomData } from "../ResultTypes";
 import { prettyNumber } from "../RunnerUtilities";
 import { timer } from "../timer";
 
@@ -49,7 +49,17 @@ export function benchmarkCustom(options: CustomBenchmarkOptions): Test {
 		};
 
 		const startTime = timer.now();
-		await options.run(reporter);
+
+		try {
+			await options.run(reporter);
+		} catch (error) {
+			const toBenchmarkError: BenchmarkError = { error: (error as Error).message };
+
+			// Instead of throwing the error, we emit it as an event so the test can continue.
+			test.emit("benchmark end", toBenchmarkError);
+			throw error;
+		}
+
 		const elapsedSeconds = timer.toSeconds(startTime, timer.now());
 
 		const results: BenchmarkData = {

tools/benchmark/src/test/CustomOptionBenchmark.test.ts

@@ -3,7 +3,7 @@
  * Licensed under the MIT License.
  */
 
-import { benchmarkCustom } from "..";
+import { benchmarkCustom, type BenchmarkError } from "..";
 import { BenchmarkType } from "../Configuration";
 
 describe("`benchmarkCustom` function", () => {
@@ -17,3 +17,30 @@ describe("`benchmarkCustom` function", () => {
 		});
 	});
 });
+
+describe("BenchmarkCustom error handling", () => {
+	const expectedErrorMessage = "INTENTIONAL error to test error handling";
+	let benchmarkEndPayloadIsCorrect: boolean = false;
+
+	const testObject = benchmarkCustom({
+		title: `test`,
+		type: BenchmarkType.Measurement,
+		run: async () => {
+			throw new Error(expectedErrorMessage);
+		},
+	});
+
+	testObject.on("benchmark end", (benchmarkError: BenchmarkError) => {
+		if (benchmarkError.error === expectedErrorMessage) {
+			benchmarkEndPayloadIsCorrect = true;
+		}
+	});
+
+	afterEach(function () {
+		if (benchmarkEndPayloadIsCorrect) {
+			this.skip();
+		} else {
+			throw new Error("The benchmark end payload was not correct.");
+		}
+	});
+});