Error handling in custom benchmark

examples/benchmarks/tablebench/package.json

@@ -43,6 +43,7 @@
 		"test": "npm run test:mocha",
 		"test:benchmark:report": "mocha --exit --perfMode --parentProcess --fgrep @Benchmark --reporter @fluid-tools/benchmark/dist/MochaReporter.js --timeout 60000",
 		"test:customBenchmarks": "mocha --config ./.mocharc.customBenchmarks.cjs",
+		"test:customBenchmarks:verbose": "cross-env FLUID_TEST_VERBOSE=1 npm run test:customBenchmarks",
 		"test:mocha": "npm run test:mocha:esm",
 		"test:mocha:esm": "mocha --exit",
 		"test:mocha:verbose": "cross-env FLUID_TEST_VERBOSE=1 npm run test:mocha",

packages/dds/tree/package.json

@@ -87,6 +87,7 @@
 		"test:benchmark:report": "mocha --exit --perfMode --parentProcess --fgrep @Benchmark --reporter @fluid-tools/benchmark/dist/MochaReporter.js --timeout 60000",
 		"test:coverage": "c8 npm test",
 		"test:customBenchmarks": "mocha --config ./.mocharc.customBenchmarks.cjs",
+		"test:customBenchmarks:verbose": "cross-env FLUID_TEST_VERBOSE=1 npm run test:customBenchmarks",
 		"test:memory": "mocha --config ./src/test/memory/.mocharc.cjs",
 		"test:memory-profiling:report": "mocha --config ./src/test/memory/.mocharc.cjs",
 		"test:mocha": "npm run test:mocha:esm && echo skipping cjs to avoid overhead - npm run test:mocha:cjs",

tools/benchmark/api-report/benchmark.alpha.api.md

@@ -55,7 +55,7 @@ export class BenchmarkReporter {
     constructor(outputDirectory?: string);
     recordResultsSummary(): void;
     recordSuiteResults(suiteName: string): void;
-    recordTestResult(suiteName: string, testName: string, result: Readonly<BenchmarkResult>): void;
+    recordTestResult(suiteName: string, testName: string, result: BenchmarkResult): void;
 }
 
 // @public

tools/benchmark/api-report/benchmark.beta.api.md

@@ -55,7 +55,7 @@ export class BenchmarkReporter {
     constructor(outputDirectory?: string);
     recordResultsSummary(): void;
     recordSuiteResults(suiteName: string): void;
-    recordTestResult(suiteName: string, testName: string, result: Readonly<BenchmarkResult>): void;
+    recordTestResult(suiteName: string, testName: string, result: BenchmarkResult): void;
 }
 
 // @public

tools/benchmark/api-report/benchmark.public.api.md

@@ -55,7 +55,7 @@ export class BenchmarkReporter {
     constructor(outputDirectory?: string);
     recordResultsSummary(): void;
     recordSuiteResults(suiteName: string): void;
-    recordTestResult(suiteName: string, testName: string, result: Readonly<BenchmarkResult>): void;
+    recordTestResult(suiteName: string, testName: string, result: BenchmarkResult): void;
 }
 
 // @public

tools/benchmark/src/MochaReporter.ts

@@ -8,7 +8,7 @@ import { Runner, Suite, Test } from "mocha";
 
 import { isChildProcess, ReporterOptions } from "./Configuration";
 import { BenchmarkReporter } from "./Reporter";
-import { BenchmarkResult } from "./ResultTypes";
+import { BenchmarkResult, type BenchmarkError } from "./ResultTypes";
 // TODO: this file should be moved in with the mocha specific stuff, but is left where it is for now to avoid breaking users of this reporter.
 // Since it's not moved yet, it needs this lint suppression to do this import:
 // eslint-disable-next-line import/no-internal-modules
@@ -66,7 +66,10 @@ module.exports = class {
 				if (test.state !== "passed") {
 					// The mocha test failed after reporting benchmark data.
 					// This may indicate the benchmark did not measure what was intended, so mark as aborted.
-					const error = `Test ${test.title} in ${suite} completed with status '${test.state}' after reporting data.`;
+					const error =
+						(benchmark as BenchmarkError).error === undefined
+							? `Test ${test.title} in ${suite} completed with status '${test.state}' after reporting data.`
+							: (benchmark as BenchmarkError).error;
 					console.error(chalk.red(error));
 					benchmark = { error };
 				}

tools/benchmark/src/Reporter.ts

@@ -97,11 +97,7 @@ export class BenchmarkReporter {
 	 * Appends a prettified version of the results of a benchmark instance provided to the provided
 	 * BenchmarkResults object.
 	 */
-	public recordTestResult(
-		suiteName: string,
-		testName: string,
-		result: Readonly<BenchmarkResult>,
-	): void {
+	public recordTestResult(suiteName: string, testName: string, result: BenchmarkResult): void {
 		let results = this.inProgressSuites.get(suiteName);
 		if (results === undefined) {
 			results = {
@@ -120,21 +116,19 @@ export class BenchmarkReporter {
 		} else {
 			table.cell("status", `${pad(4)}${chalk.green("✔")}`);
 		}
+
 		table.cell("name", chalk.italic(testName));
-		table.cell("total time (s)", prettyNumber((result as BenchmarkData).elapsedSeconds, 2));
 
-		// Additional if-statement to display columns in more readble order.
 		if (isResultError(result)) {
 			table.cell("error", result.error);
-		}
+		} else {
+			table.cell("total time (s)", prettyNumber(result.elapsedSeconds, 2));
 
-		// Using this utility to print the data means missing fields don't crash and extra fields are reported.
-		// This is useful if this reporter is given unexpected data (such as from a memory test).
-		// It can also be used as a way to add extensible data formatting in the future.
-		const customData = (result as BenchmarkData).customData;
-		for (const [key, val] of Object.entries(customData)) {
-			const displayValue = val.formattedValue;
-			table.cell(key, displayValue, Table.padLeft);
+			const customData = result.customData;
+			for (const [key, val] of Object.entries(customData)) {
+				const displayValue = val.formattedValue;
+				table.cell(key, displayValue, Table.padLeft);
+			}
 		}
 
 		table.newRow();

tools/benchmark/src/mocha/customOutputRunner.ts

@@ -5,8 +5,13 @@
 
 import { Test } from "mocha";
 
-import type { BenchmarkDescription, MochaExclusiveOptions, Titled } from "../Configuration";
-import type { BenchmarkData, CustomData } from "../ResultTypes";
+import {
+	isInPerformanceTestingMode,
+	type BenchmarkDescription,
+	type MochaExclusiveOptions,
+	type Titled,
+} from "../Configuration";
+import type { BenchmarkData, BenchmarkError, CustomData } from "../ResultTypes";
 import { prettyNumber } from "../RunnerUtilities";
 import { timer } from "../timer";
 
@@ -49,7 +54,23 @@ export function benchmarkCustom(options: CustomBenchmarkOptions): Test {
 		};
 
 		const startTime = timer.now();
-		await options.run(reporter);
+
+		try {
+			await options.run(reporter);
+		} catch (error) {
+			const benchmarkError: BenchmarkError = { error: (error as Error).message };
+
+			// When running benchmarks in perf mode, the event is necessary for the custom reporter to keep track of the benchmark result.
+			// When running them without perf mode, we need to re-throw the error for Mocha to do its normal thing when a test throws.
+			test.emit("benchmark end", benchmarkError);
+
+			if (isInPerformanceTestingMode) {
+				return;
+			}
+
+			throw error;
+		}
+
 		const elapsedSeconds = timer.toSeconds(startTime, timer.now());
 
 		const results: BenchmarkData = {