Skip to content

Commit

Permalink
refactor(bench): simplify benchmarks
Browse files Browse the repository at this point in the history
  • Loading branch information
@yifanwww
yifanwww committed Dec 2, 2024
1 parent e7cbfaf commit 0ea925b
Show file tree
Hide file tree
Showing 25 changed files with 830 additions and 1,210 deletions.
49 changes: 25 additions & 24 deletions packages/benchmark/src/factories/Err.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,12 +5,13 @@ import type { Err as NTErr } from 'neverthrow';
import { err as ntErr } from 'neverthrow';
import { Bench, hrtimeNow } from 'tinybench';

import { formatTinybenchTask } from '../helpers/tinybench.js';
import { formatNum, logTestCases } from '../utils.js';

const N = 100_000;

logTestCases([
['@rustresult/result Err', Err('error message')],
['rustresult Err', Err('error message')],
['neverthrow err', ntErr('error message')],
['effect Effect.fail', Effect.fail('error message')],
['effect Exit.fail', Exit.fail('error message')],
Expand All @@ -20,40 +21,40 @@ console.log('Loop N:', formatNum(N));

const bench = new Bench({ now: hrtimeNow });
bench
.add('@rustresult/result Err', () => {
let result: Result<never, string>;
.add('rustresult Err', () => {
let ret: Result<never, string>;
for (let i = 0; i < N; i++) {
result = Err('error message');
ret = Err('error message');
}
return result!;
return ret!;
})
.add('neverthrow err', () => {
let result: NTErr<never, string>;
let ret: NTErr<never, string>;
for (let i = 0; i < N; i++) {
result = ntErr('error message');
ret = ntErr('error message');
}
return result!;
return ret!;
})
.add('effect Effect.fail', () => {
let effect: Effect.Effect<never, string>;
let ret: Effect.Effect<never, string>;
for (let i = 0; i < N; i++) {
effect = Effect.fail('error message');
ret = Effect.fail('error message');
}
return effect!;
return ret!;
})
.add('effect Exit.fail', () => {
let effect: Exit.Exit<never, string>;
let ret: Exit.Exit<never, string>;
for (let i = 0; i < N; i++) {
effect = Exit.fail('error message');
ret = Exit.fail('error message');
}
return effect!;
return ret!;
});
await bench.run();
console.table(bench.table());
console.table(bench.table(formatTinybenchTask));

/*
> @rustresult/result Err:
> rustresult Err:
RustlikeResult {
_type: 'err',
_value: undefined,
Expand All @@ -78,13 +79,13 @@ Err { error: 'error message' }
}
Loop N: 100,000
┌─────────┬──────────────────────────┬──────────────────────┬─────────────────────┬────────────────────────────┬───────────────────────────┬─────────┐
│ (index) │ Task name │ Latency average (ns) │ Latency median (ns) │ Throughput average (ops/s) │ Throughput median (ops/s) │ Samples │
├─────────┼──────────────────────────┼──────────────────────┼─────────────────────┼────────────────────────────┼───────────────────────────┼─────────┤
│ 0 │ '@rustresult/result Err' │ '388064.42 ± 0.55%' │ '362199.90' │ '2622 ± 0.48%' │ '2761' │ 2577 │
│ 1 │ 'neverthrow err' │ '259314.39 ± 0.48%' │ '239699.96' │ '3934 ± 0.41%' │ '4172' │ 3857 │
│ 2 │ 'effect Effect.fail' │ '1115973.58 ± 0.39%' │ '1110100.03' │ '899 ± 0.36%' │ '901' │ 897 │
│ 3 │ 'effect Exit.fail' │ '1088751.47 ± 0.37%' │ '1082700.01' │ '921 ± 0.35%' │ '924' │ 919 │
└─────────┴──────────────────────────┴──────────────────────┴─────────────────────┴────────────────────────────┴───────────────────────────┴─────────┘
┌─────────┬──────────────────────┬──────────────────────┬──────────────┬────────────────┬───────────────┬─────────┐
│ (index) │ task │ mean (ns) │ median (ns) │ mean (op/s) │ median (op/s) │ samples │
├─────────┼──────────────────────┼──────────────────────┼──────────────┼────────────────┼───────────────┼─────────┤
│ 0 │ 'rustresult Err' │ '393274.20 ± 0.53%' │ '371400.00' │ '2585 ± 0.48%' │ '2693' │ 2543 │
│ 1 │ 'neverthrow err' │ '272867.02 ± 0.89%' │ '248600.01' │ '3781 ± 0.49%' │ '4023' │ 3666 │
│ 2 │ 'effect Effect.fail' │ '1162188.39 ± 0.46%' │ '1148000.00' │ '864 ± 0.44%' │ '871' │ 861 │
│ 3 │ 'effect Exit.fail' │ '1127193.58 ± 0.47%' │ '1110300.00' │ '891 ± 0.44%' │ '901' │ 888 │
└─────────┴──────────────────────┴──────────────────────┴──────────────┴────────────────┴───────────────┴─────────┘
*/
49 changes: 25 additions & 24 deletions packages/benchmark/src/factories/Ok.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,12 +5,13 @@ import type { Ok as NTOk } from 'neverthrow';
import { ok as ntOk } from 'neverthrow';
import { Bench, hrtimeNow } from 'tinybench';

import { formatTinybenchTask } from '../helpers/tinybench.js';
import { formatNum, logTestCases } from '../utils.js';

const N = 100_000;

logTestCases([
['@rustresult/result Ok', Ok(1)],
['rustresult Ok', Ok(1)],
['neverthrow ok', ntOk(1)],
['effect Effect.succeed', Effect.succeed(1)],
['effect Exit.succeed', Exit.succeed(1)],
Expand All @@ -20,40 +21,40 @@ console.log('Loop N:', formatNum(N));

const bench = new Bench({ now: hrtimeNow });
bench
.add('@rustresult/result Ok', () => {
let result: Result<number, never>;
.add('rustresult Ok', () => {
let ret: Result<number, never>;
for (let i = 0; i < N; i++) {
result = Ok(1);
ret = Ok(1);
}
return result!;
return ret!;
})
.add('neverthrow ok', () => {
let result: NTOk<number, never>;
let ret: NTOk<number, never>;
for (let i = 0; i < N; i++) {
result = ntOk(1);
ret = ntOk(1);
}
return result!;
return ret!;
})
.add('effect Effect.succeed', () => {
let effect: Effect.Effect<number, never>;
let ret: Effect.Effect<number, never>;
for (let i = 0; i < N; i++) {
effect = Effect.succeed(1);
ret = Effect.succeed(1);
}
return effect!;
return ret!;
})
.add('effect Exit.succeed', () => {
let effect: Exit.Exit<number, never>;
let ret: Exit.Exit<number, never>;
for (let i = 0; i < N; i++) {
effect = Exit.succeed(1);
ret = Exit.succeed(1);
}
return effect!;
return ret!;
});
await bench.run();
console.table(bench.table());
console.table(bench.table(formatTinybenchTask));

/*
> @rustresult/result Ok:
> rustresult Ok:
RustlikeResult { _type: 'ok', _value: 1, _error: undefined }
> neverthrow ok:
Expand All @@ -66,13 +67,13 @@ Ok { value: 1 }
{ _id: 'Exit', _tag: 'Success', value: 1 }
Loop N: 100,000
┌─────────┬─────────────────────────┬──────────────────────┬─────────────────────┬────────────────────────────┬───────────────────────────┬─────────┐
│ (index) │ Task name │ Latency average (ns) │ Latency median (ns) │ Throughput average (ops/s) │ Throughput median (ops/s) │ Samples │
├─────────┼─────────────────────────┼──────────────────────┼─────────────────────┼────────────────────────────┼───────────────────────────┼─────────┤
│ 0 │ '@rustresult/result Ok' │ '386097.84 ± 0.45%' │ '362499.95' │ '2623 ± 0.42%' │ '2759' │ 2591 │
│ 1 │ 'neverthrow ok' │ '259904.26 ± 0.48%' │ '241099.95' │ '3926 ± 0.42%' │ '4148' │ 3848 │
│ 2 │ 'effect Effect.succeed' │ '670446.05 ± 0.51%' │ '675599.99 ± 0.06' │ '1505 ± 0.47%' │ '1480' │ 1492 │
│ 3 │ 'effect Exit.succeed' │ '670574.87 ± 0.48%' │ '675899.98' │ '1503 ± 0.44%' │ '1480' │ 1492 │
└─────────┴─────────────────────────┴──────────────────────┴─────────────────────┴────────────────────────────┴───────────────────────────┴─────────┘
┌─────────┬─────────────────────────┬─────────────────────┬──────────────────────┬────────────────┬───────────────┬─────────┐
│ (index) │ task │ mean (ns) │ median (ns) │ mean (op/s) │ median (op/s) │ samples │
├─────────┼─────────────────────────┼─────────────────────┼──────────────────────┼────────────────┼───────────────┼─────────┤
│ 0 │ 'rustresult Ok' │ '395020.66 ± 0.49%' │ '371800.01' │ '2568 ± 0.45%' │ '2690' │ 2532 │
│ 1 │ 'neverthrow ok' │ '269472.49 ± 0.52%' │ '248399.97' │ '3795 ± 0.44%' │ '4026' │ 3711 │
│ 2 │ 'effect Effect.succeed' │ '684643.12 ± 0.47%' │ '690699.99' │ '1472 ± 0.45%' │ '1448' │ 1461 │
│ 3 │ 'effect Exit.succeed' │ '678484.60 ± 0.45%' │ '684549.99 ± 150.02' │ '1485 ± 0.43%' │ '1461' │ 1474 │
└─────────┴─────────────────────────┴─────────────────────┴──────────────────────┴────────────────┴───────────────┴─────────┘
*/
14 changes: 14 additions & 0 deletions packages/benchmark/src/helpers/tinybench.ts
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,14 @@
import type { Task } from 'tinybench';

const mToNs = (ms: number) => ms * 1e6;

export function formatTinybenchTask(task: Task) {
return {
task: task.name,
'mean (ns)': `${mToNs(task.result!.latency.mean).toFixed(2)} \xb1 ${task.result!.latency.rme.toFixed(2)}%`,
'median (ns)': `${mToNs(task.result!.latency.p50!).toFixed(2)}${Number.parseFloat(mToNs(task.result!.latency.mad!).toFixed(2)) > 0 ? ` \xb1 ${mToNs(task.result!.latency.mad!).toFixed(2)}` : ''}`,
'mean (op/s)': `${task.result!.throughput.mean.toFixed(0)} \xb1 ${task.result!.throughput.rme.toFixed(2)}%`,
'median (op/s)': `${task.result!.throughput.p50!.toFixed(0)}${Number.parseInt(task.result!.throughput.mad!.toFixed(0), 10) > 0 ? ` \xb1 ${task.result!.throughput.mad!.toFixed(0)}` : ''}`,
samples: task.result!.latency.samples.length,
};
}
Loading

0 comments on commit 0ea925b

Please sign in to comment.