diff --git a/README.md b/README.md index fcb60f1..b5ed980 100644 --- a/README.md +++ b/README.md @@ -40,12 +40,10 @@ SELECT * WHERE { By default, it generates CSV output in a form similar to: ```csv -name;id;error;errorDescription;failures;hash;httpRequests;httpRequestsMax;httpRequestsMin;replication;results;resultsMax;resultsMin;time;timeMax;timeMin;timestamps;timestampsMax;timestampsMin -C1;0;false;;0;d632b8166f912f4afd062d64186f2dc6;1766.5;2271;1262;2;6;6;6;1364;1398;1330;1363.5 1363.5 1363.5 1364 1364 1364;1398 1398 1398 1398 1398 1398;1329 1329 1329 1330 1330 1330 -C1;1;false;;0;e00f199d535cd1710bf9be67f04f39e4;1803.5;2308;1299;2;4;4;4;212;214;210;211.5 212 212 212;213 214 214 214;210 210 210 210 -C1;2;false;;0;c4499554f796e968a069e67a8f5d9d1c;1834.5;2339;1330;2;1;1;1;175.5;176;175;175.5;176;175 -C1;3;false;;0;6c0a9fe8be642ee232c10c9996912b97;2279.5;2784;1775;2;14;14;14;1747;1796;1698;1746 1746 1746 1746 1746 1746 1746.5 1746.5 1746.5 1746.5 1747 1747 1747 1747;1795 1795 1795 1795 1795 1795 1795 1795 1795 1795 1796 1796 1796 1796;1697 1697 1697 1697 1697 1697 1698 1698 1698 1698 1698 1698 1698 1698 -C1;4;false;;0;7536d3a2c1abc2a9ac92b1860efa3282;2522.5;3027;2018;2;8;8;8;1360;1373;1347;1355.5 1355.5 1355.5 1355.5 1359 1359 1359 1359;1372 1372 1372 1372 1372 1372 1372 1372;1339 1339 1339 1339 1346 1346 1346 1346 +name;id;error;errorDescription;failures;hash;replication;results;resultsMax;resultsMin;time;timeMax;timeMin;times;timestamps;timestampsMax;timestampsMin;timestampsStd;timeStd +C1;0;false;;0;6e0f167d2eb0e61af0673275ee8f935f;5;5;5;5;25.8;33;20;28 33 26 22 20;25.4 25.4 25.4 25.4 25.4;32 32 32 32 32;20 20 20 20 20;4.176122603564219 4.176122603564219 4.176122603564219 4.176122603564219 4.176122603564219;4.578209256903839 +C1;1;false;;0;3e279701df97583c2f296ac0c2e5b877;5;5;5;5;38.6;90;20;27 28 28 20 90;38.4 38.4 38.6 38.6 38.6;89 89 90 90 90;20 20 20 20 20;25.476263462289754 25.476263462289754 25.873538606073968 25.873538606073968 25.873538606073968;25.873538606073968 +C1;2;false;;0;4783aeaa4ce9950eafd3a623e1a537f6;5;5;5;5;35.8;80;20;28 26 80 20 25;35.8 35.8 35.8 35.8 35.8;80 80 80 80 80;20 20 20 20 20;22.25668438918969 22.25668438918969 22.25668438918969 22.25668438918969 22.25668438918969;22.25668438918969 ``` ## Installation @@ -55,6 +53,22 @@ npm install sparql-benchmark-runner ``` ## Usage +`sparql-benchmark-runner` can be used from the CLI with the the following options. + +``` +Options: + --version Show version number [boolean] + --endpoint URL of the SPARQL endpoint to send queries to + [string] [required] + --queries Directory of the queries [string] [required] + --replication Number of replication runs [number] [default: 5] + --warmup Number of warmup runs [number] [default: 1] + --output Destination for the output CSV file + [string] [default: "./output.csv"] + --timeout Timeout value in seconds to use for individual queries [number] + ----help +``` +An example input is the following. ```bash sparql-benchmark-runner \ @@ -113,12 +127,12 @@ docker run \ --rm \ --interactive \ --tty \ - --volume $(pwd)/output.csv:/tmp/output.csv \ - --volume $(pwd)/queries:/tmp/queries \ + --volume $(pwd)/output.csv:/output.csv \ + --volume $(pwd)/queries:/queries \ comunica/sparql-benchmark-runner \ --endpoint https://dbpedia.org/sparql \ - --queries /tmp/queries \ - --output /tmp/output.csv \ + --queries /queries \ + --output /output.csv \ --replication 5 \ --warmup 1 ``` diff --git a/lib/Result.ts b/lib/Result.ts index 335df5a..83d9fd2 100644 --- a/lib/Result.ts +++ b/lib/Result.ts @@ -18,14 +18,20 @@ export interface IResult extends IResultMetadata { /** * Aggregate result for multiple executions of a query. + * timestamps from IResult has the meaning of average timestamps. + * The timestamps are the arrival of a results of the query whereas the times + * are the query execution times. */ export interface IAggregateResult extends IResult { resultsMin: number; resultsMax: number; timeMin: number; timeMax: number; + timeStd: number; + times: number[]; timestampsMin: number[]; timestampsMax: number[]; + timestampsStd: number[]; replication: number; failures: number; } diff --git a/lib/ResultAggregator.ts b/lib/ResultAggregator.ts index 3caae6d..8491b63 100644 --- a/lib/ResultAggregator.ts +++ b/lib/ResultAggregator.ts @@ -33,6 +33,8 @@ export class ResultAggregator implements IResultAggregator { time: 0, timeMax: Number.NEGATIVE_INFINITY, timeMin: Number.POSITIVE_INFINITY, + timeStd: 0, + times: [], failures: 0, replication: resultGroup.length, results: 0, @@ -42,6 +44,7 @@ export class ResultAggregator implements IResultAggregator { timestamps: [], timestampsMax: [], timestampsMin: [], + timestampsStd: [], }; let inconsistentResults = false; let successfulExecutions = 0; @@ -50,6 +53,7 @@ export class ResultAggregator implements IResultAggregator { let maxNumTimestamp = 0; for (const result of resultGroup) { if (result.error) { + aggregate.times.push(Number.NaN); aggregate.error = result.error; aggregate.failures++; // If no results and error we don't register @@ -58,6 +62,7 @@ export class ResultAggregator implements IResultAggregator { } } else { successfulExecutions++; + aggregate.times.push(result.time); aggregate.time += result.time; aggregate.results += result.results; aggregate.resultsMax = Math.max(aggregate.resultsMax, result.results); @@ -92,8 +97,16 @@ export class ResultAggregator implements IResultAggregator { aggregate.timestamps = timestampsProcessed.timestampsAverage; aggregate.timestampsMin = timestampsProcessed.timestampsMin; aggregate.timestampsMax = timestampsProcessed.timestampsMax; + aggregate.timestampsStd = timestampsProcessed.timestampsStd; } + for (const { time, error } of resultGroup) { + if (!error) { + aggregate.timeStd += (time - aggregate.time) ** 2; + } + } + aggregate.timeStd = Math.sqrt(aggregate.timeStd / successfulExecutions); + // Convert all possible leftover infinity / -infinity back to 0 for backward compatibility aggregate.resultsMin = Number.isFinite(aggregate.resultsMin) ? aggregate.resultsMin : 0; aggregate.resultsMax = Number.isFinite(aggregate.resultsMax) ? aggregate.resultsMax : 0; @@ -106,10 +119,11 @@ export class ResultAggregator implements IResultAggregator { } public averageTimeStamps(timestampsAll: number[][], maxNumTimestamps: number): IProcessedTimestamps { - const timestampsSum: number[] = Array.from({ length: maxNumTimestamps }).fill(0); - const timestampsMax: number[] = Array.from({ length: maxNumTimestamps }).fill(Number.NEGATIVE_INFINITY); - const timestampsMin: number[] = Array.from({ length: maxNumTimestamps }).fill(Number.POSITIVE_INFINITY); - const nObsTimestamp: number[] = Array.from({ length: maxNumTimestamps }).fill(0); + const timestampsSum: number[] = Array.from({ length: maxNumTimestamps }).fill(0); + const timestampsMax: number[] = Array.from({ length: maxNumTimestamps }).fill(Number.NEGATIVE_INFINITY); + const timestampsMin: number[] = Array.from({ length: maxNumTimestamps }).fill(Number.POSITIVE_INFINITY); + const nObsTimestamp: number[] = Array.from({ length: maxNumTimestamps }).fill(0); + const timestampsStd: number[] = Array.from({ length: maxNumTimestamps }).fill(0); for (const timestamps of timestampsAll) { for (const [ j, ts ] of timestamps.entries()) { @@ -119,10 +133,24 @@ export class ResultAggregator implements IResultAggregator { nObsTimestamp[j]++; } } + + const timestampsAverage = timestampsSum.map((ts, i) => ts / nObsTimestamp[i]); + + for (const timestamps of timestampsAll) { + for (const [ j, ts ] of timestamps.entries()) { + timestampsStd[j] += (ts - timestampsAverage[j]) ** 2; + } + } + + for (let i = 0; i < timestampsStd.length; i++) { + timestampsStd[i] = Math.sqrt(timestampsStd[i] / nObsTimestamp[i]); + } + return { timestampsMax, timestampsMin, - timestampsAverage: timestampsSum.map((ts, i) => ts / nObsTimestamp[i]), + timestampsAverage, + timestampsStd, }; } @@ -146,4 +174,5 @@ export interface IProcessedTimestamps { timestampsMax: number[]; timestampsMin: number[]; timestampsAverage: number[]; + timestampsStd: number[]; } diff --git a/lib/ResultAggregatorComunica.ts b/lib/ResultAggregatorComunica.ts index 4d61f9a..778ab99 100644 --- a/lib/ResultAggregatorComunica.ts +++ b/lib/ResultAggregatorComunica.ts @@ -30,6 +30,15 @@ export class ResultAggregatorComunica extends ResultAggregator { groupedAggregates[key][0].httpRequests = requestsSum / successfulExecutions; groupedAggregates[key][0].httpRequestsMax = requestsMax; groupedAggregates[key][0].httpRequestsMin = requestsMin; + groupedAggregates[key][0].httpRequestsStd = 0; + + for (const { httpRequests, error } of resultGroup) { + if (!error) { + groupedAggregates[key][0].httpRequestsStd += (httpRequests - groupedAggregates[key][0].httpRequests) ** 2; + } + } + groupedAggregates[key][0].httpRequestsStd = + Math.sqrt(groupedAggregates[key][0].httpRequestsStd / successfulExecutions); } } return aggregateResults; diff --git a/test/ResultAggregator-test.ts b/test/ResultAggregator-test.ts index b743f3e..f4eeaa5 100644 --- a/test/ResultAggregator-test.ts +++ b/test/ResultAggregator-test.ts @@ -8,193 +8,218 @@ describe('ResultAggregator', () => { let results: IResult[]; let noResults: IResult[]; - beforeEach(() => { - results = [ - { + describe('aggregateResults', () => { + beforeEach(() => { + results = [ + { + name: 'a', + id: '0', + time: 30, + results: 3, + hash: 'a', + timestamps: [ 10, 20, 30 ], + }, + { + name: 'a', + id: '0', + time: 40, + results: 3, + hash: 'a', + timestamps: [ 20, 30, 40 ], + }, + { + name: 'a', + id: '0', + time: 50, + error: exampleError, + results: 1, + hash: 'b', + timestamps: [ 30 ], + }, + { + name: 'a', + id: '0', + time: 50, + results: 4, + hash: 'c', + timestamps: [ 20, 30, 40, 50 ], + }, + { + name: 'a', + id: '0', + time: 30, + results: 3, + hash: 'b', + timestamps: [ 10, 20, 30 ], + }, + ]; + noResults = [ + { + name: 'a', + id: '0', + time: 0, + error: exampleError, + results: 0, + hash: 'a', + timestamps: [ ], + }, + { + name: 'a', + id: '0', + time: 0, + error: exampleError, + results: 0, + hash: 'a', + timestamps: [ ], + }, + ]; + }); + + it('produces the aggregate across one result', () => { + const resultInput = [ results[0] ]; + const expected: IAggregateResult[] = [{ name: 'a', id: '0', time: 30, + timeMax: 30, + timeMin: 30, + timeStd: 0, + failures: 0, + replication: 1, results: 3, + resultsMax: 3, + resultsMin: 3, hash: 'a', timestamps: [ 10, 20, 30 ], - }, - { + timestampsMax: [ 10, 20, 30 ], + timestampsMin: [ 10, 20, 30 ], + timestampsStd: [ 0, 0, 0 ], + times: resultInput.map(result => result.time), + }]; + expect(aggregator.aggregateResults(resultInput)).toEqual(expected); + }); + + it('produces the aggregate across multiple results', () => { + const resultInput = results.slice(0, 2); + const expected: IAggregateResult[] = [{ name: 'a', id: '0', - time: 40, + time: 35, + timeMax: 40, + timeMin: 30, + timeStd: 5, + failures: 0, + replication: 2, results: 3, + resultsMax: 3, + resultsMin: 3, hash: 'a', - timestamps: [ 20, 30, 40 ], - }, - { + timestamps: [ 15, 25, 35 ], + timestampsMax: [ 20, 30, 40 ], + timestampsMin: [ 10, 20, 30 ], + timestampsStd: [ 5, 5, 5 ], + times: resultInput.map(result => result.time), + }]; + expect(aggregator.aggregateResults(resultInput)).toEqual(expected); + }); + + it('produces the aggregate across multiple results with no produced results and timeout', () => { + const expected: IAggregateResult[] = [{ name: 'a', id: '0', - time: 50, error: exampleError, - results: 1, - hash: 'b', - timestamps: [ 30 ], - }, - { + time: 0, + timeMax: 0, + timeMin: 0, + timeStd: Number.NaN, + failures: 2, + replication: 2, + results: 0, + resultsMax: 0, + resultsMin: 0, + hash: '', + timestamps: [ ], + timestampsMax: [ ], + timestampsMin: [ ], + timestampsStd: [ ], + times: [ Number.NaN, Number.NaN ], + }]; + expect(aggregator.aggregateResults(noResults)).toEqual(expected); + }); + + it('produces the aggregate across multiple results with errors', () => { + const resultInput = results.slice(0, 3); + const expected: IAggregateResult[] = [{ name: 'a', id: '0', - time: 50, - results: 4, - hash: 'c', - timestamps: [ 20, 30, 40, 50 ], - }, - { + time: 35, + timeMax: 40, + timeMin: 30, + timeStd: 5, + failures: 1, + replication: 3, + error: exampleError, + results: 3, + resultsMax: 3, + resultsMin: 3, + hash: 'a', + timestamps: [ 20, 25, 35 ], + timestampsMax: [ 30, 30, 40 ], + timestampsMin: [ 10, 20, 30 ], + timestampsStd: [ 8.16496580927726, 5, 5 ], + times: resultInput.map(({ time, error }) => error ? Number.NaN : time), + }]; + expect(aggregator.aggregateResults(resultInput)).toEqual(expected); + }); + + it('produces the aggregate across multiple results with inconsistent hashes', () => { + const resultInput = [ results[0], results[4] ]; + const expected: IAggregateResult[] = [{ name: 'a', id: '0', time: 30, + timeMax: 30, + timeMin: 30, + timeStd: 0, + failures: 1, + replication: 2, + error: hashError, results: 3, - hash: 'b', - timestamps: [ 10, 20, 30 ], - }, - ]; - noResults = [ - { - name: 'a', - id: '0', - time: 0, - error: exampleError, - results: 0, + resultsMax: 3, + resultsMin: 3, hash: 'a', - timestamps: [ ], - }, - { + timestamps: [ 10, 20, 30 ], + timestampsMax: [ 10, 20, 30 ], + timestampsMin: [ 10, 20, 30 ], + timestampsStd: [ 0, 0, 0 ], + times: resultInput.map(({ time, error }) => error ? Number.NaN : time), + }]; + expect(aggregator.aggregateResults(resultInput)).toEqual(expected); + }); + + it('produces the aggregate across multiple results with different result counts', () => { + const resultInput = [ ...results.slice(0, 2), results[3] ]; + const expected: IAggregateResult[] = [{ name: 'a', id: '0', - time: 0, - error: exampleError, - results: 0, + time: 40, + timeMax: 50, + timeMin: 30, + timeStd: 8.16496580927726, + failures: 1, + replication: 3, + error: hashError, + results: 3.3333333333333335, + resultsMax: 4, + resultsMin: 3, hash: 'a', - timestamps: [ ], - }, - ]; - }); - - it('produces the aggregate across one result', () => { - const expected: IAggregateResult[] = [{ - name: 'a', - id: '0', - time: 30, - timeMax: 30, - timeMin: 30, - failures: 0, - replication: 1, - results: 3, - resultsMax: 3, - resultsMin: 3, - hash: 'a', - timestamps: [ 10, 20, 30 ], - timestampsMax: [ 10, 20, 30 ], - timestampsMin: [ 10, 20, 30 ], - }]; - expect(aggregator.aggregateResults([ results[0] ])).toEqual(expected); - }); - - it('produces the aggregate across multiple results', () => { - const expected: IAggregateResult[] = [{ - name: 'a', - id: '0', - time: 35, - timeMax: 40, - timeMin: 30, - failures: 0, - replication: 2, - results: 3, - resultsMax: 3, - resultsMin: 3, - hash: 'a', - timestamps: [ 15, 25, 35 ], - timestampsMax: [ 20, 30, 40 ], - timestampsMin: [ 10, 20, 30 ], - }]; - expect(aggregator.aggregateResults(results.slice(0, 2))).toEqual(expected); - }); - - it('produces the aggregate across multiple results with no produced results and timeout', () => { - const expected: IAggregateResult[] = [{ - name: 'a', - id: '0', - error: exampleError, - time: 0, - timeMax: 0, - timeMin: 0, - failures: 2, - replication: 2, - results: 0, - resultsMax: 0, - resultsMin: 0, - hash: '', - timestamps: [ ], - timestampsMax: [ ], - timestampsMin: [ ], - }]; - expect(aggregator.aggregateResults(noResults)).toEqual(expected); - }); - - it('produces the aggregate across multiple results with errors', () => { - const expected: IAggregateResult[] = [{ - name: 'a', - id: '0', - time: 35, - timeMax: 40, - timeMin: 30, - failures: 1, - replication: 3, - error: exampleError, - results: 3, - resultsMax: 3, - resultsMin: 3, - hash: 'a', - timestamps: [ 20, 25, 35 ], - timestampsMax: [ 30, 30, 40 ], - timestampsMin: [ 10, 20, 30 ], - }]; - expect(aggregator.aggregateResults(results.slice(0, 3))).toEqual(expected); - }); - - it('produces the aggregate across multiple results with inconsistent hashes', () => { - const expected: IAggregateResult[] = [{ - name: 'a', - id: '0', - time: 30, - timeMax: 30, - timeMin: 30, - failures: 1, - replication: 2, - error: hashError, - results: 3, - resultsMax: 3, - resultsMin: 3, - hash: 'a', - timestamps: [ 10, 20, 30 ], - timestampsMax: [ 10, 20, 30 ], - timestampsMin: [ 10, 20, 30 ], - }]; - expect(aggregator.aggregateResults([ results[0], results[4] ])).toEqual(expected); - }); - - it('produces the aggregate across multiple results with different result counts', () => { - const expected: IAggregateResult[] = [{ - name: 'a', - id: '0', - time: 40, - timeMax: 50, - timeMin: 30, - failures: 1, - replication: 3, - error: hashError, - results: 3.3333333333333335, - resultsMax: 4, - resultsMin: 3, - hash: 'a', - timestamps: [ 16.666666666666668, 26.666666666666668, 36.666666666666664, 50 ], - timestampsMax: [ 20, 30, 40, 50 ], - timestampsMin: [ 10, 20, 30, 50 ], - }]; - expect(aggregator.aggregateResults([ ...results.slice(0, 2), results[3] ])).toEqual(expected); + timestamps: [ 16.666666666666668, 26.666666666666668, 36.666666666666664, 50 ], + timestampsMax: [ 20, 30, 40, 50 ], + timestampsMin: [ 10, 20, 30, 50 ], + timestampsStd: [ 4.714045207910316, 4.714045207910316, 4.714045207910317, 0 ], + times: resultInput.map(({ time, error }) => error ? Number.NaN : time), + }]; + expect(aggregator.aggregateResults(resultInput)).toEqual(expected); + }); }); }); diff --git a/test/ResultAggregatorComunica-test.ts b/test/ResultAggregatorComunica-test.ts index 4d2ce16..cb09802 100644 --- a/test/ResultAggregatorComunica-test.ts +++ b/test/ResultAggregatorComunica-test.ts @@ -59,12 +59,14 @@ describe('ResultAggregatorComunica', () => { }); it('produces the aggregate across one result', () => { + const resultInput = [ results[0] ]; const expected: IAggregateResult[] = [{ name: 'a', id: '0', time: 30, timeMax: 30, timeMin: 30, + timeStd: 0, failures: 0, replication: 1, results: 3, @@ -74,20 +76,25 @@ describe('ResultAggregatorComunica', () => { timestamps: [ 10, 20, 30 ], timestampsMax: [ 10, 20, 30 ], timestampsMin: [ 10, 20, 30 ], + timestampsStd: [ 0, 0, 0 ], httpRequests: 10, httpRequestsMax: 10, httpRequestsMin: 10, + httpRequestsStd: 0, + times: resultInput.map(result => result.time), }]; - expect(aggregator.aggregateResults([ results[0] ])).toEqual(expected); + expect(aggregator.aggregateResults(resultInput)).toEqual(expected); }); it('produces the aggregate across multiple results', () => { + const resultInput = results.slice(0, 2); const expected: IAggregateResult[] = [{ name: 'a', id: '0', time: 35, timeMax: 40, timeMin: 30, + timeStd: 5, failures: 0, replication: 2, results: 3, @@ -97,20 +104,25 @@ describe('ResultAggregatorComunica', () => { timestamps: [ 15, 25, 35 ], timestampsMax: [ 20, 30, 40 ], timestampsMin: [ 10, 20, 30 ], + timestampsStd: [ 5, 5, 5 ], httpRequests: 15, httpRequestsMax: 20, httpRequestsMin: 10, + httpRequestsStd: 5, + times: resultInput.map(result => result.time), }]; - expect(aggregator.aggregateResults(results.slice(0, 2))).toEqual(expected); + expect(aggregator.aggregateResults(resultInput)).toEqual(expected); }); it('produces the aggregate across multiple results with errors', () => { + const resultInput = results.slice(0, 3); const expected: IAggregateResult[] = [{ name: 'a', id: '0', time: 35, timeMax: 40, timeMin: 30, + timeStd: 5, failures: 1, replication: 3, error: exampleError, @@ -121,20 +133,25 @@ describe('ResultAggregatorComunica', () => { timestamps: [ 20, 25, 35 ], timestampsMax: [ 30, 30, 40 ], timestampsMin: [ 10, 20, 30 ], + timestampsStd: [ 8.16496580927726, 5, 5 ], httpRequests: 15, httpRequestsMax: 20, httpRequestsMin: 10, + httpRequestsStd: 5, + times: resultInput.map(({ time, error }) => error ? Number.NaN : time), }]; expect(aggregator.aggregateResults(results.slice(0, 3))).toEqual(expected); }); it('produces the aggregate across multiple results with inconsistent hashes', () => { + const resultInput = [ results[0], results[4] ]; const expected: IAggregateResult[] = [{ name: 'a', id: '0', time: 30, timeMax: 30, timeMin: 30, + timeStd: 0, failures: 1, replication: 2, error: hashError, @@ -145,20 +162,25 @@ describe('ResultAggregatorComunica', () => { timestamps: [ 10, 20, 30 ], timestampsMax: [ 10, 20, 30 ], timestampsMin: [ 10, 20, 30 ], + timestampsStd: [ 0, 0, 0 ], httpRequests: 10, httpRequestsMax: 10, httpRequestsMin: 10, + httpRequestsStd: 0, + times: resultInput.map(({ time, error }) => error ? Number.NaN : time), }]; - expect(aggregator.aggregateResults([ results[0], results[4] ])).toEqual(expected); + expect(aggregator.aggregateResults(resultInput)).toEqual(expected); }); it('produces the aggregate across multiple results with different result counts', () => { + const resultInput = [ ...results.slice(0, 2), results[3] ]; const expected: IAggregateResult[] = [{ name: 'a', id: '0', time: 40, timeMax: 50, timeMin: 30, + timeStd: 8.16496580927726, failures: 1, replication: 3, error: hashError, @@ -169,9 +191,12 @@ describe('ResultAggregatorComunica', () => { timestamps: [ 16.666666666666668, 26.666666666666668, 36.666666666666664, 50 ], timestampsMax: [ 20, 30, 40, 50 ], timestampsMin: [ 10, 20, 30, 50 ], + timestampsStd: [ 4.714045207910316, 4.714045207910316, 4.714045207910317, 0 ], httpRequests: 23.333333333333332, httpRequestsMax: 40, httpRequestsMin: 10, + httpRequestsStd: 12.472191289246473, + times: resultInput.map(({ time, error }) => error ? Number.NaN : time), }]; expect(aggregator.aggregateResults([ ...results.slice(0, 2), results[3] ])).toEqual(expected); }); diff --git a/test/SparqlBenchmarkRunner-test.ts b/test/SparqlBenchmarkRunner-test.ts index 1f7aab0..9409c93 100644 --- a/test/SparqlBenchmarkRunner-test.ts +++ b/test/SparqlBenchmarkRunner-test.ts @@ -71,7 +71,18 @@ describe('SparqlBenchmarkRunner', () => { describe('run', () => { it('runs the whole query set', async() => { + runner = new SparqlBenchmarkRunner({ + endpoint, + querySets, + replication, + warmup, + logger, + // The timeout is set here to lower than default 10 seconds, or the tests will take forever + availabilityCheckTimeout: 1_000, + }); + const results = await runner.run(); + const expectedResults: IAggregateResult[] = [ { name: 'a', @@ -85,9 +96,12 @@ describe('SparqlBenchmarkRunner', () => { time: 41, timeMax: 48, timeMin: 34, + timeStd: 7, timestamps: [ 38, 39, 40 ], timestampsMax: [ 45, 46, 47 ], timestampsMin: [ 31, 32, 33 ], + timestampsStd: [ 7, 7, 7 ], + times: [ 34, 48 ], }, { name: 'a', @@ -97,6 +111,7 @@ describe('SparqlBenchmarkRunner', () => { time: 48, timeMax: 55, timeMin: 41, + timeStd: 7, results: 3, resultsMax: 3, resultsMin: 3, @@ -104,6 +119,8 @@ describe('SparqlBenchmarkRunner', () => { timestamps: [ 45, 46, 47 ], timestampsMax: [ 52, 53, 54 ], timestampsMin: [ 38, 39, 40 ], + timestampsStd: [ 7, 7, 7 ], + times: [ 41, 55 ], }, { name: 'b', @@ -113,6 +130,7 @@ describe('SparqlBenchmarkRunner', () => { time: 69, timeMax: 76, timeMin: 62, + timeStd: 7, results: 3, resultsMax: 3, resultsMin: 3, @@ -120,6 +138,8 @@ describe('SparqlBenchmarkRunner', () => { timestamps: [ 66, 67, 68 ], timestampsMax: [ 73, 74, 75 ], timestampsMin: [ 59, 60, 61 ], + timestampsStd: [ 7, 7, 7 ], + times: [ 62, 76 ], }, { name: 'b', @@ -133,9 +153,12 @@ describe('SparqlBenchmarkRunner', () => { time: 76, timeMax: 83, timeMin: 69, + timeStd: 7, timestamps: [ 73, 74, 75 ], timestampsMax: [ 80, 81, 82 ], timestampsMin: [ 66, 67, 68 ], + timestampsStd: [ 7, 7, 7 ], + times: [ 69, 83 ], }, ];