From 41c600d17dcb545c9773f554a5b85c23a6737b52 Mon Sep 17 00:00:00 2001 From: tpluscode Date: Thu, 12 Sep 2024 22:00:23 +0200 Subject: [PATCH 1/3] feat(map,filter): additional arguments --- .changeset/selfish-cougars-carry.md | 5 +++++ packages/base/filter.js | 11 ++++++---- packages/base/map.js | 18 +++++++++------ packages/base/test/filter.test.js | 24 ++++++++++++++++++++ packages/base/test/map.test.js | 34 +++++++++++++++++++++++++++++ 5 files changed, 81 insertions(+), 11 deletions(-) create mode 100644 .changeset/selfish-cougars-carry.md diff --git a/.changeset/selfish-cougars-carry.md b/.changeset/selfish-cougars-carry.md new file mode 100644 index 00000000..ec2141c5 --- /dev/null +++ b/.changeset/selfish-cougars-carry.md @@ -0,0 +1,5 @@ +--- +"barnard59-base": minor +--- + +Adds a variadic parameter to `map` and` filter` steps, which will be forwarded to their respecitve callback functions called for each chunk diff --git a/packages/base/filter.js b/packages/base/filter.js index af48e3ef..b26b9412 100644 --- a/packages/base/filter.js +++ b/packages/base/filter.js @@ -1,20 +1,23 @@ import { obj } from 'through2' /** - * @typedef {(this: import('barnard59-core').Context, chunk: T, encoding: string) => boolean | Promise} Filter + * @typedef {(this: import('barnard59-core').Context, chunk: T, encoding: string, ...args: A) => boolean | Promise} Filter * @template T + * @template {Array} A */ /** * @template T + * @template {Array} A * @this {import('barnard59-core').Context} - * @param {Filter} func + * @param {Filter} func + * @param {A} args * @return {import('stream').Transform} */ -function filter(func) { +function filter(func, ...args) { return obj((chunk, encoding, callback) => { Promise.resolve().then(() => { - return func.call(this, chunk, encoding) + return func.call(this, chunk, encoding, ...args) }).then(result => { if (result) { return callback(null, chunk) diff --git a/packages/base/map.js b/packages/base/map.js index 6b2b2cd2..6f87ef8f 100644 --- a/packages/base/map.js +++ b/packages/base/map.js @@ -1,31 +1,35 @@ import transform from 'parallel-transform' /** - * @typedef {(this: import('barnard59-core').Context, chunk: From) => Promise | To} MapCallback + * @typedef {(this: import('barnard59-core').Context, chunk: From, ...args: Args) => Promise | To} MapCallback * @template From, To + * @template {Array} Args */ /** * @typedef {{ - * map: MapCallback + * map: MapCallback * concurrency?: number * ordered?: boolean * objectMode?: boolean - * }|MapCallback} MapOptions + * }|MapCallback} MapOptions * @template From, To + * @template {Array} Args */ /** * Processes chunks with a transform function * * @this {import('barnard59-core').Context} - * @param {MapOptions} options Transform function or complex options + * @param {MapOptions} options Transform function or complex options + * @param {Args} args Additional arguments to pass to the transform function * @return {import('stream').Transform} * @template From, To + * @template {Array} Args */ -export default function map(options) { +export default function map(options, ...args) { /** - * @type {MapCallback<*, *>} + * @type {MapCallback<*, *, Args>} */ let func let concurrency = 1 @@ -42,7 +46,7 @@ export default function map(options) { return transform(concurrency, { ordered, objectMode }, (data, callback) => { Promise.resolve().then(() => { - return func.call(this, data) + return func.call(this, data, ...args) }).then(result => { callback(null, result) }).catch(callback) diff --git a/packages/base/test/filter.test.js b/packages/base/test/filter.test.js index be26eecc..f29fe781 100644 --- a/packages/base/test/filter.test.js +++ b/packages/base/test/filter.test.js @@ -5,6 +5,7 @@ import filter from '../filter.js' describe('filter', () => { it('should pass pipeline context to callback function', async () => { + // given const input = new Readable({ objectMode: true, read: () => { @@ -17,11 +18,34 @@ describe('filter', () => { variable: new Map().set('condition', 'a'), } + // when const outStream = input.pipe(filter.call(context, function (chunk) { return this.variable.get('condition') === chunk })) const output = await array(outStream) + // then deepStrictEqual(output, ['a']) }) + + it('should pass additional arguments to callback function', async () => { + // given + const input = new Readable({ + objectMode: true, + read: () => { + input.push(1) + input.push(2) + input.push(3) + input.push(null) + }, + }) + + // when + const greaterThan = (chunk, _, minValue) => chunk > minValue + const outStream = input.pipe(filter.call(context, greaterThan, 2)) + const output = await array(outStream) + + // then + deepStrictEqual(output, [3]) + }) }) diff --git a/packages/base/test/map.test.js b/packages/base/test/map.test.js index 019cb576..bcdab92b 100644 --- a/packages/base/test/map.test.js +++ b/packages/base/test/map.test.js @@ -25,6 +25,40 @@ describe('map', () => { deepStrictEqual(output, ['foo_a', 'foo_b']) }) + it('should pass optional arguments to mapper function', async () => { + const input = new Readable({ + read: () => { + input.push('a') + input.push('b') + input.push(null) + }, + }) + + const prependPrefix = (chunk, prefix) => prefix + chunk + const outStream = input.pipe(map.call(context, prependPrefix, 'foo_')) + const output = await array(outStream) + + deepStrictEqual(output, ['foo_a', 'foo_b']) + }) + + it('should pass optional arguments to mapper function when used with options object', async () => { + const input = new Readable({ + read: () => { + input.push('a') + input.push('b') + input.push(null) + }, + }) + + const prependPrefix = (chunk, prefix) => prefix + chunk + const outStream = input.pipe(map.call(context, { + map: prependPrefix, + }, 'foo_')) + const output = await array(outStream) + + deepStrictEqual(output, ['foo_a', 'foo_b']) + }) + it('accepts a function as parameter', async () => { // given const transform = letter => letter.toUpperCase() From aebdd6c26fccd95e8b1f5d7c6d68e44be0de377a Mon Sep 17 00:00:00 2001 From: tpluscode Date: Thu, 12 Sep 2024 22:04:38 +0200 Subject: [PATCH 2/3] test: dogfood parametrised map step --- test/e2e/definitions/foreach/with-variable.ttl | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/test/e2e/definitions/foreach/with-variable.ttl b/test/e2e/definitions/foreach/with-variable.ttl index 6932683e..c899f187 100644 --- a/test/e2e/definitions/foreach/with-variable.ttl +++ b/test/e2e/definitions/foreach/with-variable.ttl @@ -36,6 +36,12 @@ [ p:stepList ( - [ base:map ( "filename => this.variables.get('root') + filename"^^code:EcmaScript ) ] + [ + base:map + ( + "(filename, root) => root + filename"^^code:EcmaScript + "root"^^p:VariableName + ) + ] ) ] . From 777ce5375a4ce2b89fffcc15926cb36afbc87b0b Mon Sep 17 00:00:00 2001 From: tpluscode Date: Thu, 12 Sep 2024 22:13:14 +0200 Subject: [PATCH 3/3] test: better assertion --- test/e2e/forEach.e2e.test.js | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/test/e2e/forEach.e2e.test.js b/test/e2e/forEach.e2e.test.js index 990b5929..34373519 100644 --- a/test/e2e/forEach.e2e.test.js +++ b/test/e2e/forEach.e2e.test.js @@ -1,8 +1,9 @@ -import { deepStrictEqual, strictEqual } from 'node:assert' +import { strictEqual } from 'node:assert' import { createPipeline } from 'barnard59-core' import getStream from 'get-stream' import { pipelineDefinitionLoader } from 'barnard59-test-support/loadPipelineDefinition.js' import env from 'barnard59-env' +import { expect } from 'chai' const loadPipelineDefinition = pipelineDefinitionLoader(import.meta.url, 'definitions') @@ -36,7 +37,7 @@ describe('forEach', () => { const out = await getStream.array(pipeline.stream) - deepStrictEqual(out, [ + expect(out).to.contain.all.members([ '/root/definitions/foreach/csv-duplicate.ttl', '/root/definitions/foreach/with-handler.ttl', '/root/definitions/foreach/with-variable.ttl',