diff --git a/node_stream_zip.d.ts b/node_stream_zip.d.ts index f076c72..c0e5162 100644 --- a/node_stream_zip.d.ts +++ b/node_stream_zip.d.ts @@ -14,6 +14,13 @@ declare namespace StreamZip { */ fd?: number; + /** + * http(s) url to stream from + * The server should support HEAD request returning content-length and Range header + * @default undefined + */ + url?: string; + /** * You will be able to work with entries inside zip archive, * otherwise the only way to access them is entry event diff --git a/node_stream_zip.js b/node_stream_zip.js index d95bbef..b9082e1 100644 --- a/node_stream_zip.js +++ b/node_stream_zip.js @@ -4,6 +4,8 @@ */ let fs = require('fs'); +const https = require('https'); +const http = require('http'); const util = require('util'); const path = require('path'); const events = require('events'); @@ -136,7 +138,7 @@ const consts = { }; const StreamZip = function (config) { - let fd, fileSize, chunkSize, op, centralDirectory, closed; + let fd, fileSize, chunkSize, op, centralDirectory, closed, url; const ready = false, that = this, entries = config.storeEntries !== false ? {} : null, @@ -146,7 +148,13 @@ const StreamZip = function (config) { open(); function open() { - if (config.fd) { + if(config.url) { + url = new URL(config.url); + if(!['http:', 'https:'].includes(url.protocol)) { + throw new Error('Url should be http or https') + } + readFile(); + } else if (config.fd) { fd = config.fd; readFile(); } else { @@ -161,18 +169,31 @@ const StreamZip = function (config) { } function readFile() { - fs.fstat(fd, (err, stat) => { - if (err) { - return that.emit('error', err); - } - fileSize = stat.size; - chunkSize = config.chunkSize || Math.round(fileSize / 1000); - chunkSize = Math.max( - Math.min(chunkSize, Math.min(128 * 1024, fileSize)), - Math.min(1024, fileSize) - ); - readCentralDirectory(); - }); + if(url){ + const req = selectUrlLib(url).request(url, {method: 'HEAD'}, (res) => { + fileSize = parseInt(res.headers['content-length'], 10) + chunkSize = config.chunkSize || Math.round(fileSize / 1000); + chunkSize = Math.max( + Math.min(chunkSize, Math.min(128 * 1024, fileSize)), + Math.min(1024, fileSize) + ); + readCentralDirectory(); + }) + req.end() + } else { + fs.fstat(fd, (err, stat) => { + if (err) { + return that.emit('error', err); + } + fileSize = stat.size; + chunkSize = config.chunkSize || Math.round(fileSize / 1000); + chunkSize = Math.max( + Math.min(chunkSize, Math.min(128 * 1024, fileSize)), + Math.min(1024, fileSize) + ); + readCentralDirectory(); + }); + } } function readUntilFoundCallback(err, bytesRead) { @@ -209,7 +230,7 @@ const StreamZip = function (config) { function readCentralDirectory() { const totalReadLength = Math.min(consts.ENDHDR + consts.MAXFILECOMMENT, fileSize); op = { - win: new FileWindowBuffer(fd), + win: url ? new UrlWindowBuffer(url) : new FileWindowBuffer(fd), totalReadLength, minPos: fileSize - totalReadLength, lastPos: fileSize, @@ -311,7 +332,7 @@ const StreamZip = function (config) { function readEntries() { op = { - win: new FileWindowBuffer(fd), + win: url ? new UrlWindowBuffer(url) : new FileWindowBuffer(fd), pos: centralDirectory.offset, chunkSize, entriesLeft: centralDirectory.volumeEntries, @@ -348,7 +369,7 @@ const StreamZip = function (config) { if (!config.skipEntryNameValidation) { entry.validateName(); } - if (entries) { + if (entries && !(entry.name in entries)) { entries[entry.name] = entry; } that.emit('entry', entry); @@ -393,20 +414,29 @@ const StreamZip = function (config) { return callback(err); } const offset = dataOffset(entry); - let entryStream = new EntryDataReaderStream(fd, offset, entry.compressedSize); - if (entry.method === consts.STORED) { - // nothing to do - } else if (entry.method === consts.DEFLATED) { - entryStream = entryStream.pipe(zlib.createInflateRaw()); + let entryStream + if(url){ + const req = selectUrlLib(url).get(url, {headers: { + 'Range': `bytes=${offset}-${offset + entry.compressedSize - 1}` + }}, (res) => { + if(res.statusCode !== 206){ + callback(new Error(`${url} server doesn't support Range header`)) + } + try { + callback(null, extraEntryStreamAction(entry, res)) + } catch(err) { + callback(err) + } + }) + req.end() } else { - return callback(new Error('Unknown compression method: ' + entry.method)); - } - if (canVerifyCrc(entry)) { - entryStream = entryStream.pipe( - new EntryVerifyStream(entryStream, entry.crc, entry.size) - ); + entryStream = new EntryDataReaderStream(fd, offset, entry.compressedSize); + try { + callback(null, extraEntryStreamAction(entry, entryStream)) + } catch(err) { + callback(err) + } } - callback(null, entryStream); }, false ); @@ -460,25 +490,46 @@ const StreamZip = function (config) { if (!entry.isFile) { return callback(new Error('Entry is not file')); } - if (!fd) { + if (!fd && !url) { return callback(new Error('Archive closed')); } + if (url && sync) { + return callback(new Error('Can\'t do sync on url')); + } const buffer = Buffer.alloc(consts.LOCHDR); - new FsRead(fd, buffer, 0, buffer.length, entry.offset, (err) => { - if (err) { - return callback(err); - } - let readEx; - try { - entry.readDataHeader(buffer); - if (entry.encrypted) { - readEx = new Error('Entry encrypted'); + if(url){ + new UrlRead(url, buffer, 0, buffer.length, entry.offset, (err) => { + if (err) { + return callback(err); } - } catch (ex) { - readEx = ex; - } - callback(readEx, entry); - }).read(sync); + let readEx; + try { + entry.readDataHeader(buffer); + if (entry.encrypted) { + readEx = new Error('Entry encrypted'); + } + } catch (ex) { + readEx = ex; + } + callback(readEx, entry); + }).read(); + } else { + new FsRead(fd, buffer, 0, buffer.length, entry.offset, (err) => { + if (err) { + return callback(err); + } + let readEx; + try { + entry.readDataHeader(buffer); + if (entry.encrypted) { + readEx = new Error('Entry encrypted'); + } + } catch (ex) { + readEx = ex; + } + callback(readEx, entry); + }).read(sync); + } }; function dataOffset(entry) { @@ -490,6 +541,23 @@ const StreamZip = function (config) { return (entry.flags & 0x8) !== 0x8; } + function extraEntryStreamAction(entry, entryStream) { + if (entry.method === consts.STORED) { + // nothing to do + } else if (entry.method === consts.DEFLATED) { + entryStream = entryStream.pipe(zlib.createInflateRaw()); + } else { + throw new Error('Unknown compression method: ' + entry.method); + } + if (canVerifyCrc(entry)) { + return entryStream.pipe( + new EntryVerifyStream(entryStream, entry.crc, entry.size) + ); + } + return entryStream + } + + function extract(entry, outPath, callback) { that.stream(entry, (err, stm) => { if (err) { @@ -945,6 +1013,56 @@ class ZipEntry { } } +class UrlRead { + constructor(url, buffer, offset, length, position, callback) { + this.url = url; + this.buffer = buffer; + this.offset = offset; + this.length = length; + this.position = position; + this.callback = callback; + this.bytesRead = 0; + this.waiting = false; + } + + read() { + StreamZip.debugLog('read', this.position, this.bytesRead, this.length, this.offset); + this.waiting = true; + const req = selectUrlLib(this.url).get(this.url, {headers: { + 'Range': `bytes=${this.position + this.bytesRead}-${this.position + this.length}` + }}, (res) => { + const chunks = [] + if(res.statusCode !== 206){ + throw new Error(`${this.url} server doesn't support Range header`) + } + res.on('data', (chunk) => { + chunks.push(chunk) + }) + res.on('end', () => { + const data = Buffer.concat(chunks); + data.copy(this.buffer, this.offset + this.bytesRead) + this.readCallback(null, data.length - 1); + }) + res.on('error', (err) => { + this.readCallback(err, null); + }) + }) + req.end(); + } + + readCallback(err, bytesRead) { + if (typeof bytesRead === 'number') { + this.bytesRead += bytesRead; + } + if (err || !bytesRead || this.bytesRead === this.length) { + this.waiting = false; + return this.callback(err, this.bytesRead); + } else { + this.read(); + } + } +} + class FsRead { constructor(fd, buffer, offset, length, position, callback) { this.fd = fd; @@ -1000,6 +1118,72 @@ class FsRead { } } +class UrlWindowBuffer { + constructor(url) { + this.position = 0; + this.buffer = Buffer.alloc(0); + this.url = url; + this.urlOp = null; + } + + checkOp() { + if (this.urlOp && this.urlOp.waiting) { + throw new Error('Operation in progress'); + } + } + + read(pos, length, callback) { + this.checkOp(); + if (this.buffer.length < length) { + this.buffer = Buffer.alloc(length); + } + this.position = pos; + this.urlOp = new UrlRead(this.url, this.buffer, 0, length, this.position, callback).read(); + } + + expandLeft(length, callback) { + this.checkOp(); + this.buffer = Buffer.concat([Buffer.alloc(length), this.buffer]); + this.position -= length; + if (this.position < 0) { + this.position = 0; + } + this.urlOp = new UrlRead(this.url, this.buffer, 0, length, this.position, callback).read(); + } + + expandRight(length, callback) { + this.checkOp(); + const offset = this.buffer.length; + this.buffer = Buffer.concat([this.buffer, Buffer.alloc(length)]); + this.urlOp = new UrlRead( + this.url, + this.buffer, + offset, + length, + this.position + offset, + callback + ).read(); + } + + moveRight(length, callback, shift) { + this.checkOp(); + if (shift) { + this.buffer.copy(this.buffer, 0, shift); + } else { + shift = 0; + } + this.position += shift; + this.urlOp = new UrlRead( + this.url, + this.buffer, + this.buffer.length - shift, + shift, + this.position + this.buffer.length - shift, + callback + ).read(); + } +} + class FileWindowBuffer { constructor(fd) { this.position = 0; @@ -1179,6 +1363,10 @@ class CrcVerify { } } +function selectUrlLib(url) { + return url.protocol === 'https:' ? https : http +} + function parseZipTime(timebytes, datebytes) { const timebits = toBits(timebytes, 16); const datebits = toBits(datebytes, 16); diff --git a/package-lock.json b/package-lock.json index f0d4584..0d8eb21 100644 --- a/package-lock.json +++ b/package-lock.json @@ -5,16 +5,22 @@ "requires": true, "packages": { "": { - "version": "1.12.0", + "name": "node-stream-zip", + "version": "1.15.0", "license": "MIT", "devDependencies": { "@types/node": "^14.14.6", "eslint": "^7.19.0", "nodeunit": "^0.11.3", - "prettier": "^2.2.1" + "prettier": "^2.2.1", + "send": "^0.17.2" }, "engines": { - "node": ">=0.10.0" + "node": ">=0.12.0" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/antelle" } }, "node_modules/@babel/code-frame": { @@ -592,6 +598,7 @@ "version": "4.1.1", "resolved": "https://registry.npmjs.org/debug/-/debug-4.1.1.tgz", "integrity": "sha512-pYAIzeRo8J6KPEaJ0VWOh5Pzkbw/RetuzehGM7QRRX5he4fPHx2rdKMB256ehJCkX+XRQm16eZLqLNS8RSZXZw==", + "deprecated": "Debug versions >=3.2.0 <3.2.7 || >=4 <4.3.1 have a low-severity ReDos regression when used in a Node.js environment. It is recommended you upgrade to 3.2.7 or 4.3.1. (https://github.com/visionmedia/debug/issues/797)", "dev": true, "dependencies": { "ms": "^2.1.1" @@ -633,6 +640,21 @@ "node": ">=0.4.0" } }, + "node_modules/depd": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/depd/-/depd-1.1.2.tgz", + "integrity": "sha1-m81S4UwJd2PnSbJ0xDRu0uVgtak=", + "dev": true, + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/destroy": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/destroy/-/destroy-1.0.4.tgz", + "integrity": "sha1-l4hXRCxEdJ5CBmE+N5RiBYJqvYA=", + "dev": true + }, "node_modules/diff": { "version": "1.4.0", "resolved": "https://registry.npmjs.org/diff/-/diff-1.4.0.tgz", @@ -674,11 +696,18 @@ "safer-buffer": "^2.1.0" } }, + "node_modules/ee-first": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/ee-first/-/ee-first-1.1.1.tgz", + "integrity": "sha1-WQxhFWsK4vTwJVcyoViyZrxWsh0=", + "dev": true + }, "node_modules/ejs": { "version": "2.7.4", "resolved": "https://registry.npmjs.org/ejs/-/ejs-2.7.4.tgz", "integrity": "sha512-7vmuyh5+kuUyJKePhQfRQBhXV5Ce+RnaeeQArKu1EAMpL3WbgMt5WG6uQZpEVvYSSsxMXRKOewtDk9RaTKXRlA==", "dev": true, + "hasInstallScript": true, "engines": { "node": ">=0.10.0" } @@ -689,6 +718,15 @@ "integrity": "sha512-CwBLREIQ7LvYFB0WyRvwhq5N5qPhc6PMjD6bYggFlI5YyDgl+0vxq5VHbMOFqLg7hfWzmu8T5Z1QofhmTIhItA==", "dev": true }, + "node_modules/encodeurl": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/encodeurl/-/encodeurl-1.0.2.tgz", + "integrity": "sha1-rT/0yG7C0CkyL1oCw6mmBslbP1k=", + "dev": true, + "engines": { + "node": ">= 0.8" + } + }, "node_modules/enquirer": { "version": "2.3.6", "resolved": "https://registry.npmjs.org/enquirer/-/enquirer-2.3.6.tgz", @@ -716,6 +754,12 @@ "integrity": "sha512-Um/+FxMr9CISWh0bi5Zv0iOD+4cFh5qLeks1qhAopKVAJw3drgKbKySikp7wGhDL0HPeaja0P5ULZrxLkniUVg==", "dev": true }, + "node_modules/escape-html": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/escape-html/-/escape-html-1.0.3.tgz", + "integrity": "sha1-Aljq5NPQwJdN4cFpGI7wBR0dGYg=", + "dev": true + }, "node_modules/escape-string-regexp": { "version": "1.0.5", "resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-1.0.5.tgz", @@ -1098,6 +1142,15 @@ "node": ">=0.10.0" } }, + "node_modules/etag": { + "version": "1.8.1", + "resolved": "https://registry.npmjs.org/etag/-/etag-1.8.1.tgz", + "integrity": "sha1-Qa4u62XvpiJorr/qg6x9eSmbCIc=", + "dev": true, + "engines": { + "node": ">= 0.6" + } + }, "node_modules/events-to-array": { "version": "1.1.2", "resolved": "https://registry.npmjs.org/events-to-array/-/events-to-array-1.1.2.tgz", @@ -1242,6 +1295,15 @@ "node": ">= 0.12" } }, + "node_modules/fresh": { + "version": "0.5.2", + "resolved": "https://registry.npmjs.org/fresh/-/fresh-0.5.2.tgz", + "integrity": "sha1-PYyt2Q2XZWn6g1qx+OSyOhBWBac=", + "dev": true, + "engines": { + "node": ">= 0.6" + } + }, "node_modules/fs-exists-cached": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/fs-exists-cached/-/fs-exists-cached-1.0.0.tgz", @@ -1299,6 +1361,9 @@ }, "engines": { "node": "*" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" } }, "node_modules/glob-parent": { @@ -1341,6 +1406,7 @@ "version": "5.1.3", "resolved": "https://registry.npmjs.org/har-validator/-/har-validator-5.1.3.tgz", "integrity": "sha512-sNvOCzEQNr/qrvJgc3UG/kD4QtlHycrzwS+6mfTrrSq97BvaYcPZZI1ZSqGSPR73Cxn4LKTD4PttRwfU7jWq5g==", + "deprecated": "this library is no longer supported", "dev": true, "dependencies": { "ajv": "^6.5.5", @@ -1383,6 +1449,22 @@ "integrity": "sha512-a4u9BeERWGu/S8JiWEAQcdrg9v4QArtP9keViQjGMdff20fBdd8waotXaNmODqBe6uZ3Nafi7K/ho4gCQHV3Ig==", "dev": true }, + "node_modules/http-errors": { + "version": "1.8.1", + "resolved": "https://registry.npmjs.org/http-errors/-/http-errors-1.8.1.tgz", + "integrity": "sha512-Kpk9Sm7NmI+RHhnj6OIWDI1d6fIoFAtFt9RLaTMRlg/8w49juAStsrBgp0Dp4OdxdVbRIeKhtCUvoi/RuAhO4g==", + "dev": true, + "dependencies": { + "depd": "~1.1.2", + "inherits": "2.0.4", + "setprototypeof": "1.2.0", + "statuses": ">= 1.5.0 < 2", + "toidentifier": "1.0.1" + }, + "engines": { + "node": ">= 0.6" + } + }, "node_modules/http-signature": { "version": "1.2.0", "resolved": "https://registry.npmjs.org/http-signature/-/http-signature-1.2.0.tgz", @@ -1838,6 +1920,18 @@ "node": ">=0.10.0" } }, + "node_modules/mime": { + "version": "1.6.0", + "resolved": "https://registry.npmjs.org/mime/-/mime-1.6.0.tgz", + "integrity": "sha512-x0Vn8spI+wuJ1O6S7gnbaQg8Pxh4NNHb7KSINmEWKiPE4RKOplvijn+NkmYmmRgP68mc70j2EbeTFRsrswaQeg==", + "dev": true, + "bin": { + "mime": "cli.js" + }, + "engines": { + "node": ">=4" + } + }, "node_modules/mime-db": { "version": "1.43.0", "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.43.0.tgz", @@ -1927,6 +2021,7 @@ "version": "0.11.3", "resolved": "https://registry.npmjs.org/nodeunit/-/nodeunit-0.11.3.tgz", "integrity": "sha512-gDNxrDWpx07BxYNO/jn1UrGI1vNhDQZrIFphbHMcTCDc5mrrqQBWfQMXPHJ5WSgbFwD1D6bv4HOsqtTrPG03AA==", + "deprecated": "you are strongly encouraged to use other testing options", "dev": true, "dependencies": { "ejs": "^2.5.2", @@ -1996,6 +2091,18 @@ "node": "*" } }, + "node_modules/on-finished": { + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/on-finished/-/on-finished-2.3.0.tgz", + "integrity": "sha1-IPEzZIGwg811M3mSoWlxqi2QaUc=", + "dev": true, + "dependencies": { + "ee-first": "1.1.1" + }, + "engines": { + "node": ">= 0.8" + } + }, "node_modules/once": { "version": "1.4.0", "resolved": "https://registry.npmjs.org/once/-/once-1.4.0.tgz", @@ -2065,6 +2172,9 @@ }, "engines": { "node": ">=6" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" } }, "node_modules/p-locate": { @@ -2276,6 +2386,15 @@ "node": ">=0.6" } }, + "node_modules/range-parser": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/range-parser/-/range-parser-1.2.1.tgz", + "integrity": "sha512-Hrgsx+orqoygnmhFbKaHE6c296J+HTAQXoxEF6gNupROmmGJRoyzfG3ccAveqCBrwr/2yxQ5BVd/GTl5agOwSg==", + "dev": true, + "engines": { + "node": ">= 0.6" + } + }, "node_modules/read-pkg": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/read-pkg/-/read-pkg-3.0.0.tgz", @@ -2354,6 +2473,7 @@ "version": "2.88.0", "resolved": "https://registry.npmjs.org/request/-/request-2.88.0.tgz", "integrity": "sha512-NAqBSrijGLZdM0WZNsInLJpkJokL72XYjUpnB0iwsRgxh7dB6COrHnTBNwN0E+lHDAJzu7kLAkDeY08z2/A0hg==", + "deprecated": "request has been deprecated, see https://github.com/request/request/issues/3142", "dev": true, "dependencies": { "aws-sign2": "~0.7.0", @@ -2412,6 +2532,9 @@ "dev": true, "dependencies": { "path-parse": "^1.0.6" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" } }, "node_modules/resolve-from": { @@ -2456,12 +2579,63 @@ "semver": "bin/semver" } }, + "node_modules/send": { + "version": "0.17.2", + "resolved": "https://registry.npmjs.org/send/-/send-0.17.2.tgz", + "integrity": "sha512-UJYB6wFSJE3G00nEivR5rgWp8c2xXvJ3OPWPhmuteU0IKj8nKbG3DrjiOmLwpnHGYWAVwA69zmTm++YG0Hmwww==", + "dev": true, + "dependencies": { + "debug": "2.6.9", + "depd": "~1.1.2", + "destroy": "~1.0.4", + "encodeurl": "~1.0.2", + "escape-html": "~1.0.3", + "etag": "~1.8.1", + "fresh": "0.5.2", + "http-errors": "1.8.1", + "mime": "1.6.0", + "ms": "2.1.3", + "on-finished": "~2.3.0", + "range-parser": "~1.2.1", + "statuses": "~1.5.0" + }, + "engines": { + "node": ">= 0.8.0" + } + }, + "node_modules/send/node_modules/debug": { + "version": "2.6.9", + "resolved": "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz", + "integrity": "sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA==", + "dev": true, + "dependencies": { + "ms": "2.0.0" + } + }, + "node_modules/send/node_modules/debug/node_modules/ms": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz", + "integrity": "sha1-VgiurfwAvmwpAd9fmGF4jeDVl8g=", + "dev": true + }, + "node_modules/send/node_modules/ms": { + "version": "2.1.3", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", + "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==", + "dev": true + }, "node_modules/set-blocking": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/set-blocking/-/set-blocking-2.0.0.tgz", "integrity": "sha1-BF+XgtARrppoA93TgrJDkrPYkPc=", "dev": true }, + "node_modules/setprototypeof": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/setprototypeof/-/setprototypeof-1.2.0.tgz", + "integrity": "sha512-E5LDX7Wrp85Kil5bhZv46j8jOeboKq5JMmYM3gVGdGH8xFpPWXUMsNrlODCrkoxMEeNi/XZIwuRvY4XNwYMJpw==", + "dev": true + }, "node_modules/shebang-command": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/shebang-command/-/shebang-command-2.0.0.tgz", @@ -2662,6 +2836,15 @@ "node": ">=0.10.0" } }, + "node_modules/statuses": { + "version": "1.5.0", + "resolved": "https://registry.npmjs.org/statuses/-/statuses-1.5.0.tgz", + "integrity": "sha1-Fhx9rBd2Wf2YEfQ3cfqZOBR4Yow=", + "dev": true, + "engines": { + "node": ">= 0.6" + } + }, "node_modules/string_decoder": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.1.1.tgz", @@ -2891,7 +3074,6 @@ "escape-string-regexp": "^1.0.3", "glob": "^7.0.5", "js-yaml": "^3.3.1", - "readable-stream": "^2.1.5", "tap-parser": "^5.1.0", "unicode-length": "^1.0.0" }, @@ -2924,8 +3106,7 @@ "dev": true, "dependencies": { "events-to-array": "^1.0.1", - "js-yaml": "^3.2.7", - "readable-stream": "^2" + "js-yaml": "^3.2.7" }, "bin": { "tap-parser": "bin/cmd.js" @@ -2984,6 +3165,15 @@ "node": ">=4" } }, + "node_modules/toidentifier": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/toidentifier/-/toidentifier-1.0.1.tgz", + "integrity": "sha512-o5sSPKEkg/DIQNmH43V0/uerLrpzVedkUh8tGNvaeXpfpuwjKenlSox/2O/BTlZUtEe+JG7s5YhEz608PlAHRA==", + "dev": true, + "engines": { + "node": ">=0.6" + } + }, "node_modules/tough-cookie": { "version": "2.4.3", "resolved": "https://registry.npmjs.org/tough-cookie/-/tough-cookie-2.4.3.tgz", @@ -3027,6 +3217,9 @@ }, "engines": { "node": ">=6.0.0" + }, + "peerDependencies": { + "typescript": ">=2.7" } }, "node_modules/ts-node/node_modules/diff": { @@ -3153,6 +3346,7 @@ "version": "3.3.3", "resolved": "https://registry.npmjs.org/uuid/-/uuid-3.3.3.tgz", "integrity": "sha512-pW0No1RGHgzlpHJO1nsVrHKpOEIxkGg1xB+v0ZmdNH5OAeAwzAVrCnI2/6Mtx+Uys6iaylxa+D3g4j63IKKjSQ==", + "deprecated": "Please upgrade to version 7 or higher. Older versions may use Math.random() in certain circumstances, which is known to be problematic. See https://v8.dev/blog/math-random for details.", "dev": true, "bin": { "uuid": "bin/uuid" @@ -3830,6 +4024,18 @@ "integrity": "sha1-3zrhmayt+31ECqrgsp4icrJOxhk=", "dev": true }, + "depd": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/depd/-/depd-1.1.2.tgz", + "integrity": "sha1-m81S4UwJd2PnSbJ0xDRu0uVgtak=", + "dev": true + }, + "destroy": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/destroy/-/destroy-1.0.4.tgz", + "integrity": "sha1-l4hXRCxEdJ5CBmE+N5RiBYJqvYA=", + "dev": true + }, "diff": { "version": "1.4.0", "resolved": "https://registry.npmjs.org/diff/-/diff-1.4.0.tgz", @@ -3861,6 +4067,12 @@ "safer-buffer": "^2.1.0" } }, + "ee-first": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/ee-first/-/ee-first-1.1.1.tgz", + "integrity": "sha1-WQxhFWsK4vTwJVcyoViyZrxWsh0=", + "dev": true + }, "ejs": { "version": "2.7.4", "resolved": "https://registry.npmjs.org/ejs/-/ejs-2.7.4.tgz", @@ -3873,6 +4085,12 @@ "integrity": "sha512-CwBLREIQ7LvYFB0WyRvwhq5N5qPhc6PMjD6bYggFlI5YyDgl+0vxq5VHbMOFqLg7hfWzmu8T5Z1QofhmTIhItA==", "dev": true }, + "encodeurl": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/encodeurl/-/encodeurl-1.0.2.tgz", + "integrity": "sha1-rT/0yG7C0CkyL1oCw6mmBslbP1k=", + "dev": true + }, "enquirer": { "version": "2.3.6", "resolved": "https://registry.npmjs.org/enquirer/-/enquirer-2.3.6.tgz", @@ -3897,6 +4115,12 @@ "integrity": "sha512-Um/+FxMr9CISWh0bi5Zv0iOD+4cFh5qLeks1qhAopKVAJw3drgKbKySikp7wGhDL0HPeaja0P5ULZrxLkniUVg==", "dev": true }, + "escape-html": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/escape-html/-/escape-html-1.0.3.tgz", + "integrity": "sha1-Aljq5NPQwJdN4cFpGI7wBR0dGYg=", + "dev": true + }, "escape-string-regexp": { "version": "1.0.5", "resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-1.0.5.tgz", @@ -4177,6 +4401,12 @@ "integrity": "sha512-kVscqXk4OCp68SZ0dkgEKVi6/8ij300KBWTJq32P/dYeWTSwK41WyTxalN1eRmA5Z9UU/LX9D7FWSmV9SAYx6g==", "dev": true }, + "etag": { + "version": "1.8.1", + "resolved": "https://registry.npmjs.org/etag/-/etag-1.8.1.tgz", + "integrity": "sha1-Qa4u62XvpiJorr/qg6x9eSmbCIc=", + "dev": true + }, "events-to-array": { "version": "1.1.2", "resolved": "https://registry.npmjs.org/events-to-array/-/events-to-array-1.1.2.tgz", @@ -4296,6 +4526,12 @@ "mime-types": "^2.1.12" } }, + "fresh": { + "version": "0.5.2", + "resolved": "https://registry.npmjs.org/fresh/-/fresh-0.5.2.tgz", + "integrity": "sha1-PYyt2Q2XZWn6g1qx+OSyOhBWBac=", + "dev": true + }, "fs-exists-cached": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/fs-exists-cached/-/fs-exists-cached-1.0.0.tgz", @@ -4413,6 +4649,19 @@ "integrity": "sha512-a4u9BeERWGu/S8JiWEAQcdrg9v4QArtP9keViQjGMdff20fBdd8waotXaNmODqBe6uZ3Nafi7K/ho4gCQHV3Ig==", "dev": true }, + "http-errors": { + "version": "1.8.1", + "resolved": "https://registry.npmjs.org/http-errors/-/http-errors-1.8.1.tgz", + "integrity": "sha512-Kpk9Sm7NmI+RHhnj6OIWDI1d6fIoFAtFt9RLaTMRlg/8w49juAStsrBgp0Dp4OdxdVbRIeKhtCUvoi/RuAhO4g==", + "dev": true, + "requires": { + "depd": "~1.1.2", + "inherits": "2.0.4", + "setprototypeof": "1.2.0", + "statuses": ">= 1.5.0 < 2", + "toidentifier": "1.0.1" + } + }, "http-signature": { "version": "1.2.0", "resolved": "https://registry.npmjs.org/http-signature/-/http-signature-1.2.0.tgz", @@ -4787,6 +5036,12 @@ } } }, + "mime": { + "version": "1.6.0", + "resolved": "https://registry.npmjs.org/mime/-/mime-1.6.0.tgz", + "integrity": "sha512-x0Vn8spI+wuJ1O6S7gnbaQg8Pxh4NNHb7KSINmEWKiPE4RKOplvijn+NkmYmmRgP68mc70j2EbeTFRsrswaQeg==", + "dev": true + }, "mime-db": { "version": "1.43.0", "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.43.0.tgz", @@ -4923,6 +5178,15 @@ "integrity": "sha512-fexhUFFPTGV8ybAtSIGbV6gOkSv8UtRbDBnAyLQw4QPKkgNlsH2ByPGtMUqdWkos6YCRmAqViwgZrJc/mRDzZQ==", "dev": true }, + "on-finished": { + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/on-finished/-/on-finished-2.3.0.tgz", + "integrity": "sha1-IPEzZIGwg811M3mSoWlxqi2QaUc=", + "dev": true, + "requires": { + "ee-first": "1.1.1" + } + }, "once": { "version": "1.4.0", "resolved": "https://registry.npmjs.org/once/-/once-1.4.0.tgz", @@ -5139,6 +5403,12 @@ "integrity": "sha512-N5ZAX4/LxJmF+7wN74pUD6qAh9/wnvdQcjq9TZjevvXzSUo7bfmw91saqMjzGS2xq91/odN2dW/WOl7qQHNDGA==", "dev": true }, + "range-parser": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/range-parser/-/range-parser-1.2.1.tgz", + "integrity": "sha512-Hrgsx+orqoygnmhFbKaHE6c296J+HTAQXoxEF6gNupROmmGJRoyzfG3ccAveqCBrwr/2yxQ5BVd/GTl5agOwSg==", + "dev": true + }, "read-pkg": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/read-pkg/-/read-pkg-3.0.0.tgz", @@ -5288,12 +5558,64 @@ "integrity": "sha512-sauaDf/PZdVgrLTNYHRtpXa1iRiKcaebiKQ1BJdpQlWH2lCvexQdX55snPFyK7QzpudqbCI0qXFfOasHdyNDGQ==", "dev": true }, + "send": { + "version": "0.17.2", + "resolved": "https://registry.npmjs.org/send/-/send-0.17.2.tgz", + "integrity": "sha512-UJYB6wFSJE3G00nEivR5rgWp8c2xXvJ3OPWPhmuteU0IKj8nKbG3DrjiOmLwpnHGYWAVwA69zmTm++YG0Hmwww==", + "dev": true, + "requires": { + "debug": "2.6.9", + "depd": "~1.1.2", + "destroy": "~1.0.4", + "encodeurl": "~1.0.2", + "escape-html": "~1.0.3", + "etag": "~1.8.1", + "fresh": "0.5.2", + "http-errors": "1.8.1", + "mime": "1.6.0", + "ms": "2.1.3", + "on-finished": "~2.3.0", + "range-parser": "~1.2.1", + "statuses": "~1.5.0" + }, + "dependencies": { + "debug": { + "version": "2.6.9", + "resolved": "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz", + "integrity": "sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA==", + "dev": true, + "requires": { + "ms": "2.0.0" + }, + "dependencies": { + "ms": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz", + "integrity": "sha1-VgiurfwAvmwpAd9fmGF4jeDVl8g=", + "dev": true + } + } + }, + "ms": { + "version": "2.1.3", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", + "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==", + "dev": true + } + } + }, "set-blocking": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/set-blocking/-/set-blocking-2.0.0.tgz", "integrity": "sha1-BF+XgtARrppoA93TgrJDkrPYkPc=", "dev": true }, + "setprototypeof": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/setprototypeof/-/setprototypeof-1.2.0.tgz", + "integrity": "sha512-E5LDX7Wrp85Kil5bhZv46j8jOeboKq5JMmYM3gVGdGH8xFpPWXUMsNrlODCrkoxMEeNi/XZIwuRvY4XNwYMJpw==", + "dev": true + }, "shebang-command": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/shebang-command/-/shebang-command-2.0.0.tgz", @@ -5457,6 +5779,12 @@ "integrity": "sha512-MTX+MeG5U994cazkjd/9KNAapsHnibjMLnfXodlkXw76JEea0UiNzrqidzo1emMwk7w5Qhc9jd4Bn9TBb1MFwA==", "dev": true }, + "statuses": { + "version": "1.5.0", + "resolved": "https://registry.npmjs.org/statuses/-/statuses-1.5.0.tgz", + "integrity": "sha1-Fhx9rBd2Wf2YEfQ3cfqZOBR4Yow=", + "dev": true + }, "string_decoder": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.1.1.tgz", @@ -5721,6 +6049,12 @@ "integrity": "sha1-3F5pjL0HkmW8c+A3doGk5Og/YW4=", "dev": true }, + "toidentifier": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/toidentifier/-/toidentifier-1.0.1.tgz", + "integrity": "sha512-o5sSPKEkg/DIQNmH43V0/uerLrpzVedkUh8tGNvaeXpfpuwjKenlSox/2O/BTlZUtEe+JG7s5YhEz608PlAHRA==", + "dev": true + }, "tough-cookie": { "version": "2.4.3", "resolved": "https://registry.npmjs.org/tough-cookie/-/tough-cookie-2.4.3.tgz", diff --git a/package.json b/package.json index 5fd74e0..b404d69 100644 --- a/package.json +++ b/package.json @@ -38,7 +38,8 @@ "@types/node": "^14.14.6", "eslint": "^7.19.0", "nodeunit": "^0.11.3", - "prettier": "^2.2.1" + "prettier": "^2.2.1", + "send": "^0.17.2" }, "funding": { "type": "github", diff --git a/test/tests.js b/test/tests.js index a2ac1bf..89b80c7 100644 --- a/test/tests.js +++ b/test/tests.js @@ -1,4 +1,6 @@ const fs = require('fs'); +const http = require('http') +const send = require('send') const path = require('path'); const StreamZip = require('../node_stream_zip.js'); @@ -204,6 +206,31 @@ module.exports.ok['fd'] = function (test) { }); }; +module.exports.ok['url'] = function (test) { + const server = http.createServer((req, res) => { + send(req, 'test/ok/normal.zip').pipe(res) + }) + + server.listen(8000) + const zip = new StreamZip({ url: 'http://127.0.0.1:8000/normal.zip' }); + zip.on('ready', () => { + const entries = zip.entries(); + const entry = entries['doc/changelog-foot.html']; + test.ok(entry); + const entryBeforeOpen = Object.assign({}, entry); + zip.openEntry( + entry, + (err, entryAfterOpen) => { + test.equal(err, undefined); + test.notDeepEqual(entryBeforeOpen, entryAfterOpen); + test.done(); + server.close() + }, + false + ); + }); +}; + module.exports.ok['encoding-utf8'] = function (test) { test.expect(1); const zip = new StreamZip({ file: 'test/special/utf8.zip' }); @@ -395,6 +422,31 @@ module.exports.parallel['streaming 100 files'] = function (test) { }); }; +module.exports.parallel['streaming 100 files from url'] = async function (test) { + const num = 100; + const server = http.createServer((req, res) => { + send(req, 'test/ok/normal.zip').pipe(res) + }) + + server.listen(8000) + const zip = new StreamZip.async({ url: 'http://127.0.0.1:8000/normal.zip' }); + let extracted = 0; + const files = [ + 'doc/changelog-foot.html', + 'doc/sh_javascript.min.js', + 'BSDmakefile', + 'README.md', + ]; + for (let i = 0; i < num; i++) { + const file = files[Math.floor(Math.random() * files.length)]; + await zip.extract(file, testPathTmp + i) + if (++extracted === num) { + test.done(); + server.close() + } + } +}; + module.exports['callback exception'] = function (test) { test.expect(3); const zip = new StreamZip({ file: 'test/special/tiny.zip' });