add support for testing multiple versions of supported modules (DataD…

…og#243)

* add support for testing multiple versions of supported modules

* fix older express 4.x minor versions not instrumented properly

.circleci/config.yml

@@ -7,9 +7,7 @@ docker-base: &docker-base
     - &mysql
       image: mysql:5.7
       environment:
-        - MYSQL_ROOT_PASSWORD=rootpass
-        - MYSQL_PASSWORD=userpass
-        - MYSQL_USER=user
+        - MYSQL_ALLOW_EMPTY_PASSWORD=yes
         - MYSQL_DATABASE=db
     - &redis
       image: redis:4.0-alpine
@@ -26,16 +24,16 @@ build-node-base: &node-base
     - checkout
     - run:
         name: Versions
-        command: npm version
+        command: yarn versions
     - run:
         name: Install dependencies
-        command: npm install
+        command: yarn install
     - run:
         name: Test
-        command: npm test
+        command: yarn test
     - run:
         name: Benchmark
-        command: npm run bench
+        command: yarn bench
 jobs:
   lint:
     docker:
@@ -45,27 +43,27 @@ jobs:
       - checkout
       - run:
           name: Versions
-          command: npm version
+          command: yarn versions
       - run:
           name: Install dependencies
-          command: npm install
+          command: yarn install
       - run:
           name: Lint
-          command: npm run lint
+          command: yarn lint
   test-memory-leaks:
     <<: *docker-base
     working_directory: ~/dd-trace-js
     steps:
       - checkout
       - run:
           name: Versions
-          command: npm version
+          command: yarn versions
       - run:
           name: Install dependencies
-          command: npm install
+          command: yarn install
       - run:
           name: Test
-          command: npm run leak
+          command: yarn leak
   build-node-4:
     <<: *node-base
     docker:

.eslintignore

@@ -1,2 +1,3 @@
 coverage
 out
+test/plugins/versions

.gitignore

@@ -103,4 +103,5 @@ typings/
 package-lock.json
 yarn.lock
 out
+test/plugins/versions
 !test/node_modules

LICENSE-3rdparty.csv

@@ -46,6 +46,7 @@ dev,nyc,ISC,Copyright 2015 Contributors
 dev,pg,MIT,Copyright 2010-2018 Brian Carlson
 dev,proxyquire,MIT,Copyright 2013 Thorsten Lorenz
 dev,redis,MIT,Copyright 2016 NodeRedis
+dev,require-dir,MIT,2012-2015 Aseem Kishore
 dev,retry,MIT,Copyright 2011 Tim Koschützki Felix Geisendörfer
 dev,semver,ISC,Copyright Isaac Z. Schlueter and Contributors
 dev,sinon,BSD-3-Clause,Copyright 2010-2017 Christian Johansen

README.md

@@ -24,10 +24,12 @@ Before contributing to this open source project, read our [CONTRIBUTING.md](http
 Since this project supports multiple Node versions, using a version
 manager such as [nvm](https://github.com/creationix/nvm) is recommended.
 
-To get started once you have a Node version installed, run:
+We use [yarn](https://yarnpkg.com/) for its workspace functionality, so make sure to install that as well.
+
+To get started once you have Node and yarn installed, run:
 
 ```sh
-$ npm install
+$ yarn
 ```
 
 ### Testing
@@ -45,21 +47,21 @@ $ docker-compose up -d
 To run the unit tests, use:
 
 ```sh
-$ npm test
+$ yarn test
 ```
 
 To run the unit tests continuously in watch mode while developing, use:
 
 ```sh
-$ npm run tdd
+$ yarn tdd
 ```
 
 #### Memory Leaks
 
 To run the memory leak tests, use:
 
 ```sh
-$ npm run leak
+$ yarn leak
 ```
 
 Please note that memory leak tests only run on Node `>=8`.
@@ -72,7 +74,7 @@ conform to our coding standards.
 To run the linter, use:
 
 ```sh
-$ npm run lint
+$ yarn lint
 ```
 
 ### Continuous Integration
@@ -99,5 +101,5 @@ in the `benchmark/index.js` module so that we can keep track of the
 most efficient algorithm. To run your benchmark, just:
 
 ```sh
-$ npm run bench
+$ yarn bench
 ```

docker-compose.yml

@@ -7,9 +7,7 @@ services:
   mysql:
     image: mysql:5.7
     environment:
-      - MYSQL_ROOT_PASSWORD=rootpass
-      - MYSQL_PASSWORD=userpass
-      - MYSQL_USER=user
+      - MYSQL_ALLOW_EMPTY_PASSWORD=yes
       - MYSQL_DATABASE=db
     ports:
       - "127.0.0.1:3306:3306"

package.json

@@ -8,8 +8,8 @@
     "jsdoc": "gulp jsdoc",
     "jsdoc:watch": "gulp jsdoc:watch",
     "lint": "eslint . && node scripts/check_licenses.js",
-    "tdd": "mocha --watch",
-    "test": "nyc --reporter text --reporter lcov mocha 'test/**/*.spec.js'",
+    "tdd": "node ./scripts/install_plugin_modules && mocha --watch",
+    "test": "node ./scripts/install_plugin_modules && nyc --reporter text --reporter lcov mocha 'test/**/*.spec.js'",
     "leak": "node --no-warnings ./node_modules/.bin/tape 'test/leak/**/*.js'"
   },
   "repository": {
@@ -82,6 +82,7 @@
     "pg": "^6.4.2",
     "proxyquire": "^1.8.0",
     "redis": "^2.8.0",
+    "require-dir": "^1.0.0",
     "retry": "^0.10.1",
     "sinon": "^4.2.1",
     "sinon-chai": "^2.14.0",

scripts/install_plugin_modules.js

@@ -0,0 +1,108 @@
+'use strict'
+
+const fs = require('fs')
+const path = require('path')
+const requireDir = require('require-dir')
+const crypto = require('crypto')
+const semver = require('semver')
+const exec = require('./helpers/exec')
+const plugins = requireDir('../src/plugins')
+
+const workspaces = new Set()
+
+run()
+
+function run () {
+  assertVersions()
+  assertWorkspace()
+  install()
+}
+
+function assertVersions () {
+  Object.keys(plugins).filter(key => key !== 'index').forEach(key => {
+    [].concat(plugins[key]).forEach(instrumentation => {
+      [].concat(instrumentation.versions).forEach(version => {
+        if (version) {
+          assertModules(instrumentation.name, version)
+          assertModules(instrumentation.name, semver.coerce(version).version)
+        }
+      })
+    })
+  })
+}
+
+function assertModules (name, version) {
+  addFolder(name, version)
+  assertFolder(name, version)
+  assertPackage(name, version)
+  assertIndex(name, version)
+}
+
+function assertFolder (name, version) {
+  if (!fs.existsSync(folder())) {
+    fs.mkdirSync(folder())
+  }
+
+  if (!fs.existsSync(folder(name, version))) {
+    fs.mkdirSync(folder(name, version))
+  }
+}
+
+function assertPackage (name, version) {
+  fs.writeFileSync(filename(name, version, 'package.json'), JSON.stringify({
+    name: [name, sha1(version)].filter(val => val).join('-'),
+    version: '1.0.0',
+    license: 'BSD-3-Clause',
+    private: true,
+    dependencies: {
+      [name]: version
+    }
+  }, null, 2) + '\n')
+}
+
+function assertIndex (name, version) {
+  const index = `'use strict'
+
+module.exports = {
+  get (id) { return require(id || '${name}') },
+  version () { return require('${name}/package.json').version }
+}
+`
+  fs.writeFileSync(filename(name, version, 'index.js'), index)
+}
+
+function assertWorkspace () {
+  fs.writeFileSync(filename(null, null, 'package.json'), JSON.stringify({
+    name: 'versions',
+    version: '1.0.0',
+    license: 'BSD-3-Clause',
+    private: true,
+    workspaces: Array.from(workspaces)
+  }, null, 2) + '\n')
+}
+
+function install () {
+  exec('yarn', { cwd: folder() })
+}
+
+function addFolder (name, version) {
+  const basename = [name, version].filter(val => val).join('@')
+  workspaces.add(basename)
+}
+
+function folder (name, version) {
+  const basename = [name, version].filter(val => val).join('@')
+  return path.join(__dirname, '..', 'test', 'plugins', 'versions', basename)
+}
+
+function filename (name, version, file) {
+  return path.join(folder(name, version), file)
+}
+
+function sha1 (str) {
+  if (!str) return
+
+  const shasum = crypto.createHash('sha1')
+  shasum.update(str)
+  return shasum.digest('hex')
+}

scripts/publish_docs.js

@@ -1,6 +1,5 @@
 'use strict'
 
-const fs = require('fs')
 const exec = require('./helpers/exec')
 const title = require('./helpers/title')
 
@@ -12,15 +11,10 @@ if (!msg) {
   throw new Error('Please provide a reason for the change. Example: node scripts/publish_docs.js "fix typo"')
 }
 
-if (fs.existsSync('yarn.lock')) {
-  exec('yarn')
-} else {
-  exec('npm install')
-}
-
+exec('yarn install')
 exec('rm -rf ./out')
 exec('git clone -b gh-pages --single-branch [email protected]:DataDog/dd-trace-js.git out')
-exec('npm run jsdoc')
+exec('yarn jsdoc')
 exec('git add -A', { cwd: './out' })
 exec(`git commit -m "${msg}"`, { cwd: './out' })
 exec('git push', { cwd: './out' })

scripts/release.js

@@ -5,13 +5,12 @@ const title = require('./helpers/title')
 
 title(`Publishing package to the npm registry`)
 
-exec('npm whoami')
 exec('git checkout master')
 exec('git pull')
 
 const pkg = require('../package.json')
 
 exec(`git tag v${pkg.version}`)
 exec(`git push origin refs/tags/v${pkg.version}`)
-exec('npm publish')
+exec('yarn publish')
 exec(`node scripts/publish_docs.js "v${pkg.version}"`)

src/plugins/express.js

@@ -84,6 +84,8 @@ function createWrapProcessParams (tracer, config) {
     return function processParamsWithTrace (layer, called, req, res, done) {
       const matchers = layer._datadog_matchers
 
+      req = done ? req : called
+
       if (matchers) {
         // Try to guess which path actually matched
         for (let i = 0; i < matchers.length; i++) {
@@ -108,15 +110,16 @@ function createWrapRouterMethod (tracer) {
       const matchers = extractMatchers(fn)
 
       this.stack.slice(offset).forEach(layer => {
-        const handleRequest = layer.handle_request
-        const handleError = layer.handle_error
-
-        layer.handle_request = (req, res, next) => {
-          return handleRequest.call(layer, req, res, wrapNext(tracer, layer, req, next))
-        }
+        const handle = layer.handle
 
-        layer.handle_error = (error, req, res, next) => {
-          return handleError.call(layer, error, req, res, wrapNext(tracer, layer, req, next))
+        if (handle.length === 4) {
+          layer.handle = (error, req, res, next) => {
+            return handle.call(layer, error, req, res, wrapNext(tracer, layer, req, next))
+          }
+        } else {
+          layer.handle = (req, res, next) => {
+            return handle.call(layer, req, res, wrapNext(tracer, layer, req, next))
+          }
         }
 
         layer._datadog_matchers = matchers
@@ -138,7 +141,7 @@ function wrapNext (tracer, layer, req, next) {
   req._datadog.scope = tracer.scopeManager().activate(req._datadog.span)
 
   return function (error) {
-    if (!error && layer.path && !layer.regexp.fast_star) {
+    if (!error && layer.path && !isFastStar(layer)) {
       req._datadog.paths.pop()
     }
 
@@ -161,6 +164,14 @@ function extractMatchers (fn) {
   }))
 }
 
+function isFastStar (layer) {
+  if (layer.regexp.fast_star !== undefined) {
+    return layer.regexp.fast_star
+  }
+
+  return layer._datadog_matchers.some(matcher => matcher.path === '*')
+}
+
 function flatten (arr) {
   return arr.reduce((acc, val) => Array.isArray(val) ? acc.concat(flatten(val)) : acc.concat(val), [])
 }

test/.eslintrc.json

@@ -10,7 +10,8 @@
     "sinon": true,
     "proxyquire": true,
     "nock": true,
-    "wrapIt": true
+    "wrapIt": true,
+    "withVersions": true
   },
   "rules": {
     "no-unused-expressions": 0

test/leak/plugins/mysql.js

@@ -11,8 +11,7 @@ const profile = require('../../profile')
 test('mysql plugin should not leak', t => {
   const connection = mysql.createConnection({
     host: 'localhost',
-    user: 'user',
-    password: 'userpass',
+    user: 'root',
     database: 'db'
   })