From 094782c9b1182f756d028aebb0e18e83894a31f8 Mon Sep 17 00:00:00 2001 From: Ida Liu <119438987+ida613@users.noreply.github.com> Date: Tue, 27 Feb 2024 11:35:35 -0500 Subject: [PATCH] Active tracing configuration (#3928) Co-authored-by: Thomas Hunter II Co-authored-by: Igor Unanua Co-authored-by: simon-id Co-authored-by: Attila Szegedi --- packages/datadog-core/src/utils/src/get.js | 11 + packages/datadog-core/src/utils/src/has.js | 14 + .../src}/utils/src/kebabcase.js | 0 .../{ => datadog-core/src}/utils/src/pick.js | 0 packages/datadog-core/src/utils/src/set.js | 16 + .../{ => datadog-core/src}/utils/src/uniq.js | 0 .../datadog-core/test/utils/src/get.spec.js | 22 + .../datadog-core/test/utils/src/has.spec.js | 22 + .../datadog-core/test/utils/src/set.spec.js | 15 + .../datadog-instrumentations/src/amqplib.js | 2 +- packages/datadog-plugin-graphql/src/index.js | 2 +- packages/datadog-plugin-grpc/src/util.js | 2 +- packages/dd-trace/src/config.js | 914 +++++++++--------- .../src/opentracing/propagation/text_map.js | 2 +- packages/dd-trace/src/plugins/util/web.js | 2 +- packages/dd-trace/src/telemetry/index.js | 56 +- packages/dd-trace/test/config.spec.js | 177 ++++ .../dd-trace/test/telemetry/index.spec.js | 19 - 18 files changed, 758 insertions(+), 518 deletions(-) create mode 100644 packages/datadog-core/src/utils/src/get.js create mode 100644 packages/datadog-core/src/utils/src/has.js rename packages/{ => datadog-core/src}/utils/src/kebabcase.js (100%) rename packages/{ => datadog-core/src}/utils/src/pick.js (100%) create mode 100644 packages/datadog-core/src/utils/src/set.js rename packages/{ => datadog-core/src}/utils/src/uniq.js (100%) create mode 100644 packages/datadog-core/test/utils/src/get.spec.js create mode 100644 packages/datadog-core/test/utils/src/has.spec.js create mode 100644 packages/datadog-core/test/utils/src/set.spec.js diff --git a/packages/datadog-core/src/utils/src/get.js b/packages/datadog-core/src/utils/src/get.js new file mode 100644 index 00000000000..f5913c3016c --- /dev/null +++ b/packages/datadog-core/src/utils/src/get.js @@ -0,0 +1,11 @@ +'use strict' + +module.exports = (object, path) => { + const pathArr = path.split('.') + let val = object + for (const p of pathArr) { + if (val === undefined) return val + val = val[p] + } + return val +} diff --git a/packages/datadog-core/src/utils/src/has.js b/packages/datadog-core/src/utils/src/has.js new file mode 100644 index 00000000000..fafc7cc775f --- /dev/null +++ b/packages/datadog-core/src/utils/src/has.js @@ -0,0 +1,14 @@ +'use strict' + +module.exports = (object, path) => { + const pathArr = path.split('.') + let property = object + for (const n of pathArr) { + if (property.hasOwnProperty(n)) { + property = property[n] + } else { + return false + } + } + return true +} diff --git a/packages/utils/src/kebabcase.js b/packages/datadog-core/src/utils/src/kebabcase.js similarity index 100% rename from packages/utils/src/kebabcase.js rename to packages/datadog-core/src/utils/src/kebabcase.js diff --git a/packages/utils/src/pick.js b/packages/datadog-core/src/utils/src/pick.js similarity index 100% rename from packages/utils/src/pick.js rename to packages/datadog-core/src/utils/src/pick.js diff --git a/packages/datadog-core/src/utils/src/set.js b/packages/datadog-core/src/utils/src/set.js new file mode 100644 index 00000000000..e6b9fc7f12a --- /dev/null +++ b/packages/datadog-core/src/utils/src/set.js @@ -0,0 +1,16 @@ +'use strict' + +module.exports = (object, path, value) => { + const pathArr = path.split('.') + let property = object + let i + for (i = 0; i < pathArr.length - 1; i++) { + const n = pathArr[i] + if (property.hasOwnProperty(n)) { + property = property[n] + } else { + property[n] = property = {} + } + } + property[pathArr[i]] = value +} diff --git a/packages/utils/src/uniq.js b/packages/datadog-core/src/utils/src/uniq.js similarity index 100% rename from packages/utils/src/uniq.js rename to packages/datadog-core/src/utils/src/uniq.js diff --git a/packages/datadog-core/test/utils/src/get.spec.js b/packages/datadog-core/test/utils/src/get.spec.js new file mode 100644 index 00000000000..4815b9bf3ee --- /dev/null +++ b/packages/datadog-core/test/utils/src/get.spec.js @@ -0,0 +1,22 @@ +'use strict' + +require('../../../../dd-trace/test/setup/tap') + +const { expect } = require('chai') +const get = require('../../../src/utils/src/get') + +describe('get', () => { + const obj = { + 'a': { + 'b': 'c' + } + } + + it('should return value at path', () => { + expect(get(obj, 'a.b')).to.be.equal('c') + }) + + it('should return undefined if path does not exist', () => { + expect(get(obj, 'd')).to.be.undefined + }) +}) diff --git a/packages/datadog-core/test/utils/src/has.spec.js b/packages/datadog-core/test/utils/src/has.spec.js new file mode 100644 index 00000000000..54130547ceb --- /dev/null +++ b/packages/datadog-core/test/utils/src/has.spec.js @@ -0,0 +1,22 @@ +'use strict' + +require('../../../../dd-trace/test/setup/tap') + +const { expect } = require('chai') +const has = require('../../../src/utils/src/has') + +describe('has', () => { + const obj = { + 'a': { + 'b': 'c' + } + } + + it('should true if path exists', () => { + expect(has(obj, 'a.b')).to.be.true + }) + + it('should return false if path does not exist', () => { + expect(has(obj, 'd')).to.be.false + }) +}) diff --git a/packages/datadog-core/test/utils/src/set.spec.js b/packages/datadog-core/test/utils/src/set.spec.js new file mode 100644 index 00000000000..02f9c7c19d7 --- /dev/null +++ b/packages/datadog-core/test/utils/src/set.spec.js @@ -0,0 +1,15 @@ +'use strict' + +require('../../../../dd-trace/test/setup/tap') + +const { expect } = require('chai') +const set = require('../../../src/utils/src/set') + +describe('set', () => { + const obj = {} + + it('should set value at path', () => { + set(obj, 'a.b', 'c') + expect(obj.a.b).to.be.equal('c') + }) +}) diff --git a/packages/datadog-instrumentations/src/amqplib.js b/packages/datadog-instrumentations/src/amqplib.js index 2e6835aad8d..a5d07ebb4ab 100644 --- a/packages/datadog-instrumentations/src/amqplib.js +++ b/packages/datadog-instrumentations/src/amqplib.js @@ -5,7 +5,7 @@ const { addHook, AsyncResource } = require('./helpers/instrument') -const kebabCase = require('../../utils/src/kebabcase') +const kebabCase = require('../../datadog-core/src/utils/src/kebabcase') const shimmer = require('../../datadog-shimmer') const startCh = channel('apm:amqplib:command:start') diff --git a/packages/datadog-plugin-graphql/src/index.js b/packages/datadog-plugin-graphql/src/index.js index da0dd01635f..1e530fb94d0 100644 --- a/packages/datadog-plugin-graphql/src/index.js +++ b/packages/datadog-plugin-graphql/src/index.js @@ -1,6 +1,6 @@ 'use strict' -const pick = require('../../utils/src/pick') +const pick = require('../../datadog-core/src/utils/src/pick') const CompositePlugin = require('../../dd-trace/src/plugins/composite') const log = require('../../dd-trace/src/log') const GraphQLExecutePlugin = require('./execute') diff --git a/packages/datadog-plugin-grpc/src/util.js b/packages/datadog-plugin-grpc/src/util.js index f21599a2a1d..1c1937e7ea7 100644 --- a/packages/datadog-plugin-grpc/src/util.js +++ b/packages/datadog-plugin-grpc/src/util.js @@ -1,6 +1,6 @@ 'use strict' -const pick = require('../../utils/src/pick') +const pick = require('../../datadog-core/src/utils/src/pick') const log = require('../../dd-trace/src/log') module.exports = { diff --git a/packages/dd-trace/src/config.js b/packages/dd-trace/src/config.js index ecaf522e0bb..b03d2d55fdc 100644 --- a/packages/dd-trace/src/config.js +++ b/packages/dd-trace/src/config.js @@ -8,6 +8,9 @@ const log = require('./log') const pkg = require('./pkg') const coalesce = require('koalas') const tagger = require('./tagger') +const get = require('../../datadog-core/src/utils/src/get') +const has = require('../../datadog-core/src/utils/src/has') +const set = require('../../datadog-core/src/utils/src/set') const { isTrue, isFalse } = require('./util') const { GIT_REPOSITORY_URL, GIT_COMMIT_SHA } = require('./plugins/util/tags') const { getGitMetadataFromGitProperties, removeUserSensitiveInfo } = require('./git_properties') @@ -20,6 +23,13 @@ const fromEntries = Object.fromEntries || (entries => // eslint-disable-next-line max-len const qsRegex = '(?:p(?:ass)?w(?:or)?d|pass(?:_?phrase)?|secret|(?:api_?|private_?|public_?|access_?|secret_?)key(?:_?id)?|token|consumer_?(?:id|key|secret)|sign(?:ed|ature)?|auth(?:entication|orization)?)(?:(?:\\s|%20)*(?:=|%3D)[^&]+|(?:"|%22)(?:\\s|%20)*(?::|%3A)(?:\\s|%20)*(?:"|%22)(?:%2[^2]|%[^2]|[^"%])+(?:"|%22))|bearer(?:\\s|%20)+[a-z0-9\\._\\-]+|token(?::|%3A)[a-z0-9]{13}|gh[opsu]_[0-9a-zA-Z]{36}|ey[I-L](?:[\\w=-]|%3D)+\\.ey[I-L](?:[\\w=-]|%3D)+(?:\\.(?:[\\w.+\\/=-]|%3D|%2F|%2B)+)?|[\\-]{5}BEGIN(?:[a-z\\s]|%20)+PRIVATE(?:\\s|%20)KEY[\\-]{5}[^\\-]+[\\-]{5}END(?:[a-z\\s]|%20)+PRIVATE(?:\\s|%20)KEY|ssh-rsa(?:\\s|%20)*(?:[a-z0-9\\/\\.+]|%2F|%5C|%2B){100,}' +const defaultWafObfuscatorKeyRegex = `(?i)(?:p(?:ass)?w(?:or)?d|pass(?:_?phrase)?|secret|(?:api_?|private_?\ +|public_?)key)|token|consumer_?(?:id|key|secret)|sign(?:ed|ature)|bearer|authorization` +const defaultWafObfuscatorValueRegex = +`(?i)(?:p(?:ass)?w(?:or)?d|pass(?:_?phrase)?|secret|(?:api_?|private_?|public_?|access_?|secret_?)key(?:_?id)?|to\ +ken|consumer_?(?:id|key|secret)|sign(?:ed|ature)?|auth(?:entication|orization)?)(?:\\s*=[^;]|"\\s*:\\s*"[^"]+")|bearer\ +\\s+[a-z0-9\\._\\-]+|token:[a-z0-9]{13}|gh[opsu]_[0-9a-zA-Z]{36}|ey[I-L][\\w=-]+\\.ey[I-L][\\w=-]+(?:\\.[\\w.+\\/=-]+)?\ +|[\\-]{5}BEGIN[a-z\\s]+PRIVATE\\sKEY[\\-]{5}[^\\-]+[\\-]{5}END[a-z\\s]+PRIVATE\\sKEY|ssh-rsa\\s*[a-z0-9\\/\\.+]{100,}` function maybeFile (filepath) { if (!filepath) return @@ -94,6 +104,11 @@ function propagationStyle (key, option, defaultValue) { class Config { constructor (options) { options = options || {} + options = this.options = { + ...options, + appsec: options.appsec != null ? options.appsec : options.experimental?.appsec, + iastOptions: options.experimental?.iast + } // Configure the logger first so it can be used to warn about other configs this.debug = isTrue(coalesce( @@ -110,70 +125,6 @@ class Config { log.use(this.logger) log.toggle(this.debug, this.logLevel, this) - const DD_PROFILING_ENABLED = coalesce( - options.profiling, // TODO: remove when enabled by default - process.env.DD_EXPERIMENTAL_PROFILING_ENABLED, - process.env.DD_PROFILING_ENABLED, - false - ) - const DD_PROFILING_EXPORTERS = coalesce( - process.env.DD_PROFILING_EXPORTERS, - 'agent' - ) - const DD_PROFILING_SOURCE_MAP = process.env.DD_PROFILING_SOURCE_MAP - const DD_RUNTIME_METRICS_ENABLED = coalesce( - options.runtimeMetrics, // TODO: remove when enabled by default - process.env.DD_RUNTIME_METRICS_ENABLED, - false - ) - const DD_DBM_PROPAGATION_MODE = coalesce( - options.dbmPropagationMode, - process.env.DD_DBM_PROPAGATION_MODE, - 'disabled' - ) - const DD_DATA_STREAMS_ENABLED = coalesce( - options.dsmEnabled, - process.env.DD_DATA_STREAMS_ENABLED, - false - ) - const DD_AGENT_HOST = coalesce( - options.hostname, - process.env.DD_AGENT_HOST, - process.env.DD_TRACE_AGENT_HOSTNAME, - '127.0.0.1' - ) - const DD_TRACE_AGENT_PORT = coalesce( - options.port, - process.env.DD_TRACE_AGENT_PORT, - '8126' - ) - const DD_TRACE_AGENT_URL = coalesce( - options.url, - process.env.DD_TRACE_AGENT_URL, - process.env.DD_TRACE_URL, - null - ) - const DD_IS_CIVISIBILITY = coalesce( - options.isCiVisibility, - false - ) - const DD_CIVISIBILITY_AGENTLESS_URL = process.env.DD_CIVISIBILITY_AGENTLESS_URL - - const DD_CIVISIBILITY_ITR_ENABLED = coalesce( - process.env.DD_CIVISIBILITY_ITR_ENABLED, - true - ) - - const DD_CIVISIBILITY_MANUAL_API_ENABLED = coalesce( - process.env.DD_CIVISIBILITY_MANUAL_API_ENABLED, - false - ) - - const DD_CIVISIBILITY_EARLY_FLAKE_DETECTION_ENABLED = coalesce( - process.env.DD_CIVISIBILITY_EARLY_FLAKE_DETECTION_ENABLED, - true - ) - const DD_TRACE_MEMCACHED_COMMAND_ENABLED = coalesce( process.env.DD_TRACE_MEMCACHED_COMMAND_ENABLED, false @@ -185,75 +136,12 @@ class Config { process.env.DD_SERVICE_MAPPING.split(',').map(x => x.trim().split(':')) ) : {} ) - const DD_TRACE_STARTUP_LOGS = coalesce( - options.startupLogs, - process.env.DD_TRACE_STARTUP_LOGS, - false - ) - - const DD_OPENAI_LOGS_ENABLED = coalesce( - options.openAiLogsEnabled, - process.env.DD_OPENAI_LOGS_ENABLED, - false - ) const DD_API_KEY = coalesce( process.env.DATADOG_API_KEY, process.env.DD_API_KEY ) - const inAWSLambda = process.env.AWS_LAMBDA_FUNCTION_NAME !== undefined - - const isGCPFunction = getIsGCPFunction() - const isAzureFunctionConsumptionPlan = getIsAzureFunctionConsumptionPlan() - - const inServerlessEnvironment = inAWSLambda || isGCPFunction || isAzureFunctionConsumptionPlan - - const isJestWorker = !!process.env.JEST_WORKER_ID - - const DD_INSTRUMENTATION_TELEMETRY_ENABLED = coalesce( - process.env.DD_TRACE_TELEMETRY_ENABLED, // for backward compatibility - process.env.DD_INSTRUMENTATION_TELEMETRY_ENABLED, // to comply with instrumentation telemetry specs - !(inServerlessEnvironment || isJestWorker) - ) - const DD_TELEMETRY_HEARTBEAT_INTERVAL = process.env.DD_TELEMETRY_HEARTBEAT_INTERVAL - ? Math.floor(parseFloat(process.env.DD_TELEMETRY_HEARTBEAT_INTERVAL) * 1000) - : 60000 - const DD_OPENAI_SPAN_CHAR_LIMIT = process.env.DD_OPENAI_SPAN_CHAR_LIMIT - ? parseInt(process.env.DD_OPENAI_SPAN_CHAR_LIMIT) - : 128 - const DD_TELEMETRY_DEBUG = coalesce( - process.env.DD_TELEMETRY_DEBUG, - false - ) - const DD_TELEMETRY_METRICS_ENABLED = coalesce( - process.env.DD_TELEMETRY_METRICS_ENABLED, - true - ) - const DD_TRACE_AGENT_PROTOCOL_VERSION = coalesce( - options.protocolVersion, - process.env.DD_TRACE_AGENT_PROTOCOL_VERSION, - '0.4' - ) - const DD_TRACE_PARTIAL_FLUSH_MIN_SPANS = coalesce( - parseInt(options.flushMinSpans), - parseInt(process.env.DD_TRACE_PARTIAL_FLUSH_MIN_SPANS), - 1000 - ) - const DD_TRACE_OBFUSCATION_QUERY_STRING_REGEXP = coalesce( - process.env.DD_TRACE_OBFUSCATION_QUERY_STRING_REGEXP, - qsRegex - ) - const DD_TRACE_CLIENT_IP_ENABLED = coalesce( - options.clientIpEnabled, - process.env.DD_TRACE_CLIENT_IP_ENABLED && isTrue(process.env.DD_TRACE_CLIENT_IP_ENABLED), - false - ) - const DD_TRACE_CLIENT_IP_HEADER = coalesce( - options.clientIpHeader, - process.env.DD_TRACE_CLIENT_IP_HEADER, - null - ) // TODO: Remove the experimental env vars as a major? const DD_TRACE_B3_ENABLED = coalesce( options.experimental && options.experimental.b3, @@ -289,238 +177,36 @@ class Config { process.env.DD_TRACE_PROPAGATION_EXTRACT_FIRST, false ) - const DD_TRACE_RUNTIME_ID_ENABLED = coalesce( - options.experimental && options.experimental.runtimeId, - process.env.DD_TRACE_EXPERIMENTAL_RUNTIME_ID_ENABLED, - false - ) - const DD_TRACE_EXPORTER = coalesce( - options.experimental && options.experimental.exporter, - process.env.DD_TRACE_EXPERIMENTAL_EXPORTER - ) - const DD_TRACE_GET_RUM_DATA_ENABLED = coalesce( - options.experimental && options.experimental.enableGetRumData, - process.env.DD_TRACE_EXPERIMENTAL_GET_RUM_DATA_ENABLED, - false - ) - const DD_TRACE_SPAN_ATTRIBUTE_SCHEMA = validateNamingVersion( - coalesce( - options.spanAttributeSchema, - process.env.DD_TRACE_SPAN_ATTRIBUTE_SCHEMA - ) - ) - const DD_TRACE_PEER_SERVICE_MAPPING = coalesce( - options.peerServiceMapping, - process.env.DD_TRACE_PEER_SERVICE_MAPPING ? fromEntries( - process.env.DD_TRACE_PEER_SERVICE_MAPPING.split(',').map(x => x.trim().split(':')) - ) : {} - ) - - const peerServiceSet = ( - options.hasOwnProperty('spanComputePeerService') || - process.env.hasOwnProperty('DD_TRACE_PEER_SERVICE_DEFAULTS_ENABLED') - ) - const peerServiceValue = coalesce( - options.spanComputePeerService, - process.env.DD_TRACE_PEER_SERVICE_DEFAULTS_ENABLED - ) - - const DD_TRACE_PEER_SERVICE_DEFAULTS_ENABLED = ( - DD_TRACE_SPAN_ATTRIBUTE_SCHEMA === 'v0' - // In v0, peer service is computed only if it is explicitly set to true - ? peerServiceSet && isTrue(peerServiceValue) - // In >v0, peer service is false only if it is explicitly set to false - : (peerServiceSet ? !isFalse(peerServiceValue) : true) - ) - - const DD_TRACE_REMOVE_INTEGRATION_SERVICE_NAMES_ENABLED = coalesce( - options.spanRemoveIntegrationFromService, - isTrue(process.env.DD_TRACE_REMOVE_INTEGRATION_SERVICE_NAMES_ENABLED) - ) - const DD_TRACE_X_DATADOG_TAGS_MAX_LENGTH = coalesce( - process.env.DD_TRACE_X_DATADOG_TAGS_MAX_LENGTH, - '512' - ) - - const DD_TRACE_STATS_COMPUTATION_ENABLED = coalesce( - options.stats, - process.env.DD_TRACE_STATS_COMPUTATION_ENABLED, - isGCPFunction || isAzureFunctionConsumptionPlan - ) - - // the tracer generates 128 bit IDs by default as of v5 - const DD_TRACE_128_BIT_TRACEID_GENERATION_ENABLED = coalesce( - options.traceId128BitGenerationEnabled, - process.env.DD_TRACE_128_BIT_TRACEID_GENERATION_ENABLED, - true - ) - - const DD_TRACE_128_BIT_TRACEID_LOGGING_ENABLED = coalesce( - options.traceId128BitLoggingEnabled, - process.env.DD_TRACE_128_BIT_TRACEID_LOGGING_ENABLED, - false - ) - let appsec = options.appsec != null ? options.appsec : options.experimental && options.experimental.appsec - - if (typeof appsec === 'boolean') { - appsec = { - enabled: appsec + if (typeof options.appsec === 'boolean') { + options.appsec = { + enabled: options.appsec } - } else if (appsec == null) { - appsec = {} + } else if (options.appsec == null) { + options.appsec = {} } - const DD_APPSEC_ENABLED = coalesce( - appsec.enabled, - process.env.DD_APPSEC_ENABLED && isTrue(process.env.DD_APPSEC_ENABLED) - ) - const DD_APPSEC_RULES = coalesce( - appsec.rules, - process.env.DD_APPSEC_RULES - ) - const DD_APPSEC_TRACE_RATE_LIMIT = coalesce( - parseInt(appsec.rateLimit), - parseInt(process.env.DD_APPSEC_TRACE_RATE_LIMIT), - 100 - ) - const DD_APPSEC_WAF_TIMEOUT = coalesce( - parseInt(appsec.wafTimeout), - parseInt(process.env.DD_APPSEC_WAF_TIMEOUT), - 5e3 // µs - ) - const DD_APPSEC_OBFUSCATION_PARAMETER_KEY_REGEXP = coalesce( - appsec.obfuscatorKeyRegex, - process.env.DD_APPSEC_OBFUSCATION_PARAMETER_KEY_REGEXP, - `(?i)(?:p(?:ass)?w(?:or)?d|pass(?:_?phrase)?|secret|(?:api_?|private_?|public_?)key)|token|consumer_?(?:id|key|se\ -cret)|sign(?:ed|ature)|bearer|authorization` - ) - const DD_APPSEC_OBFUSCATION_PARAMETER_VALUE_REGEXP = coalesce( - appsec.obfuscatorValueRegex, - process.env.DD_APPSEC_OBFUSCATION_PARAMETER_VALUE_REGEXP, - `(?i)(?:p(?:ass)?w(?:or)?d|pass(?:_?phrase)?|secret|(?:api_?|private_?|public_?|access_?|secret_?)key(?:_?id)?|to\ -ken|consumer_?(?:id|key|secret)|sign(?:ed|ature)?|auth(?:entication|orization)?)(?:\\s*=[^;]|"\\s*:\\s*"[^"]+")|bearer\ -\\s+[a-z0-9\\._\\-]+|token:[a-z0-9]{13}|gh[opsu]_[0-9a-zA-Z]{36}|ey[I-L][\\w=-]+\\.ey[I-L][\\w=-]+(?:\\.[\\w.+\\/=-]+)?\ -|[\\-]{5}BEGIN[a-z\\s]+PRIVATE\\sKEY[\\-]{5}[^\\-]+[\\-]{5}END[a-z\\s]+PRIVATE\\sKEY|ssh-rsa\\s*[a-z0-9\\/\\.+]{100,}` - ) - const DD_APPSEC_HTTP_BLOCKED_TEMPLATE_HTML = coalesce( - maybeFile(appsec.blockedTemplateHtml), - maybeFile(process.env.DD_APPSEC_HTTP_BLOCKED_TEMPLATE_HTML) - ) - const DD_APPSEC_HTTP_BLOCKED_TEMPLATE_JSON = coalesce( - maybeFile(appsec.blockedTemplateJson), - maybeFile(process.env.DD_APPSEC_HTTP_BLOCKED_TEMPLATE_JSON) - ) const DD_APPSEC_GRAPHQL_BLOCKED_TEMPLATE_JSON = coalesce( - maybeFile(appsec.blockedTemplateGraphql), + maybeFile(options.appsec.blockedTemplateGraphql), maybeFile(process.env.DD_APPSEC_GRAPHQL_BLOCKED_TEMPLATE_JSON) ) const DD_APPSEC_AUTOMATED_USER_EVENTS_TRACKING = coalesce( - appsec.eventTracking && appsec.eventTracking.mode, + options.appsec.eventTracking && options.appsec.eventTracking.mode, process.env.DD_APPSEC_AUTOMATED_USER_EVENTS_TRACKING, 'safe' ).toLowerCase() const DD_API_SECURITY_ENABLED = coalesce( - appsec?.apiSecurity?.enabled, + options.appsec?.apiSecurity?.enabled, process.env.DD_API_SECURITY_ENABLED && isTrue(process.env.DD_API_SECURITY_ENABLED), process.env.DD_EXPERIMENTAL_API_SECURITY_ENABLED && isTrue(process.env.DD_EXPERIMENTAL_API_SECURITY_ENABLED), true ) const DD_API_SECURITY_REQUEST_SAMPLE_RATE = coalesce( - appsec?.apiSecurity?.requestSampling, + options.appsec?.apiSecurity?.requestSampling, parseFloat(process.env.DD_API_SECURITY_REQUEST_SAMPLE_RATE), 0.1 ) - const remoteConfigOptions = options.remoteConfig || {} - const DD_REMOTE_CONFIGURATION_ENABLED = coalesce( - process.env.DD_REMOTE_CONFIGURATION_ENABLED && isTrue(process.env.DD_REMOTE_CONFIGURATION_ENABLED), - !inServerlessEnvironment - ) - const DD_REMOTE_CONFIG_POLL_INTERVAL_SECONDS = coalesce( - parseFloat(remoteConfigOptions.pollInterval), - parseFloat(process.env.DD_REMOTE_CONFIG_POLL_INTERVAL_SECONDS), - 5 // seconds - ) - - const iastOptions = options?.experimental?.iast - const DD_IAST_ENABLED = coalesce( - iastOptions && - (iastOptions === true || iastOptions.enabled === true), - process.env.DD_IAST_ENABLED, - false - ) - const DD_TELEMETRY_LOG_COLLECTION_ENABLED = coalesce( - process.env.DD_TELEMETRY_LOG_COLLECTION_ENABLED, - DD_IAST_ENABLED - ) - - const DD_TELEMETRY_DEPENDENCY_COLLECTION_ENABLED = coalesce( - process.env.DD_TELEMETRY_DEPENDENCY_COLLECTION_ENABLED, - true - ) - - const defaultIastRequestSampling = 30 - const iastRequestSampling = coalesce( - parseInt(iastOptions?.requestSampling), - parseInt(process.env.DD_IAST_REQUEST_SAMPLING), - defaultIastRequestSampling - ) - const DD_IAST_REQUEST_SAMPLING = iastRequestSampling < 0 || - iastRequestSampling > 100 ? defaultIastRequestSampling : iastRequestSampling - - const DD_IAST_MAX_CONCURRENT_REQUESTS = coalesce( - parseInt(iastOptions?.maxConcurrentRequests), - parseInt(process.env.DD_IAST_MAX_CONCURRENT_REQUESTS), - 2 - ) - - const DD_IAST_MAX_CONTEXT_OPERATIONS = coalesce( - parseInt(iastOptions?.maxContextOperations), - parseInt(process.env.DD_IAST_MAX_CONTEXT_OPERATIONS), - 2 - ) - - const DD_IAST_DEDUPLICATION_ENABLED = coalesce( - iastOptions?.deduplicationEnabled, - process.env.DD_IAST_DEDUPLICATION_ENABLED && isTrue(process.env.DD_IAST_DEDUPLICATION_ENABLED), - true - ) - - const DD_IAST_REDACTION_ENABLED = coalesce( - iastOptions?.redactionEnabled, - !isFalse(process.env.DD_IAST_REDACTION_ENABLED), - true - ) - - const DD_IAST_REDACTION_NAME_PATTERN = coalesce( - iastOptions?.redactionNamePattern, - process.env.DD_IAST_REDACTION_NAME_PATTERN, - null - ) - - const DD_IAST_REDACTION_VALUE_PATTERN = coalesce( - iastOptions?.redactionValuePattern, - process.env.DD_IAST_REDACTION_VALUE_PATTERN, - null - ) - - const DD_IAST_TELEMETRY_VERBOSITY = coalesce( - iastOptions?.telemetryVerbosity, - process.env.DD_IAST_TELEMETRY_VERBOSITY, - 'INFORMATION' - ) - - const DD_CIVISIBILITY_GIT_UPLOAD_ENABLED = coalesce( - process.env.DD_CIVISIBILITY_GIT_UPLOAD_ENABLED, - true - ) - - const DD_TRACE_GIT_METADATA_ENABLED = coalesce( - process.env.DD_TRACE_GIT_METADATA_ENABLED, - true - ) - // 0: disabled, 1: logging, 2: garbage collection + logging const DD_TRACE_SPAN_LEAK_DEBUG = coalesce( process.env.DD_TRACE_SPAN_LEAK_DEBUG, @@ -540,10 +226,7 @@ ken|consumer_?(?:id|key|secret)|sign(?:ed|ature)?|auth(?:entication|orization)?) null ) - const ingestion = options.ingestion || {} - const dogstatsd = coalesce(options.dogstatsd, {}) const sampler = { - rateLimit: coalesce(options.rateLimit, process.env.DD_TRACE_RATE_LIMIT, ingestion.rateLimit), rules: coalesce( options.samplingRules, safeJsonParse(process.env.DD_TRACE_SAMPLING_RULES), @@ -566,73 +249,16 @@ ken|consumer_?(?:id|key|secret)|sign(?:ed|ature)?|auth(?:entication|orization)?) }) } - const defaultFlushInterval = inAWSLambda ? 0 : 2000 - - this.dbmPropagationMode = DD_DBM_PROPAGATION_MODE - this.dsmEnabled = isTrue(DD_DATA_STREAMS_ENABLED) - this.openAiLogsEnabled = DD_OPENAI_LOGS_ENABLED + // TODO: refactor this.apiKey = DD_API_KEY - this.url = DD_CIVISIBILITY_AGENTLESS_URL ? new URL(DD_CIVISIBILITY_AGENTLESS_URL) - : getAgentUrl(DD_TRACE_AGENT_URL, options) - this.site = coalesce(options.site, process.env.DD_SITE, 'datadoghq.com') - this.hostname = DD_AGENT_HOST || (this.url && this.url.hostname) - this.port = String(DD_TRACE_AGENT_PORT || (this.url && this.url.port)) - this.flushInterval = coalesce(parseInt(options.flushInterval, 10), defaultFlushInterval) - this.flushMinSpans = DD_TRACE_PARTIAL_FLUSH_MIN_SPANS - this.queryStringObfuscation = DD_TRACE_OBFUSCATION_QUERY_STRING_REGEXP - this.clientIpEnabled = DD_TRACE_CLIENT_IP_ENABLED - this.clientIpHeader = DD_TRACE_CLIENT_IP_HEADER - this.plugins = !!coalesce(options.plugins, true) this.serviceMapping = DD_SERVICE_MAPPING - this.dogstatsd = { - hostname: coalesce(dogstatsd.hostname, process.env.DD_DOGSTATSD_HOSTNAME, this.hostname), - port: String(coalesce(dogstatsd.port, process.env.DD_DOGSTATSD_PORT, 8125)) - } - this.runtimeMetrics = isTrue(DD_RUNTIME_METRICS_ENABLED) this.tracePropagationStyle = { inject: DD_TRACE_PROPAGATION_STYLE_INJECT, extract: DD_TRACE_PROPAGATION_STYLE_EXTRACT } this.tracePropagationExtractFirst = isTrue(DD_TRACE_PROPAGATION_EXTRACT_FIRST) - this.experimental = { - runtimeId: isTrue(DD_TRACE_RUNTIME_ID_ENABLED), - exporter: DD_TRACE_EXPORTER, - enableGetRumData: isTrue(DD_TRACE_GET_RUM_DATA_ENABLED) - } this.sampler = sampler - this.reportHostname = isTrue(coalesce(options.reportHostname, process.env.DD_TRACE_REPORT_HOSTNAME, false)) - this.scope = process.env.DD_TRACE_SCOPE - this.profiling = { - enabled: isTrue(DD_PROFILING_ENABLED), - sourceMap: !isFalse(DD_PROFILING_SOURCE_MAP), - exporters: DD_PROFILING_EXPORTERS - } - this.spanAttributeSchema = DD_TRACE_SPAN_ATTRIBUTE_SCHEMA - this.spanComputePeerService = DD_TRACE_PEER_SERVICE_DEFAULTS_ENABLED - this.spanRemoveIntegrationFromService = DD_TRACE_REMOVE_INTEGRATION_SERVICE_NAMES_ENABLED - this.peerServiceMapping = DD_TRACE_PEER_SERVICE_MAPPING - this.lookup = options.lookup - this.startupLogs = isTrue(DD_TRACE_STARTUP_LOGS) - this.telemetry = { - enabled: isTrue(DD_INSTRUMENTATION_TELEMETRY_ENABLED), - heartbeatInterval: DD_TELEMETRY_HEARTBEAT_INTERVAL, - debug: isTrue(DD_TELEMETRY_DEBUG), - logCollection: isTrue(DD_TELEMETRY_LOG_COLLECTION_ENABLED), - metrics: isTrue(DD_TELEMETRY_METRICS_ENABLED), - dependencyCollection: DD_TELEMETRY_DEPENDENCY_COLLECTION_ENABLED - } - this.protocolVersion = DD_TRACE_AGENT_PROTOCOL_VERSION - this.tagsHeaderMaxLength = parseInt(DD_TRACE_X_DATADOG_TAGS_MAX_LENGTH) this.appsec = { - enabled: DD_APPSEC_ENABLED, - rules: DD_APPSEC_RULES, - customRulesProvided: !!DD_APPSEC_RULES, - rateLimit: DD_APPSEC_TRACE_RATE_LIMIT, - wafTimeout: DD_APPSEC_WAF_TIMEOUT, - obfuscatorKeyRegex: DD_APPSEC_OBFUSCATION_PARAMETER_KEY_REGEXP, - obfuscatorValueRegex: DD_APPSEC_OBFUSCATION_PARAMETER_VALUE_REGEXP, - blockedTemplateHtml: DD_APPSEC_HTTP_BLOCKED_TEMPLATE_HTML, - blockedTemplateJson: DD_APPSEC_HTTP_BLOCKED_TEMPLATE_JSON, blockedTemplateGraphql: DD_APPSEC_GRAPHQL_BLOCKED_TEMPLATE_JSON, eventTracking: { enabled: ['extended', 'safe'].includes(DD_APPSEC_AUTOMATED_USER_EVENTS_TRACKING), @@ -645,49 +271,10 @@ ken|consumer_?(?:id|key|secret)|sign(?:ed|ature)?|auth(?:entication|orization)?) } } - this.remoteConfig = { - enabled: DD_REMOTE_CONFIGURATION_ENABLED, - pollInterval: DD_REMOTE_CONFIG_POLL_INTERVAL_SECONDS - } - this.iast = { - enabled: isTrue(DD_IAST_ENABLED), - requestSampling: DD_IAST_REQUEST_SAMPLING, - maxConcurrentRequests: DD_IAST_MAX_CONCURRENT_REQUESTS, - maxContextOperations: DD_IAST_MAX_CONTEXT_OPERATIONS, - deduplicationEnabled: DD_IAST_DEDUPLICATION_ENABLED, - redactionEnabled: DD_IAST_REDACTION_ENABLED, - redactionNamePattern: DD_IAST_REDACTION_NAME_PATTERN, - redactionValuePattern: DD_IAST_REDACTION_VALUE_PATTERN, - telemetryVerbosity: DD_IAST_TELEMETRY_VERBOSITY - } - - this.isCiVisibility = isTrue(DD_IS_CIVISIBILITY) - - this.isIntelligentTestRunnerEnabled = this.isCiVisibility && isTrue(DD_CIVISIBILITY_ITR_ENABLED) - this.isGitUploadEnabled = this.isCiVisibility && - (this.isIntelligentTestRunnerEnabled && !isFalse(DD_CIVISIBILITY_GIT_UPLOAD_ENABLED)) - - this.gitMetadataEnabled = isTrue(DD_TRACE_GIT_METADATA_ENABLED) - this.isManualApiEnabled = this.isCiVisibility && isTrue(DD_CIVISIBILITY_MANUAL_API_ENABLED) - this.isEarlyFlakeDetectionEnabled = this.isCiVisibility && isTrue(DD_CIVISIBILITY_EARLY_FLAKE_DETECTION_ENABLED) - - this.openaiSpanCharLimit = DD_OPENAI_SPAN_CHAR_LIMIT - // Requires an accompanying DD_APM_OBFUSCATION_MEMCACHED_KEEP_COMMAND=true in the agent this.memcachedCommandEnabled = isTrue(DD_TRACE_MEMCACHED_COMMAND_ENABLED) - - this.stats = { - enabled: isTrue(DD_TRACE_STATS_COMPUTATION_ENABLED) - } - - this.traceId128BitGenerationEnabled = isTrue(DD_TRACE_128_BIT_TRACEID_GENERATION_ENABLED) - this.traceId128BitLoggingEnabled = isTrue(DD_TRACE_128_BIT_TRACEID_LOGGING_ENABLED) - - this.isGCPFunction = isGCPFunction - this.isAzureFunctionConsumptionPlan = isAzureFunctionConsumptionPlan - + this.isAzureFunctionConsumptionPlan = getIsAzureFunctionConsumptionPlan() this.spanLeakDebug = Number(DD_TRACE_SPAN_LEAK_DEBUG) - this.installSignature = { id: DD_INSTRUMENTATION_INSTALL_ID, time: DD_INSTRUMENTATION_INSTALL_TIME, @@ -697,6 +284,7 @@ ken|consumer_?(?:id|key|secret)|sign(?:ed|ature)?|auth(?:entication|orization)?) this._applyDefaults() this._applyEnvironment() this._applyOptions(options) + this._applyCalculated() this._applyRemote({}) this._merge() @@ -755,9 +343,19 @@ ken|consumer_?(?:id|key|secret)|sign(?:ed|ature)?|auth(?:entication|orization)?) this._applyOptions(options) } + // TODO: test + this._applyCalculated() this._merge() } + _isInServerlessEnvironment () { + const inAWSLambda = process.env.AWS_LAMBDA_FUNCTION_NAME !== undefined + const isGCPFunction = getIsGCPFunction() + const isAzureFunctionConsumptionPlan = getIsAzureFunctionConsumptionPlan() + return inAWSLambda || isGCPFunction || isAzureFunctionConsumptionPlan + } + + // for _merge to work, every config value must have a default value _applyDefaults () { const { AWS_LAMBDA_FUNCTION_NAME, @@ -775,27 +373,159 @@ ken|consumer_?(?:id|key|secret)|sign(?:ed|ature)?|auth(?:entication|orization)?) const defaults = this._defaults = {} - this._setValue(defaults, 'service', service) + this._setValue(defaults, 'appsec.blockedTemplateHtml', undefined) + this._setValue(defaults, 'appsec.blockedTemplateJson', undefined) + this._setValue(defaults, 'appsec.customRulesProvided', false) + this._setValue(defaults, 'appsec.enabled', undefined) + this._setValue(defaults, 'appsec.obfuscatorKeyRegex', defaultWafObfuscatorKeyRegex) + this._setValue(defaults, 'appsec.obfuscatorValueRegex', defaultWafObfuscatorValueRegex) + this._setValue(defaults, 'appsec.rateLimit', 100) + this._setValue(defaults, 'appsec.rules', undefined) + this._setValue(defaults, 'appsec.wafTimeout', 5e3) // µs + this._setValue(defaults, 'clientIpEnabled', false) + this._setValue(defaults, 'clientIpHeader', null) + this._setValue(defaults, 'dbmPropagationMode', 'disabled') + this._setValue(defaults, 'dogstatsd.hostname', '127.0.0.1') + this._setValue(defaults, 'dogstatsd.port', '8125') + this._setValue(defaults, 'dsmEnabled', false) this._setValue(defaults, 'env', undefined) - this._setValue(defaults, 'version', pkg.version) - this._setUnit(defaults, 'sampleRate', undefined) - this._setBoolean(defaults, 'logInjection', false) - this._setArray(defaults, 'headerTags', []) + this._setValue(defaults, 'experimental.enableGetRumData', false) + this._setValue(defaults, 'experimental.exporter', undefined) + this._setValue(defaults, 'experimental.runtimeId', false) + this._setValue(defaults, 'flushInterval', 2000) + this._setValue(defaults, 'flushMinSpans', 1000) + this._setValue(defaults, 'gitMetadataEnabled', true) + this._setValue(defaults, 'headerTags', []) + this._setValue(defaults, 'hostname', '127.0.0.1') + this._setValue(defaults, 'iast.deduplicationEnabled', true) + this._setValue(defaults, 'iast.enabled', false) + this._setValue(defaults, 'iast.maxConcurrentRequests', 2) + this._setValue(defaults, 'iast.maxContextOperations', 2) + this._setValue(defaults, 'iast.redactionEnabled', true) + this._setValue(defaults, 'iast.redactionNamePattern', null) + this._setValue(defaults, 'iast.redactionValuePattern', null) + this._setValue(defaults, 'iast.requestSampling', 30) + this._setValue(defaults, 'iast.telemetryVerbosity', 'INFORMATION') + this._setValue(defaults, 'isCiVisibility', false) + this._setValue(defaults, 'isEarlyFlakeDetectionEnabled', false) + this._setValue(defaults, 'isGCPFunction', false) + this._setValue(defaults, 'isGitUploadEnabled', false) + this._setValue(defaults, 'isIntelligentTestRunnerEnabled', false) + this._setValue(defaults, 'isManualApiEnabled', false) + this._setValue(defaults, 'logInjection', false) + this._setValue(defaults, 'lookup', undefined) + this._setValue(defaults, 'openAiLogsEnabled', false) + this._setValue(defaults, 'openaiSpanCharLimit', 128) + this._setValue(defaults, 'peerServiceMapping', {}) + this._setValue(defaults, 'plugins', true) + this._setValue(defaults, 'port', '8126') + this._setValue(defaults, 'profiling.enabled', false) + this._setValue(defaults, 'profiling.exporters', 'agent') + this._setValue(defaults, 'profiling.sourceMap', true) + this._setValue(defaults, 'protocolVersion', '0.4') + this._setValue(defaults, 'queryStringObfuscation', qsRegex) + this._setValue(defaults, 'remoteConfig.enabled', true) + this._setValue(defaults, 'remoteConfig.pollInterval', 5) // seconds + this._setValue(defaults, 'reportHostname', false) + this._setValue(defaults, 'runtimeMetrics', false) + this._setValue(defaults, 'sampleRate', undefined) + this._setValue(defaults, 'sampler.rateLimit', undefined) + this._setValue(defaults, 'scope', undefined) + this._setValue(defaults, 'service', service) + this._setValue(defaults, 'site', 'datadoghq.com') + this._setValue(defaults, 'spanAttributeSchema', 'v0') + this._setValue(defaults, 'spanComputePeerService', false) + this._setValue(defaults, 'spanRemoveIntegrationFromService', false) + this._setValue(defaults, 'startupLogs', false) + this._setValue(defaults, 'stats.enabled', false) this._setValue(defaults, 'tags', {}) - this._setBoolean(defaults, 'tracing', true) + this._setValue(defaults, 'tagsHeaderMaxLength', 512) + this._setValue(defaults, 'telemetry.debug', false) + this._setValue(defaults, 'telemetry.dependencyCollection', true) + this._setValue(defaults, 'telemetry.enabled', true) + this._setValue(defaults, 'telemetry.heartbeatInterval', 60000) + this._setValue(defaults, 'telemetry.logCollection', false) + this._setValue(defaults, 'telemetry.metrics', true) + this._setValue(defaults, 'traceId128BitGenerationEnabled', true) + this._setValue(defaults, 'traceId128BitLoggingEnabled', false) + this._setValue(defaults, 'tracing', true) + this._setValue(defaults, 'url', undefined) + this._setValue(defaults, 'version', pkg.version) } _applyEnvironment () { const { + AWS_LAMBDA_FUNCTION_NAME, + DD_AGENT_HOST, + DD_APPSEC_ENABLED, + DD_APPSEC_HTTP_BLOCKED_TEMPLATE_HTML, + DD_APPSEC_HTTP_BLOCKED_TEMPLATE_JSON, + DD_APPSEC_OBFUSCATION_PARAMETER_KEY_REGEXP, + DD_APPSEC_OBFUSCATION_PARAMETER_VALUE_REGEXP, + DD_APPSEC_RULES, + DD_APPSEC_TRACE_RATE_LIMIT, + DD_APPSEC_WAF_TIMEOUT, + DD_DATA_STREAMS_ENABLED, + DD_DBM_PROPAGATION_MODE, + DD_DOGSTATSD_HOSTNAME, + DD_DOGSTATSD_PORT, DD_ENV, + DD_EXPERIMENTAL_PROFILING_ENABLED, + JEST_WORKER_ID, + DD_IAST_DEDUPLICATION_ENABLED, + DD_IAST_ENABLED, + DD_IAST_MAX_CONCURRENT_REQUESTS, + DD_IAST_MAX_CONTEXT_OPERATIONS, + DD_IAST_REDACTION_ENABLED, + DD_IAST_REDACTION_NAME_PATTERN, + DD_IAST_REDACTION_VALUE_PATTERN, + DD_IAST_REQUEST_SAMPLING, + DD_IAST_TELEMETRY_VERBOSITY, + DD_INSTRUMENTATION_TELEMETRY_ENABLED, DD_LOGS_INJECTION, + DD_OPENAI_LOGS_ENABLED, + DD_OPENAI_SPAN_CHAR_LIMIT, + DD_PROFILING_ENABLED, + DD_PROFILING_EXPORTERS, + DD_PROFILING_SOURCE_MAP, + DD_REMOTE_CONFIGURATION_ENABLED, + DD_REMOTE_CONFIG_POLL_INTERVAL_SECONDS, + DD_RUNTIME_METRICS_ENABLED, DD_SERVICE, DD_SERVICE_NAME, + DD_SITE, DD_TAGS, + DD_TELEMETRY_DEBUG, + DD_TELEMETRY_DEPENDENCY_COLLECTION_ENABLED, + DD_TELEMETRY_HEARTBEAT_INTERVAL, + DD_TELEMETRY_LOG_COLLECTION_ENABLED, + DD_TELEMETRY_METRICS_ENABLED, + DD_TRACE_128_BIT_TRACEID_GENERATION_ENABLED, + DD_TRACE_128_BIT_TRACEID_LOGGING_ENABLED, + DD_TRACE_AGENT_HOSTNAME, + DD_TRACE_AGENT_PORT, + DD_TRACE_AGENT_PROTOCOL_VERSION, + DD_TRACE_CLIENT_IP_ENABLED, + DD_TRACE_CLIENT_IP_HEADER, + DD_TRACE_EXPERIMENTAL_EXPORTER, + DD_TRACE_EXPERIMENTAL_GET_RUM_DATA_ENABLED, + DD_TRACE_EXPERIMENTAL_RUNTIME_ID_ENABLED, + DD_TRACE_GIT_METADATA_ENABLED, DD_TRACE_GLOBAL_TAGS, DD_TRACE_HEADER_TAGS, + DD_TRACE_OBFUSCATION_QUERY_STRING_REGEXP, + DD_TRACE_PARTIAL_FLUSH_MIN_SPANS, + DD_TRACE_PEER_SERVICE_MAPPING, + DD_TRACE_RATE_LIMIT, + DD_TRACE_REMOVE_INTEGRATION_SERVICE_NAMES_ENABLED, + DD_TRACE_REPORT_HOSTNAME, DD_TRACE_SAMPLE_RATE, + DD_TRACE_SCOPE, + DD_TRACE_SPAN_ATTRIBUTE_SCHEMA, + DD_TRACE_STARTUP_LOGS, DD_TRACE_TAGS, + DD_TRACE_TELEMETRY_ENABLED, + DD_TRACE_X_DATADOG_TAGS_MAX_LENGTH, DD_TRACING_ENABLED, DD_VERSION } = process.env @@ -807,31 +537,283 @@ ken|consumer_?(?:id|key|secret)|sign(?:ed|ature)?|auth(?:entication|orization)?) tagger.add(tags, DD_TRACE_TAGS) tagger.add(tags, DD_TRACE_GLOBAL_TAGS) - this._setString(env, 'service', DD_SERVICE || DD_SERVICE_NAME || tags.service) + this._setValue(env, 'appsec.blockedTemplateHtml', maybeFile(DD_APPSEC_HTTP_BLOCKED_TEMPLATE_HTML)) + this._setValue(env, 'appsec.blockedTemplateJson', maybeFile(DD_APPSEC_HTTP_BLOCKED_TEMPLATE_JSON)) + if (DD_APPSEC_RULES) this._setBoolean(env, 'appsec.customRulesProvided', true) + this._setBoolean(env, 'appsec.enabled', DD_APPSEC_ENABLED) + this._setString(env, 'appsec.obfuscatorKeyRegex', DD_APPSEC_OBFUSCATION_PARAMETER_KEY_REGEXP) + this._setString(env, 'appsec.obfuscatorValueRegex', DD_APPSEC_OBFUSCATION_PARAMETER_VALUE_REGEXP) + this._setValue(env, 'appsec.rateLimit', maybeInt(DD_APPSEC_TRACE_RATE_LIMIT)) + this._setString(env, 'appsec.rules', DD_APPSEC_RULES) + this._setValue(env, 'appsec.wafTimeout', maybeInt(DD_APPSEC_WAF_TIMEOUT)) + this._setBoolean(env, 'clientIpEnabled', DD_TRACE_CLIENT_IP_ENABLED) + this._setString(env, 'clientIpHeader', DD_TRACE_CLIENT_IP_HEADER) + this._setString(env, 'dbmPropagationMode', DD_DBM_PROPAGATION_MODE) + this._setString(env, 'dogstatsd.hostname', DD_DOGSTATSD_HOSTNAME) + this._setString(env, 'dogstatsd.port', DD_DOGSTATSD_PORT) + this._setBoolean(env, 'dsmEnabled', DD_DATA_STREAMS_ENABLED) this._setString(env, 'env', DD_ENV || tags.env) - this._setString(env, 'version', DD_VERSION || tags.version) - this._setUnit(env, 'sampleRate', DD_TRACE_SAMPLE_RATE) - this._setBoolean(env, 'logInjection', DD_LOGS_INJECTION) + this._setBoolean(env, 'experimental.enableGetRumData', DD_TRACE_EXPERIMENTAL_GET_RUM_DATA_ENABLED) + this._setString(env, 'experimental.exporter', DD_TRACE_EXPERIMENTAL_EXPORTER) + this._setBoolean(env, 'experimental.runtimeId', DD_TRACE_EXPERIMENTAL_RUNTIME_ID_ENABLED) + if (AWS_LAMBDA_FUNCTION_NAME) this._setValue(env, 'flushInterval', 0) + this._setValue(env, 'flushMinSpans', maybeInt(DD_TRACE_PARTIAL_FLUSH_MIN_SPANS)) + this._setBoolean(env, 'gitMetadataEnabled', DD_TRACE_GIT_METADATA_ENABLED) this._setArray(env, 'headerTags', DD_TRACE_HEADER_TAGS) + this._setString(env, 'hostname', coalesce(DD_AGENT_HOST, DD_TRACE_AGENT_HOSTNAME)) + this._setBoolean(env, 'iast.deduplicationEnabled', DD_IAST_DEDUPLICATION_ENABLED) + this._setBoolean(env, 'iast.enabled', DD_IAST_ENABLED) + this._setValue(env, 'iast.maxConcurrentRequests', maybeInt(DD_IAST_MAX_CONCURRENT_REQUESTS)) + this._setValue(env, 'iast.maxContextOperations', maybeInt(DD_IAST_MAX_CONTEXT_OPERATIONS)) + this._setBoolean(env, 'iast.redactionEnabled', DD_IAST_REDACTION_ENABLED && !isFalse(DD_IAST_REDACTION_ENABLED)) + this._setString(env, 'iast.redactionNamePattern', DD_IAST_REDACTION_NAME_PATTERN) + this._setString(env, 'iast.redactionValuePattern', DD_IAST_REDACTION_VALUE_PATTERN) + const iastRequestSampling = maybeInt(DD_IAST_REQUEST_SAMPLING) + if (iastRequestSampling > -1 && iastRequestSampling < 101) { + this._setValue(env, 'iast.requestSampling', iastRequestSampling) + } + this._setString(env, 'iast.telemetryVerbosity', DD_IAST_TELEMETRY_VERBOSITY) + this._setBoolean(env, 'isGCPFunction', getIsGCPFunction()) + this._setBoolean(env, 'logInjection', DD_LOGS_INJECTION) + this._setBoolean(env, 'openAiLogsEnabled', DD_OPENAI_LOGS_ENABLED) + this._setValue(env, 'openaiSpanCharLimit', maybeInt(DD_OPENAI_SPAN_CHAR_LIMIT)) + if (DD_TRACE_PEER_SERVICE_MAPPING) { + this._setValue(env, 'peerServiceMapping', fromEntries( + process.env.DD_TRACE_PEER_SERVICE_MAPPING.split(',').map(x => x.trim().split(':')) + )) + } + this._setString(env, 'port', DD_TRACE_AGENT_PORT) + this._setBoolean(env, 'profiling.enabled', coalesce(DD_EXPERIMENTAL_PROFILING_ENABLED, DD_PROFILING_ENABLED)) + this._setString(env, 'profiling.exporters', DD_PROFILING_EXPORTERS) + this._setBoolean(env, 'profiling.sourceMap', DD_PROFILING_SOURCE_MAP && !isFalse(DD_PROFILING_SOURCE_MAP)) + this._setString(env, 'protocolVersion', DD_TRACE_AGENT_PROTOCOL_VERSION) + this._setString(env, 'queryStringObfuscation', DD_TRACE_OBFUSCATION_QUERY_STRING_REGEXP) + this._setBoolean(env, 'remoteConfig.enabled', coalesce( + DD_REMOTE_CONFIGURATION_ENABLED, + !this._isInServerlessEnvironment() + )) + this._setValue(env, 'remoteConfig.pollInterval', maybeFloat(DD_REMOTE_CONFIG_POLL_INTERVAL_SECONDS)) + this._setBoolean(env, 'reportHostname', DD_TRACE_REPORT_HOSTNAME) + this._setBoolean(env, 'runtimeMetrics', DD_RUNTIME_METRICS_ENABLED) + this._setUnit(env, 'sampleRate', DD_TRACE_SAMPLE_RATE) + this._setValue(env, 'sampler.rateLimit', DD_TRACE_RATE_LIMIT) + this._setString(env, 'scope', DD_TRACE_SCOPE) + this._setString(env, 'service', DD_SERVICE || DD_SERVICE_NAME || tags.service) + this._setString(env, 'site', DD_SITE) + if (DD_TRACE_SPAN_ATTRIBUTE_SCHEMA) { + this._setString(env, 'spanAttributeSchema', validateNamingVersion(DD_TRACE_SPAN_ATTRIBUTE_SCHEMA)) + } + this._setBoolean(env, 'spanRemoveIntegrationFromService', DD_TRACE_REMOVE_INTEGRATION_SERVICE_NAMES_ENABLED) + this._setBoolean(env, 'startupLogs', DD_TRACE_STARTUP_LOGS) this._setTags(env, 'tags', tags) + this._setValue(env, 'tagsHeaderMaxLength', DD_TRACE_X_DATADOG_TAGS_MAX_LENGTH) + this._setBoolean(env, 'telemetry.enabled', coalesce( + DD_TRACE_TELEMETRY_ENABLED, // for backward compatibility + DD_INSTRUMENTATION_TELEMETRY_ENABLED, // to comply with instrumentation telemetry specs + !(this._isInServerlessEnvironment() || JEST_WORKER_ID) + )) + this._setBoolean(env, 'telemetry.debug', DD_TELEMETRY_DEBUG) + this._setBoolean(env, 'telemetry.dependencyCollection', DD_TELEMETRY_DEPENDENCY_COLLECTION_ENABLED) + this._setValue(env, 'telemetry.heartbeatInterval', maybeInt(Math.floor(DD_TELEMETRY_HEARTBEAT_INTERVAL * 1000))) + this._setBoolean(env, 'telemetry.logCollection', coalesce(DD_TELEMETRY_LOG_COLLECTION_ENABLED, DD_IAST_ENABLED)) + this._setBoolean(env, 'telemetry.metrics', DD_TELEMETRY_METRICS_ENABLED) + this._setBoolean(env, 'traceId128BitGenerationEnabled', DD_TRACE_128_BIT_TRACEID_GENERATION_ENABLED) + this._setBoolean(env, 'traceId128BitLoggingEnabled', DD_TRACE_128_BIT_TRACEID_LOGGING_ENABLED) this._setBoolean(env, 'tracing', DD_TRACING_ENABLED) + this._setString(env, 'version', DD_VERSION || tags.version) } _applyOptions (options) { const opts = this._options = this._options || {} const tags = {} - options = Object.assign({ ingestion: {} }, options, opts) + options = this.options = Object.assign({ ingestion: {} }, options, opts) tagger.add(tags, options.tags) - this._setString(opts, 'service', options.service || tags.service) + this._setValue(opts, 'appsec.blockedTemplateHtml', maybeFile(options.appsec.blockedTemplateHtml)) + this._setValue(opts, 'appsec.blockedTemplateJson', maybeFile(options.appsec.blockedTemplateJson)) + if (options.appsec.rules) this._setBoolean(opts, 'appsec.customRulesProvided', true) + this._setBoolean(opts, 'appsec.enabled', options.appsec.enabled) + this._setString(opts, 'appsec.obfuscatorKeyRegex', options.appsec.obfuscatorKeyRegex) + this._setString(opts, 'appsec.obfuscatorValueRegex', options.appsec.obfuscatorValueRegex) + this._setValue(opts, 'appsec.rateLimit', maybeInt(options.appsec.rateLimit)) + this._setString(opts, 'appsec.rules', options.appsec.rules) + this._setValue(opts, 'appsec.wafTimeout', maybeInt(options.appsec.wafTimeout)) + this._setBoolean(opts, 'clientIpEnabled', options.clientIpEnabled) + this._setString(opts, 'clientIpHeader', options.clientIpHeader) + this._setString(opts, 'dbmPropagationMode', options.dbmPropagationMode) + if (options.dogstatsd) { + this._setString(opts, 'dogstatsd.hostname', options.dogstatsd.hostname) + this._setString(opts, 'dogstatsd.port', options.dogstatsd.port) + } + this._setBoolean(opts, 'dsmEnabled', options.dsmEnabled) this._setString(opts, 'env', options.env || tags.env) - this._setString(opts, 'version', options.version || tags.version) - this._setUnit(opts, 'sampleRate', coalesce(options.sampleRate, options.ingestion.sampleRate)) - this._setBoolean(opts, 'logInjection', options.logInjection) + this._setBoolean(opts, 'experimental.enableGetRumData', + options.experimental && options.experimental.enableGetRumData) + this._setString(opts, 'experimental.exporter', options.experimental && options.experimental.exporter) + this._setBoolean(opts, 'experimental.runtimeId', options.experimental && options.experimental.runtimeId) + this._setValue(opts, 'flushInterval', maybeInt(options.flushInterval)) + this._setValue(opts, 'flushMinSpans', maybeInt(options.flushMinSpans)) this._setArray(opts, 'headerTags', options.headerTags) + this._setString(opts, 'hostname', options.hostname) + this._setBoolean(opts, 'iast.deduplicationEnabled', options.iastOptions && options.iastOptions.deduplicationEnabled) + this._setBoolean(opts, 'iast.enabled', + options.iastOptions && (options.iastOptions === true || options.iastOptions.enabled === true)) + const iastRequestSampling = maybeInt(options.iastOptions?.requestSampling) + this._setValue(opts, 'iast.maxConcurrentRequests', + maybeInt(options.iastOptions?.maxConcurrentRequests)) + this._setValue(opts, 'iast.maxContextOperations', + maybeInt(options.iastOptions && options.iastOptions.maxContextOperations)) + this._setBoolean(opts, 'iast.redactionEnabled', options.iastOptions && options.iastOptions.redactionEnabled) + this._setString(opts, 'iast.redactionNamePattern', options.iastOptions?.redactionNamePattern) + this._setString(opts, 'iast.redactionValuePattern', options.iastOptions?.redactionValuePattern) + if (iastRequestSampling > -1 && iastRequestSampling < 101) { + this._setValue(opts, 'iast.requestSampling', iastRequestSampling) + } + this._setString(opts, 'iast.telemetryVerbosity', options.iastOptions && options.iastOptions.telemetryVerbosity) + this._setBoolean(opts, 'isCiVisibility', options.isCiVisibility) + this._setBoolean(opts, 'logInjection', options.logInjection) + this._setString(opts, 'lookup', options.lookup) + this._setBoolean(opts, 'openAiLogsEnabled', options.openAiLogsEnabled) + this._setValue(opts, 'peerServiceMapping', options.peerServiceMapping) + this._setBoolean(opts, 'plugins', options.plugins) + this._setString(opts, 'port', options.port) + this._setBoolean(opts, 'profiling.enabled', options.profiling) + this._setString(opts, 'protocolVersion', options.protocolVersion) + if (options.remoteConfig) { + this._setValue(opts, 'remoteConfig.pollInterval', maybeFloat(options.remoteConfig.pollInterval)) + } + this._setBoolean(opts, 'reportHostname', options.reportHostname) + this._setBoolean(opts, 'runtimeMetrics', options.runtimeMetrics) + this._setUnit(opts, 'sampleRate', coalesce(options.sampleRate, options.ingestion.sampleRate)) + const ingestion = options.ingestion || {} + this._setValue(opts, 'sampler.rateLimit', coalesce(options.rateLimit, ingestion.rateLimit)) + this._setString(opts, 'service', options.service || tags.service) + this._setString(opts, 'site', options.site) + if (options.spanAttributeSchema) { + this._setString(opts, 'spanAttributeSchema', validateNamingVersion(options.spanAttributeSchema)) + } + this._setBoolean(opts, 'spanRemoveIntegrationFromService', options.spanRemoveIntegrationFromService) + this._setBoolean(opts, 'startupLogs', options.startupLogs) this._setTags(opts, 'tags', tags) + this._setBoolean(opts, 'telemetry.logCollection', options.iastOptions && + (options.iastOptions === true || options.iastOptions.enabled === true)) + this._setBoolean(opts, 'traceId128BitGenerationEnabled', options.traceId128BitGenerationEnabled) + this._setBoolean(opts, 'traceId128BitLoggingEnabled', options.traceId128BitLoggingEnabled) + this._setString(opts, 'version', options.version || tags.version) + } + + _isCiVisibility () { + return coalesce( + this.options.isCiVisibility, + this._defaults['isCiVisibility'] + ) + } + + _isCiVisibilityItrEnabled () { + return coalesce( + process.env.DD_CIVISIBILITY_ITR_ENABLED, + true + ) + } + + _getHostname () { + const DD_CIVISIBILITY_AGENTLESS_URL = process.env.DD_CIVISIBILITY_AGENTLESS_URL + const url = DD_CIVISIBILITY_AGENTLESS_URL ? new URL(DD_CIVISIBILITY_AGENTLESS_URL) + : getAgentUrl(this._getTraceAgentUrl(), this.options) + const DD_AGENT_HOST = coalesce( + this.options.hostname, + process.env.DD_AGENT_HOST, + process.env.DD_TRACE_AGENT_HOSTNAME, + '127.0.0.1' + ) + return DD_AGENT_HOST || (url && url.hostname) + } + + _getSpanComputePeerService () { + const DD_TRACE_SPAN_ATTRIBUTE_SCHEMA = validateNamingVersion( + coalesce( + this.options.spanAttributeSchema, + process.env.DD_TRACE_SPAN_ATTRIBUTE_SCHEMA + ) + ) + + const peerServiceSet = ( + this.options.hasOwnProperty('spanComputePeerService') || + process.env.hasOwnProperty('DD_TRACE_PEER_SERVICE_DEFAULTS_ENABLED') + ) + const peerServiceValue = coalesce( + this.options.spanComputePeerService, + process.env.DD_TRACE_PEER_SERVICE_DEFAULTS_ENABLED + ) + + const spanComputePeerService = ( + DD_TRACE_SPAN_ATTRIBUTE_SCHEMA === 'v0' + // In v0, peer service is computed only if it is explicitly set to true + ? peerServiceSet && isTrue(peerServiceValue) + // In >v0, peer service is false only if it is explicitly set to false + : (peerServiceSet ? !isFalse(peerServiceValue) : true) + ) + + return spanComputePeerService + } + + _isCiVisibilityGitUploadEnabled () { + return coalesce( + process.env.DD_CIVISIBILITY_GIT_UPLOAD_ENABLED, + true + ) + } + + _isCiVisibilityManualApiEnabled () { + return isTrue(coalesce( + process.env.DD_CIVISIBILITY_MANUAL_API_ENABLED, + false + )) + } + + _isTraceStatsComputationEnabled () { + return coalesce( + this.options.stats, + process.env.DD_TRACE_STATS_COMPUTATION_ENABLED, + getIsGCPFunction() || getIsAzureFunctionConsumptionPlan() + ) + } + + _getTraceAgentUrl () { + return coalesce( + this.options.url, + process.env.DD_TRACE_AGENT_URL, + process.env.DD_TRACE_URL, + null + ) + } + + // handles values calculated from a mixture of options and env vars + _applyCalculated () { + const calc = this._calculated = {} + + const { + DD_CIVISIBILITY_AGENTLESS_URL + } = process.env + + if (DD_CIVISIBILITY_AGENTLESS_URL) { + this._setValue(calc, 'url', new URL(DD_CIVISIBILITY_AGENTLESS_URL)) + } else { + this._setValue(calc, 'url', getAgentUrl(this._getTraceAgentUrl(), this.options)) + } + if (this._isCiVisibility()) { + this._setBoolean(calc, 'isEarlyFlakeDetectionEnabled', + coalesce(process.env.DD_CIVISIBILITY_EARLY_FLAKE_DETECTION_ENABLED, true)) + this._setBoolean(calc, 'isIntelligentTestRunnerEnabled', isTrue(this._isCiVisibilityItrEnabled())) + this._setBoolean(calc, 'isManualApiEnabled', this._isCiVisibilityManualApiEnabled()) + } + this._setString(calc, 'dogstatsd.hostname', this._getHostname()) + this._setBoolean(calc, 'isGitUploadEnabled', + calc['isIntelligentTestRunnerEnabled'] && !isFalse(this._isCiVisibilityGitUploadEnabled())) + this._setBoolean(calc, 'spanComputePeerService', this._getSpanComputePeerService()) + this._setBoolean(calc, 'stats.enabled', this._isTraceStatsComputationEnabled()) } _applyRemote (options) { @@ -890,7 +872,7 @@ ken|consumer_?(?:id|key|secret)|sign(?:ed|ature)?|auth(?:entication|orization)?) } _setString (obj, name, value) { - obj[name] = value || undefined // unset for empty strings + obj[name] = value ? String(value) : undefined // unset for empty strings } _setTags (obj, name, value) { @@ -908,9 +890,12 @@ ken|consumer_?(?:id|key|secret)|sign(?:ed|ature)?|auth(?:entication|orization)?) // TODO: Report origin changes and errors to telemetry. // TODO: Deeply merge configurations. // TODO: Move change tracking to telemetry. + // for telemetry reporting, `name`s in `containers` need to be keys from: + // eslint-disable-next-line max-len + // https://github.com/DataDog/dd-go/blob/prod/trace/apps/tracer-telemetry-intake/telemetry-payload/static/config_norm_rules.json _merge () { - const containers = [this._remote, this._options, this._env, this._defaults] - const origins = ['remote_config', 'code', 'env_var', 'default'] + const containers = [this._remote, this._options, this._env, this._calculated, this._defaults] + const origins = ['remote_config', 'code', 'env_var', 'calculated', 'default'] const changes = [] for (const name in this._defaults) { @@ -919,9 +904,10 @@ ken|consumer_?(?:id|key|secret)|sign(?:ed|ature)?|auth(?:entication|orization)?) const origin = origins[i] if ((container[name] !== null && container[name] !== undefined) || container === this._defaults) { - if (this[name] === container[name] && this.hasOwnProperty(name)) break + if (get(this, name) === container[name] && has(this, name)) break - const value = this[name] = container[name] + const value = container[name] + set(this, name, value) changes.push({ name, value, origin }) @@ -931,11 +917,19 @@ ken|consumer_?(?:id|key|secret)|sign(?:ed|ature)?|auth(?:entication|orization)?) } this.sampler.sampleRate = this.sampleRate - updateConfig(changes, this) } } +function maybeInt (number) { + const parsed = parseInt(number) + return isNaN(parsed) ? undefined : parsed +} +function maybeFloat (number) { + const parsed = parseFloat(number) + return isNaN(parsed) ? undefined : parsed +} + function getAgentUrl (url, options) { if (url) return new URL(url) diff --git a/packages/dd-trace/src/opentracing/propagation/text_map.js b/packages/dd-trace/src/opentracing/propagation/text_map.js index 6c118406fe0..abe1835a0a2 100644 --- a/packages/dd-trace/src/opentracing/propagation/text_map.js +++ b/packages/dd-trace/src/opentracing/propagation/text_map.js @@ -1,6 +1,6 @@ 'use strict' -const pick = require('../../../../utils/src/pick') +const pick = require('../../../../datadog-core/src/utils/src/pick') const id = require('../../id') const DatadogSpanContext = require('../span_context') const log = require('../../log') diff --git a/packages/dd-trace/src/plugins/util/web.js b/packages/dd-trace/src/plugins/util/web.js index f145e27ef56..3e8bc787c72 100644 --- a/packages/dd-trace/src/plugins/util/web.js +++ b/packages/dd-trace/src/plugins/util/web.js @@ -1,6 +1,6 @@ 'use strict' -const uniq = require('../../../../utils/src/uniq') +const uniq = require('../../../../datadog-core/src/utils/src/uniq') const analyticsSampler = require('../../analytics_sampler') const FORMAT_HTTP_HEADERS = 'http_headers' const log = require('../../log') diff --git a/packages/dd-trace/src/telemetry/index.js b/packages/dd-trace/src/telemetry/index.js index c03f6f4bda0..aa79ec7e23f 100644 --- a/packages/dd-trace/src/telemetry/index.js +++ b/packages/dd-trace/src/telemetry/index.js @@ -20,6 +20,7 @@ let heartbeatTimeout let heartbeatInterval let extendedInterval let integrations +let configWithOrigin = [] let retryData = null const extendedHeartbeatPayload = {} @@ -96,23 +97,6 @@ function getProducts (config) { return products } -function flatten (input, result = [], prefix = [], traversedObjects = null) { - traversedObjects = traversedObjects || new WeakSet() - if (traversedObjects.has(input)) { - return - } - traversedObjects.add(input) - for (const [key, value] of Object.entries(input)) { - if (typeof value === 'object' && value !== null) { - flatten(value, result, [...prefix, key], traversedObjects) - } else { - // TODO: add correct origin value - result.push({ name: [...prefix, key].join('.'), value, origin: 'unknown' }) - } - } - return result -} - function getInstallSignature (config) { const { installSignature: sig } = config if (sig && (sig.id || sig.time || sig.type)) { @@ -127,7 +111,7 @@ function getInstallSignature (config) { function appStarted (config) { const app = { products: getProducts(config), - configuration: flatten(config) + configuration: configWithOrigin } const installSignature = getInstallSignature(config) if (installSignature) { @@ -305,13 +289,18 @@ function updateIntegrations () { sendData(config, application, host, reqType, payload, updateRetryData) } +function formatMapForTelemetry (map) { + // format from an object to a string map in order for + // telemetry intake to accept the configuration + return map + ? Object.entries(map).map(([key, value]) => `${key}:${value}`).join(',') + : '' +} + function updateConfig (changes, config) { if (!config.telemetry.enabled) return if (changes.length === 0) return - // Hack to make system tests happy until we ship telemetry v2 - if (process.env.DD_INTERNAL_TELEMETRY_V2_ENABLED !== '1') return - const application = createAppObject(config) const host = createHostObject() @@ -325,24 +314,23 @@ function updateConfig (changes, config) { const configuration = [] for (const change of changes) { - if (!names.hasOwnProperty(change.name)) continue - - const name = names[change.name] + const name = names[change.name] || change.name const { origin, value } = change - const entry = { name, origin, value } + const entry = { name, value, origin } - if (Array.isArray(value)) { - entry.value = value.join(',') - } else if (name === 'DD_TAGS') { - entry.value = Object.entries(value).map(([key, value]) => `${key}:${value}`) - } + if (Array.isArray(value)) entry.value = value.join(',') + if (entry.name === 'DD_TAGS') entry.value = formatMapForTelemetry(entry.value) + if (entry.name === 'url' && entry.value) entry.value = entry.value.toString() + if (entry.name === 'peerServiceMapping' || entry.name === 'tags') entry.value = formatMapForTelemetry(entry.value) configuration.push(entry) } - - const { reqType, payload } = createPayload('app-client-configuration-change', { configuration }) - - sendData(config, application, host, reqType, payload, updateRetryData) + if (!configWithOrigin.length) { + configWithOrigin = configuration + } else { + const { reqType, payload } = createPayload('app-client-configuration-change', { configuration }) + sendData(config, application, host, reqType, payload, updateRetryData) + } } module.exports = { diff --git a/packages/dd-trace/test/config.spec.js b/packages/dd-trace/test/config.spec.js index 500ab936d36..e45b015488b 100644 --- a/packages/dd-trace/test/config.spec.js +++ b/packages/dd-trace/test/config.spec.js @@ -4,6 +4,7 @@ require('./setup/tap') const { expect } = require('chai') const { readFileSync } = require('fs') +const sinon = require('sinon') describe('Config', () => { let Config @@ -15,6 +16,7 @@ describe('Config', () => { let existsSyncParam let existsSyncReturn let osType + let updateConfig const RECOMMENDED_JSON_PATH = require.resolve('../src/appsec/recommended.json') const RULES_JSON_PATH = require.resolve('./fixtures/config/appsec-rules.json') @@ -39,6 +41,8 @@ describe('Config', () => { error: sinon.spy() } + updateConfig = sinon.stub() + env = process.env process.env = {} fs = { @@ -57,12 +61,14 @@ describe('Config', () => { Config = proxyquire('../src/config', { './pkg': pkg, './log': log, + './telemetry': { updateConfig }, fs, os }) }) afterEach(() => { + updateConfig.reset() process.env = env existsSyncParam = undefined }) @@ -124,6 +130,78 @@ describe('Config', () => { expect(config).to.have.nested.property('installSignature.id', null) expect(config).to.have.nested.property('installSignature.time', null) expect(config).to.have.nested.property('installSignature.type', null) + + expect(updateConfig).to.be.calledOnce + + expect(updateConfig.getCall(0).args[0]).to.deep.include( + { name: 'service', value: 'node', origin: 'default' }, + { name: 'logInjection', value: false, origin: 'default' }, + { name: 'headerTags', value: [], origin: 'default' }, + { name: 'tracing', value: true, origin: 'default' }, + { name: 'dbmPropagationMode', value: 'disabled', origin: 'default' }, + { name: 'dsmEnabled', value: false, origin: 'default' }, + { name: 'openAiLogsEnabled', value: false, origin: 'default' }, + { name: 'url', value: undefined, origin: 'default' }, + { name: 'site', value: 'datadoghq.com', origin: 'default' }, + { name: 'hostname', value: '127.0.0.1', origin: 'default' }, + { name: 'port', value: '8126', origin: 'default' }, + { name: 'debug', value: false, origin: 'default' }, + { name: 'protocolVersion', value: '0.4', origin: 'default' }, + { name: 'dogstatsd.port', value: '8125', origin: 'default' }, + { name: 'flushInterval', value: 2000, origin: 'default' }, + { name: 'flushMinSpans', value: 1000, origin: 'default' }, + { name: 'clientIpEnabled', value: false, origin: 'default' }, + { name: 'clientIpHeader', value: null, origin: 'default' }, + { name: 'sampleRate', value: undefined, origin: 'default' }, + { name: 'runtimeMetrics', value: false, origin: 'default' }, + { name: 'plugins', value: true, origin: 'default' }, + { name: 'reportHostname', value: false, origin: 'default' }, + { name: 'scope', value: undefined, origin: 'default' }, + { name: 'logLevel', value: 'debug', origin: 'default' }, + { name: 'traceId128BitGenerationEnabled', value: false, origin: 'default' }, + { name: 'traceId128BitLoggingEnabled', value: false, origin: 'default' }, + { name: 'spanAttributeSchema', value: 'v0', origin: 'default' }, + { name: 'spanRemoveIntegrationFromService', value: false, origin: 'default' }, + { name: 'peerServiceMapping', value: '', origin: 'default' }, + { name: 'tracePropagationStyle.extract', value: ['datadog', 'tracecontext'], origin: 'default' }, + { name: 'experimental.runtimeId', value: false, origin: 'default' }, + { name: 'experimental.exporter', value: undefined, origin: 'default' }, + { name: 'experimental.enableGetRumData', value: false, origin: 'default' }, + { name: 'reportHostname', value: false, origin: 'default' }, + { name: 'profiling.enabled', value: false, origin: 'default' }, + { name: 'profiling.sourceMap', value: true, origin: 'default' }, + { name: 'profiling.exporters', value: 'agent', origin: 'default' }, + { name: 'startupLogs', value: false, origin: 'default' }, + { name: 'telemetry.enabled', value: true, origin: 'default' }, + { name: 'telemetry.heartbeatInterval', value: 60000, origin: 'default' }, + { name: 'telemetry.debug', value: false, origin: 'default' }, + { name: 'telemetry.metrics', value: false, origin: 'default' }, + { name: 'telemetry.dependencyCollection', value: true, origin: 'default' }, + { name: 'tagsHeaderMaxLength', value: 512, origin: 'default' }, + { name: 'appsec.enabled', value: undefined, origin: 'default' }, + { name: 'appsec.rules', value: undefined, origin: 'default' }, + { name: 'appsec.customRulesProvided', value: false, origin: 'default' }, + { name: 'appsec.rateLimit', value: 100, origin: 'default' }, + { name: 'appsec.wafTimeout', value: 5e3, origin: 'default' }, + { name: 'appsec.blockedTemplateHtml', value: undefined, origin: 'default' }, + { name: 'appsec.blockedTemplateJson', value: undefined, origin: 'default' }, + { name: 'appsec.eventTracking.enabled', value: true, origin: 'default' }, + { name: 'appsec.eventTracking.mode', value: 'safe', origin: 'default' }, + { name: 'remoteConfig.enabled', value: true, origin: 'default' }, + { name: 'remoteConfig.pollInterval', value: 5, origin: 'default' }, + { name: 'iast.enabled', value: false, origin: 'default' }, + { name: 'iast.requestSampling', value: 30, origin: 'default' }, + { name: 'iast.maxConcurrentRequests', value: 2, origin: 'default' }, + { name: 'iast.maxContextOperations', value: 2, origin: 'default' }, + { name: 'iast.deduplicationEnabled', value: true, origin: 'default' }, + { name: 'iast.redactionEnabled', value: true, origin: 'default' }, + { name: 'iast.telemetryVerbosity', value: 'INFORMATION', origin: 'default' }, + { name: 'isCiVisibility', value: false, origin: 'default' }, + { name: 'gitMetadataEnabled', value: true, origin: 'default' }, + { name: 'openaiSpanCharLimit', value: 128, origin: 'default' }, + { name: 'traceId128BitGenerationEnabled', value: false, origin: 'default' }, + { name: 'traceId128BitLoggingEnabled', value: false, origin: 'default' } + ) }) it('should support logging', () => { @@ -319,6 +397,54 @@ describe('Config', () => { type: 'k8s_single_step', time: '1703188212' }) + + expect(updateConfig).to.be.calledOnce + + expect(updateConfig.getCall(0).args[0]).to.deep.include( + { name: 'tracing', value: false, origin: 'env_var' }, + { name: 'debug', value: true, origin: 'env_var' }, + { name: 'protocolVersion', value: '0.5', origin: 'env_var' }, + { name: 'hostname', value: 'agent', origin: 'env_var' }, + { name: 'dogstatsd.hostname', value: 'dsd-agent', origin: 'env_var' }, + { name: 'dogstatsd.port', value: '5218', origin: 'env_var' }, + { name: 'service', value: 'service', origin: 'env_var' }, + { name: 'version', value: '1.0.0', origin: 'env_var' }, + { name: 'clientIpEnabled', value: true, origin: 'env_var' }, + { name: 'clientIpHeader', value: 'x-true-client-ip', origin: 'env_var' }, + { name: 'runtimeMetrics', value: true, origin: 'env_var' }, + { name: 'reportHostname', value: true, origin: 'env_var' }, + { name: 'env', value: 'test', origin: 'env_var' }, + { name: 'sampleRate', value: 0.5, origin: 'env_var' }, + { name: 'traceId128BitGenerationEnabled', value: true, origin: 'env_var' }, + { name: 'traceId128BitLoggingEnabled', value: true, origin: 'env_var' }, + { name: 'spanAttributeSchema', value: 'v1', origin: 'env_var' }, + { name: 'spanComputePeerService', value: true, origin: 'env_var' }, + { name: 'sampler.rateLimit', value: '-1', origin: 'env_var' }, + { name: 'spanRemoveIntegrationFromService', value: true, origin: 'env_var' }, + { name: 'peerServiceMapping', value: 'c:cc,d:dd', origin: 'env_var' }, + { name: 'tracePropagationStyle.extract', value: ['b3', 'tracecontext'], origin: 'env_var' }, + { name: 'experimental.runtimeId', value: true, origin: 'env_var' }, + { name: 'experimental.exporter', value: 'log', origin: 'env_var' }, + { name: 'experimental.enableGetRumData', value: true, origin: 'env_var' }, + { name: 'appsec.enabled', value: true, origin: 'env_var' }, + { name: 'appsec.rules', value: RULES_JSON_PATH, origin: 'env_var' }, + { name: 'appsec.customRulesProvided', value: true, origin: 'env_var' }, + { name: 'appsec.rateLimit', value: 42, origin: 'env_var' }, + { name: 'appsec.wafTimeout', value: 42, origin: 'env_var' }, + { name: 'appsec.blockedTemplateHtml', value: BLOCKED_TEMPLATE_HTML, origin: 'env_var' }, + { name: 'appsec.blockedTemplateJson', value: BLOCKED_TEMPLATE_JSON, origin: 'env_var' }, + { name: 'appsec.eventTracking.enabled', value: true, origin: 'env_var' }, + { name: 'appsec.eventTracking.mode', value: 'extended', origin: 'env_var' }, + { name: 'remoteConfig.enabled', value: false, origin: 'calculated' }, + { name: 'remoteConfig.pollInterval', value: 42, origin: 'env_var' }, + { name: 'iast.enabled', value: true, origin: 'env_var' }, + { name: 'iast.requestSampling', value: 40, origin: 'env_var' }, + { name: 'iast.maxConcurrentRequests', value: 3, origin: 'env_var' }, + { name: 'iast.maxContextOperations', value: 4, origin: 'env_var' }, + { name: 'iast.deduplicationEnabled', value: false, origin: 'env_var' }, + { name: 'iast.redactionEnabled', value: false, origin: 'env_var' }, + { name: 'iast.telemetryVerbosity', value: 'DEBUG', origin: 'env_var' } + ) }) it('should ignore empty strings', () => { @@ -529,6 +655,48 @@ describe('Config', () => { a: 'aa', b: 'bb' }) + + expect(updateConfig).to.be.calledOnce + + expect(updateConfig.getCall(0).args[0]).to.deep.include( + { name: 'protocolVersion', value: '0.5', origin: 'code' }, + { name: 'site', value: 'datadoghq.eu', origin: 'code' }, + { name: 'hostname', value: 'agent', origin: 'code' }, + { name: 'port', value: '6218', origin: 'code' }, + { name: 'dogstatsd.hostname', value: 'agent-dsd', origin: 'calculated' }, + { name: 'dogstatsd.port', value: '5218', origin: 'code' }, + { name: 'service', value: 'service', origin: 'code' }, + { name: 'version', value: '0.1.0', origin: 'code' }, + { name: 'env', value: 'test', origin: 'code' }, + { name: 'sampleRate', value: 0.5, origin: 'code' }, + { name: 'clientIpEnabled', value: true, origin: 'code' }, + { name: 'clientIpHeader', value: 'x-true-client-ip', origin: 'code' }, + { name: 'flushInterval', value: 5000, origin: 'code' }, + { name: 'flushMinSpans', value: 500, origin: 'code' }, + { name: 'runtimeMetrics', value: true, origin: 'code' }, + { name: 'reportHostname', value: true, origin: 'code' }, + { name: 'plugins', value: false, origin: 'code' }, + { name: 'logLevel', value: logLevel, origin: 'code' }, + { name: 'traceId128BitGenerationEnabled', value: true, origin: 'code' }, + { name: 'traceId128BitLoggingEnabled', value: true, origin: 'code' }, + { name: 'spanRemoveIntegrationFromService', value: true, origin: 'code' }, + { name: 'spanComputePeerService', value: true, origin: 'code' }, + { name: 'peerServiceMapping', value: 'd:dd', origin: 'code' }, + { name: 'tracePropagationStyle.extract', value: ['datadog'], origin: 'calculated' }, + { name: 'experimental.runtimeId', value: true, origin: 'code' }, + { name: 'experimental.exporter', value: 'log', origin: 'code' }, + { name: 'experimental.enableGetRumData', value: true, origin: 'code' }, + { name: 'appsec.enabled', value: false, origin: 'code' }, + { name: 'remoteConfig.pollInterval', value: 42, origin: 'code' }, + { name: 'iast.enabled', value: true, origin: 'code' }, + { name: 'iast.requestSampling', value: 50, origin: 'code' }, + { name: 'iast.maxConcurrentRequests', value: 4, origin: 'code' }, + { name: 'iast.maxContextOperations', value: 5, origin: 'code' }, + { name: 'iast.deduplicationEnabled', value: false, origin: 'code' }, + { name: 'iast.redactionEnabled', value: false, origin: 'code' }, + { name: 'iast.telemetryVerbosity', value: 'DEBUG', origin: 'code' }, + { name: 'sampler.sampleRate', value: 0.5, origin: 'code' } + ) }) it('should initialize from the options with url taking precedence', () => { @@ -1113,6 +1281,15 @@ describe('Config', () => { expect(config.remoteConfig.enabled).to.be.false }) + it('should send empty array when remote config is called on empty options', () => { + const config = new Config() + + config.configure({}, true) + + expect(updateConfig).to.be.calledTwice + expect(updateConfig.getCall(1).args[0]).to.deep.equal([]) + }) + it('should ignore invalid iast.requestSampling', () => { const config = new Config({ experimental: { diff --git a/packages/dd-trace/test/telemetry/index.spec.js b/packages/dd-trace/test/telemetry/index.spec.js index e5480e429b1..c66818b41fa 100644 --- a/packages/dd-trace/test/telemetry/index.spec.js +++ b/packages/dd-trace/test/telemetry/index.spec.js @@ -96,25 +96,6 @@ describe('telemetry', () => { appsec: { enabled: true }, profiler: { version: tracerVersion, enabled: true } }) - expect(payload).to.have.property('configuration').that.deep.equal([ - { name: 'telemetry.enabled', value: true, origin: 'unknown' }, - { name: 'telemetry.heartbeatInterval', value: DEFAULT_HEARTBEAT_INTERVAL, origin: 'unknown' }, - { name: 'hostname', value: 'localhost', origin: 'unknown' }, - { name: 'port', value: traceAgent.address().port, origin: 'unknown' }, - { name: 'service', value: 'test service', origin: 'unknown' }, - { name: 'version', value: '1.2.3-beta4', origin: 'unknown' }, - { name: 'env', value: 'preprod', origin: 'unknown' }, - { name: 'tags.runtime-id', value: '1a2b3c', origin: 'unknown' }, - { name: 'circularObject.child.field', value: 'child_value', origin: 'unknown' }, - { name: 'circularObject.field', value: 'parent_value', origin: 'unknown' }, - { name: 'appsec.enabled', value: true, origin: 'unknown' }, - { name: 'profiling.enabled', value: true, origin: 'unknown' }, - { name: 'peerServiceMapping.service_1', value: 'remapped_service_1', origin: 'unknown' }, - { name: 'peerServiceMapping.service_2', value: 'remapped_service_2', origin: 'unknown' }, - { name: 'installSignature.id', value: '68e75c48-57ca-4a12-adfc-575c4b05fcbe', origin: 'unknown' }, - { name: 'installSignature.type', value: 'k8s_single_step', origin: 'unknown' }, - { name: 'installSignature.time', value: '1703188212', origin: 'unknown' } - ]) expect(payload).to.have.property('install_signature').that.deep.equal({ install_id: '68e75c48-57ca-4a12-adfc-575c4b05fcbe', install_type: 'k8s_single_step',