chore(release): pull main into develop post release v1.79.1

CHANGELOG.md

@@ -2,6 +2,16 @@
 
 All notable changes to this project will be documented in this file. See [standard-version](https://github.com/conventional-changelog/standard-version) for commit guidelines.
 
+### [1.79.1](https://github.com/rudderlabs/rudder-transformer/compare/v1.79.0...v1.79.1) (2024-09-24)
+
+
+### Bug Fixes
+
+* allow users context traits and underscore divide numbers configuration ([#3752](https://github.com/rudderlabs/rudder-transformer/issues/3752)) ([386d2ab](https://github.com/rudderlabs/rudder-transformer/commit/386d2ab88c0fe72dc47ba119be08ad1c0cd6d51b))
+* populate users fields for sentAt, timestamp and originalTimestamp ([#3753](https://github.com/rudderlabs/rudder-transformer/issues/3753)) ([f50effe](https://github.com/rudderlabs/rudder-transformer/commit/f50effeeabdb888f82451c225a80971dbe6532b6))
+* prefer event check vs config check for vdm ([#3754](https://github.com/rudderlabs/rudder-transformer/issues/3754)) ([b2c1a18](https://github.com/rudderlabs/rudder-transformer/commit/b2c1a1893dfb957ac7a24c000b33cd254ef54b6c))
+* support different lookup fields and custom_attributes for rETL events  ([#3751](https://github.com/rudderlabs/rudder-transformer/issues/3751)) ([10d914e](https://github.com/rudderlabs/rudder-transformer/commit/10d914e25203bd6ae95801c2a98c17690bd2d6ef))
+
 ## [1.79.0](https://github.com/rudderlabs/rudder-transformer/compare/v1.78.0...v1.79.0) (2024-09-20)
 
 

package.json

@@ -1,6 +1,6 @@
 {
   "name": "rudder-transformer",
-  "version": "1.79.0",
+  "version": "1.79.1",
   "description": "",
   "homepage": "https://github.com/rudderlabs/rudder-transformer#readme",
   "bugs": {

src/cdk/v2/destinations/intercom/procWorkflow.yaml

@@ -77,8 +77,8 @@ steps:
     template: |
       const payload = .message.context.mappedToDestination ? $.outputs.rEtlPayload : $.outputs.identifyTransformationForLatestVersion;
       payload.name = $.getName(.message);
-      payload.custom_attributes = .message.context.traits || {};
-      payload.custom_attributes = $.filterCustomAttributes(payload, "user", .destination);
+      payload.custom_attributes = (.message.context.mappedToDestination ? .message.traits.custom_attributes : .message.context.traits) || {};
+      payload.custom_attributes = $.filterCustomAttributes(payload, "user", .destination, .message);
       payload.external_id = !payload.external_id && .destination.Config.sendAnonymousId && .message.anonymousId ? .message.anonymousId : payload.external_id;
       $.context.payload = payload;
       $.assert($.context.payload.external_id || $.context.payload.email, "Either email or userId is required for Identify call");
@@ -114,7 +114,7 @@ steps:
        update_last_request_at: typeof .destination.Config.updateLastRequestAt === 'boolean' ? .destination.Config.updateLastRequestAt : true
       }
       payload.companies = $.getCompaniesList(payload);
-      payload.custom_attributes = !.message.context.mappedToDestination ? $.filterCustomAttributes(payload, "user", .destination);
+      payload.custom_attributes = !.message.context.mappedToDestination ? $.filterCustomAttributes(payload, "user", .destination,.message);
       payload.user_id = !payload.user_id && .destination.Config.sendAnonymousId && .message.anonymousId ? .message.anonymousId : payload.user_id;
       $.context.payload = payload;
       $.assert($.context.payload.user_id || $.context.payload.email, "Either of `email` or `userId` is required for Identify call");
@@ -175,7 +175,7 @@ steps:
           $.assert(.message.groupId, "groupId is required for group call");
           const payload = .message.context.mappedToDestination ? $.outputs.rEtlPayload : $.outputs.groupTransformation;
           payload.custom_attributes = .message.traits || {};
-          payload.custom_attributes = $.filterCustomAttributes(payload, "company", .destination);
+          payload.custom_attributes = $.filterCustomAttributes(payload, "company", .destination,.message);
           $.context.payload = payload;
       - name: whenSearchContactFound
         condition: $.isDefinedAndNotNull($.outputs.searchContact)
@@ -214,7 +214,7 @@ steps:
         ...payload,
         custom_attributes : $.getFieldValueFromMessage(.message, "traits") || {}
       }
-      payload.custom_attributes = $.filterCustomAttributes(payload, "company", .destination);
+      payload.custom_attributes = $.filterCustomAttributes(payload, "company", .destination, .message);
       response.body.JSON = $.removeUndefinedAndNullValues(payload);
       response.endpoint = $.getBaseEndpoint(.destination) + "/" + "companies";
       response.headers = $.getHeaders(.destination, $.outputs.apiVersion);

src/cdk/v2/destinations/intercom/utils.js

@@ -233,20 +233,26 @@ const attachUserAndCompany = (message, Config) => {
  * @param {*} type
  * @returns
  */
-const filterCustomAttributes = (payload, type, destination) => {
+const filterCustomAttributes = (payload, type, destination, message) => {
   let ReservedAttributesList;
   let { apiVersion } = destination.Config;
   apiVersion = isDefinedAndNotNull(apiVersion) ? apiVersion : 'v2';
+  // we are discarding the lookup field from custom attributes
+  const lookupField = getLookUpField(message);
   if (type === 'user') {
-    ReservedAttributesList =
-      apiVersion === 'v1'
+    ReservedAttributesList = [
+      ...(apiVersion === 'v1'
         ? ReservedAttributes.v1UserAttributes
-        : ReservedAttributes.v2UserAttributes;
+        : ReservedAttributes.v2UserAttributes),
+      lookupField,
+    ];
   } else {
-    ReservedAttributesList =
-      apiVersion === 'v1'
+    ReservedAttributesList = [
+      ...(apiVersion === 'v1'
         ? ReservedAttributes.v1CompanyAttributes
-        : ReservedAttributes.v2CompanyAttributes;
+        : ReservedAttributes.v2CompanyAttributes),
+      lookupField !== 'email' && lookupField,
+    ];
   }
   let customAttributes = { ...get(payload, 'custom_attributes') };
   if (customAttributes) {
@@ -270,7 +276,10 @@ const filterCustomAttributes = (payload, type, destination) => {
  */
 const searchContact = async (message, destination, metadata) => {
   const lookupField = getLookUpField(message);
-  const lookupFieldValue = getFieldValueFromMessage(message, lookupField);
+  let lookupFieldValue = getFieldValueFromMessage(message, lookupField);
+  if (!lookupFieldValue) {
+    lookupFieldValue = message?.context?.traits?.[lookupField];
+  }
   const data = JSON.stringify({
     query: {
       operator: 'AND',

src/v0/destinations/fb_custom_audience/recordTransform.js

@@ -269,10 +269,11 @@ const processRecordInputsV2 = (groupedRecordInputs) => {
 
 function processRecordInputs(groupedRecordInputs) {
   const event = groupedRecordInputs[0];
-  if (isEventSentByVDMV2Flow(event)) {
-    return processRecordInputsV2(groupedRecordInputs);
+  // First check for rETL flow and second check for ES flow
+  if (isEventSentByVDMV1Flow(event) || !isEventSentByVDMV2Flow(event)) {
+    return processRecordInputsV1(groupedRecordInputs);
   }
-  return processRecordInputsV1(groupedRecordInputs);
+  return processRecordInputsV2(groupedRecordInputs);
 }
 
 module.exports = {

src/warehouse/index.js

@@ -643,6 +643,18 @@ function processWarehouseMessage(message, options) {
   const skipReservedKeywordsEscaping =
     options.integrationOptions.skipReservedKeywordsEscaping || false;
 
+  // underscoreDivideNumbers when set to false, if a column has a format like "_v_3_", it will be formatted to "_v3_"
+  // underscoreDivideNumbers when set to true, if a column has a format like "_v_3_", we keep it like that
+  // For older destinations, it will come as true and for new destinations this config will not be present which means we will treat it as false.
+  options.underscoreDivideNumbers = options.destConfig?.underscoreDivideNumbers || false;
+
+  // allowUsersContextTraits when set to true, if context.traits.* is present, it will be added as context_traits_* and *,
+  // e.g., for context.traits.name, context_traits_name and name will be added to the user's table.
+  // allowUsersContextTraits when set to false, if context.traits.* is present, it will be added only as context_traits_*
+  // e.g., for context.traits.name, only context_traits_name will be added to the user's table.
+  // For older destinations, it will come as true, and for new destinations this config will not be present, which means we will treat it as false.
+  const allowUsersContextTraits = options.destConfig?.allowUsersContextTraits || false;
+
   addJsonKeysToOptions(options);
 
   if (isBlank(message.messageId)) {
@@ -898,16 +910,18 @@ function processWarehouseMessage(message, options) {
         `${eventType + '_userProperties_'}`,
         2,
       );
-      setDataFromInputAndComputeColumnTypes(
-        utils,
-        eventType,
-        commonProps,
-        message.context ? message.context.traits : {},
-        commonColumnTypes,
-        options,
-        `${eventType + '_context_traits_'}`,
-        3,
-      );
+      if (allowUsersContextTraits) {
+        setDataFromInputAndComputeColumnTypes(
+          utils,
+          eventType,
+          commonProps,
+          message.context ? message.context.traits : {},
+          commonColumnTypes,
+          options,
+          `${eventType + '_context_traits_'}`,
+          3,
+        );
+      }
       setDataFromInputAndComputeColumnTypes(
         utils,
         eventType,
@@ -987,11 +1001,23 @@ function processWarehouseMessage(message, options) {
 
       const usersEvent = { ...commonProps };
       const usersColumnTypes = {};
+      let userColumnMappingRules = whUserColumnMappingRules;
+      if (!isDataLakeProvider(options.provider)) {
+        userColumnMappingRules = {
+          ...userColumnMappingRules,
+          ...{
+            sent_at: 'sentAt',
+            timestamp: 'timestamp',
+            original_timestamp: 'originalTimestamp',
+          },
+        };
+      }
+
       setDataFromColumnMappingAndComputeColumnTypes(
         utils,
         usersEvent,
         message,
-        whUserColumnMappingRules,
+        userColumnMappingRules,
         usersColumnTypes,
         options,
       );

src/warehouse/snakecase/snakecase.js

@@ -0,0 +1,37 @@
+const { toString } = require('lodash');
+const { unicodeWords, unicodeWordsWithNumbers } = require('./unicodeWords');
+
+const hasUnicodeWord = RegExp.prototype.test.bind(
+  /[a-z][A-Z]|[A-Z]{2}[a-z]|[0-9][a-zA-Z]|[a-zA-Z][0-9]|[^a-zA-Z0-9 ]/,
+);
+
+/** Used to match words composed of alphanumeric characters. */
+// eslint-disable-next-line no-control-regex
+const reAsciiWord = /[^\x00-\x2f\x3a-\x40\x5b-\x60\x7b-\x7f]+/g;
+
+function asciiWords(string) {
+  return string.match(reAsciiWord);
+}
+
+function words(string) {
+  const result = hasUnicodeWord(string) ? unicodeWords(string) : asciiWords(string);
+  return result || [];
+}
+
+function wordsWithNumbers(string) {
+  const result = hasUnicodeWord(string) ? unicodeWordsWithNumbers(string) : asciiWords(string);
+  return result || [];
+}
+
+const snakeCase = (string) =>
+  words(toString(string).replace(/['\u2019]/g, '')).reduce(
+    (result, word, index) => result + (index ? '_' : '') + word.toLowerCase(),
+    '',
+  );
+const snakeCaseWithNumbers = (string) =>
+  wordsWithNumbers(toString(string).replace(/['\u2019]/g, '')).reduce(
+    (result, word, index) => result + (index ? '_' : '') + word.toLowerCase(),
+    '',
+  );
+
+module.exports = { words, wordsWithNumbers, snakeCase, snakeCaseWithNumbers };

src/warehouse/snakecase/unicodeWords.js

@@ -0,0 +1,94 @@
+/** Used to compose unicode character classes. */
+const rsAstralRange = '\\ud800-\\udfff';
+const rsComboMarksRange = '\\u0300-\\u036f';
+const reComboHalfMarksRange = '\\ufe20-\\ufe2f';
+const rsComboSymbolsRange = '\\u20d0-\\u20ff';
+const rsComboMarksExtendedRange = '\\u1ab0-\\u1aff';
+const rsComboMarksSupplementRange = '\\u1dc0-\\u1dff';
+const rsComboRange =
+  rsComboMarksRange +
+  reComboHalfMarksRange +
+  rsComboSymbolsRange +
+  rsComboMarksExtendedRange +
+  rsComboMarksSupplementRange;
+const rsDingbatRange = '\\u2700-\\u27bf';
+const rsLowerRange = 'a-z\\xdf-\\xf6\\xf8-\\xff';
+const rsMathOpRange = '\\xac\\xb1\\xd7\\xf7';
+const rsNonCharRange = '\\x00-\\x2f\\x3a-\\x40\\x5b-\\x60\\x7b-\\xbf';
+const rsPunctuationRange = '\\u2000-\\u206f';
+const rsSpaceRange =
+  ' \\t\\x0b\\f\\xa0\\ufeff\\n\\r\\u2028\\u2029\\u1680\\u180e\\u2000\\u2001\\u2002\\u2003\\u2004\\u2005\\u2006\\u2007\\u2008\\u2009\\u200a\\u202f\\u205f\\u3000';
+const rsUpperRange = 'A-Z\\xc0-\\xd6\\xd8-\\xde';
+const rsVarRange = '\\ufe0e\\ufe0f';
+const rsBreakRange = rsMathOpRange + rsNonCharRange + rsPunctuationRange + rsSpaceRange;
+
+/** Used to compose unicode capture groups. */
+const rsApos = "['\u2019]";
+const rsBreak = `[${rsBreakRange}]`;
+const rsCombo = `[${rsComboRange}]`;
+const rsDigit = '\\d';
+const rsDingbat = `[${rsDingbatRange}]`;
+const rsLower = `[${rsLowerRange}]`;
+const rsMisc = `[^${rsAstralRange}${rsBreakRange + rsDigit + rsDingbatRange + rsLowerRange + rsUpperRange}]`;
+const rsFitz = '\\ud83c[\\udffb-\\udfff]';
+const rsModifier = `(?:${rsCombo}|${rsFitz})`;
+const rsNonAstral = `[^${rsAstralRange}]`;
+const rsRegional = '(?:\\ud83c[\\udde6-\\uddff]){2}';
+const rsSurrPair = '[\\ud800-\\udbff][\\udc00-\\udfff]';
+const rsUpper = `[${rsUpperRange}]`;
+const rsZWJ = '\\u200d';
+
+/** Used to compose unicode regexes. */
+const rsMiscLower = `(?:${rsLower}|${rsMisc})`;
+const rsMiscUpper = `(?:${rsUpper}|${rsMisc})`;
+const rsOptContrLower = `(?:${rsApos}(?:d|ll|m|re|s|t|ve))?`;
+const rsOptContrUpper = `(?:${rsApos}(?:D|LL|M|RE|S|T|VE))?`;
+const reOptMod = `${rsModifier}?`;
+const rsOptVar = `[${rsVarRange}]?`;
+const rsOptJoin = `(?:${rsZWJ}(?:${[rsNonAstral, rsRegional, rsSurrPair].join('|')})${rsOptVar + reOptMod})*`;
+const rsOrdLower = '\\d*(?:1st|2nd|3rd|(?![123])\\dth)(?=\\b|[A-Z_])';
+const rsOrdUpper = '\\d*(?:1ST|2ND|3RD|(?![123])\\dTH)(?=\\b|[a-z_])';
+const rsSeq = rsOptVar + reOptMod + rsOptJoin;
+const rsEmoji = `(?:${[rsDingbat, rsRegional, rsSurrPair].join('|')})${rsSeq}`;
+
+const reUnicodeWords = RegExp(
+  [
+    `${rsUpper}?${rsLower}+${rsOptContrLower}(?=${[rsBreak, rsUpper, '$'].join('|')})`, // Regular words, lowercase letters followed by optional contractions
+    `${rsMiscUpper}+${rsOptContrUpper}(?=${[rsBreak, rsUpper + rsMiscLower, '$'].join('|')})`, // Miscellaneous uppercase characters with optional contractions
+    `${rsUpper}?${rsMiscLower}+${rsOptContrLower}`, // Miscellaneous lowercase sequences with optional contractions
+    `${rsUpper}+${rsOptContrUpper}`, // All uppercase words with optional contractions (e.g., "THIS")
+    rsOrdUpper, // Ordinals for uppercase (e.g., "1ST", "2ND")
+    rsOrdLower, // Ordinals for lowercase (e.g., "1st", "2nd")
+    `${rsDigit}+`, // Pure digits (e.g., "123")
+    rsEmoji, // Emojis (e.g., 😀, ❤️)
+  ].join('|'),
+  'g',
+);
+
+const reUnicodeWordsWithNumbers = RegExp(
+  [
+    `${rsUpper}?${rsLower}+${rsDigit}+`, // Lowercase letters followed by digits (e.g., "abc123")
+    `${rsUpper}+${rsDigit}+`, // Uppercase letters followed by digits (e.g., "ABC123")
+    `${rsDigit}+${rsUpper}?${rsLower}+`, // Digits followed by lowercase letters (e.g., "123abc")
+    `${rsDigit}+${rsUpper}+`, // Digits followed by uppercase letters (e.g., "123ABC")
+    `${rsUpper}?${rsLower}+${rsOptContrLower}(?=${[rsBreak, rsUpper, '$'].join('|')})`, // Regular words, lowercase letters followed by optional contractions
+    `${rsMiscUpper}+${rsOptContrUpper}(?=${[rsBreak, rsUpper + rsMiscLower, '$'].join('|')})`, // Miscellaneous uppercase characters with optional contractions
+    `${rsUpper}?${rsMiscLower}+${rsOptContrLower}`, // Miscellaneous lowercase sequences with optional contractions
+    `${rsUpper}+${rsOptContrUpper}`, // All uppercase words with optional contractions (e.g., "THIS")
+    rsOrdUpper, // Ordinals for uppercase (e.g., "1ST", "2ND")
+    rsOrdLower, // Ordinals for lowercase (e.g., "1st", "2nd")
+    `${rsDigit}+`, // Pure digits (e.g., "123")
+    rsEmoji, // Emojis (e.g., 😀, ❤️)
+  ].join('|'),
+  'g',
+);
+
+function unicodeWords(string) {
+  return string.match(reUnicodeWords);
+}
+
+function unicodeWordsWithNumbers(string) {
+  return string.match(reUnicodeWordsWithNumbers);
+}
+
+module.exports = { unicodeWords, unicodeWordsWithNumbers };

src/warehouse/util.js

@@ -1,7 +1,5 @@
-const _ = require('lodash');
 const get = require('get-value');
 
-const v0 = require('./v0/util');
 const v1 = require('./v1/util');
 const { PlatformError, InstrumentationError } = require('@rudderstack/integrations-lib');
 const { isBlank } = require('./config/helpers');
@@ -112,14 +110,7 @@ function validTimestamp(input) {
 }
 
 function getVersionedUtils(schemaVersion) {
-  switch (schemaVersion) {
-    case 'v0':
-      return v0;
-    case 'v1':
-      return v1;
-    default:
-      return v1;
-  }
+  return v1;
 }
 
 function isRudderSourcesEvent(event) {