Adding IT for BigInt unsigned in custom transformations

GoogleCloudPlatform · Dec 8, 2024 · e0f203e · e0f203e
1 parent 91b5be3
commit e0f203e
Show file tree

Hide file tree

Showing 5 changed files with 13 additions and 4 deletions.
diff --git a/...e-cloud-platform/src/main/java/org/apache/beam/it/gcp/spanner/SpannerResourceManager.java b/...e-cloud-platform/src/main/java/org/apache/beam/it/gcp/spanner/SpannerResourceManager.java
@@ -382,7 +382,7 @@ public ImmutableList<Struct> runQuery(String query) {
       }
       ImmutableList<Struct> tableRecords = tableRecordsBuilder.build();
 
-      LOG.info("Loaded {} rows from {}", tableRecords.size(), query);
+      LOG.info("Loaded {} rows from {}\n initial record: {}", tableRecords.size(), query, (tableRecords.size() < 2)? tableRecords : tableRecords.get(0));
       return tableRecords;
     } catch (Exception e) {
       throw new SpannerResourceManagerException("Failed to read query " + query, e);

diff --git a/...t/java/com/google/cloud/teleport/v2/templates/MySQLCustomTransformationsNonShardedIT.java b/...t/java/com/google/cloud/teleport/v2/templates/MySQLCustomTransformationsNonShardedIT.java
@@ -126,14 +126,15 @@ public void simpleTest() throws Exception {
     row.put("tinyblob_column", "V29ybWQ=");
     row.put("tinytext_column", "This is tinytext append");
     row.put("json_column", "{\"k1\":\"v1\",\"k2\":\"v2\"}");
+    row.put("bigint_unsigned_column", 12346);
 
     events.add(row);
 
     SpannerAsserts.assertThatStructs(
             spannerResourceManager.runQuery(
                 "SELECT varchar_column, tinyint_column, text_column, date_column, int_column, bigint_column, float_column, double_column, decimal_column, datetime_column, timestamp_column, time_column, year_column, blob_column, enum_column, bool_column, varbinary_column, bit_column, binary_column, char_column, longblob_column,"
                     + "longtext_column, mediumblob_column, mediumint_column, mediumtext_column, set_column, smallint_column,"
-                    + "tinyblob_column, tinytext_column, json_column FROM AllDatatypeTransformation"))
+                    + "tinyblob_column, tinytext_column, json_column bigint_unsigned_column FROM AllDatatypeTransformation"))
         .hasRecordsUnorderedCaseInsensitiveColumns(events);
   }
 }
diff --git a/v2/sourcedb-to-spanner/src/test/resources/CustomTransformationsNonShardedIT/mysql-schema.sql b/v2/sourcedb-to-spanner/src/test/resources/CustomTransformationsNonShardedIT/mysql-schema.sql
@@ -29,6 +29,7 @@ CREATE TABLE AllDatatypeTransformation (
     tinyblob_column TINYBLOB,
     tinytext_column TINYTEXT,
     json_column JSON,
+    bigint_unsigned_column BIGINT UNSIGNED,
     PRIMARY KEY (int_column)
 );
 
@@ -40,14 +41,14 @@ INSERT INTO AllDatatypeTransformation (
     timestamp_column, time_column, year_column, blob_column, enum_column,
     bool_column, varbinary_column, bit_column, binary_column, char_column, longblob_column,
     longtext_column, mediumblob_column, mediumint_column, mediumtext_column, set_column, smallint_column,
-    tinyblob_column, tinytext_column, json_column
+    tinyblob_column, tinytext_column, json_column, bigint_unsigned_column
 )
 VALUES (
     'id1', 12, 'This is a text value', '2024-06-21', 100,
     134567890, 3.14159, 2.71828, 12345.6789, '2024-06-21 17:10:01',
     '2022-12-31 23:59:58', '17:00:00', '2024', x'7835383030', '2',
     false, x'7835383030000000000000000000000000000000', 42,x'7835383030000000000000000000000000000000',
     'a', x'7835383030', 'This is longtext', x'7835383030', 2000, 'This is mediumtext',
-    'v1,v2', 10, x'7835383030', 'This is tinytext', '{"k1": "v1"}'
+    'v1,v2', 10, x'7835383030', 'This is tinytext', '{"k1": "v1"}', 12345
 );
 
diff --git a/...urcedb-to-spanner/src/test/resources/CustomTransformationsNonShardedIT/spanner-schema.sql b/...urcedb-to-spanner/src/test/resources/CustomTransformationsNonShardedIT/spanner-schema.sql
@@ -29,4 +29,5 @@ CREATE TABLE AllDatatypeTransformation (
     tinyblob_column BYTES(MAX),
     tinytext_column STRING(MAX),
     json_column JSON,
+    bigint_unsigned_column NUMERIC
 ) PRIMARY KEY (varchar_column);
diff --git a/v2/spanner-custom-shard/src/main/java/com/custom/CustomTransformationWithShardForBulkIT.java b/v2/spanner-custom-shard/src/main/java/com/custom/CustomTransformationWithShardForBulkIT.java
@@ -19,6 +19,7 @@
 import com.google.cloud.teleport.v2.spanner.utils.ISpannerMigrationTransformer;
 import com.google.cloud.teleport.v2.spanner.utils.MigrationTransformationRequest;
 import com.google.cloud.teleport.v2.spanner.utils.MigrationTransformationResponse;
+import java.math.BigInteger;
 import java.text.SimpleDateFormat;
 import java.time.LocalTime;
 import java.time.format.DateTimeFormatter;
@@ -135,6 +136,11 @@ public MigrationTransformationResponse toSpannerRow(MigrationTransformationReque
       if (row.containsKey("json_column")) {
         row.put("json_column", "{\"k1\": \"v1\", \"k2\": \"v2\"}");
       }
+      if (row.containsKey("bigint_unsigned_column")) {
+        BigInteger bigInteger = new BigInteger((String) row.get("bigint_unsigned_column"));
+        bigInteger = bigInteger.add(BigInteger.ONE);
+        row.put("bigint_unsigned_column", bigInteger.toString());
+      }
       MigrationTransformationResponse response = new MigrationTransformationResponse(row, false);
       return response;
     }