From e0f203e04f40c03e0d860d8e21aabc071f287fd5 Mon Sep 17 00:00:00 2001 From: Vardhan Thigle Date: Fri, 6 Dec 2024 10:52:04 +0000 Subject: [PATCH] Adding IT for BigInt unsigned in custom transformations --- .../apache/beam/it/gcp/spanner/SpannerResourceManager.java | 2 +- .../templates/MySQLCustomTransformationsNonShardedIT.java | 3 ++- .../CustomTransformationsNonShardedIT/mysql-schema.sql | 5 +++-- .../CustomTransformationsNonShardedIT/spanner-schema.sql | 1 + .../com/custom/CustomTransformationWithShardForBulkIT.java | 6 ++++++ 5 files changed, 13 insertions(+), 4 deletions(-) diff --git a/it/google-cloud-platform/src/main/java/org/apache/beam/it/gcp/spanner/SpannerResourceManager.java b/it/google-cloud-platform/src/main/java/org/apache/beam/it/gcp/spanner/SpannerResourceManager.java index 73079eb00e..da30fde5b9 100644 --- a/it/google-cloud-platform/src/main/java/org/apache/beam/it/gcp/spanner/SpannerResourceManager.java +++ b/it/google-cloud-platform/src/main/java/org/apache/beam/it/gcp/spanner/SpannerResourceManager.java @@ -382,7 +382,7 @@ public ImmutableList runQuery(String query) { } ImmutableList tableRecords = tableRecordsBuilder.build(); - LOG.info("Loaded {} rows from {}", tableRecords.size(), query); + LOG.info("Loaded {} rows from {}\n initial record: {}", tableRecords.size(), query, (tableRecords.size() < 2)? tableRecords : tableRecords.get(0)); return tableRecords; } catch (Exception e) { throw new SpannerResourceManagerException("Failed to read query " + query, e); diff --git a/v2/sourcedb-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/MySQLCustomTransformationsNonShardedIT.java b/v2/sourcedb-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/MySQLCustomTransformationsNonShardedIT.java index 3d9ee9e209..1fac70e7bc 100644 --- a/v2/sourcedb-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/MySQLCustomTransformationsNonShardedIT.java +++ b/v2/sourcedb-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/MySQLCustomTransformationsNonShardedIT.java @@ -126,6 +126,7 @@ public void simpleTest() throws Exception { row.put("tinyblob_column", "V29ybWQ="); row.put("tinytext_column", "This is tinytext append"); row.put("json_column", "{\"k1\":\"v1\",\"k2\":\"v2\"}"); + row.put("bigint_unsigned_column", 12346); events.add(row); @@ -133,7 +134,7 @@ public void simpleTest() throws Exception { spannerResourceManager.runQuery( "SELECT varchar_column, tinyint_column, text_column, date_column, int_column, bigint_column, float_column, double_column, decimal_column, datetime_column, timestamp_column, time_column, year_column, blob_column, enum_column, bool_column, varbinary_column, bit_column, binary_column, char_column, longblob_column," + "longtext_column, mediumblob_column, mediumint_column, mediumtext_column, set_column, smallint_column," - + "tinyblob_column, tinytext_column, json_column FROM AllDatatypeTransformation")) + + "tinyblob_column, tinytext_column, json_column bigint_unsigned_column FROM AllDatatypeTransformation")) .hasRecordsUnorderedCaseInsensitiveColumns(events); } } diff --git a/v2/sourcedb-to-spanner/src/test/resources/CustomTransformationsNonShardedIT/mysql-schema.sql b/v2/sourcedb-to-spanner/src/test/resources/CustomTransformationsNonShardedIT/mysql-schema.sql index 312201d404..57cc990154 100644 --- a/v2/sourcedb-to-spanner/src/test/resources/CustomTransformationsNonShardedIT/mysql-schema.sql +++ b/v2/sourcedb-to-spanner/src/test/resources/CustomTransformationsNonShardedIT/mysql-schema.sql @@ -29,6 +29,7 @@ CREATE TABLE AllDatatypeTransformation ( tinyblob_column TINYBLOB, tinytext_column TINYTEXT, json_column JSON, + bigint_unsigned_column BIGINT UNSIGNED, PRIMARY KEY (int_column) ); @@ -40,7 +41,7 @@ INSERT INTO AllDatatypeTransformation ( timestamp_column, time_column, year_column, blob_column, enum_column, bool_column, varbinary_column, bit_column, binary_column, char_column, longblob_column, longtext_column, mediumblob_column, mediumint_column, mediumtext_column, set_column, smallint_column, - tinyblob_column, tinytext_column, json_column + tinyblob_column, tinytext_column, json_column, bigint_unsigned_column ) VALUES ( 'id1', 12, 'This is a text value', '2024-06-21', 100, @@ -48,6 +49,6 @@ VALUES ( '2022-12-31 23:59:58', '17:00:00', '2024', x'7835383030', '2', false, x'7835383030000000000000000000000000000000', 42,x'7835383030000000000000000000000000000000', 'a', x'7835383030', 'This is longtext', x'7835383030', 2000, 'This is mediumtext', - 'v1,v2', 10, x'7835383030', 'This is tinytext', '{"k1": "v1"}' + 'v1,v2', 10, x'7835383030', 'This is tinytext', '{"k1": "v1"}', 12345 ); diff --git a/v2/sourcedb-to-spanner/src/test/resources/CustomTransformationsNonShardedIT/spanner-schema.sql b/v2/sourcedb-to-spanner/src/test/resources/CustomTransformationsNonShardedIT/spanner-schema.sql index 98fcea7d04..ea5653311c 100644 --- a/v2/sourcedb-to-spanner/src/test/resources/CustomTransformationsNonShardedIT/spanner-schema.sql +++ b/v2/sourcedb-to-spanner/src/test/resources/CustomTransformationsNonShardedIT/spanner-schema.sql @@ -29,4 +29,5 @@ CREATE TABLE AllDatatypeTransformation ( tinyblob_column BYTES(MAX), tinytext_column STRING(MAX), json_column JSON, + bigint_unsigned_column NUMERIC ) PRIMARY KEY (varchar_column); \ No newline at end of file diff --git a/v2/spanner-custom-shard/src/main/java/com/custom/CustomTransformationWithShardForBulkIT.java b/v2/spanner-custom-shard/src/main/java/com/custom/CustomTransformationWithShardForBulkIT.java index a5bf28cc3b..a92b8640a8 100644 --- a/v2/spanner-custom-shard/src/main/java/com/custom/CustomTransformationWithShardForBulkIT.java +++ b/v2/spanner-custom-shard/src/main/java/com/custom/CustomTransformationWithShardForBulkIT.java @@ -19,6 +19,7 @@ import com.google.cloud.teleport.v2.spanner.utils.ISpannerMigrationTransformer; import com.google.cloud.teleport.v2.spanner.utils.MigrationTransformationRequest; import com.google.cloud.teleport.v2.spanner.utils.MigrationTransformationResponse; +import java.math.BigInteger; import java.text.SimpleDateFormat; import java.time.LocalTime; import java.time.format.DateTimeFormatter; @@ -135,6 +136,11 @@ public MigrationTransformationResponse toSpannerRow(MigrationTransformationReque if (row.containsKey("json_column")) { row.put("json_column", "{\"k1\": \"v1\", \"k2\": \"v2\"}"); } + if (row.containsKey("bigint_unsigned_column")) { + BigInteger bigInteger = new BigInteger((String) row.get("bigint_unsigned_column")); + bigInteger = bigInteger.add(BigInteger.ONE); + row.put("bigint_unsigned_column", bigInteger.toString()); + } MigrationTransformationResponse response = new MigrationTransformationResponse(row, false); return response; }