From ca69a97809822a576a9a2668ef2eb4010390c96e Mon Sep 17 00:00:00 2001 From: cadl Date: Thu, 25 Aug 2022 17:40:31 +0800 Subject: [PATCH 1/2] fix: build error primarykey --- dbt/include/spark/macros/adapters.sql | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/dbt/include/spark/macros/adapters.sql b/dbt/include/spark/macros/adapters.sql index 8c2a7749c..70961f514 100644 --- a/dbt/include/spark/macros/adapters.sql +++ b/dbt/include/spark/macros/adapters.sql @@ -31,7 +31,8 @@ {%- set options = config.get('options') -%} {%- if config.get('file_format') == 'hudi' -%} {%- set unique_key = config.get('unique_key') -%} - {%- set expected_options_primarykey = unique_key|join(',') if unique_key is sequence else unique_key -%} + {%- set expected_options_primarykey = unique_key|join(',') if unique_key is sequence and (unique_key is not string and unique_key is not mapping) else unique_key -%} + {%- if unique_key is not none and options is none -%} {%- set options = {'primaryKey': expected_options_primarykey} -%} {%- elif unique_key is not none and options is not none and 'primaryKey' not in options -%} From b2e038dbaebe29d0c6105f069e3aaa7cd52ef96d Mon Sep 17 00:00:00 2001 From: cadl Date: Thu, 25 Aug 2022 17:47:36 +0800 Subject: [PATCH 2/2] test: add sequence unique_key test case --- tests/unit/test_macros.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/tests/unit/test_macros.py b/tests/unit/test_macros.py index 220a74db7..21da92f2a 100644 --- a/tests/unit/test_macros.py +++ b/tests/unit/test_macros.py @@ -80,12 +80,25 @@ def test_macros_create_table_as_hudi_options(self): sql = self.__run_macro(template, 'spark__create_table_as', False, 'my_table', 'select 1 as id').strip() self.assertEqual(sql, 'create table my_table using hudi options (primaryKey "id" ) as select 1 as id') + self.config['file_format'] = 'hudi' + self.config['unique_key'] = ['a', 'b'] + self.config['options'] = {} + sql = self.__run_macro(template, 'spark__create_table_as', False, 'my_table', 'select 1 as a, 1 as b').strip() + self.assertEqual(sql, 'create table my_table using hudi options (primaryKey "a,b" ) as select 1 as a, 1 as b') + + self.config['file_format'] = 'hudi' + self.config['unique_key'] = ['a', 'b'] + self.config['options'] = {'primaryKey': 'a,b'} + sql = self.__run_macro(template, 'spark__create_table_as', False, 'my_table', 'select 1 as a, 1 as b').strip() + self.assertEqual(sql, 'create table my_table using hudi options (primaryKey "a,b" ) as select 1 as a, 1 as b') + self.config['file_format'] = 'hudi' self.config['unique_key'] = 'uuid' self.config['options'] = {'primaryKey': 'id'} sql = self.__run_macro(template, 'spark__create_table_as', False, 'my_table', 'select 1 as id') self.assertIn('mock.raise_compiler_error()', sql) + def test_macros_create_table_as_partition(self): template = self.__get_template('adapters.sql')