From bc9b17b88fb84a6fbad1a81e73b139bc5debb485 Mon Sep 17 00:00:00 2001 From: Mikael Ene Date: Fri, 1 May 2020 09:43:10 +0200 Subject: [PATCH 01/19] Fixed macronames. --- .../materializations/incremental/incremental.sql | 2 +- .../sqlserver/macros/materializations/seed/seed.sql | 2 +- .../macros/materializations/snapshot/snapshot.sql | 10 +++++----- .../macros/materializations/snapshot/strategies.sql | 2 +- dbt/include/sqlserver/macros/schema_tests.sql | 8 ++++---- setup.py | 4 ++-- 6 files changed, 14 insertions(+), 14 deletions(-) diff --git a/dbt/include/sqlserver/macros/materializations/incremental/incremental.sql b/dbt/include/sqlserver/macros/materializations/incremental/incremental.sql index fb638e8d..7852be94 100644 --- a/dbt/include/sqlserver/macros/materializations/incremental/incremental.sql +++ b/dbt/include/sqlserver/macros/materializations/incremental/incremental.sql @@ -1,4 +1,4 @@ - {% macro dbt__incremental_delete(target_relation, tmp_relation) -%} +{% macro dbt__incremental_delete(target_relation, tmp_relation) -%} {%- set unique_key = config.require('unique_key') -%} delete diff --git a/dbt/include/sqlserver/macros/materializations/seed/seed.sql b/dbt/include/sqlserver/macros/materializations/seed/seed.sql index 9b84ab4d..a496bccd 100644 --- a/dbt/include/sqlserver/macros/materializations/seed/seed.sql +++ b/dbt/include/sqlserver/macros/materializations/seed/seed.sql @@ -1,4 +1,4 @@ -{% macro basic_load_csv_rows(model, batch_size, agate_table) %} +{% macro sqlserver__basic_load_csv_rows(model, batch_size, agate_table) %} {% set cols_sql = get_seed_column_quoted_csv(model, agate_table.column_names) %} {% set bindings = [] %} diff --git a/dbt/include/sqlserver/macros/materializations/snapshot/snapshot.sql b/dbt/include/sqlserver/macros/materializations/snapshot/snapshot.sql index fff31d5e..4d1eb79a 100644 --- a/dbt/include/sqlserver/macros/materializations/snapshot/snapshot.sql +++ b/dbt/include/sqlserver/macros/materializations/snapshot/snapshot.sql @@ -1,4 +1,4 @@ -{% macro snapshot_staging_table_inserts(strategy, source_sql, target_relation) -%} +{% macro sqlserver__snapshot_staging_table_inserts(strategy, source_sql, target_relation) -%} select 'insert' as dbt_change_type, @@ -37,7 +37,7 @@ {%- endmacro %} -{% macro snapshot_staging_table_updates(strategy, source_sql, target_relation) -%} +{% macro sqlserver__snapshot_staging_table_updates(strategy, source_sql, target_relation) -%} select 'update' as dbt_change_type, @@ -73,7 +73,7 @@ {%- endmacro %} -{% macro build_snapshot_staging_table(strategy, sql, target_relation) %} +{% macro sqlserver__build_snapshot_staging_table(strategy, sql, target_relation) %} {% set tmp_relation = make_temp_relation(target_relation) %} {% set inserts_select = snapshot_staging_table_inserts(strategy, sql, target_relation) %} @@ -94,7 +94,7 @@ {% endmacro %} -{% materialization snapshot, default %} +{% materialization sqlserver__snapshot, default %} {%- set config = model['config'] -%} {%- set target_database = config.get('target_database') -%} @@ -184,7 +184,7 @@ {% endmacro %} -{% macro build_snapshot_table(strategy, sql) %} +{% macro sqlserver__build_snapshot_table(strategy, sql) %} select *, COALESCE({{ strategy.scd_id }}, NULL) as dbt_scd_id, diff --git a/dbt/include/sqlserver/macros/materializations/snapshot/strategies.sql b/dbt/include/sqlserver/macros/materializations/snapshot/strategies.sql index bfe143ba..f98e1183 100644 --- a/dbt/include/sqlserver/macros/materializations/snapshot/strategies.sql +++ b/dbt/include/sqlserver/macros/materializations/snapshot/strategies.sql @@ -4,7 +4,7 @@ {% endfor %}), 2) {% endmacro %} -{% macro snapshot_check_strategy(node, snapshotted_rel, current_rel, config, target_exists) %} +{% macro sqlserver__snapshot_check_strategy(node, snapshotted_rel, current_rel, config, target_exists) %} {% set check_cols_config = config['check_cols'] %} {% set primary_key = config['unique_key'] %} {% set updated_at = snapshot_get_time() %} diff --git a/dbt/include/sqlserver/macros/schema_tests.sql b/dbt/include/sqlserver/macros/schema_tests.sql index 7cc8d464..9c40e4e9 100644 --- a/dbt/include/sqlserver/macros/schema_tests.sql +++ b/dbt/include/sqlserver/macros/schema_tests.sql @@ -1,4 +1,4 @@ -{% macro test_unique(model) %} +{% macro sqlserver__test_unique(model) %} {% set column_name = kwargs.get('column_name', kwargs.get('arg')) %} @@ -18,7 +18,7 @@ from ( {% endmacro %} -{% macro test_not_null(model) %} +{% macro sqlserver__test_not_null(model) %} {% set column_name = kwargs.get('column_name', kwargs.get('arg')) %} @@ -29,7 +29,7 @@ where {{ column_name }} is null {% endmacro %} -{% macro test_accepted_values(model, values) %} +{% macro sqlserver__test_accepted_values(model, values) %} {% set column_name = kwargs.get('column_name', kwargs.get('field')) %} @@ -63,7 +63,7 @@ from validation_errors {% endmacro %} -{% macro test_relationships(model, to, field) %} +{% macro sqlserver__test_relationships(model, to, field) %} {% set column_name = kwargs.get('column_name', kwargs.get('from')) %} diff --git a/setup.py b/setup.py index bdcd96df..9d24bf10 100644 --- a/setup.py +++ b/setup.py @@ -3,7 +3,7 @@ from distutils.core import setup package_name = "dbt-sqlserver" -package_version = "0.15.2" +package_version = "0.16.0" description = """A sqlserver adpter plugin for dbt (data build tool)""" setup( @@ -25,7 +25,7 @@ ] }, install_requires=[ - 'dbt-core>=0.15.0', + 'dbt-core>=0.16.0', 'pyodbc>=4.0.27', ] ) From 0b088eca63ccbac390317ba8d921903229b2766a Mon Sep 17 00:00:00 2001 From: Mikael Ene Date: Fri, 12 Jun 2020 08:19:10 +0200 Subject: [PATCH 02/19] Bumped to v0.17.0 --- setup.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/setup.py b/setup.py index 9d24bf10..695253e8 100644 --- a/setup.py +++ b/setup.py @@ -3,7 +3,7 @@ from distutils.core import setup package_name = "dbt-sqlserver" -package_version = "0.16.0" +package_version = "0.17.0" description = """A sqlserver adpter plugin for dbt (data build tool)""" setup( @@ -25,7 +25,7 @@ ] }, install_requires=[ - 'dbt-core>=0.16.0', + 'dbt-core>=0.17.0', 'pyodbc>=4.0.27', ] ) From ec074f2109fe07e943be384fe827bb8ea6f2b2f4 Mon Sep 17 00:00:00 2001 From: Mikael Ene Date: Fri, 12 Jun 2020 09:27:42 +0200 Subject: [PATCH 03/19] Bumped to v0.17.0 --- dbt/include/sqlserver/macros/adapters.sql | 20 +++-- dbt/include/sqlserver/macros/schema_tests.sql | 81 ------------------- 2 files changed, 12 insertions(+), 89 deletions(-) delete mode 100644 dbt/include/sqlserver/macros/schema_tests.sql diff --git a/dbt/include/sqlserver/macros/adapters.sql b/dbt/include/sqlserver/macros/adapters.sql index 0343e14d..12f88223 100644 --- a/dbt/include/sqlserver/macros/adapters.sql +++ b/dbt/include/sqlserver/macros/adapters.sql @@ -1,4 +1,8 @@ -{% macro sqlserver__list_relations_without_caching(information_schema, schema) %} +{% macro sqlserver__information_schema_name(database) -%} + information_schema +{%- endmacro %} + +{% macro sqlserver__list_relations_without_caching(schema_relation) %} {% call statement('list_relations_without_caching', fetch_result=True) -%} select table_catalog as [database], @@ -8,9 +12,9 @@ when table_type = 'VIEW' then 'view' else table_type end as table_type - from {{ information_schema }}.tables + from information_schema.tables where table_schema like '{{ schema }}' - and table_catalog like '{{ information_schema.database.lower() }}' + and table_catalog like '{{ schema_relation.database.lower() }}' {% endcall %} {{ return(load_result('list_relations_without_caching').table) }} {% endmacro %} @@ -23,19 +27,19 @@ {{ return(load_result('list_schemas').table) }} {% endmacro %} -{% macro sqlserver__create_schema(database_name, schema_name) -%} +{% macro sqlserver__create_schema(relation) -%} {% call statement('create_schema') -%} - USE {{ database_name }} - IF NOT EXISTS (SELECT * FROM sys.schemas WHERE name = {{ schema_name | replace('"', "'") }}) + USE {{ relation.database }} + IF NOT EXISTS (SELECT * FROM sys.schemas WHERE name = '{{ relation.without_identifier().schema }}') BEGIN - EXEC('CREATE SCHEMA {{ schema_name | replace('"', "") }}') + EXEC('CREATE SCHEMA {{ relation.without_identifier().schema }}') END {% endcall %} {% endmacro %} {% macro sqlserver__drop_schema(database_name, schema_name) -%} {% call statement('drop_schema') -%} - drop schema if exists {{database_name}}.{{schema_name}} + drop schema if exists {{ rrelation.without_identifier().schema }} {% endcall %} {% endmacro %} diff --git a/dbt/include/sqlserver/macros/schema_tests.sql b/dbt/include/sqlserver/macros/schema_tests.sql deleted file mode 100644 index 9c40e4e9..00000000 --- a/dbt/include/sqlserver/macros/schema_tests.sql +++ /dev/null @@ -1,81 +0,0 @@ -{% macro sqlserver__test_unique(model) %} - -{% set column_name = kwargs.get('column_name', kwargs.get('arg')) %} - -select count(*) as validation_errors -from ( - - select - {{ column_name }} as validation_errors - - from {{ model }} - where {{ column_name }} is not null - group by {{ column_name }} - having count(*) > 1 - -) validation_errors - -{% endmacro %} - - -{% macro sqlserver__test_not_null(model) %} - -{% set column_name = kwargs.get('column_name', kwargs.get('arg')) %} - -select count(*) as validation_errors -from {{ model }} -where {{ column_name }} is null - -{% endmacro %} - - -{% macro sqlserver__test_accepted_values(model, values) %} - -{% set column_name = kwargs.get('column_name', kwargs.get('field')) %} - -with all_values as ( - - select distinct - {{ column_name }} as value_field - - from {{ model }} - -), - -validation_errors as ( - - select - value_field - - from all_values - where value_field not in ( - {% for value in values -%} - - '{{ value }}' {% if not loop.last -%} , {%- endif %} - - {%- endfor %} - ) -) - -select count(*) as validation_errors -from validation_errors - -{% endmacro %} - - -{% macro sqlserver__test_relationships(model, to, field) %} - -{% set column_name = kwargs.get('column_name', kwargs.get('from')) %} - - -select count(*) as validation_errors -from ( - select {{ column_name }} as id from {{ model }} -) as child -left join ( - select {{ field }} as id from {{ to }} -) as parent on parent.id = child.id -where child.id is not null - and parent.id is null - -{% endmacro %} \ No newline at end of file From 59855d76ac96db807a934ac68fcc23113d52cb3d Mon Sep 17 00:00:00 2001 From: Mikael Ene Date: Fri, 12 Jun 2020 09:48:24 +0200 Subject: [PATCH 04/19] Bumped to v0.17.0 --- README.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/README.md b/README.md index 22e84761..09527aff 100644 --- a/README.md +++ b/README.md @@ -6,6 +6,8 @@ Passing all tests in [dbt-integration-tests](https://github.com/fishtown-analyti Only supports dbt 0.14 and newer! - For dbt 0.14.x use dbt-sqlserver 0.14.x - For dbt 0.15.x use dbt-sqlserver 0.15.x +- dbt 0.16.x is unsupported +- For dbt 0.17.x use dbt-sqlserver 0.17.x Easiest install is to use pip: @@ -98,6 +100,11 @@ Example of applying Unique clustered index on two columns, Ordinary index on one ## Changelog +### v0.17.0 + +#### New Features: +- Adds support for dbt 0.17.x + ### v0.15.2 #### Fixes: From 5f7f50303bd2b79c55415f28415daada36f3c86a Mon Sep 17 00:00:00 2001 From: "mikael.ene" Date: Fri, 12 Jun 2020 10:20:30 +0200 Subject: [PATCH 05/19] Fixed a bug in relations_without_caching --- dbt/include/sqlserver/macros/adapters.sql | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dbt/include/sqlserver/macros/adapters.sql b/dbt/include/sqlserver/macros/adapters.sql index 12f88223..1ca65331 100644 --- a/dbt/include/sqlserver/macros/adapters.sql +++ b/dbt/include/sqlserver/macros/adapters.sql @@ -13,8 +13,8 @@ else table_type end as table_type from information_schema.tables - where table_schema like '{{ schema }}' - and table_catalog like '{{ schema_relation.database.lower() }}' + where table_schema like '{{ schema_relation.schema }}' + and table_catalog like '{{ schema_relation.database }}' {% endcall %} {{ return(load_result('list_relations_without_caching').table) }} {% endmacro %} From 822ef23ec8ab7bf982f601bb3567e7373e73dc55 Mon Sep 17 00:00:00 2001 From: Mikael Ene Date: Mon, 15 Jun 2020 07:55:52 +0200 Subject: [PATCH 06/19] Fix in seeds macro. Solves #30 --- dbt/include/sqlserver/macros/materializations/seed/seed.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dbt/include/sqlserver/macros/materializations/seed/seed.sql b/dbt/include/sqlserver/macros/materializations/seed/seed.sql index a496bccd..64e6e68a 100644 --- a/dbt/include/sqlserver/macros/materializations/seed/seed.sql +++ b/dbt/include/sqlserver/macros/materializations/seed/seed.sql @@ -34,5 +34,5 @@ {% endmacro %} {% macro sqlserver__load_csv_rows(model, agate_table) %} - {{ return(basic_load_csv_rows(model, 200, agate_table) )}} + {{ return(sqlserver__basic_load_csv_rows(model, 200, agate_table) )}} {% endmacro %} \ No newline at end of file From 15df5265b30f09368f463ad4e9b1169b3cecee7e Mon Sep 17 00:00:00 2001 From: "mikael.ene" Date: Mon, 15 Jun 2020 08:33:07 +0200 Subject: [PATCH 07/19] Added [] for USE database sql --- dbt/include/sqlserver/macros/adapters.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dbt/include/sqlserver/macros/adapters.sql b/dbt/include/sqlserver/macros/adapters.sql index 1ca65331..704848c2 100644 --- a/dbt/include/sqlserver/macros/adapters.sql +++ b/dbt/include/sqlserver/macros/adapters.sql @@ -29,7 +29,7 @@ {% macro sqlserver__create_schema(relation) -%} {% call statement('create_schema') -%} - USE {{ relation.database }} + USE [{{ relation.database }}] IF NOT EXISTS (SELECT * FROM sys.schemas WHERE name = '{{ relation.without_identifier().schema }}') BEGIN EXEC('CREATE SCHEMA {{ relation.without_identifier().schema }}') From d20ee56557a274312e619bdf25e577adb9f8d6a2 Mon Sep 17 00:00:00 2001 From: "mikael.ene" Date: Tue, 23 Jun 2020 13:03:16 +0200 Subject: [PATCH 08/19] Added [] for USE database sql --- dbt/include/sqlserver/macros/adapters.sql | 7 +- .../materializations/snapshot/snapshot.sql | 210 +----------------- .../materializations/snapshot/strategies.sql | 78 +++---- 3 files changed, 49 insertions(+), 246 deletions(-) diff --git a/dbt/include/sqlserver/macros/adapters.sql b/dbt/include/sqlserver/macros/adapters.sql index 704848c2..c8f21215 100644 --- a/dbt/include/sqlserver/macros/adapters.sql +++ b/dbt/include/sqlserver/macros/adapters.sql @@ -168,4 +168,9 @@ path={"identifier": tmp_identifier}) -%} {% do return(tmp_relation) %} -{% endmacro %} \ No newline at end of file +{% endmacro %} + +{% macro sqlserver__snapshot_string_as_time(timestamp) -%} + {%- set result = "CONVERT(DATETIME2, '" ~ timestamp ~ "')" -%} + {{ return(result) }} +{%- endmacro %} \ No newline at end of file diff --git a/dbt/include/sqlserver/macros/materializations/snapshot/snapshot.sql b/dbt/include/sqlserver/macros/materializations/snapshot/snapshot.sql index 4d1eb79a..81cdfe1f 100644 --- a/dbt/include/sqlserver/macros/materializations/snapshot/snapshot.sql +++ b/dbt/include/sqlserver/macros/materializations/snapshot/snapshot.sql @@ -1,206 +1,4 @@ -{% macro sqlserver__snapshot_staging_table_inserts(strategy, source_sql, target_relation) -%} - - select - 'insert' as dbt_change_type, - source_data.* - - from ( - - select *, - COALESCE({{ strategy.scd_id }}, NULL) as dbt_scd_id, - COALESCE({{ strategy.unique_key }}, NULL) as dbt_unique_key, - COALESCE({{ strategy.updated_at }}, NULL) as dbt_updated_at, - COALESCE({{ strategy.updated_at }}, NULL) as dbt_valid_from, - nullif({{ strategy.updated_at }}, {{ strategy.updated_at }}) as dbt_valid_to - - from ( - {{ source_sql }} - ) snapshot_query - ) source_data - left outer join ( - - select *, - {{ strategy.unique_key }} as dbt_unique_key - - from {{ target_relation }} - - ) snapshotted_data on snapshotted_data.dbt_unique_key = source_data.dbt_unique_key - where snapshotted_data.dbt_unique_key is null - or ( - snapshotted_data.dbt_unique_key is not null - and snapshotted_data.dbt_valid_to is null - and ( - {{ strategy.row_changed }} - ) - ) - -{%- endmacro %} - - -{% macro sqlserver__snapshot_staging_table_updates(strategy, source_sql, target_relation) -%} - - select - 'update' as dbt_change_type, - snapshotted_data.dbt_scd_id, - source_data.dbt_valid_from as dbt_valid_to - - from ( - - select - *, - COALESCE({{ strategy.scd_id }}, NULL) as dbt_scd_id, - COALESCE({{ strategy.unique_key }}, NULL) as dbt_unique_key, - COALESCE({{ strategy.updated_at }}, NULL) as dbt_updated_at, - COALESCE({{ strategy.updated_at }}, NULL) as dbt_valid_from - - from ( - {{ source_sql }} - ) snapshot_query - ) source_data - join ( - - select *, - {{ strategy.unique_key }} as dbt_unique_key - - from {{ target_relation }} - -) snapshotted_data on snapshotted_data.dbt_unique_key = source_data.dbt_unique_key - where snapshotted_data.dbt_valid_to is null - and ( - {{ strategy.row_changed }} - ) - -{%- endmacro %} - - -{% macro sqlserver__build_snapshot_staging_table(strategy, sql, target_relation) %} - {% set tmp_relation = make_temp_relation(target_relation) %} - - {% set inserts_select = snapshot_staging_table_inserts(strategy, sql, target_relation) %} - {% set updates_select = snapshot_staging_table_updates(strategy, sql, target_relation) %} - - {% call statement('build_snapshot_staging_relation_inserts') %} - {{ create_table_as(False, tmp_relation, inserts_select) }} - {% endcall %} - - {% call statement('build_snapshot_staging_relation_updates') %} - insert into {{ tmp_relation }} (dbt_change_type, dbt_scd_id, dbt_valid_to) - select dbt_change_type, dbt_scd_id, dbt_valid_to from ( - {{ updates_select }} - ) dbt_sbq; - {% endcall %} - - {% do return(tmp_relation) %} -{% endmacro %} - - -{% materialization sqlserver__snapshot, default %} - {%- set config = model['config'] -%} - - {%- set target_database = config.get('target_database') -%} - {%- set target_schema = config.get('target_schema') -%} - {%- set target_table = model.get('alias', model.get('name')) -%} - - {%- set strategy_name = config.get('strategy') -%} - {%- set unique_key = config.get('unique_key') %} - - {% if not adapter.check_schema_exists(target_database, target_schema) %} - {% do create_schema(target_database, target_schema) %} - {% endif %} - - {% set target_relation_exists, target_relation = get_or_create_relation( - database=target_database, - schema=target_schema, - identifier=target_table, - type='table') -%} - - {%- if not target_relation.is_table -%} - {% do exceptions.relation_wrong_type(target_relation, 'table') %} - {%- endif -%} - - {% set strategy_macro = strategy_dispatch(strategy_name) %} - {% set strategy = strategy_macro(model, "snapshotted_data", "source_data", config) %} - - {% if not target_relation_exists %} - - {% set build_sql = build_snapshot_table(strategy, model['injected_sql']) %} - {% call statement('main') -%} - {{ create_table_as(False, target_relation, build_sql) }} - {% endcall %} - - {% else %} - - {{ adapter.valid_snapshot_target(target_relation) }} - - {% set staging_table = build_snapshot_staging_table(strategy, sql, target_relation) %} - - {% do adapter.expand_target_column_types(from_relation=staging_table, - to_relation=target_relation) %} - - {% set missing_columns = adapter.get_missing_columns(staging_table, target_relation) - | rejectattr('name', 'equalto', 'dbt_change_type') - | rejectattr('name', 'equalto', 'DBT_CHANGE_TYPE') - | rejectattr('name', 'equalto', 'dbt_unique_key') - | rejectattr('name', 'equalto', 'DBT_UNIQUE_KEY') - | list %} - - {% do create_columns(target_relation, missing_columns) %} - - {% set source_columns = adapter.get_columns_in_relation(staging_table) - | rejectattr('name', 'equalto', 'dbt_change_type') - | rejectattr('name', 'equalto', 'DBT_CHANGE_TYPE') - | rejectattr('name', 'equalto', 'dbt_unique_key') - | rejectattr('name', 'equalto', 'DBT_UNIQUE_KEY') - | list %} - - {% set quoted_source_columns = [] %} - {% for column in source_columns %} - {% do quoted_source_columns.append(adapter.quote(column.name)) %} - {% endfor %} - - {% call statement('main') %} - {{ snapshot_merge_sql( - target = target_relation, - source = staging_table, - insert_cols = quoted_source_columns - ) - }} - {{ drop_relation(staging_table) }} - {% endcall %} - - {% endif %} - - {{ adapter.commit() }} - - {% if staging_table is defined %} - {% do post_snapshot(staging_table) %} - {% endif %} - -{% endmaterialization %} - -{% macro sqlserver__post_snapshot(staging_relation) %} - -- Clean up the snapshot temp table - {% do drop_relation(staging_relation) %} -{% endmacro %} - - -{% macro sqlserver__build_snapshot_table(strategy, sql) %} - - select *, - COALESCE({{ strategy.scd_id }}, NULL) as dbt_scd_id, - COALESCE({{ strategy.updated_at }}, NULL) as dbt_updated_at, - COALESCE({{ strategy.updated_at }}, NULL) as dbt_valid_from, - nullif({{ strategy.updated_at }}, {{ strategy.updated_at }}) as dbt_valid_to - from ( - {{ sql }} - ) sbq - -{% endmacro %} - -{% macro sqlserver__create_columns(relation, columns) %} - {% for column in columns %} - {% call statement() %} - alter table {{ relation }} add "{{ column.name }}" {{ column.data_type }}; - {% endcall %} - {% endfor %} -{% endmacro %} \ No newline at end of file +{% macro sqlserver__post_snapshot(staging_relation) %} + -- Clean up the snapshot temp table + {% do drop_relation(staging_relation) %} +{% endmacro %} diff --git a/dbt/include/sqlserver/macros/materializations/snapshot/strategies.sql b/dbt/include/sqlserver/macros/materializations/snapshot/strategies.sql index f98e1183..4da82c9d 100644 --- a/dbt/include/sqlserver/macros/materializations/snapshot/strategies.sql +++ b/dbt/include/sqlserver/macros/materializations/snapshot/strategies.sql @@ -1,40 +1,40 @@ -{% macro sqlserver__snapshot_hash_arguments(args) %} - CONVERT(VARCHAR(32), HashBytes('MD5', {% for arg in args %} - coalesce(cast({{ arg }} as varchar ), '') {% if not loop.last %} + '|' + {% endif %} - {% endfor %}), 2) -{% endmacro %} - -{% macro sqlserver__snapshot_check_strategy(node, snapshotted_rel, current_rel, config, target_exists) %} - {% set check_cols_config = config['check_cols'] %} - {% set primary_key = config['unique_key'] %} - {% set updated_at = snapshot_get_time() %} - - {% if check_cols_config == 'all' %} - {% set check_cols = get_columns_in_query(node['injected_sql']) %} - {% elif check_cols_config is iterable and (check_cols_config | length) > 0 %} - {% set check_cols = check_cols_config %} - {% else %} - {% do exceptions.raise_compiler_error("Invalid value for 'check_cols': " ~ check_cols_config) %} - {% endif %} - - {% set row_changed_expr -%} - ( - {% for col in check_cols %} - {{ snapshotted_rel }}.{{ col }} != {{ current_rel }}.{{ col }} - or - ({{ snapshotted_rel }}.{{ col }} is null) and not ({{ current_rel }}.{{ col }} is null) - {%- if not loop.last %} or {% endif %} - - {% endfor %} - ) - {%- endset %} - - {% set scd_id_expr = snapshot_hash_arguments([primary_key, updated_at]) %} - - {% do return({ - "unique_key": primary_key, - "updated_at": updated_at, - "row_changed": row_changed_expr, - "scd_id": scd_id_expr - }) %} +{% macro sqlserver__snapshot_hash_arguments(args) %} + CONVERT(VARCHAR(32), HashBytes('MD5', {% for arg in args %} + coalesce(cast({{ arg }} as varchar ), '') {% if not loop.last %} + '|' + {% endif %} + {% endfor %}), 2) +{% endmacro %} + +{% macro sqlserver__snapshot_check_strategy(node, snapshotted_rel, current_rel, config, target_exists) %} + {% set check_cols_config = config['check_cols'] %} + {% set primary_key = config['unique_key'] %} + {% set updated_at = snapshot_get_time() %} + + {% if check_cols_config == 'all' %} + {% set check_cols = get_columns_in_query(node['injected_sql']) %} + {% elif check_cols_config is iterable and (check_cols_config | length) > 0 %} + {% set check_cols = check_cols_config %} + {% else %} + {% do exceptions.raise_compiler_error("Invalid value for 'check_cols': " ~ check_cols_config) %} + {% endif %} + + {% set row_changed_expr -%} + ( + {% for col in check_cols %} + {{ snapshotted_rel }}.{{ col }} != {{ current_rel }}.{{ col }} + or + ({{ snapshotted_rel }}.{{ col }} is null) and not ({{ current_rel }}.{{ col }} is null) + {%- if not loop.last %} or {% endif %} + + {% endfor %} + ) + {%- endset %} + + {% set scd_id_expr = sqlserver__snapshot_hash_arguments([primary_key, updated_at]) %} + + {% do return({ + "unique_key": primary_key, + "updated_at": updated_at, + "row_changed": row_changed_expr, + "scd_id": scd_id_expr + }) %} {% endmacro %} \ No newline at end of file From 681ceca7b9329cb0097e95c23612449eaa503543 Mon Sep 17 00:00:00 2001 From: "mikael.ene" Date: Tue, 25 Aug 2020 13:46:59 +0200 Subject: [PATCH 09/19] Fixed some errors from previous merge --- dbt/include/sqlserver/macros/adapters.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dbt/include/sqlserver/macros/adapters.sql b/dbt/include/sqlserver/macros/adapters.sql index c8f21215..dadd6da3 100644 --- a/dbt/include/sqlserver/macros/adapters.sql +++ b/dbt/include/sqlserver/macros/adapters.sql @@ -39,7 +39,7 @@ {% macro sqlserver__drop_schema(database_name, schema_name) -%} {% call statement('drop_schema') -%} - drop schema if exists {{ rrelation.without_identifier().schema }} + drop schema if exists {{ relation.without_identifier().schema }} {% endcall %} {% endmacro %} From a4ae7a7e8d3640967a2986f1bc70185961c23a94 Mon Sep 17 00:00:00 2001 From: "mikael.ene" Date: Tue, 25 Aug 2020 15:30:38 +0200 Subject: [PATCH 10/19] Updated README.md and set supported version in setup.py --- README.md | 17 ++++++++++++----- setup.py | 2 +- 2 files changed, 13 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index 09527aff..282855f5 100644 --- a/README.md +++ b/README.md @@ -7,7 +7,8 @@ Only supports dbt 0.14 and newer! - For dbt 0.14.x use dbt-sqlserver 0.14.x - For dbt 0.15.x use dbt-sqlserver 0.15.x - dbt 0.16.x is unsupported -- For dbt 0.17.x use dbt-sqlserver 0.17.x +- dbt 0.17.x is unsupported +- dbt 0.18.x is unsupported - development in progress Easiest install is to use pip: @@ -16,7 +17,7 @@ Easiest install is to use pip: On Ubuntu make sure you have the ODBC header files before installing sudo apt install unixodbc-dev - + ## Configure your profile Configure your dbt profile for using SQL Server authentication or Integrated Security: ##### SQL Server authentication @@ -100,10 +101,16 @@ Example of applying Unique clustered index on two columns, Ordinary index on one ## Changelog -### v0.17.0 +### v0.15.3.1 -#### New Features: -- Adds support for dbt 0.17.x +#### Fixes: +- Snapshots did not work on dbt v0.15.1 to v0.15.3 + +### v0.15.3 + +#### Fixes: +- Fix output of sql in the log files. +- Limited the version of dbt to 0.15, since later versions are unsupported. ### v0.15.2 diff --git a/setup.py b/setup.py index 695253e8..29f03d02 100644 --- a/setup.py +++ b/setup.py @@ -25,7 +25,7 @@ ] }, install_requires=[ - 'dbt-core>=0.17.0', + 'dbt-core==0.18.0rc1', 'pyodbc>=4.0.27', ] ) From d3aa13c5e6616c39257c801503aae96745a5ff01 Mon Sep 17 00:00:00 2001 From: "mikael.ene" Date: Tue, 25 Aug 2020 15:43:05 +0200 Subject: [PATCH 11/19] Fix for snapshots --- .../sqlserver/macros/materializations/snapshot/strategies.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dbt/include/sqlserver/macros/materializations/snapshot/strategies.sql b/dbt/include/sqlserver/macros/materializations/snapshot/strategies.sql index 4da82c9d..6145ee06 100644 --- a/dbt/include/sqlserver/macros/materializations/snapshot/strategies.sql +++ b/dbt/include/sqlserver/macros/materializations/snapshot/strategies.sql @@ -4,7 +4,7 @@ {% endfor %}), 2) {% endmacro %} -{% macro sqlserver__snapshot_check_strategy(node, snapshotted_rel, current_rel, config, target_exists) %} +{% macro snapshot_check_strategy(node, snapshotted_rel, current_rel, config, target_exists) %} {% set check_cols_config = config['check_cols'] %} {% set primary_key = config['unique_key'] %} {% set updated_at = snapshot_get_time() %} From 57b3c8760440da0308ca0b10c774005df5c1fa30 Mon Sep 17 00:00:00 2001 From: mikaelene Date: Wed, 26 Aug 2020 15:46:16 +0200 Subject: [PATCH 12/19] Updated version in setup.py, applied fix for connections.py port --- dbt/adapters/sqlserver/connections.py | 3 +-- setup.py | 2 +- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/dbt/adapters/sqlserver/connections.py b/dbt/adapters/sqlserver/connections.py index d1a4cc52..9a3d2a2c 100644 --- a/dbt/adapters/sqlserver/connections.py +++ b/dbt/adapters/sqlserver/connections.py @@ -87,8 +87,7 @@ def open(cls, connection): try: con_str = [] con_str.append(f"DRIVER={{{credentials.driver}}}") - con_str.append(f"SERVER={credentials.host}") - con_str.append(f"PORT={credentials.port}") + con_str.append(f"SERVER={credentials.host},{credentials.port}") con_str.append(f"Database={credentials.database}") if not getattr(credentials, 'windows_login', False): diff --git a/setup.py b/setup.py index 29f03d02..da15ef80 100644 --- a/setup.py +++ b/setup.py @@ -3,7 +3,7 @@ from distutils.core import setup package_name = "dbt-sqlserver" -package_version = "0.17.0" +package_version = "0.18.0" description = """A sqlserver adpter plugin for dbt (data build tool)""" setup( From 377e0d144a5030bd10e17823399bc0a6dfdcdcb2 Mon Sep 17 00:00:00 2001 From: Anders Swanson Date: Thu, 3 Sep 2020 10:13:02 -0700 Subject: [PATCH 13/19] hotfix for v.0.16 change (see dbt PR #2037) --- dbt/include/sqlserver/macros/catalog.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dbt/include/sqlserver/macros/catalog.sql b/dbt/include/sqlserver/macros/catalog.sql index c40f7ae6..62dbed2b 100644 --- a/dbt/include/sqlserver/macros/catalog.sql +++ b/dbt/include/sqlserver/macros/catalog.sql @@ -1,5 +1,5 @@ -{% macro sqlserver__get_catalog(information_schemas) -%} +{% macro sqlserver__get_catalog(information_schema, schemas) -%} {%- call statement('catalog', fetch_result=True) -%} From 88432012d54da33e601203e37fb4b28fc449357e Mon Sep 17 00:00:00 2001 From: mikaelene Date: Thu, 10 Sep 2020 21:36:57 +0200 Subject: [PATCH 14/19] Updated install_requires to the now released version of dbt 0.18.0 --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index da15ef80..f068c1c3 100644 --- a/setup.py +++ b/setup.py @@ -25,7 +25,7 @@ ] }, install_requires=[ - 'dbt-core==0.18.0rc1', + 'dbt-core>=0.18.0', 'pyodbc>=4.0.27', ] ) From 1e15f0910e8132e8267876e1979285155efc7ff9 Mon Sep 17 00:00:00 2001 From: Langsbo Anders Date: Thu, 10 Sep 2020 21:59:59 +0200 Subject: [PATCH 15/19] added support for named instance --- dbt/adapters/sqlserver/connections.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/dbt/adapters/sqlserver/connections.py b/dbt/adapters/sqlserver/connections.py index 9a3d2a2c..9d9b0484 100644 --- a/dbt/adapters/sqlserver/connections.py +++ b/dbt/adapters/sqlserver/connections.py @@ -87,7 +87,14 @@ def open(cls, connection): try: con_str = [] con_str.append(f"DRIVER={{{credentials.driver}}}") - con_str.append(f"SERVER={credentials.host},{credentials.port}") + + if "\\" in credentials.host: + # if there is a backslash \ in the host name the host is a sql-server named instance + # in this case then port number has to be omitted + con_str.append(f"SERVER={credentials.host}") + else: + con_str.append(f"SERVER={credentials.host},{credentials.port}") + con_str.append(f"Database={credentials.database}") if not getattr(credentials, 'windows_login', False): From 223fab3c2cd3332783369a447c36ed4b90515756 Mon Sep 17 00:00:00 2001 From: "mikael.ene" Date: Fri, 11 Sep 2020 10:25:34 +0200 Subject: [PATCH 16/19] There was a type in the last PR. This fixes that. --- dbt/include/sqlserver/macros/catalog.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dbt/include/sqlserver/macros/catalog.sql b/dbt/include/sqlserver/macros/catalog.sql index 62dbed2b..1534ce7c 100644 --- a/dbt/include/sqlserver/macros/catalog.sql +++ b/dbt/include/sqlserver/macros/catalog.sql @@ -1,5 +1,5 @@ -{% macro sqlserver__get_catalog(information_schema, schemas) -%} +{% macro sqlserver__get_catalog(information_schemas, schemas) -%} {%- call statement('catalog', fetch_result=True) -%} From 57b45961f5f67b666b7f095835e38d629f2d6ed2 Mon Sep 17 00:00:00 2001 From: mikaelene Date: Tue, 15 Sep 2020 09:16:06 +0200 Subject: [PATCH 17/19] Set install_requires to approximate version of dbt 0.18.0 --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index f068c1c3..2f9aafa0 100644 --- a/setup.py +++ b/setup.py @@ -25,7 +25,7 @@ ] }, install_requires=[ - 'dbt-core>=0.18.0', + 'dbt-core~=0.18.0', 'pyodbc>=4.0.27', ] ) From 2aae108cbb06cc00a452fe48cc95093ea139c9ed Mon Sep 17 00:00:00 2001 From: mikaelene Date: Tue, 15 Sep 2020 09:20:15 +0200 Subject: [PATCH 18/19] Fixed typo and mail address in setup.py --- setup.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/setup.py b/setup.py index 2f9aafa0..dea82334 100644 --- a/setup.py +++ b/setup.py @@ -4,7 +4,7 @@ package_name = "dbt-sqlserver" package_version = "0.18.0" -description = """A sqlserver adpter plugin for dbt (data build tool)""" +description = """A sqlserver adapter plugin for dbt (data build tool)""" setup( name=package_name, @@ -12,8 +12,9 @@ description=description, long_description=description, long_description_content_type='text/markdown', + license='MIT', author="Mikael Ene", - author_email="mikael.ene@gmail.com", + author_email="mikael.ene@eneanalytics.com", url="https://github.com/mikaelene/dbt-sqlserver", packages=find_packages(), package_data={ From a831ed4bdb986f237db25df763ca400f05e78407 Mon Sep 17 00:00:00 2001 From: mikaelene Date: Tue, 15 Sep 2020 09:37:28 +0200 Subject: [PATCH 19/19] Updated README.md --- README.md | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 282855f5..f29d355e 100644 --- a/README.md +++ b/README.md @@ -4,11 +4,12 @@ Passing all tests in [dbt-integration-tests](https://github.com/fishtown-analytics/dbt-integration-tests/). Only supports dbt 0.14 and newer! -- For dbt 0.14.x use dbt-sqlserver 0.14.x -- For dbt 0.15.x use dbt-sqlserver 0.15.x -- dbt 0.16.x is unsupported +- For dbt 0.18.x use dbt-sqlserver 0.18.x - dbt 0.17.x is unsupported -- dbt 0.18.x is unsupported - development in progress +- dbt 0.16.x is unsupported +- For dbt 0.15.x use dbt-sqlserver 0.15.x +- For dbt 0.14.x use dbt-sqlserver 0.14.x + Easiest install is to use pip: @@ -101,6 +102,10 @@ Example of applying Unique clustered index on two columns, Ordinary index on one ## Changelog +### v0.18.0 +#### New Features: +- Adds support for dbt v0.18.0 + ### v0.15.3.1 #### Fixes: