diff --git a/dbt_internal_packages/dbt-adapters/dbt_project.yml b/dbt_internal_packages/dbt-adapters/dbt_project.yml new file mode 100644 index 0000000..764e319 --- /dev/null +++ b/dbt_internal_packages/dbt-adapters/dbt_project.yml @@ -0,0 +1 @@ +name: dbt diff --git a/dbt_internal_packages/dbt-adapters/macros/adapters/apply_grants.sql b/dbt_internal_packages/dbt-adapters/macros/adapters/apply_grants.sql new file mode 100644 index 0000000..5319cf9 --- /dev/null +++ b/dbt_internal_packages/dbt-adapters/macros/adapters/apply_grants.sql @@ -0,0 +1,180 @@ +{# ------- BOOLEAN MACROS --------- #} + +{# + -- COPY GRANTS + -- When a relational object (view or table) is replaced in this database, + -- do previous grants carry over to the new object? This may depend on: + -- whether we use alter-rename-swap versus CREATE OR REPLACE + -- user-supplied configuration (e.g. copy_grants on Snowflake) + -- By default, play it safe, assume TRUE: that grants ARE copied over. + -- This means dbt will first "show" current grants and then calculate diffs. + -- It may require an additional query than is strictly necessary, + -- but better safe than sorry. +#} + +-- funcsign: () -> bool +{% macro copy_grants() %} + {{ return(adapter.dispatch('copy_grants', 'dbt')()) }} +{% endmacro %} + +-- funcsign: () -> bool +{% macro default__copy_grants() %} + {{ return(True) }} +{% endmacro %} + + +{# + -- SUPPORT MULTIPLE GRANTEES PER DCL STATEMENT + -- Does this database support 'grant {privilege} to {grantee_1}, {grantee_2}, ...' + -- Or must these be separate statements: + -- `grant {privilege} to {grantee_1}`; + -- `grant {privilege} to {grantee_2}`; + -- By default, pick the former, because it's what we prefer when available. +#} + +-- funcsign: () -> bool +{% macro support_multiple_grantees_per_dcl_statement() %} + {{ return(adapter.dispatch('support_multiple_grantees_per_dcl_statement', 'dbt')()) }} +{% endmacro %} + +-- funcsign: () -> bool +{%- macro default__support_multiple_grantees_per_dcl_statement() -%} + {{ return(True) }} +{%- endmacro -%} + +-- funcsign: (optional[relation], optional[bool]) -> bool +{% macro should_revoke(existing_relation, full_refresh_mode=True) %} + + {% if not existing_relation %} + {#-- The table doesn't already exist, so no grants to copy over --#} + {{ return(False) }} + {% elif full_refresh_mode %} + {#-- The object is being REPLACED -- whether grants are copied over depends on the value of user config --#} + {{ return(copy_grants()) }} + {% else %} + {#-- The table is being merged/upserted/inserted -- grants will be carried over --#} + {{ return(True) }} + {% endif %} + +{% endmacro %} + +{# ------- DCL STATEMENT TEMPLATES --------- #} + +-- funcsign: (relation) -> string +{% macro get_show_grant_sql(relation) %} + {{ return(adapter.dispatch("get_show_grant_sql", "dbt")(relation)) }} +{% endmacro %} + +-- funcsign: (relation) -> string +{% macro default__get_show_grant_sql(relation) %} + show grants on {{ relation.render() }} +{% endmacro %} + +-- funcsign: (relation, string, list[string]) -> string +{% macro get_grant_sql(relation, privilege, grantees) %} + {{ return(adapter.dispatch('get_grant_sql', 'dbt')(relation, privilege, grantees)) }} +{% endmacro %} + +-- funcsign: (relation, string, list[string]) -> string +{%- macro default__get_grant_sql(relation, privilege, grantees) -%} + grant {{ privilege }} on {{ relation.render() }} to {{ grantees | join(', ') }} +{%- endmacro -%} + + +-- funcsign: (relation, string, list[string]) -> string +{% macro get_revoke_sql(relation, privilege, grantees) %} + {{ return(adapter.dispatch('get_revoke_sql', 'dbt')(relation, privilege, grantees)) }} +{% endmacro %} + +-- funcsign: (relation, string, list[string]) -> string +{%- macro default__get_revoke_sql(relation, privilege, grantees) -%} + revoke {{ privilege }} on {{ relation.render() }} from {{ grantees | join(', ') }} +{%- endmacro -%} + + +{# ------- RUNTIME APPLICATION --------- #} +-- funcsign: (relation, dict[string, list[string]], (relation, string, list[string]) -> string) -> list[string] +{% macro get_dcl_statement_list(relation, grant_config, get_dcl_macro) %} + {{ return(adapter.dispatch('get_dcl_statement_list', 'dbt')(relation, grant_config, get_dcl_macro)) }} +{% endmacro %} + +-- funcsign: (relation, dict[string, list[string]], (relation, string, list[string]) -> string) -> list[string] +{%- macro default__get_dcl_statement_list(relation, grant_config, get_dcl_macro) -%} + {# + -- Unpack grant_config into specific privileges and the set of users who need them granted/revoked. + -- Depending on whether this database supports multiple grantees per statement, pass in the list of + -- all grantees per privilege, or (if not) template one statement per privilege-grantee pair. + -- `get_dcl_macro` will be either `get_grant_sql` or `get_revoke_sql` + #} + {%- set dcl_statements = [] -%} + {%- for privilege, grantees in grant_config.items() %} + {%- if support_multiple_grantees_per_dcl_statement() and grantees -%} + {%- set dcl = get_dcl_macro(relation, privilege, grantees) -%} + {%- do dcl_statements.append(dcl) -%} + {%- else -%} + {%- for grantee in grantees -%} + {% set dcl = get_dcl_macro(relation, privilege, [grantee]) %} + {%- do dcl_statements.append(dcl) -%} + {% endfor -%} + {%- endif -%} + {%- endfor -%} + {{ return(dcl_statements) }} +{%- endmacro %} + +-- funcsign: (list[string]) -> string +{% macro call_dcl_statements(dcl_statement_list) %} + {{ return(adapter.dispatch("call_dcl_statements", "dbt")(dcl_statement_list)) }} +{% endmacro %} + +-- funcsign: (list[string]) -> string +{% macro default__call_dcl_statements(dcl_statement_list) %} + {# + -- By default, supply all grant + revoke statements in a single semicolon-separated block, + -- so that they're all processed together. + + -- Some databases do not support this. Those adapters will need to override this macro + -- to run each statement individually. + #} + {% call statement('grants') %} + {% for dcl_statement in dcl_statement_list %} + {{ dcl_statement }}; + {% endfor %} + {% endcall %} +{% endmacro %} + + +-- funcsign: (relation, optional[dict[string, list[string]]], bool) -> string +{% macro apply_grants(relation, grant_config, should_revoke) %} + {{ return(adapter.dispatch("apply_grants", "dbt")(relation, grant_config, should_revoke)) }} +{% endmacro %} + +-- funcsign: (relation, optional[dict[string, list[string]]], bool) -> string +{% macro default__apply_grants(relation, grant_config, should_revoke) %} + {#-- If grant_config is {} or None, this is a no-op --#} + {% if grant_config %} + {% if should_revoke %} + {#-- We think previous grants may have carried over --#} + {#-- Show current grants and calculate diffs --#} + {% set current_grants_table = run_query(get_show_grant_sql(relation)) %} + {% set current_grants_dict = adapter.standardize_grants_dict(current_grants_table) %} + {% set needs_granting = diff_of_two_dicts(grant_config, current_grants_dict) %} + {% set needs_revoking = diff_of_two_dicts(current_grants_dict, grant_config) %} + {% if not (needs_granting or needs_revoking) %} + {{ log('On ' ~ relation.render() ~': All grants are in place, no revocation or granting needed.')}} + {% endif %} + {% else %} + {#-- We don't think there's any chance of previous grants having carried over. --#} + {#-- Jump straight to granting what the user has configured. --#} + {% set needs_revoking = {} %} + {% set needs_granting = grant_config %} + {% endif %} + {% if needs_granting or needs_revoking %} + {% set revoke_statement_list = get_dcl_statement_list(relation, needs_revoking, get_revoke_sql) %} + {% set grant_statement_list = get_dcl_statement_list(relation, needs_granting, get_grant_sql) %} + {% set dcl_statement_list = revoke_statement_list + grant_statement_list %} + {% if dcl_statement_list %} + {{ call_dcl_statements(dcl_statement_list) }} + {% endif %} + {% endif %} + {% endif %} +{% endmacro %} diff --git a/dbt_internal_packages/dbt-adapters/macros/adapters/columns.sql b/dbt_internal_packages/dbt-adapters/macros/adapters/columns.sql new file mode 100644 index 0000000..91c8ea0 --- /dev/null +++ b/dbt_internal_packages/dbt-adapters/macros/adapters/columns.sql @@ -0,0 +1,151 @@ +-- funcsign: (relation) -> optional[agate_table] +{% macro get_columns_in_relation(relation) -%} + {{ return(adapter.dispatch('get_columns_in_relation', 'dbt')(relation)) }} +{% endmacro %} + +-- funcsign: (relation) -> optional[agate_table] +{% macro default__get_columns_in_relation(relation) -%} + {{ exceptions.raise_not_implemented( + 'get_columns_in_relation macro not implemented for adapter '+adapter.type()) }} +{% endmacro %} + +{# helper for adapter-specific implementations of get_columns_in_relation #} +-- funcsign: (agate_table) -> list[api.column] +{% macro sql_convert_columns_in_relation(table) -%} + {% set columns = [] %} + {% for row in table %} + {% do columns.append(api.Column(*row)) %} + {% endfor %} + {{ return(columns) }} +{% endmacro %} + +-- funcsign: (string, optional[string]) -> string +{% macro get_empty_subquery_sql(select_sql, select_sql_header=none) -%} + {{ return(adapter.dispatch('get_empty_subquery_sql', 'dbt')(select_sql, select_sql_header)) }} +{% endmacro %} + +{# + Builds a query that results in the same schema as the given select_sql statement, without necessitating a data scan. + Useful for running a query in a 'pre-flight' context, such as model contract enforcement (assert_columns_equivalent macro). +#} +-- funcsign: (string, optional[string]) -> string +{% macro default__get_empty_subquery_sql(select_sql, select_sql_header=none) %} + {%- if select_sql_header is not none -%} + {{ select_sql_header }} + {%- endif -%} + select * from ( + {{ select_sql }} + ) as __dbt_sbq + where false + limit 0 +{% endmacro %} + +-- funcsign: (dict[string, api.column]) -> string +{% macro get_empty_schema_sql(columns) -%} + {{ return(adapter.dispatch('get_empty_schema_sql', 'dbt')(columns)) }} +{% endmacro %} + +-- funcsign: (dict[string, api.column]) -> string +{% macro default__get_empty_schema_sql(columns) %} + {%- set col_err = [] -%} + {%- set col_naked_numeric = [] -%} + select + {% for i in columns %} + {%- set col = columns[i] -%} + {%- if col['data_type'] is not defined -%} + {%- do col_err.append(col['name']) -%} + {#-- If this column's type is just 'numeric' then it is missing precision/scale, raise a warning --#} + {#-- TYPE CHECK: col['data_type'] is optional[string] but user have this constraint --#} + {%- elif col['data_type'].strip().lower() in ('numeric', 'decimal', 'number') -%} + {%- do col_naked_numeric.append(col['name']) -%} + {%- endif -%} + {% set col_name = adapter.quote(col['name']) if col.get('quote') else col['name'] %} + {{ cast('null', col['data_type']) }} as {{ col_name }}{{ ", " if not loop.last }} + {%- endfor -%} + {%- if (col_err | length) > 0 -%} + {{ exceptions.column_type_missing(column_names=col_err) }} + {%- elif (col_naked_numeric | length) > 0 -%} + {{ exceptions.warn("Detected columns with numeric type and unspecified precision/scale, this can lead to unintended rounding: " ~ col_naked_numeric ~ "`") }} + {%- endif -%} +{% endmacro %} + +-- funcsign: (string, optional[string]) -> list[base_column] +{% macro get_column_schema_from_query(select_sql, select_sql_header=none) -%} + {% set columns = [] %} + {# -- Using an 'empty subquery' here to get the same schema as the given select_sql statement, without necessitating a data scan.#} + {% set sql = get_empty_subquery_sql(select_sql, select_sql_header) %} + {% set column_schema = adapter.get_column_schema_from_query(sql) %} + {{ return(column_schema) }} +{% endmacro %} + +-- here for back compat +-- funcsign: (string) -> list[string] +{% macro get_columns_in_query(select_sql) -%} + {{ return(adapter.dispatch('get_columns_in_query', 'dbt')(select_sql)) }} +{% endmacro %} + +-- funcsign: (string) -> list[string] +{% macro default__get_columns_in_query(select_sql) %} + {% call statement('get_columns_in_query', fetch_result=True, auto_begin=False) -%} + {{ get_empty_subquery_sql(select_sql) }} + {% endcall %} + {{ return(load_result('get_columns_in_query').table.columns | map(attribute='name') | list) }} +{% endmacro %} + +-- funcsign: (relation, string, string) -> string +{% macro alter_column_type(relation, column_name, new_column_type) -%} + {{ return(adapter.dispatch('alter_column_type', 'dbt')(relation, column_name, new_column_type)) }} +{% endmacro %} + +-- funcsign: (relation, string, string) -> string +{% macro default__alter_column_type(relation, column_name, new_column_type) -%} + {# + 1. Create a new column (w/ temp name and correct type) + 2. Copy data over to it + 3. Drop the existing column (cascade!) + 4. Rename the new column to existing column + #} + {%- set tmp_column = column_name + "__dbt_alter" -%} + + {% call statement('alter_column_type') %} + alter table {{ relation.render() }} add column {{ adapter.quote(tmp_column) }} {{ new_column_type }}; + update {{ relation.render() }} set {{ adapter.quote(tmp_column) }} = {{ adapter.quote(column_name) }}; + alter table {{ relation.render() }} drop column {{ adapter.quote(column_name) }} cascade; + alter table {{ relation.render() }} rename column {{ adapter.quote(tmp_column) }} to {{ adapter.quote(column_name) }} + {% endcall %} + +{% endmacro %} + + +-- funcsign: (relation, optional[list[base_column]], optional[list[base_column]]) -> string +{% macro alter_relation_add_remove_columns(relation, add_columns = none, remove_columns = none) -%} + {{ return(adapter.dispatch('alter_relation_add_remove_columns', 'dbt')(relation, add_columns, remove_columns)) }} +{% endmacro %} + +-- funcsign: (relation, optional[list[base_column]], optional[list[base_column]]) -> string +{% macro default__alter_relation_add_remove_columns(relation, add_columns, remove_columns) %} + + {% if add_columns is none %} + {% set add_columns = [] %} + {% endif %} + {% if remove_columns is none %} + {% set remove_columns = [] %} + {% endif %} + + {% set sql -%} + + alter {{ relation.type }} {{ relation.render() }} + + {% for column in add_columns %} + add column {{ column.name }} {{ column.data_type }}{{ ',' if not loop.last }} + {% endfor %}{{ ',' if add_columns and remove_columns }} + + {% for column in remove_columns %} + drop column {{ column.name }}{{ ',' if not loop.last }} + {% endfor %} + + {%- endset -%} + + {% do run_query(sql) %} + +{% endmacro %} diff --git a/dbt_internal_packages/dbt-adapters/macros/adapters/freshness.sql b/dbt_internal_packages/dbt-adapters/macros/adapters/freshness.sql new file mode 100644 index 0000000..4ef7ac0 --- /dev/null +++ b/dbt_internal_packages/dbt-adapters/macros/adapters/freshness.sql @@ -0,0 +1,36 @@ +-- funcsign: (string, string, optional[string]) -> any +{% macro collect_freshness(source, loaded_at_field, filter) %} + {{ return(adapter.dispatch('collect_freshness', 'dbt')(source, loaded_at_field, filter))}} +{% endmacro %} + +-- funcsign: (string, string, optional[string]) -> any +{% macro default__collect_freshness(source, loaded_at_field, filter) %} + {% call statement('collect_freshness', fetch_result=True, auto_begin=False) -%} + select + max({{ loaded_at_field }}) as max_loaded_at, + {{ current_timestamp() }} as snapshotted_at + from {{ source }} + {% if filter %} + where {{ filter }} + {% endif %} + {% endcall %} + {{ return(load_result('collect_freshness')) }} +{% endmacro %} + +-- funcsign: (string, string) -> any +{% macro collect_freshness_custom_sql(source, loaded_at_query) %} + {{ return(adapter.dispatch('collect_freshness_custom_sql', 'dbt')(source, loaded_at_query))}} +{% endmacro %} + +-- funcsign: (string, string) -> any +{% macro default__collect_freshness_custom_sql(source, loaded_at_query) %} + {% call statement('collect_freshness_custom_sql', fetch_result=True, auto_begin=False) -%} + with source_query as ( + {{ loaded_at_query }} + ) + select + (select * from source_query) as max_loaded_at, + {{ current_timestamp() }} as snapshotted_at + {% endcall %} + {{ return(load_result('collect_freshness_custom_sql')) }} +{% endmacro %} diff --git a/dbt_internal_packages/dbt-adapters/macros/adapters/indexes.sql b/dbt_internal_packages/dbt-adapters/macros/adapters/indexes.sql new file mode 100644 index 0000000..8f2d0a7 --- /dev/null +++ b/dbt_internal_packages/dbt-adapters/macros/adapters/indexes.sql @@ -0,0 +1,46 @@ +-- funcsign: (relation, dict[string, string]) -> optional[string] +{% macro get_create_index_sql(relation, index_dict) -%} + {{ return(adapter.dispatch('get_create_index_sql', 'dbt')(relation, index_dict)) }} +{% endmacro %} + +-- funcsign: (relation, dict[string, string]) -> optional[string] +{% macro default__get_create_index_sql(relation, index_dict) -%} + {% do return(None) %} +{% endmacro %} + +-- funcsign: (relation) -> string +{% macro create_indexes(relation) -%} + {{ adapter.dispatch('create_indexes', 'dbt')(relation) }} +{%- endmacro %} + +-- funcsign: (relation) -> string +{% macro default__create_indexes(relation) -%} + {%- set _indexes = config.get('indexes', default=[]) -%} + + {% for _index_dict in _indexes %} + {% set create_index_sql = get_create_index_sql(relation, _index_dict) %} + {% if create_index_sql %} + {% do run_query(create_index_sql) %} + {% endif %} + {% endfor %} +{% endmacro %} + +-- funcsign: (relation, string) -> string +{% macro get_drop_index_sql(relation, index_name) -%} + {{ adapter.dispatch('get_drop_index_sql', 'dbt')(relation, index_name) }} +{%- endmacro %} + +-- funcsign: (relation, string) -> string +{% macro default__get_drop_index_sql(relation, index_name) -%} + {{ exceptions.raise_compiler_error("`get_drop_index_sql has not been implemented for this adapter.") }} +{%- endmacro %} + +-- funcsign: (relation) -> string +{% macro get_show_indexes_sql(relation) -%} + {{ adapter.dispatch('get_show_indexes_sql', 'dbt')(relation) }} +{%- endmacro %} + +-- funcsign: (relation) -> string +{% macro default__get_show_indexes_sql(relation) -%} + {{ exceptions.raise_compiler_error("`get_show_indexes_sql has not been implemented for this adapter.") }} +{%- endmacro %} diff --git a/dbt_internal_packages/dbt-adapters/macros/adapters/metadata.sql b/dbt_internal_packages/dbt-adapters/macros/adapters/metadata.sql new file mode 100644 index 0000000..8de322f --- /dev/null +++ b/dbt_internal_packages/dbt-adapters/macros/adapters/metadata.sql @@ -0,0 +1,123 @@ +-- funcsign: (relation, list[relation]) -> list[relation] +{% macro get_catalog_relations(dbschema, relations) -%} + {{ return(adapter.dispatch('get_catalog_relations', 'dbt')(dbschema, relations)) }} +{%- endmacro %} + +-- funcsign: (relation, list[relation]) -> list[relation] +{% macro default__get_catalog_relations(dbschema, relations) -%} + {% set typename = adapter.type() %} + {% set msg -%} + get_catalog_relations not implemented for {{ typename }} + {%- endset %} + + {{ exceptions.raise_compiler_error(msg) }} +{%- endmacro %} + +-- funcsign: (relation, list[string]) -> agate_table +{% macro get_catalog(dbschema, schemas) -%} + {{ return(adapter.dispatch('get_catalog', 'dbt')(dbschema, schemas)) }} +{%- endmacro %} + +-- funcsign: (relation, list[string]) -> agate_table +{% macro default__get_catalog(dbschema, schemas) -%} + + {% set typename = adapter.type() %} + {% set msg -%} + get_catalog not implemented for {{ typename }} + {%- endset %} + + {{ exceptions.raise_compiler_error(msg) }} +{% endmacro %} + + +-- funcsign: (string) -> string +{% macro information_schema_name(database) %} + {{ return(adapter.dispatch('information_schema_name', 'dbt')(database)) }} +{% endmacro %} + +-- funcsign: (string) -> string +{% macro default__information_schema_name(database) -%} + {%- if database -%} + {{ database }}.INFORMATION_SCHEMA + {%- else -%} + INFORMATION_SCHEMA + {%- endif -%} +{%- endmacro %} + + +-- funcsign: (string) -> agate_table +{% macro list_schemas(database) -%} + {{ return(adapter.dispatch('list_schemas', 'dbt')(database)) }} +{% endmacro %} + +-- funcsign: (string) -> agate_table +{% macro default__list_schemas(database) -%} + {% set sql %} + select distinct schema_name + from {{ information_schema_name(database) }}.SCHEMATA + where catalog_name ilike '{{ database }}' + {% endset %} + {{ return(run_query(sql)) }} +{% endmacro %} + + +-- funcsign: (information_schema, string) -> agate_table +{% macro check_schema_exists(information_schema, schema) -%} + {{ return(adapter.dispatch('check_schema_exists', 'dbt')(information_schema, schema)) }} +{% endmacro %} + +-- funcsign: (information_schema, string) -> agate_table +{% macro default__check_schema_exists(information_schema, schema) -%} + {% set sql -%} + select count(*) + from {{ information_schema.replace(information_schema_view='SCHEMATA') }} + where catalog_name='{{ information_schema.database }}' + and schema_name='{{ schema }}' + {%- endset %} + {{ return(run_query(sql)) }} +{% endmacro %} + + +-- funcsign: (relation) -> list[relation] +{% macro list_relations_without_caching(schema_relation) %} + {{ return(adapter.dispatch('list_relations_without_caching', 'dbt')(schema_relation)) }} +{% endmacro %} + +-- funcsign: (relation) -> list[relation] +{% macro default__list_relations_without_caching(schema_relation) %} + {{ exceptions.raise_not_implemented( + 'list_relations_without_caching macro not implemented for adapter '+adapter.type()) }} +{% endmacro %} + +-- funcsign: (relation) -> agate_table +{% macro get_catalog_for_single_relation(relation) %} + {{ return(adapter.dispatch('get_catalog_for_single_relation', 'dbt')(relation)) }} +{% endmacro %} + +-- funcsign: (relation) -> agate_table +{% macro default__get_catalog_for_single_relation(relation) %} + {{ exceptions.raise_not_implemented( + 'get_catalog_for_single_relation macro not implemented for adapter '+adapter.type()) }} +{% endmacro %} + +-- funcsign: () -> list[relation] +{% macro get_relations() %} + {{ return(adapter.dispatch('get_relations', 'dbt')()) }} +{% endmacro %} + +-- funcsign: () -> list[relation] +{% macro default__get_relations() %} + {{ exceptions.raise_not_implemented( + 'get_relations macro not implemented for adapter '+adapter.type()) }} +{% endmacro %} + +-- funcsign: (information_schema, list[relation]) -> agate_table +{% macro get_relation_last_modified(information_schema, relations) %} + {{ return(adapter.dispatch('get_relation_last_modified', 'dbt')(information_schema, relations)) }} +{% endmacro %} + +-- funcsign: (information_schema, list[relation]) -> agate_table +{% macro default__get_relation_last_modified(information_schema, relations) %} + {{ exceptions.raise_not_implemented( + 'get_relation_last_modified macro not implemented for adapter ' + adapter.type()) }} +{% endmacro %} diff --git a/dbt_internal_packages/dbt-adapters/macros/adapters/persist_docs.sql b/dbt_internal_packages/dbt-adapters/macros/adapters/persist_docs.sql new file mode 100644 index 0000000..44ba230 --- /dev/null +++ b/dbt_internal_packages/dbt-adapters/macros/adapters/persist_docs.sql @@ -0,0 +1,37 @@ +-- funcsign: (relation, dict[string, api.column]) -> string +{% macro alter_column_comment(relation, column_dict) -%} + {{ return(adapter.dispatch('alter_column_comment', 'dbt')(relation, column_dict)) }} +{% endmacro %} + +-- funcsign: (relation, dict[string, api.column]) -> string +{% macro default__alter_column_comment(relation, column_dict) -%} + {{ exceptions.raise_not_implemented( + 'alter_column_comment macro not implemented for adapter '+adapter.type()) }} +{% endmacro %} + +-- funcsign: (relation, string) -> string +{% macro alter_relation_comment(relation, relation_comment) -%} + {{ return(adapter.dispatch('alter_relation_comment', 'dbt')(relation, relation_comment)) }} +{% endmacro %} + +-- funcsign: (relation, string) -> string +{% macro default__alter_relation_comment(relation, relation_comment) -%} + {{ exceptions.raise_not_implemented( + 'alter_relation_comment macro not implemented for adapter '+adapter.type()) }} +{% endmacro %} + +-- funcsign: (relation, model, optional[bool], optional[bool]) -> string +{% macro persist_docs(relation, model, for_relation=true, for_columns=true) -%} + {{ return(adapter.dispatch('persist_docs', 'dbt')(relation, model, for_relation, for_columns)) }} +{% endmacro %} + +-- funcsign: (relation, model, optional[bool], optional[bool]) -> string +{% macro default__persist_docs(relation, model, for_relation, for_columns) -%} + {% if for_relation and config.persist_relation_docs() and model.description %} + {% do run_query(alter_relation_comment(relation, model.description)) %} + {% endif %} + + {% if for_columns and config.persist_column_docs() and model.columns %} + {% do run_query(alter_column_comment(relation, model.columns)) %} + {% endif %} +{% endmacro %} diff --git a/dbt_internal_packages/dbt-adapters/macros/adapters/relation.sql b/dbt_internal_packages/dbt-adapters/macros/adapters/relation.sql new file mode 100644 index 0000000..374a053 --- /dev/null +++ b/dbt_internal_packages/dbt-adapters/macros/adapters/relation.sql @@ -0,0 +1,95 @@ +-- funcsign: (relation, optional[string]) -> relation +{% macro make_intermediate_relation(base_relation, suffix='__dbt_tmp') %} + {{ return(adapter.dispatch('make_intermediate_relation', 'dbt')(base_relation, suffix)) }} +{% endmacro %} + +-- funcsign: (relation, string) -> relation +{% macro default__make_intermediate_relation(base_relation, suffix) %} + {{ return(default__make_temp_relation(base_relation, suffix)) }} +{% endmacro %} + +-- funcsign: (relation, optional[string]) -> relation +{% macro make_temp_relation(base_relation, suffix='__dbt_tmp') %} + {#-- This ensures microbatch batches get unique temp relations to avoid clobbering --#} + {% if suffix == '__dbt_tmp' and model.batch %} + {#-- TYPE CHECK: model.batch is optional --#} + {% set suffix = suffix ~ '_' ~ model.batch.id %} + {% endif %} + + {{ return(adapter.dispatch('make_temp_relation', 'dbt')(base_relation, suffix)) }} +{% endmacro %} + +-- funcsign: (relation, string) -> relation +{% macro default__make_temp_relation(base_relation, suffix) %} + {%- set temp_identifier = base_relation.identifier ~ suffix -%} + {%- set temp_relation = base_relation.incorporate( + path={"identifier": temp_identifier}) -%} + + {{ return(temp_relation) }} +{% endmacro %} + +-- funcsign: (relation, string, optional[string]) -> relation +{% macro make_backup_relation(base_relation, backup_relation_type, suffix='__dbt_backup') %} + {{ return(adapter.dispatch('make_backup_relation', 'dbt')(base_relation, backup_relation_type, suffix)) }} +{% endmacro %} + +-- funcsign: (relation, string, string) -> relation +{% macro default__make_backup_relation(base_relation, backup_relation_type, suffix) %} + {%- set backup_identifier = base_relation.identifier ~ suffix -%} + {%- set backup_relation = base_relation.incorporate( + path={"identifier": backup_identifier}, + type=backup_relation_type + ) -%} + {{ return(backup_relation) }} +{% endmacro %} + +-- funcsign: (relation) -> string +{% macro truncate_relation(relation) -%} + {{ return(adapter.dispatch('truncate_relation', 'dbt')(relation)) }} +{% endmacro %} + +-- funcsign: (relation) -> string +{% macro default__truncate_relation(relation) -%} + {% call statement('truncate_relation') -%} + truncate table {{ relation.render() }} + {%- endcall %} +{% endmacro %} + +-- funcsign: (string, string, string, string) -> tuple[bool, relation] +{% macro get_or_create_relation(database, schema, identifier, type) -%} + {{ return(adapter.dispatch('get_or_create_relation', 'dbt')(database, schema, identifier, type)) }} +{% endmacro %} + +-- funcsign: (string, string, string, string) -> tuple[bool, relation] +{% macro default__get_or_create_relation(database, schema, identifier, type) %} + {%- set target_relation = adapter.get_relation(database=database, schema=schema, identifier=identifier) %} + + {% if target_relation %} + {% do return((true, target_relation)) %} + {% endif %} + + {%- set new_relation = api.Relation.create( + database=database, + schema=schema, + identifier=identifier, + type=type + ) -%} + {% do return((false, new_relation)) %} +{% endmacro %} + + +-- a user-friendly interface into adapter.get_relation +-- funcsign: (relation) -> optional[relation] +{% macro load_cached_relation(relation) %} + {% do return(adapter.get_relation( + database=relation.database, + schema=relation.schema, + identifier=relation.identifier + )) -%} +{% endmacro %} + +-- old name for backwards compatibility +-- funcsign: (relation) -> optional[relation] +{% macro load_relation(relation) %} + {{ return(load_cached_relation(relation)) }} +{% endmacro %} diff --git a/dbt_internal_packages/dbt-adapters/macros/adapters/schema.sql b/dbt_internal_packages/dbt-adapters/macros/adapters/schema.sql new file mode 100644 index 0000000..ac92b4f --- /dev/null +++ b/dbt_internal_packages/dbt-adapters/macros/adapters/schema.sql @@ -0,0 +1,23 @@ +-- funcsign: (relation) -> string +{% macro create_schema(relation) -%} + {{ adapter.dispatch('create_schema', 'dbt')(relation) }} +{% endmacro %} + +-- funcsign: (relation) -> string +{% macro default__create_schema(relation) -%} + {%- call statement('create_schema') -%} + create schema if not exists {{ relation.without_identifier() }} + {% endcall %} +{% endmacro %} + +-- funcsign: (relation) -> string +{% macro drop_schema(relation) -%} + {{ adapter.dispatch('drop_schema', 'dbt')(relation) }} +{% endmacro %} + +-- funcsign: (relation) -> string +{% macro default__drop_schema(relation) -%} + {%- call statement('drop_schema') -%} + drop schema if exists {{ relation.without_identifier() }} cascade + {% endcall %} +{% endmacro %} diff --git a/dbt_internal_packages/dbt-adapters/macros/adapters/show.sql b/dbt_internal_packages/dbt-adapters/macros/adapters/show.sql new file mode 100644 index 0000000..69a4c8b --- /dev/null +++ b/dbt_internal_packages/dbt-adapters/macros/adapters/show.sql @@ -0,0 +1,29 @@ +{# + We expect a syntax error if dbt show is invoked both with a --limit flag to show + and with a limit predicate embedded in its inline query. No special handling is + provided out-of-box. +#} +-- funcsign: (string, string, optional[integer]) -> string +{% macro get_show_sql(compiled_code, sql_header, limit) -%} + {%- if sql_header is not none -%} + {{ sql_header }} + {%- endif %} + {{ get_limit_subquery_sql(compiled_code, limit) }} +{% endmacro %} + +{# + Not necessarily a true subquery anymore. Now, merely a query subordinate + to the calling macro. +#} +-- funcsign: (string, optional[integer]) -> string +{%- macro get_limit_subquery_sql(sql, limit) -%} + {{ adapter.dispatch('get_limit_sql', 'dbt')(sql, limit) }} +{%- endmacro -%} + +-- funcsign: (string, optional[integer]) -> string +{% macro default__get_limit_sql(sql, limit) %} + {{ sql }} + {% if limit is not none %} + limit {{ limit }} + {%- endif -%} +{% endmacro %} diff --git a/dbt_internal_packages/dbt-adapters/macros/adapters/timestamps.sql b/dbt_internal_packages/dbt-adapters/macros/adapters/timestamps.sql new file mode 100644 index 0000000..b868a78 --- /dev/null +++ b/dbt_internal_packages/dbt-adapters/macros/adapters/timestamps.sql @@ -0,0 +1,61 @@ +-- funcsign: () -> string +{%- macro current_timestamp() -%} + {{ adapter.dispatch('current_timestamp', 'dbt')() }} +{%- endmacro -%} + +-- funcsign: () -> string +{% macro default__current_timestamp() -%} + {{ exceptions.raise_not_implemented( + 'current_timestamp macro not implemented for adapter ' + adapter.type()) }} +{%- endmacro %} + +-- funcsign: () -> string +{%- macro snapshot_get_time() -%} + {{ adapter.dispatch('snapshot_get_time', 'dbt')() }} +{%- endmacro -%} + +-- funcsign: () -> string +{% macro default__snapshot_get_time() %} + {{ current_timestamp() }} +{% endmacro %} + +-- funcsign: () -> optional[string] +{% macro get_snapshot_get_time_data_type() %} + {% set snapshot_time = adapter.dispatch('snapshot_get_time', 'dbt')() %} + {% set time_data_type_sql = 'select ' ~ snapshot_time ~ ' as dbt_snapshot_time' %} + {% set snapshot_time_column_schema = get_column_schema_from_query(time_data_type_sql) %} + {% set time_data_type = snapshot_time_column_schema[0].dtype %} + {{ return(time_data_type or none) }} +{% endmacro %} + +--------------------------------------------- + +/* {# + DEPRECATED: DO NOT USE IN NEW PROJECTS + + This is ONLY to handle the fact that Snowflake + Postgres had functionally + different implementations of {{ dbt.current_timestamp }} + {{ dbt_utils.current_timestamp }} + + If you had a project or package that called {{ dbt_utils.current_timestamp() }}, you should + continue to use this macro to guarantee identical behavior on those two databases. +#} */ + +-- funcsign: () -> string +{% macro current_timestamp_backcompat() %} + {{ return(adapter.dispatch('current_timestamp_backcompat', 'dbt')()) }} +{% endmacro %} + +-- funcsign: () -> string +{% macro default__current_timestamp_backcompat() %} + current_timestamp::timestamp +{% endmacro %} + +-- funcsign: () -> string +{% macro current_timestamp_in_utc_backcompat() %} + {{ return(adapter.dispatch('current_timestamp_in_utc_backcompat', 'dbt')()) }} +{% endmacro %} + +-- funcsign: () -> string +{% macro default__current_timestamp_in_utc_backcompat() %} + {{ return(adapter.dispatch('current_timestamp_backcompat', 'dbt')()) }} +{% endmacro %} diff --git a/dbt_internal_packages/dbt-adapters/macros/adapters/validate_sql.sql b/dbt_internal_packages/dbt-adapters/macros/adapters/validate_sql.sql new file mode 100644 index 0000000..119745f --- /dev/null +++ b/dbt_internal_packages/dbt-adapters/macros/adapters/validate_sql.sql @@ -0,0 +1,12 @@ +-- funcsign: (string) -> agate_table +{% macro validate_sql(sql) -%} + {{ return(adapter.dispatch('validate_sql', 'dbt')(sql)) }} +{% endmacro %} + +-- funcsign: (string) -> agate_table +{% macro default__validate_sql(sql) -%} + {% call statement('validate_sql') -%} + explain {{ sql }} + {% endcall %} + {{ return(load_result('validate_sql')) }} +{% endmacro %} diff --git a/dbt_internal_packages/dbt-adapters/macros/etc/datetime.sql b/dbt_internal_packages/dbt-adapters/macros/etc/datetime.sql new file mode 100644 index 0000000..72583f1 --- /dev/null +++ b/dbt_internal_packages/dbt-adapters/macros/etc/datetime.sql @@ -0,0 +1,66 @@ +-- funcsign: (string, string) -> timestamp +{% macro convert_datetime(date_str, date_fmt) %} + + {% set error_msg -%} + The provided partition date '{{ date_str }}' does not match the expected format '{{ date_fmt }}' + {%- endset %} + + {# Deviation from Core: `try_or_compiler_error` now takes in the function to call in two arguments: + # the (optional) package containing the function, and the name of the function as a string. + #} + {% set res = try_or_compiler_error(error_msg, modules.datetime.datetime, "strptime", date_str.strip(), date_fmt) %} + {{ return(res) }} + +{% endmacro %} + +-- funcsign: (string, optional[string], optional[string], optional[string]) -> list[timestamp] +{% macro dates_in_range(start_date_str, end_date_str=none, in_fmt="%Y%m%d", out_fmt="%Y%m%d") %} + {% set end_date_str = start_date_str if end_date_str is none else end_date_str %} + + {% set start_date = convert_datetime(start_date_str, in_fmt) %} + {% set end_date = convert_datetime(end_date_str, in_fmt) %} + + {% set day_count = (end_date - start_date).days %} + {% if day_count < 0 %} + {% set msg -%} + Partition start date is after the end date ({{ start_date }}, {{ end_date }}) + {%- endset %} + + {{ exceptions.raise_compiler_error(msg, model) }} + {% endif %} + + {% set date_list = [] %} + {% for i in range(0, day_count + 1) %} + {% set the_date = (modules.datetime.timedelta(days=i) + start_date) %} + {% if not out_fmt %} + {% set _ = date_list.append(the_date) %} + {% else %} + {% set _ = date_list.append(the_date.strftime(out_fmt)) %} + {% endif %} + {% endfor %} + + {{ return(date_list) }} +{% endmacro %} + +-- funcsign: (string, optional[string]) -> list[timestamp] +{% macro partition_range(raw_partition_date, date_fmt='%Y%m%d') %} + {% set partition_range = (raw_partition_date | string).split(",") %} + + {% if (partition_range | length) == 1 %} + {% set start_date = partition_range[0] %} + {% set end_date = none %} + {% elif (partition_range | length) == 2 %} + {% set start_date = partition_range[0] %} + {% set end_date = partition_range[1] %} + {% else %} + {{ exceptions.raise_compiler_error("Invalid partition time. Expected format: {Start Date}[,{End Date}]. Got: " ~ raw_partition_date) }} + {% endif %} + + {{ return(dates_in_range(start_date, end_date, in_fmt=date_fmt)) }} +{% endmacro %} + +-- funcsign: () -> string +{% macro py_current_timestring() %} + {% set dt = modules.datetime.datetime.now() %} + {% do return(dt.strftime("%Y%m%d%H%M%S%f")) %} +{% endmacro %} diff --git a/dbt_internal_packages/dbt-adapters/macros/etc/statement.sql b/dbt_internal_packages/dbt-adapters/macros/etc/statement.sql new file mode 100644 index 0000000..f0bb604 --- /dev/null +++ b/dbt_internal_packages/dbt-adapters/macros/etc/statement.sql @@ -0,0 +1,55 @@ +{#-- +The macro override naming method (spark__statement) only works for macros which are called with adapter.dispatch. For macros called directly, you can just redefine them. +--#} +-- funcsign: (optional[string], optional[bool], optional[bool], optional[string]) -> string +{%- macro statement(name=None, fetch_result=False, auto_begin=True, language='sql') -%} + {%- if execute: -%} + {%- set compiled_code = caller() -%} + + {%- if name == 'main' -%} + {{ log('Writing runtime {} for node "{}"'.format(language, model['unique_id'])) }} + {{ write(compiled_code) }} + {%- endif -%} + {%- if language == 'sql'-%} + {%- set res, table = adapter.execute(compiled_code, auto_begin=auto_begin, fetch=fetch_result) -%} + {%- elif language == 'python' -%} + {%- set res = submit_python_job(model, compiled_code) -%} + {#-- TODO: What should table be for python models? --#} + {%- set table = None -%} + {%- else -%} + {% do exceptions.raise_compiler_error("statement macro didn't get supported language") %} + {%- endif -%} + + {%- if name is not none -%} + {{ store_result(name, response=res, agate_table=table) }} + {%- endif -%} + + {%- endif -%} +{%- endmacro %} + +-- ai +-- funcsign: (optional[string], optional[string], optional[string], optional[string], optional[agate_table]) -> string +{% macro noop_statement(name=None, message=None, code=None, rows_affected=None, res=None) -%} + {%- set sql = caller() -%} + + {%- if name == 'main' -%} + {{ log('Writing runtime SQL for node "{}"'.format(model['unique_id'])) }} + {{ write(sql) }} + {%- endif -%} + + {%- if name is not none -%} + {{ store_raw_result(name, message=message, code=code, rows_affected=rows_affected, agate_table=res) }} + {%- endif -%} + +{%- endmacro %} + + +{# a user-friendly interface into statements #} +-- funcsign: (string) -> agate_table +{% macro run_query(sql) %} + {% call statement("run_query_statement", fetch_result=true, auto_begin=false) %} + {{ sql }} + {% endcall %} + + {% do return(load_result("run_query_statement").table) %} +{% endmacro %} diff --git a/dbt_internal_packages/dbt-adapters/macros/generic_test_sql/accepted_values.sql b/dbt_internal_packages/dbt-adapters/macros/generic_test_sql/accepted_values.sql new file mode 100644 index 0000000..a61db1a --- /dev/null +++ b/dbt_internal_packages/dbt-adapters/macros/generic_test_sql/accepted_values.sql @@ -0,0 +1,28 @@ +-- funcsign: (model, string, list[string], bool) -> string +{% macro default__test_accepted_values(model, column_name, values, quote=True) %} + +with all_values as ( + + select + {{ column_name }} as value_field, + count(*) as n_records + + from {{ model }} + group by {{ column_name }} + +) + +select * +from all_values +where value_field not in ( + {% for value in values -%} + {% if quote -%} + '{{ value }}' + {%- else -%} + {{ value }} + {%- endif -%} + {%- if not loop.last -%},{%- endif %} + {%- endfor %} +) + +{% endmacro %} diff --git a/dbt_internal_packages/dbt-adapters/macros/generic_test_sql/not_null.sql b/dbt_internal_packages/dbt-adapters/macros/generic_test_sql/not_null.sql new file mode 100644 index 0000000..73e3401 --- /dev/null +++ b/dbt_internal_packages/dbt-adapters/macros/generic_test_sql/not_null.sql @@ -0,0 +1,9 @@ +{% macro default__test_not_null(model, column_name) %} + +{% set column_list = '*' if should_store_failures() else column_name %} + +select {{ column_list }} +from {{ model }} +where {{ column_name }} is null + +{% endmacro %} diff --git a/dbt_internal_packages/dbt-adapters/macros/generic_test_sql/relationships.sql b/dbt_internal_packages/dbt-adapters/macros/generic_test_sql/relationships.sql new file mode 100644 index 0000000..db779a4 --- /dev/null +++ b/dbt_internal_packages/dbt-adapters/macros/generic_test_sql/relationships.sql @@ -0,0 +1,23 @@ +{% macro default__test_relationships(model, column_name, to, field) %} + +with child as ( + select {{ column_name }} as from_field + from {{ model }} + where {{ column_name }} is not null +), + +parent as ( + select {{ field }} as to_field + from {{ to }} +) + +select + from_field + +from child +left join parent + on child.from_field = parent.to_field + +where parent.to_field is null + +{% endmacro %} diff --git a/dbt_internal_packages/dbt-adapters/macros/generic_test_sql/unique.sql b/dbt_internal_packages/dbt-adapters/macros/generic_test_sql/unique.sql new file mode 100644 index 0000000..ed18c5c --- /dev/null +++ b/dbt_internal_packages/dbt-adapters/macros/generic_test_sql/unique.sql @@ -0,0 +1,12 @@ +{% macro default__test_unique(model, column_name) %} + +select + {{ column_name }} as unique_field, + count(*) as n_records + +from {{ model }} +where {{ column_name }} is not null +group by {{ column_name }} +having count(*) > 1 + +{% endmacro %} diff --git a/dbt_internal_packages/dbt-adapters/macros/get_custom_name/get_custom_alias.sql b/dbt_internal_packages/dbt-adapters/macros/get_custom_name/get_custom_alias.sql new file mode 100644 index 0000000..7350e41 --- /dev/null +++ b/dbt_internal_packages/dbt-adapters/macros/get_custom_name/get_custom_alias.sql @@ -0,0 +1,38 @@ + +{# + Renders a alias name given a custom alias name. If the custom + alias name is none, then the resulting alias is just the filename of the + model. If an alias override is specified, then that is used. + + This macro can be overriden in projects to define different semantics + for rendering a alias name. + + Arguments: + custom_alias_name: The custom alias name specified for a model, or none + node: The available node that an alias is being generated for, or none + +#} + +-- funcsign: (optional[string], optional[node]) -> string +{% macro generate_alias_name(custom_alias_name=none, node=none) -%} + {% do return(adapter.dispatch('generate_alias_name', 'dbt')(custom_alias_name, node)) %} +{%- endmacro %} + +-- funcsign: (optional[string], optional[node]) -> string +{% macro default__generate_alias_name(custom_alias_name=none, node=none) -%} + + {%- if custom_alias_name -%} + + {{ custom_alias_name | trim }} + + {%- elif node -%} + + {% if node.version -%} + {{ node.name ~ "_v" ~ (node.version | replace(".", "_")) }} + {%- else -%} + {{ node.name }} + {%- endif -%} + + {%- endif -%} + +{%- endmacro %} diff --git a/dbt_internal_packages/dbt-adapters/macros/get_custom_name/get_custom_database.sql b/dbt_internal_packages/dbt-adapters/macros/get_custom_name/get_custom_database.sql new file mode 100644 index 0000000..647e7b0 --- /dev/null +++ b/dbt_internal_packages/dbt-adapters/macros/get_custom_name/get_custom_database.sql @@ -0,0 +1,34 @@ +{# + Renders a database name given a custom database name. If the custom + database name is none, then the resulting database is just the "database" + value in the specified target. If a database override is specified, then + the custom database name is used instead of the default "database" value. + + This macro can be overriden in projects to define different semantics + for rendering a database name. + + Arguments: + custom_database_name: The custom database name specified for a model, or none + node: The node the database is being generated for + +#} + +-- funcsign: (optional[string], optional[node]) -> string +{% macro generate_database_name(custom_database_name=none, node=none) -%} + {% do return(adapter.dispatch('generate_database_name', 'dbt')(custom_database_name, node)) %} +{%- endmacro %} + +-- funcsign: (optional[string], optional[node]) -> string +{% macro default__generate_database_name(custom_database_name=none, node=none) -%} + {%- set default_database = target.database -%} + {%- if custom_database_name is none -%} + + {{ default_database }} + + {%- else -%} + + {{ custom_database_name }} + + {%- endif -%} + +{%- endmacro %} diff --git a/dbt_internal_packages/dbt-adapters/macros/get_custom_name/get_custom_schema.sql b/dbt_internal_packages/dbt-adapters/macros/get_custom_name/get_custom_schema.sql new file mode 100644 index 0000000..7ee5563 --- /dev/null +++ b/dbt_internal_packages/dbt-adapters/macros/get_custom_name/get_custom_schema.sql @@ -0,0 +1,63 @@ + +{# + Renders a schema name given a custom schema name. If the custom + schema name is none, then the resulting schema is just the "schema" + value in the specified target. If a schema override is specified, then + the resulting schema is the default schema concatenated with the + custom schema. + + This macro can be overriden in projects to define different semantics + for rendering a schema name. + + Arguments: + custom_schema_name: The custom schema name specified for a model, or none + node: The node the schema is being generated for + +#} +-- funcsign: (optional[string], optional[node]) -> string +{% macro generate_schema_name(custom_schema_name=none, node=none) -%} + {{ return(adapter.dispatch('generate_schema_name', 'dbt')(custom_schema_name, node)) }} +{% endmacro %} + +-- funcsign: (optional[string], optional[node]) -> string +{% macro default__generate_schema_name(custom_schema_name, node) -%} + + {%- set default_schema = target.schema -%} + {%- if custom_schema_name is none -%} + + {{ default_schema }} + + {%- else -%} + + {{ default_schema }}_{{ custom_schema_name | trim }} + + {%- endif -%} + +{%- endmacro %} + + +{# + Renders a schema name given a custom schema name. In production, this macro + will render out the overriden schema name for a model. Otherwise, the default + schema specified in the active target is used. + + Arguments: + custom_schema_name: The custom schema name specified for a model, or none + node: The node the schema is being generated for + +#} +-- funcsign: (optional[string], optional[node]) -> string +{% macro generate_schema_name_for_env(custom_schema_name, node) -%} + + {%- set default_schema = target.schema -%} + {%- if target.name == 'prod' and custom_schema_name is not none -%} + + {{ custom_schema_name | trim }} + + {%- else -%} + + {{ default_schema }} + + {%- endif -%} + +{%- endmacro %} diff --git a/dbt_internal_packages/dbt-adapters/macros/internal/test/names.sql b/dbt_internal_packages/dbt-adapters/macros/internal/test/names.sql new file mode 100644 index 0000000..0c1cf36 --- /dev/null +++ b/dbt_internal_packages/dbt-adapters/macros/internal/test/names.sql @@ -0,0 +1,29 @@ +-- funcsign: () -> string +{% macro t_database_name() %} + {{ return (adapter.dispatch("t_database_name")()) }} +{% endmacro %} + +-- funcsign: () -> string +{% macro default__t_database_name() %} + {{ return(env_var('DBT_DB_NAME')) }} +{% endmacro %} + +-- funcsign: () -> string +{% macro bigquery__t_database_name() %} + {{ return(env_var('GOOGLE_CLOUD_PROJECT')) }} +{% endmacro %} + +-- funcsign: () -> string +{% macro databricks__t_database_name() %} + {{ return(env_var('DATABRICKS_CATALOG')) }} +{% endmacro %} + +-- funcsign: () -> string +{% macro redshift__t_database_name() %} + {{ return(env_var('REDSHIFT_DATABASE')) }} +{% endmacro %} + +-- funcsign: () -> string +{% macro t_schema_name() %} + {{ return(target.schema) }} +{% endmacro %} diff --git a/dbt_internal_packages/dbt-adapters/macros/materializations/configs.sql b/dbt_internal_packages/dbt-adapters/macros/materializations/configs.sql new file mode 100644 index 0000000..e206465 --- /dev/null +++ b/dbt_internal_packages/dbt-adapters/macros/materializations/configs.sql @@ -0,0 +1,23 @@ +-- funcsign: (config) -> string +{% macro set_sql_header(config) -%} + {{ config.set('sql_header', caller()) }} +{%- endmacro %} + +-- funcsign: () -> bool +{% macro should_full_refresh() %} + {% set config_full_refresh = config.get('full_refresh') %} + {% if config_full_refresh is none %} + {% set config_full_refresh = flags.FULL_REFRESH %} + {% endif %} + {% do return(config_full_refresh) %} +{% endmacro %} + +-- ai +-- funcsign: () -> bool +{% macro should_store_failures() %} + {% set config_store_failures = config.get('store_failures') %} + {% if config_store_failures is none %} + {% set config_store_failures = flags.STORE_FAILURES %} + {% endif %} + {% do return(config_store_failures) %} +{% endmacro %} diff --git a/dbt_internal_packages/dbt-adapters/macros/materializations/functions/function.sql b/dbt_internal_packages/dbt-adapters/macros/materializations/functions/function.sql new file mode 100644 index 0000000..e31137a --- /dev/null +++ b/dbt_internal_packages/dbt-adapters/macros/materializations/functions/function.sql @@ -0,0 +1,16 @@ +{% materialization function, default %} + {% set existing_relation = load_cached_relation(this) %} + {% set target_relation = this.incorporate(type=this.Function) %} + + {{ run_hooks(pre_hooks) }} + + {% set function_type_macro = get_function_macro('scalar', 'sql') %} + {% set build_sql = function_type_macro(target_relation) %} + + {{ function_execute_build_sql(build_sql, existing_relation, target_relation) }} + + {{ run_hooks(post_hooks) }} + + {{ return({'relations': [target_relation]}) }} + +{% endmaterialization %} diff --git a/dbt_internal_packages/dbt-adapters/macros/materializations/functions/helpers.sql b/dbt_internal_packages/dbt-adapters/macros/materializations/functions/helpers.sql new file mode 100644 index 0000000..fa02018 --- /dev/null +++ b/dbt_internal_packages/dbt-adapters/macros/materializations/functions/helpers.sql @@ -0,0 +1,34 @@ +{% macro function_execute_build_sql(build_sql, existing_relation, target_relation) %} + {{ return(adapter.dispatch('function_execute_build_sql', 'dbt')(build_sql, existing_relation, target_relation)) }} +{% endmacro %} + +{% macro default__function_execute_build_sql(build_sql, existing_relation, target_relation) %} + + {% set grant_config = config.get('grants') %} + + {% call statement(name="main") %} + {{ build_sql }} + {% endcall %} + + {% set should_revoke = should_revoke(existing_relation, full_refresh_mode=True) %} + {% do apply_grants(target_relation, grant_config, should_revoke=should_revoke) %} + + {% do persist_docs(target_relation, model) %} + + {{ adapter.commit() }} + +{% endmacro %} + + +{% macro get_function_macro(function_type, function_language) %} + {{ return(adapter.dispatch('get_function_macro', 'dbt')(function_type, function_language)) }} +{% endmacro %} + +{% macro default__get_function_macro(function_type, function_language) %} + {% set macro_name = function_type ~ "_function_" ~ function_language %} + {% if not macro_name in context %} + {{ exceptions.raise_not_implemented(function_language ~ ' ' ~ function_type ~ ' function not implemented for adapter ' ~adapter.type()) }} + {% endif %} + {% set macro = context[macro_name] %} + {{ return(macro) }} +{% endmacro %} diff --git a/dbt_internal_packages/dbt-adapters/macros/materializations/functions/scalar.sql b/dbt_internal_packages/dbt-adapters/macros/materializations/functions/scalar.sql new file mode 100644 index 0000000..d1cc780 --- /dev/null +++ b/dbt_internal_packages/dbt-adapters/macros/materializations/functions/scalar.sql @@ -0,0 +1,38 @@ +{% macro scalar_function_sql(target_relation) %} + {{ return(adapter.dispatch('scalar_function_sql', 'dbt')(target_relation)) }} +{% endmacro %} + +{% macro default__scalar_function_sql(target_relation) %} + {{ scalar_function_create_replace_signature_sql(target_relation) }} + {{ scalar_function_body_sql() }}; +{% endmacro %} + +{% macro scalar_function_create_replace_signature_sql(target_relation) %} + {{ return(adapter.dispatch('scalar_function_create_replace_signature_sql', 'dbt')(target_relation)) }} +{% endmacro %} + +{% macro default__scalar_function_create_replace_signature_sql(target_relation) %} + CREATE OR REPLACE FUNCTION {{ target_relation.render() }} ({{ formatted_scalar_function_args_sql()}}) RETURNS {{ model.returns.data_type }} AS +{% endmacro %} + +{% macro formatted_scalar_function_args_sql() %} + {{ return(adapter.dispatch('formatted_scalar_function_args_sql', 'dbt')()) }} +{% endmacro %} + +{% macro default__formatted_scalar_function_args_sql() %} + {% set args = [] %} + {% for arg in model.arguments -%} + {%- do args.append(arg.name ~ ' ' ~ arg.data_type) -%} + {%- endfor %} + {{ args | join(', ') }} +{% endmacro %} + +{% macro scalar_function_body_sql() %} + {{ return(adapter.dispatch('scalar_function_body_sql', 'dbt')()) }} +{% endmacro %} + +{% macro default__scalar_function_body_sql() %} + $$ + {{ compiled_code }} + $$ LANGUAGE SQL +{% endmacro %} diff --git a/dbt_internal_packages/dbt-adapters/macros/materializations/hooks.sql b/dbt_internal_packages/dbt-adapters/macros/materializations/hooks.sql new file mode 100644 index 0000000..52cade2 --- /dev/null +++ b/dbt_internal_packages/dbt-adapters/macros/materializations/hooks.sql @@ -0,0 +1,36 @@ +-- funcsign: (list[hook], optional[bool]) -> string +{% macro run_hooks(hooks, inside_transaction=True) %} + {% for hook in hooks | selectattr('transaction', 'equalto', inside_transaction) %} + {% if not inside_transaction and loop.first %} + {% call statement(auto_begin=inside_transaction) %} + commit; + {% endcall %} + {% endif %} + {% set rendered = render(hook.get('sql')) | trim %} + {% if (rendered | length) > 0 %} + {% call statement(auto_begin=inside_transaction) %} + {{ rendered }} + {% endcall %} + {% endif %} + {% endfor %} +{% endmacro %} + +-- funcsign: (string, bool) -> string +{% macro make_hook_config(sql, inside_transaction) %} + {{ tojson({"sql": sql, "transaction": inside_transaction}) }} +{% endmacro %} + +-- funcsign: (string) -> string +{% macro before_begin(sql) %} + {{ make_hook_config(sql, inside_transaction=False) }} +{% endmacro %} + +-- funcsign: (string) -> string +{% macro in_transaction(sql) %} + {{ make_hook_config(sql, inside_transaction=True) }} +{% endmacro %} + +-- funcsign: (string) -> string +{% macro after_commit(sql) %} + {{ make_hook_config(sql, inside_transaction=False) }} +{% endmacro %} diff --git a/dbt_internal_packages/dbt-adapters/macros/materializations/models/clone/can_clone_table.sql b/dbt_internal_packages/dbt-adapters/macros/materializations/models/clone/can_clone_table.sql new file mode 100644 index 0000000..8784ea1 --- /dev/null +++ b/dbt_internal_packages/dbt-adapters/macros/materializations/models/clone/can_clone_table.sql @@ -0,0 +1,9 @@ +-- funcsign: () -> bool +{% macro can_clone_table() %} + {{ return(adapter.dispatch('can_clone_table', 'dbt')()) }} +{% endmacro %} + +-- funcsign: () -> bool +{% macro default__can_clone_table() %} + {{ return(False) }} +{% endmacro %} diff --git a/dbt_internal_packages/dbt-adapters/macros/materializations/models/clone/clone.sql b/dbt_internal_packages/dbt-adapters/macros/materializations/models/clone/clone.sql new file mode 100644 index 0000000..9686d7c --- /dev/null +++ b/dbt_internal_packages/dbt-adapters/macros/materializations/models/clone/clone.sql @@ -0,0 +1,67 @@ +{%- materialization clone, default -%} + + {%- set relations = {'relations': []} -%} + + {%- if not defer_relation -%} + -- nothing to do + {{ log("No relation found in state manifest for " ~ model.unique_id, info=True) }} + {{ return(relations) }} + {%- endif -%} + + {%- set existing_relation = load_cached_relation(this) -%} + + {%- if existing_relation and not flags.FULL_REFRESH -%} + -- noop! + {{ log("Relation " ~ existing_relation ~ " already exists", info=True) }} + {{ return(relations) }} + {%- endif -%} + + {%- set other_existing_relation = load_cached_relation(defer_relation) -%} + + -- If this is a database that can do zero-copy cloning of tables, and the other relation is a table, then this will be a table + -- Otherwise, this will be a view + + {% set can_clone_table = can_clone_table() %} + + {%- if other_existing_relation and other_existing_relation.type == 'table' and can_clone_table -%} + + {%- set target_relation = this.incorporate(type='table') -%} + {% if existing_relation is not none and not existing_relation.is_table %} + {{ log("Dropping relation " ~ existing_relation.render() ~ " because it is of type " ~ existing_relation.type) }} + {{ drop_relation_if_exists(existing_relation) }} + {% endif %} + + -- as a general rule, data platforms that can clone tables can also do atomic 'create or replace' + {% call statement('main') %} + {% if target_relation and defer_relation and target_relation == defer_relation %} + {{ log("Target relation and defer relation are the same, skipping clone for relation: " ~ target_relation.render()) }} + {% else %} + {{ create_or_replace_clone(target_relation, defer_relation) }} + {% endif %} + + {% endcall %} + + {% set should_revoke = should_revoke(existing_relation, full_refresh_mode=True) %} + {% do apply_grants(target_relation, grant_config, should_revoke=should_revoke) %} -- noqa: Unknown local variable 'grant_config' + {% do persist_docs(target_relation, model) %} + + {{ return({'relations': [target_relation]}) }} + + {%- else -%} + + {%- set target_relation = this.incorporate(type='view') -%} + + -- reuse the view materialization + -- TODO: support actual dispatch for materialization macros + -- Tracking ticket: https://github.com/dbt-labs/dbt-core/issues/7799 + {% set search_name = "materialization_view_" ~ adapter.type() %} + {% if not search_name in context %} + {% set search_name = "materialization_view_default" %} + {% endif %} + {% set materialization_macro = context[search_name] %} + {% set relations = materialization_macro() %} + {{ return(relations) }} + + {%- endif -%} + +{%- endmaterialization -%} diff --git a/dbt_internal_packages/dbt-adapters/macros/materializations/models/clone/create_or_replace_clone.sql b/dbt_internal_packages/dbt-adapters/macros/materializations/models/clone/create_or_replace_clone.sql new file mode 100644 index 0000000..75e4549 --- /dev/null +++ b/dbt_internal_packages/dbt-adapters/macros/materializations/models/clone/create_or_replace_clone.sql @@ -0,0 +1,9 @@ +-- funcsign: (relation, relation) -> string +{% macro create_or_replace_clone(this_relation, defer_relation) %} + {{ return(adapter.dispatch('create_or_replace_clone', 'dbt')(this_relation, defer_relation)) }} +{% endmacro %} + +-- funcsign: (relation, relation) -> string +{% macro default__create_or_replace_clone(this_relation, defer_relation) %} + create or replace table {{ this_relation.render() }} clone {{ defer_relation.render() }} +{% endmacro %} diff --git a/dbt_internal_packages/dbt-adapters/macros/materializations/models/incremental/column_helpers.sql b/dbt_internal_packages/dbt-adapters/macros/materializations/models/incremental/column_helpers.sql new file mode 100644 index 0000000..cec9e61 --- /dev/null +++ b/dbt_internal_packages/dbt-adapters/macros/materializations/models/incremental/column_helpers.sql @@ -0,0 +1,83 @@ +/* {# + Helper macros for internal use with incremental materializations. + Use with care if calling elsewhere. +#} */ + +-- funcsign: (list[string]) -> string +{% macro get_quoted_csv(column_names) %} + + {% set quoted = [] %} + {% for col in column_names -%} + {%- do quoted.append(adapter.quote(col)) -%} + {%- endfor %} + + {%- set dest_cols_csv = quoted | join(', ') -%} + {{ return(dest_cols_csv) }} + +{% endmacro %} + + +-- funcsign: (list[base_column], list[base_column]) -> list[base_column] +{% macro diff_columns(source_columns, target_columns) %} + + {% set result = [] %} + {% set source_names = source_columns | map(attribute = 'column') | list %} + {% set target_names = target_columns | map(attribute = 'column') | list %} + + {# --check whether the name attribute exists in the target - this does not perform a data type check #} + {% for sc in source_columns %} + {% if sc.name not in target_names %} + {{ result.append(sc) }} + {% endif %} + {% endfor %} + + {{ return(result) }} + +{% endmacro %} + +-- funcsign: (list[base_column], list[base_column]) -> list[dict[string, string]] +{% macro diff_column_data_types(source_columns, target_columns) %} + + {% set result = [] %} + {% for sc in source_columns %} + {% set tc = target_columns | selectattr("name", "equalto", sc.name) | list | first %} + {% if tc %} + {% if sc.data_type != tc.data_type and not sc.can_expand_to(other_column=tc) %} + {{ result.append( { 'column_name': tc.name, 'new_type': sc.data_type } ) }} + {% endif %} + {% endif %} + {% endfor %} + + {{ return(result) }} + +{% endmacro %} + +-- funcsign: (optional[list[string]], optional[list[string]], list[base_column]) -> list[string] +{% macro get_merge_update_columns(merge_update_columns, merge_exclude_columns, dest_columns) %} + {{ return(adapter.dispatch('get_merge_update_columns', 'dbt')(merge_update_columns, merge_exclude_columns, dest_columns)) }} +{% endmacro %} + +-- funcsign: (optional[list[string]], optional[list[string]], list[base_column]) -> list[string] +{% macro default__get_merge_update_columns(merge_update_columns, merge_exclude_columns, dest_columns) %} + {%- set default_cols = dest_columns | map(attribute="quoted") | list -%} + + {%- if merge_update_columns and merge_exclude_columns -%} + {{ exceptions.raise_compiler_error( + 'Model cannot specify merge_update_columns and merge_exclude_columns. Please update model to use only one config' + )}} + {%- elif merge_update_columns -%} + {%- set update_columns = merge_update_columns -%} + {%- elif merge_exclude_columns -%} + {%- set update_columns = [] -%} + {%- for column in dest_columns -%} + {% if column.column | lower not in merge_exclude_columns | map("lower") | list %} + {%- do update_columns.append(column.quoted) -%} + {% endif %} + {%- endfor -%} + {%- else -%} + {%- set update_columns = default_cols -%} + {%- endif -%} + + {{ return(update_columns) }} + +{% endmacro %} diff --git a/dbt_internal_packages/dbt-adapters/macros/materializations/models/incremental/incremental.sql b/dbt_internal_packages/dbt-adapters/macros/materializations/models/incremental/incremental.sql new file mode 100644 index 0000000..f2097fa --- /dev/null +++ b/dbt_internal_packages/dbt-adapters/macros/materializations/models/incremental/incremental.sql @@ -0,0 +1,99 @@ + +{% materialization incremental, default -%} + + -- relations + {%- set existing_relation = load_cached_relation(this) -%} + {%- set target_relation = this.incorporate(type='table') -%} + {%- set temp_relation = make_temp_relation(target_relation)-%} + {%- set intermediate_relation = make_intermediate_relation(target_relation)-%} + {%- set backup_relation_type = 'table' if existing_relation is none else existing_relation.type -%} + {%- set backup_relation = make_backup_relation(target_relation, backup_relation_type) -%} + + -- configs + {%- set unique_key = config.get('unique_key') -%} + {%- set full_refresh_mode = (should_full_refresh() or existing_relation.is_view) -%} -- noqa: existing_relation could be None + {%- set on_schema_change = incremental_validate_on_schema_change(config.get('on_schema_change'), default='ignore') -%} -- noqa: incremental_validate_on_schema_change 1st parameter is not nullable + + -- the temp_ and backup_ relations should not already exist in the database; get_relation + -- will return None in that case. Otherwise, we get a relation that we can drop + -- later, before we try to use this name for the current operation. This has to happen before + -- BEGIN, in a separate transaction + {%- set preexisting_intermediate_relation = load_cached_relation(intermediate_relation)-%} + {%- set preexisting_backup_relation = load_cached_relation(backup_relation) -%} + -- grab current tables grants config for comparision later on + {% set grant_config = config.get('grants') %} + {{ drop_relation_if_exists(preexisting_intermediate_relation) }} + {{ drop_relation_if_exists(preexisting_backup_relation) }} + + {{ run_hooks(pre_hooks, inside_transaction=False) }} + + -- `BEGIN` happens here: + {{ run_hooks(pre_hooks, inside_transaction=True) }} + + {% set to_drop = [] %} + + {% set incremental_strategy = config.get('incremental_strategy') or 'default' %} + {% set strategy_sql_macro_func = adapter.get_incremental_strategy_macro(context, incremental_strategy) %} + + {% if existing_relation is none %} + {% set build_sql = get_create_table_as_sql(False, target_relation, sql) %} + {% set relation_for_indexes = target_relation %} + {% elif full_refresh_mode %} + {% set build_sql = get_create_table_as_sql(False, intermediate_relation, sql) %} + {% set relation_for_indexes = intermediate_relation %} + {% set need_swap = true %} + {% else %} + {% do run_query(get_create_table_as_sql(True, temp_relation, sql)) %} + {% set relation_for_indexes = temp_relation %} + {% set contract_config = config.get('contract') %} + {% if not contract_config or not contract_config.enforced %} + {% do adapter.expand_target_column_types( + from_relation=temp_relation, + to_relation=target_relation) %} + {% endif %} + {#-- Process schema changes. Returns dict of changes if successful. Use source columns for upserting/merging --#} + {% set dest_columns = process_schema_changes(on_schema_change, temp_relation, existing_relation) %} + {% if not dest_columns %} + {% set dest_columns = adapter.get_columns_in_relation(existing_relation) %} + {% endif %} + + {#-- Get the incremental_strategy, the macro to use for the strategy, and build the sql --#} + {% set incremental_predicates = config.get('predicates', none) or config.get('incremental_predicates', none) %} + {% set strategy_arg_dict = ({'target_relation': target_relation, 'temp_relation': temp_relation, 'unique_key': unique_key, 'dest_columns': dest_columns, 'incremental_predicates': incremental_predicates }) %} + {% set build_sql = strategy_sql_macro_func(strategy_arg_dict) %} + + {% endif %} + + {% call statement("main") %} + {{ build_sql }} + {% endcall %} + + {% if existing_relation is none or existing_relation.is_view or should_full_refresh() %} + {% do create_indexes(relation_for_indexes) %} + {% endif %} + + {% if need_swap %} + {% do adapter.rename_relation(target_relation, backup_relation) %} + {% do adapter.rename_relation(intermediate_relation, target_relation) %} + {% do to_drop.append(backup_relation) %} + {% endif %} + + {% set should_revoke = should_revoke(existing_relation, full_refresh_mode) %} + {% do apply_grants(target_relation, grant_config, should_revoke=should_revoke) %} + + {% do persist_docs(target_relation, model) %} + + {{ run_hooks(post_hooks, inside_transaction=True) }} + + -- `COMMIT` happens here + {% do adapter.commit() %} + + {% for rel in to_drop %} + {% do adapter.drop_relation(rel) %} + {% endfor %} + + {{ run_hooks(post_hooks, inside_transaction=False) }} + + {{ return({'relations': [target_relation]}) }} + +{%- endmaterialization %} diff --git a/dbt_internal_packages/dbt-adapters/macros/materializations/models/incremental/is_incremental.sql b/dbt_internal_packages/dbt-adapters/macros/materializations/models/incremental/is_incremental.sql new file mode 100644 index 0000000..e6c8df4 --- /dev/null +++ b/dbt_internal_packages/dbt-adapters/macros/materializations/models/incremental/is_incremental.sql @@ -0,0 +1,13 @@ +-- funcsign: () -> bool +{% macro is_incremental() %} + {#-- do not run introspective queries in parsing #} + {% if not execute %} + {{ return(False) }} + {% else %} + {% set relation = adapter.get_relation(this.database, this.schema, this.table) %} + {{ return(relation is not none + and relation.type == 'table' + and model.config.materialized == 'incremental' + and not should_full_refresh()) }} + {% endif %} +{% endmacro %} diff --git a/dbt_internal_packages/dbt-adapters/macros/materializations/models/incremental/merge.sql b/dbt_internal_packages/dbt-adapters/macros/materializations/models/incremental/merge.sql new file mode 100644 index 0000000..892b1ef --- /dev/null +++ b/dbt_internal_packages/dbt-adapters/macros/materializations/models/incremental/merge.sql @@ -0,0 +1,124 @@ +-- funcsign: (string, string, string|list[string]|none, list[base_column], optional[list[string]]) -> string +{% macro get_merge_sql(target, source, unique_key, dest_columns, incremental_predicates=none) -%} + -- back compat for old kwarg name + {% set incremental_predicates = kwargs.get('predicates', incremental_predicates) %} + {{ adapter.dispatch('get_merge_sql', 'dbt')(target, source, unique_key, dest_columns, incremental_predicates) }} +{%- endmacro %} + +-- funcsign: (string, string, string|list[string]|none, list[base_column], optional[list[string]]) -> string +{% macro default__get_merge_sql(target, source, unique_key, dest_columns, incremental_predicates=none) -%} + {%- set predicates = [] if incremental_predicates is none else [] + incremental_predicates -%} + {%- set dest_cols_csv = get_quoted_csv(dest_columns | map(attribute="name")) -%} + {%- set merge_update_columns = config.get('merge_update_columns') -%} + {%- set merge_exclude_columns = config.get('merge_exclude_columns') -%} + {%- set update_columns = get_merge_update_columns(merge_update_columns, merge_exclude_columns, dest_columns) -%} + {%- set sql_header = config.get('sql_header', none) -%} + + {% if unique_key %} + {% if unique_key is sequence and unique_key is not mapping and unique_key is not string %} + {% for key in unique_key %} + {% set this_key_match %} + DBT_INTERNAL_SOURCE.{{ key }} = DBT_INTERNAL_DEST.{{ key }} + {% endset %} + {% do predicates.append(this_key_match) %} + {% endfor %} + {% else %} + {% set source_unique_key = ("DBT_INTERNAL_SOURCE." ~ unique_key) | trim %} + {% set target_unique_key = ("DBT_INTERNAL_DEST." ~ unique_key) | trim %} + {% set unique_key_match = equals(source_unique_key, target_unique_key) | trim %} + {% do predicates.append(unique_key_match) %} + {% endif %} + {% else %} + {% do predicates.append('FALSE') %} + {% endif %} + + {{ sql_header if sql_header is not none }} + + merge into {{ target }} as DBT_INTERNAL_DEST + using {{ source }} as DBT_INTERNAL_SOURCE + on {{"(" ~ predicates | join(") and (") ~ ")"}} + + {% if unique_key %} + when matched then update set + {% for column_name in update_columns -%} + {{ column_name }} = DBT_INTERNAL_SOURCE.{{ column_name }} + {%- if not loop.last %}, {%- endif %} + {%- endfor %} + {% endif %} + + when not matched then insert + ({{ dest_cols_csv }}) + values + ({{ dest_cols_csv }}) + +{% endmacro %} + +-- funcsign: (string, string, string|list[string]|none, list[base_column], optional[list[string]]) -> string +{% macro get_delete_insert_merge_sql(target, source, unique_key, dest_columns, incremental_predicates) -%} + {{ adapter.dispatch('get_delete_insert_merge_sql', 'dbt')(target, source, unique_key, dest_columns, incremental_predicates) }} +{%- endmacro %} + +-- funcsign: (string, string, string|list[string]|none, list[base_column], optional[list[string]]) -> string +{% macro default__get_delete_insert_merge_sql(target, source, unique_key, dest_columns, incremental_predicates) -%} + + {%- set dest_cols_csv = get_quoted_csv(dest_columns | map(attribute="name")) -%} + + {% if unique_key %} + {% if unique_key is string %} + {% set unique_key = [unique_key] %} + {% endif %} + + {%- set unique_key_str = unique_key|join(', ') -%} + + delete from {{ target }} as DBT_INTERNAL_DEST + where ({{ unique_key_str }}) in ( + select distinct {{ unique_key_str }} + from {{ source }} as DBT_INTERNAL_SOURCE + ) + {%- if incremental_predicates %} + {% for predicate in incremental_predicates %} + and {{ predicate }} + {% endfor %} + {%- endif -%}; + + {% endif %} + + insert into {{ target }} ({{ dest_cols_csv }}) + ( + select {{ dest_cols_csv }} + from {{ source }} + ) + +{%- endmacro %} + +-- funcsign: (string, string, list[base_column], optional[list[string]], optional[bool]) -> string +{% macro get_insert_overwrite_merge_sql(target, source, dest_columns, predicates, include_sql_header=false) -%} + {{ adapter.dispatch('get_insert_overwrite_merge_sql', 'dbt')(target, source, dest_columns, predicates, include_sql_header) }} +{%- endmacro %} + +-- funcsign: (string, string, list[base_column], optional[list[string]], optional[bool]) -> string +{% macro default__get_insert_overwrite_merge_sql(target, source, dest_columns, predicates, include_sql_header) -%} + {#-- The only time include_sql_header is True: --#} + {#-- BigQuery + insert_overwrite strategy + "static" partitions config --#} + {#-- We should consider including the sql header at the materialization level instead --#} + + {%- set predicates = [] if predicates is none else [] + predicates -%} + {%- set dest_cols_csv = get_quoted_csv(dest_columns | map(attribute="name")) -%} + {%- set sql_header = config.get('sql_header', none) -%} + + {{ sql_header if sql_header is not none and include_sql_header }} + + merge into {{ target }} as DBT_INTERNAL_DEST + using {{ source }} as DBT_INTERNAL_SOURCE + on FALSE + + when not matched by source + {% if predicates %} and {{ predicates | join(' and ') }} {% endif %} + then delete + + when not matched then insert + ({{ dest_cols_csv }}) + values + ({{ dest_cols_csv }}) + +{% endmacro %} diff --git a/dbt_internal_packages/dbt-adapters/macros/materializations/models/incremental/on_schema_change.sql b/dbt_internal_packages/dbt-adapters/macros/materializations/models/incremental/on_schema_change.sql new file mode 100644 index 0000000..200ebf7 --- /dev/null +++ b/dbt_internal_packages/dbt-adapters/macros/materializations/models/incremental/on_schema_change.sql @@ -0,0 +1,145 @@ +-- funcsign: (string, optional[string]) -> string +{% macro incremental_validate_on_schema_change(on_schema_change, default='ignore') %} + + {% if on_schema_change not in ['sync_all_columns', 'append_new_columns', 'fail', 'ignore'] %} + + {% set log_message = 'Invalid value for on_schema_change (%s) specified. Setting default value of %s.' % (on_schema_change, default) %} + {% do log(log_message) %} + + {{ return(default) }} + + {% else %} + + {{ return(on_schema_change) }} + + {% endif %} + +{% endmacro %} + +-- funcsign: (relation, relation) -> struct{schema_changed: bool, source_not_in_target: list[base_column], target_not_in_source: list[base_column], source_columns: list[base_column], target_columns: list[base_column], new_target_types: list[dict[string, string]]} +{% macro check_for_schema_changes(source_relation, target_relation) %} + + {% set schema_changed = False %} + + {%- set source_columns = adapter.get_columns_in_relation(source_relation) -%} + {%- set target_columns = adapter.get_columns_in_relation(target_relation) -%} + {%- set source_not_in_target = diff_columns(source_columns, target_columns) -%} + {%- set target_not_in_source = diff_columns(target_columns, source_columns) -%} + + {% set new_target_types = diff_column_data_types(source_columns, target_columns) %} + + {% if source_not_in_target != [] %} + {% set schema_changed = True %} + {% elif target_not_in_source != [] or new_target_types != [] %} + {% set schema_changed = True %} + {% elif new_target_types != [] %} + {% set schema_changed = True %} + {% endif %} + + {% set changes_dict = { + 'schema_changed': schema_changed, + 'source_not_in_target': source_not_in_target, + 'target_not_in_source': target_not_in_source, + 'source_columns': source_columns, + 'target_columns': target_columns, + 'new_target_types': new_target_types + } %} + + {% set msg %} + In {{ target_relation }}: + Schema changed: {{ schema_changed }} + Source columns not in target: {{ source_not_in_target }} + Target columns not in source: {{ target_not_in_source }} + New column types: {{ new_target_types }} + {% endset %} + + {% do log(msg) %} + + {{ return(changes_dict) }} + +{% endmacro %} + +-- funcsign: (string, relation, struct{schema_changed: bool, source_not_in_target: list[base_column], target_not_in_source: list[base_column], source_columns: list[base_column], target_columns: list[base_column], new_target_types: list[dict[string, string]]}) -> string +{% macro sync_column_schemas(on_schema_change, target_relation, schema_changes_dict) %} + + {%- set add_to_target_arr = schema_changes_dict['source_not_in_target'] -%} + + {%- if on_schema_change == 'append_new_columns'-%} + {%- if add_to_target_arr | length > 0 -%} + {%- do alter_relation_add_remove_columns(target_relation, add_to_target_arr, none) -%} + {%- endif -%} + + {% elif on_schema_change == 'sync_all_columns' %} + {%- set remove_from_target_arr = schema_changes_dict['target_not_in_source'] -%} + {%- set new_target_types = schema_changes_dict['new_target_types'] -%} + + {% if add_to_target_arr | length > 0 or remove_from_target_arr | length > 0 %} + {%- do alter_relation_add_remove_columns(target_relation, add_to_target_arr, remove_from_target_arr) -%} + {% endif %} + + {% if new_target_types != [] %} + {% for ntt in new_target_types %} + {% set column_name = ntt['column_name'] %} + {% set new_type = ntt['new_type'] %} + {% do alter_column_type(target_relation, column_name, new_type) %} + {% endfor %} + {% endif %} + + {% endif %} + + {% set schema_change_message %} + In {{ target_relation }}: + Schema change approach: {{ on_schema_change }} + Columns added: {{ add_to_target_arr }} + Columns removed: {{ remove_from_target_arr }} + Data types changed: {{ new_target_types }} + {% endset %} + + {% do log(schema_change_message) %} + +{% endmacro %} + +-- funcsign: (string, relation, relation) -> list[base_column] +{% macro process_schema_changes(on_schema_change, source_relation, target_relation) %} + + {% if on_schema_change == 'ignore' %} + + {{ return({}) }} -- noqa: return value should be list[base_column] + + {% else %} + + {% set schema_changes_dict = check_for_schema_changes(source_relation, target_relation) %} + + {% if schema_changes_dict['schema_changed'] %} + + {% if on_schema_change == 'fail' %} + + {% set fail_msg %} + The source and target schemas on this incremental model are out of sync! + They can be reconciled in several ways: + - set the `on_schema_change` config to either append_new_columns or sync_all_columns, depending on your situation. + - Re-run the incremental model with `full_refresh: True` to update the target schema. + - update the schema manually and re-run the process. + + Additional troubleshooting context: + Source columns not in target: {{ schema_changes_dict['source_not_in_target'] }} + Target columns not in source: {{ schema_changes_dict['target_not_in_source'] }} + New column types: {{ schema_changes_dict['new_target_types'] }} + {% endset %} + + {% do exceptions.raise_compiler_error(fail_msg) %} + + {# -- unless we ignore, run the sync operation per the config #} + {% else %} + + {% do sync_column_schemas(on_schema_change, target_relation, schema_changes_dict) %} + + {% endif %} + + {% endif %} + + {{ return(schema_changes_dict['source_columns']) }} + + {% endif %} + +{% endmacro %} diff --git a/dbt_internal_packages/dbt-adapters/macros/materializations/models/incremental/strategies.sql b/dbt_internal_packages/dbt-adapters/macros/materializations/models/incremental/strategies.sql new file mode 100644 index 0000000..8af6085 --- /dev/null +++ b/dbt_internal_packages/dbt-adapters/macros/materializations/models/incremental/strategies.sql @@ -0,0 +1,103 @@ +-- funcsign: (struct{target_relation: string, temp_relation: string, dest_columns: list[base_column], unique_key: string, incremental_predicates: optional[list[string]]}) -> string +{% macro get_incremental_append_sql(arg_dict) %} + + {{ return(adapter.dispatch('get_incremental_append_sql', 'dbt')(arg_dict)) }} + +{% endmacro %} + +-- funcsign: (struct{target_relation: string, temp_relation: string, dest_columns: list[base_column], unique_key: string, incremental_predicates: optional[list[string]]}) -> string +{% macro default__get_incremental_append_sql(arg_dict) %} + + {% do return(get_insert_into_sql(arg_dict["target_relation"], arg_dict["temp_relation"], arg_dict["dest_columns"])) %} + +{% endmacro %} + + +{# snowflake #} +-- funcsign: (struct{target_relation: string, temp_relation: string, dest_columns: list[base_column], unique_key: string, incremental_predicates: optional[list[string]]}) -> string +{% macro get_incremental_delete_insert_sql(arg_dict) %} + + {{ return(adapter.dispatch('get_incremental_delete_insert_sql', 'dbt')(arg_dict)) }} + +{% endmacro %} + +-- funcsign: (struct{target_relation: string, temp_relation: string, dest_columns: list[base_column], unique_key: string, incremental_predicates: optional[list[string]]}) -> string +{% macro default__get_incremental_delete_insert_sql(arg_dict) %} + + {% do return(get_delete_insert_merge_sql(arg_dict["target_relation"], arg_dict["temp_relation"], arg_dict["unique_key"], arg_dict["dest_columns"], arg_dict["incremental_predicates"])) %} + +{% endmacro %} + + +{# snowflake, bigquery, spark #} +-- funcsign: (struct{target_relation: string, temp_relation: string, dest_columns: list[base_column], unique_key: string, incremental_predicates: optional[list[string]]}) -> string +{% macro get_incremental_merge_sql(arg_dict) %} + + {{ return(adapter.dispatch('get_incremental_merge_sql', 'dbt')(arg_dict)) }} + +{% endmacro %} + +-- funcsign: (struct{target_relation: string, temp_relation: string, dest_columns: list[base_column], unique_key: string, incremental_predicates: optional[list[string]]}) -> string +{% macro default__get_incremental_merge_sql(arg_dict) %} + + {% do return(get_merge_sql(arg_dict["target_relation"], arg_dict["temp_relation"], arg_dict["unique_key"], arg_dict["dest_columns"], arg_dict["incremental_predicates"])) %} + +{% endmacro %} + + +-- funcsign: (struct{target_relation: string, temp_relation: string, dest_columns: list[base_column], unique_key: string, incremental_predicates: optional[list[string]]}) -> string +{% macro get_incremental_insert_overwrite_sql(arg_dict) %} + + {{ return(adapter.dispatch('get_incremental_insert_overwrite_sql', 'dbt')(arg_dict)) }} + +{% endmacro %} + +-- funcsign: (struct{target_relation: string, temp_relation: string, dest_columns: list[base_column], unique_key: string, incremental_predicates: optional[list[string]]}) -> string +{% macro default__get_incremental_insert_overwrite_sql(arg_dict) %} + + {% do return(get_insert_overwrite_merge_sql(arg_dict["target_relation"], arg_dict["temp_relation"], arg_dict["dest_columns"], arg_dict["incremental_predicates"])) %} + +{% endmacro %} + + +-- funcsign: (struct{target_relation: string, temp_relation: string, dest_columns: list[base_column], unique_key: string, incremental_predicates: optional[list[string]]}) -> string +{% macro get_incremental_default_sql(arg_dict) %} + + {{ return(adapter.dispatch('get_incremental_default_sql', 'dbt')(arg_dict)) }} + +{% endmacro %} + +-- funcsign: (struct{target_relation: string, temp_relation: string, dest_columns: list[base_column], unique_key: string, incremental_predicates: optional[list[string]]}) -> string +{% macro default__get_incremental_default_sql(arg_dict) %} + + {% do return(get_incremental_append_sql(arg_dict)) %} + +{% endmacro %} + + +-- funcsign: (struct{target_relation: string, temp_relation: string, dest_columns: list[base_column], unique_key: string, incremental_predicates: optional[list[string]]}) -> string +{% macro get_incremental_microbatch_sql(arg_dict) %} + + {{ return(adapter.dispatch('get_incremental_microbatch_sql', 'dbt')(arg_dict)) }} + +{% endmacro %} + +-- funcsign: (struct{target_relation: string, temp_relation: string, dest_columns: list[base_column], unique_key: string, incremental_predicates: optional[list[string]]}) -> string +{% macro default__get_incremental_microbatch_sql(arg_dict) %} + + {{ exceptions.raise_not_implemented('microbatch materialization strategy not implemented for adapter ' + adapter.type()) }} + +{% endmacro %} + +-- funcsign: (string, string, list[base_column]) -> string +{% macro get_insert_into_sql(target_relation, temp_relation, dest_columns) %} + + {%- set dest_cols_csv = get_quoted_csv(dest_columns | map(attribute="name")) -%} + + insert into {{ target_relation }} ({{ dest_cols_csv }}) + ( + select {{ dest_cols_csv }} + from {{ temp_relation }} + ) + +{% endmacro %} diff --git a/dbt_internal_packages/dbt-adapters/macros/materializations/models/materialized_view.sql b/dbt_internal_packages/dbt-adapters/macros/materializations/models/materialized_view.sql new file mode 100644 index 0000000..3f02050 --- /dev/null +++ b/dbt_internal_packages/dbt-adapters/macros/materializations/models/materialized_view.sql @@ -0,0 +1,123 @@ +{% materialization materialized_view, default %} + {% set existing_relation = load_cached_relation(this) %} + {% set target_relation = this.incorporate(type=this.MaterializedView) %} + {% set intermediate_relation = make_intermediate_relation(target_relation) %} + {% set backup_relation_type = target_relation.MaterializedView if existing_relation is none else existing_relation.type %} + {% set backup_relation = make_backup_relation(target_relation, backup_relation_type) %} + + {{ materialized_view_setup(backup_relation, intermediate_relation, pre_hooks) }} + + {% set build_sql = materialized_view_get_build_sql(existing_relation, target_relation, backup_relation, intermediate_relation) %} + + {% if build_sql == '' %} + {{ materialized_view_execute_no_op(target_relation) }} + {% else %} + {{ materialized_view_execute_build_sql(build_sql, existing_relation, target_relation, post_hooks) }} + {% endif %} + + {{ materialized_view_teardown(backup_relation, intermediate_relation, post_hooks) }} + + {{ return({'relations': [target_relation]}) }} + +{% endmaterialization %} + +-- funcsign: (relation, relation, list[hook]) -> string +{% macro materialized_view_setup(backup_relation, intermediate_relation, pre_hooks) %} + + -- backup_relation and intermediate_relation should not already exist in the database + -- it's possible these exist because of a previous run that exited unexpectedly + {% set preexisting_backup_relation = load_cached_relation(backup_relation) %} + {% set preexisting_intermediate_relation = load_cached_relation(intermediate_relation) %} + + -- drop the temp relations if they exist already in the database + {{ drop_relation_if_exists(preexisting_backup_relation) }} + {{ drop_relation_if_exists(preexisting_intermediate_relation) }} + + {{ run_hooks(pre_hooks, inside_transaction=False) }} + +{% endmacro %} + +-- funcsign: (optional[relation], optional[relation], list[hook]) -> string +{% macro materialized_view_teardown(backup_relation, intermediate_relation, post_hooks) %} + + -- drop the temp relations if they exist to leave the database clean for the next run + {{ drop_relation_if_exists(backup_relation) }} + {{ drop_relation_if_exists(intermediate_relation) }} + + {{ run_hooks(post_hooks, inside_transaction=False) }} + +{% endmacro %} + + +-- funcsign: (optional[relation], relation, relation, relation) -> string +{% macro materialized_view_get_build_sql(existing_relation, target_relation, backup_relation, intermediate_relation) %} + + {% set full_refresh_mode = should_full_refresh() %} + + -- determine the scenario we're in: create, full_refresh, alter, refresh data + {% if existing_relation is none %} + {% set build_sql = get_create_materialized_view_as_sql(target_relation, sql) %} + {% elif full_refresh_mode or not existing_relation.is_materialized_view %} + {% set build_sql = get_replace_sql(existing_relation, target_relation, sql) %} + {% else %} + + -- get config options + {% set on_configuration_change = config.get('on_configuration_change', 'apply') %} + {% set configuration_changes = get_materialized_view_configuration_changes(existing_relation, config) %} + + {% if configuration_changes is none %} + {% set build_sql = refresh_materialized_view(target_relation) %} + + {% elif on_configuration_change == 'apply' %} + {% set build_sql = get_alter_materialized_view_as_sql(target_relation, configuration_changes, sql, existing_relation, backup_relation, intermediate_relation) %} + {% elif on_configuration_change == 'continue' %} + {% set build_sql = '' %} + {{ exceptions.warn("Configuration changes were identified and `on_configuration_change` was set to `continue` for `" ~ target_relation.render() ~ "`") }} + {% elif on_configuration_change == 'fail' %} + {{ exceptions.raise_fail_fast_error("Configuration changes were identified and `on_configuration_change` was set to `fail` for `" ~ target_relation.render() ~ "`") }} + + {% else %} + -- this only happens if the user provides a value other than `apply`, 'skip', 'fail' + {{ exceptions.raise_compiler_error("Unexpected configuration scenario") }} + + {% endif %} + + {% endif %} + + {% do return(build_sql) %} + +{% endmacro %} + + +-- funcsign: (relation) -> string +{% macro materialized_view_execute_no_op(target_relation) %} + {% do store_raw_result( + name="main", + message="skip " ~ target_relation, + code="skip", + rows_affected="-1" + ) %} +{% endmacro %} + +-- funcsign: (string, optional[relation], relation, list[hook]) -> string +{% macro materialized_view_execute_build_sql(build_sql, existing_relation, target_relation, post_hooks) %} + + -- `BEGIN` happens here: + {{ run_hooks(pre_hooks, inside_transaction=True) }} + + {% set grant_config = config.get('grants') %} + + {% call statement(name="main") %} + {{ build_sql }} + {% endcall %} + + {% set should_revoke = should_revoke(existing_relation, full_refresh_mode=True) %} + {% do apply_grants(target_relation, grant_config, should_revoke=should_revoke) %} + + {% do persist_docs(target_relation, model) %} + + {{ run_hooks(post_hooks, inside_transaction=True) }} + + {{ adapter.commit() }} + +{% endmacro %} diff --git a/dbt_internal_packages/dbt-adapters/macros/materializations/models/table.sql b/dbt_internal_packages/dbt-adapters/macros/materializations/models/table.sql new file mode 100644 index 0000000..fb9f073 --- /dev/null +++ b/dbt_internal_packages/dbt-adapters/macros/materializations/models/table.sql @@ -0,0 +1,64 @@ +{% materialization table, default %} + + {%- set existing_relation = load_cached_relation(this) -%} + {%- set target_relation = this.incorporate(type='table') %} + {%- set intermediate_relation = make_intermediate_relation(target_relation) -%} + -- the intermediate_relation should not already exist in the database; get_relation + -- will return None in that case. Otherwise, we get a relation that we can drop + -- later, before we try to use this name for the current operation + {%- set preexisting_intermediate_relation = load_cached_relation(intermediate_relation) -%} + /* + See ../view/view.sql for more information about this relation. + */ + {%- set backup_relation_type = 'table' if existing_relation is none else existing_relation.type -%} + {%- set backup_relation = make_backup_relation(target_relation, backup_relation_type) -%} + -- as above, the backup_relation should not already exist + {%- set preexisting_backup_relation = load_cached_relation(backup_relation) -%} + -- grab current tables grants config for comparision later on + {% set grant_config = config.get('grants') %} + + -- drop the temp relations if they exist already in the database + {{ drop_relation_if_exists(preexisting_intermediate_relation) }} + {{ drop_relation_if_exists(preexisting_backup_relation) }} + + {{ run_hooks(pre_hooks, inside_transaction=False) }} + + -- `BEGIN` happens here: + {{ run_hooks(pre_hooks, inside_transaction=True) }} + + -- build model + {% call statement('main') -%} + {{ get_create_table_as_sql(False, intermediate_relation, sql) }} + {%- endcall %} + + {% do create_indexes(intermediate_relation) %} + + -- cleanup + {% if existing_relation is not none %} + /* Do the equivalent of rename_if_exists. 'existing_relation' could have been dropped + since the variable was first set. */ + {% set existing_relation = load_cached_relation(existing_relation) %} + {% if existing_relation is not none %} + {{ adapter.rename_relation(existing_relation, backup_relation) }} + {% endif %} + {% endif %} + + {{ adapter.rename_relation(intermediate_relation, target_relation) }} + + {{ run_hooks(post_hooks, inside_transaction=True) }} + + {% set should_revoke = should_revoke(existing_relation, full_refresh_mode=True) %} + {% do apply_grants(target_relation, grant_config, should_revoke=should_revoke) %} + + {% do persist_docs(target_relation, model) %} + + -- `COMMIT` happens here + {{ adapter.commit() }} + + -- finally, drop the existing/backup relation after the commit + {{ drop_relation_if_exists(backup_relation) }} + + {{ run_hooks(post_hooks, inside_transaction=False) }} + + {{ return({'relations': [target_relation]}) }} +{% endmaterialization %} diff --git a/dbt_internal_packages/dbt-adapters/macros/materializations/models/view.sql b/dbt_internal_packages/dbt-adapters/macros/materializations/models/view.sql new file mode 100644 index 0000000..59ac6c4 --- /dev/null +++ b/dbt_internal_packages/dbt-adapters/macros/materializations/models/view.sql @@ -0,0 +1,72 @@ +{%- materialization view, default -%} + + {%- set existing_relation = load_cached_relation(this) -%} + {%- set target_relation = this.incorporate(type='view') -%} + {%- set intermediate_relation = make_intermediate_relation(target_relation) -%} + + -- the intermediate_relation should not already exist in the database; get_relation + -- will return None in that case. Otherwise, we get a relation that we can drop + -- later, before we try to use this name for the current operation + {%- set preexisting_intermediate_relation = load_cached_relation(intermediate_relation) -%} + /* + This relation (probably) doesn't exist yet. If it does exist, it's a leftover from + a previous run, and we're going to try to drop it immediately. At the end of this + materialization, we're going to rename the "existing_relation" to this identifier, + and then we're going to drop it. In order to make sure we run the correct one of: + - drop view ... + - drop table ... + + We need to set the type of this relation to be the type of the existing_relation, if it exists, + or else "view" as a sane default if it does not. Note that if the existing_relation does not + exist, then there is nothing to move out of the way and subsequentally drop. In that case, + this relation will be effectively unused. + */ + {%- set backup_relation_type = 'view' if existing_relation is none else existing_relation.type -%} + {%- set backup_relation = make_backup_relation(target_relation, backup_relation_type) -%} + -- as above, the backup_relation should not already exist + {%- set preexisting_backup_relation = load_cached_relation(backup_relation) -%} + -- grab current tables grants config for comparision later on + {% set grant_config = config.get('grants') %} + + {{ run_hooks(pre_hooks, inside_transaction=False) }} + + -- drop the temp relations if they exist already in the database + {{ drop_relation_if_exists(preexisting_intermediate_relation) }} + {{ drop_relation_if_exists(preexisting_backup_relation) }} + + -- `BEGIN` happens here: + {{ run_hooks(pre_hooks, inside_transaction=True) }} + + -- build model + {% call statement('main') -%} + {{ get_create_view_as_sql(intermediate_relation, sql) }} + {%- endcall %} + + -- cleanup + -- move the existing view out of the way + {% if existing_relation is not none %} + /* Do the equivalent of rename_if_exists. 'existing_relation' could have been dropped + since the variable was first set. */ + {% set existing_relation = load_cached_relation(existing_relation) %} + {% if existing_relation is not none %} + {{ adapter.rename_relation(existing_relation, backup_relation) }} + {% endif %} + {% endif %} + {{ adapter.rename_relation(intermediate_relation, target_relation) }} + + {% set should_revoke = should_revoke(existing_relation, full_refresh_mode=True) %} + {% do apply_grants(target_relation, grant_config, should_revoke=should_revoke) %} + + {% do persist_docs(target_relation, model) %} + + {{ run_hooks(post_hooks, inside_transaction=True) }} + + {{ adapter.commit() }} + + {{ drop_relation_if_exists(backup_relation) }} + + {{ run_hooks(post_hooks, inside_transaction=False) }} + + {{ return({'relations': [target_relation]}) }} + +{%- endmaterialization -%} diff --git a/dbt_internal_packages/dbt-adapters/macros/materializations/seeds/helpers.sql b/dbt_internal_packages/dbt-adapters/macros/materializations/seeds/helpers.sql new file mode 100644 index 0000000..073c40f --- /dev/null +++ b/dbt_internal_packages/dbt-adapters/macros/materializations/seeds/helpers.sql @@ -0,0 +1,134 @@ +-- funcsign: (model, agate_table) -> string +{% macro create_csv_table(model, agate_table) -%} + {{ adapter.dispatch('create_csv_table', 'dbt')(model, agate_table) }} +{%- endmacro %} + +-- funcsign: (model, agate_table) -> string +{% macro default__create_csv_table(model, agate_table) %} + {%- set column_override = model['config'].get('column_types', {}) -%} + {%- set quote_seed_column = model['config'].get('quote_columns', None) -%} + + {% set sql %} + create table {{ this.render() }} ( + {%- for col_name in agate_table.column_names -%} + {%- set inferred_type = adapter.convert_type(agate_table, loop.index0) -%} + {%- set type = column_override.get(col_name, inferred_type) -%} + {%- set column_name = (col_name | string) -%} + {{ adapter.quote_seed_column(column_name, quote_seed_column) }} {{ type }} {%- if not loop.last -%}, {%- endif -%} + {%- endfor -%} + ) + {% endset %} + + {% call statement('_') -%} + {{ sql }} + {%- endcall %} + + {{ return(sql) }} +{% endmacro %} + +-- funcsign: (model, bool, relation, agate_table) -> string +{% macro reset_csv_table(model, full_refresh, old_relation, agate_table) -%} + {{ adapter.dispatch('reset_csv_table', 'dbt')(model, full_refresh, old_relation, agate_table) }} +{%- endmacro %} + +-- funcsign: (model, bool, relation, agate_table) -> string +{% macro default__reset_csv_table(model, full_refresh, old_relation, agate_table) %} + {% set sql = "" %} + {% if full_refresh %} + {{ adapter.drop_relation(old_relation) }} + {% set sql = create_csv_table(model, agate_table) %} + {% else %} + {{ adapter.truncate_relation(old_relation) }} + {% set sql = "truncate table " ~ old_relation.render() %} + {% endif %} + + {{ return(sql) }} +{% endmacro %} + +-- funcsign: (string, string) -> string +{% macro get_csv_sql(create_or_truncate_sql, insert_sql) %} + {{ adapter.dispatch('get_csv_sql', 'dbt')(create_or_truncate_sql, insert_sql) }} +{% endmacro %} + +-- funcsign: (string, string) -> string +{% macro default__get_csv_sql(create_or_truncate_sql, insert_sql) %} + {{ create_or_truncate_sql }}; + -- dbt seed -- + {{ insert_sql }} +{% endmacro %} + +-- funcsign: () -> string +{% macro get_binding_char() -%} + {{ adapter.dispatch('get_binding_char', 'dbt')() }} +{%- endmacro %} + +-- funcsign: () -> string +{% macro default__get_binding_char() %} + {{ return('%s') }} +{% endmacro %} + +-- funcsign: () -> integer +{% macro get_batch_size() -%} + {{ return(adapter.dispatch('get_batch_size', 'dbt')()) }} +{%- endmacro %} + +-- funcsign: () -> integer +{% macro default__get_batch_size() %} + {{ return(10000) }} +{% endmacro %} + +-- funcsign: (model, list[string]) -> string +{% macro get_seed_column_quoted_csv(model, column_names) %} + {%- set quote_seed_column = model['config'].get('quote_columns', None) -%} + {% set quoted = [] %} + {% for col in column_names -%} + {%- do quoted.append(adapter.quote_seed_column(col, quote_seed_column)) -%} + {%- endfor %} + + {%- set dest_cols_csv = quoted | join(', ') -%} + {{ return(dest_cols_csv) }} +{% endmacro %} + +-- funcsign: (model, agate_table) -> string +{% macro load_csv_rows(model, agate_table) -%} + {{ adapter.dispatch('load_csv_rows', 'dbt')(model, agate_table) }} +{%- endmacro %} + +-- funcsign: (model, agate_table) -> string +{% macro default__load_csv_rows(model, agate_table) %} + + {% set batch_size = get_batch_size() %} + + {% set cols_sql = get_seed_column_quoted_csv(model, agate_table.column_names) %} + {% set bindings = [] %} + + {% set statements = [] %} + + {% for chunk in agate_table.rows | batch(batch_size) %} + {% set bindings = [] %} + + {% for row in chunk %} + {% do bindings.extend(row) %} + {% endfor %} + + {% set sql %} + insert into {{ this.render() }} ({{ cols_sql }}) values + {% for row in chunk -%} + ({%- for column in agate_table.column_names -%} + {{ get_binding_char() }} + {%- if not loop.last%},{%- endif %} + {%- endfor -%}) + {%- if not loop.last%},{%- endif %} + {%- endfor %} + {% endset %} + + {% do adapter.add_query(sql, bindings=bindings, abridge_sql_log=True) %} + + {% if loop.index0 == 0 %} + {% do statements.append(sql) %} + {% endif %} + {% endfor %} + + {# Return SQL so we can render it out into the compiled files #} + {{ return(statements[0]) }} +{% endmacro %} diff --git a/dbt_internal_packages/dbt-adapters/macros/materializations/seeds/seed.sql b/dbt_internal_packages/dbt-adapters/macros/materializations/seeds/seed.sql new file mode 100644 index 0000000..14ee4c3 --- /dev/null +++ b/dbt_internal_packages/dbt-adapters/macros/materializations/seeds/seed.sql @@ -0,0 +1,60 @@ +{% materialization seed, default %} + + {%- set identifier = model['alias'] -%} + {%- set full_refresh_mode = (should_full_refresh()) -%} + + {%- set old_relation = adapter.get_relation(database=database, schema=schema, identifier=identifier) -%} + + {%- set exists_as_table = (old_relation is not none and old_relation.is_table) -%} + {%- set exists_as_view = (old_relation is not none and old_relation.is_view) -%} + + {%- set grant_config = config.get('grants') -%} + {%- set agate_table = load_agate_table() -%} + -- grab current tables grants config for comparison later on + + {%- do store_result('agate_table', response='OK', agate_table=agate_table) -%} + + {{ run_hooks(pre_hooks, inside_transaction=False) }} + + -- `BEGIN` happens here: + {{ run_hooks(pre_hooks, inside_transaction=True) }} + + -- build model + {% set create_table_sql = "" %} + {% if exists_as_view %} + {{ exceptions.raise_compiler_error("Cannot seed to '{}', it is a view".format(old_relation.render())) }} -- noqa: optional[relation] does not support render() + {% elif exists_as_table %} + {% set create_table_sql = reset_csv_table(model, full_refresh_mode, old_relation, agate_table) %} -- noqa: optional[relation] as arguments + {% else %} + {% set create_table_sql = create_csv_table(model, agate_table) %} + {% endif %} + + {% set code = 'CREATE' if full_refresh_mode else 'INSERT' %} + {% set rows_affected = (agate_table.rows | length) %} + {% set sql = load_csv_rows(model, agate_table) %} + + {% call noop_statement('main', code ~ ' ' ~ rows_affected, code, rows_affected) %} -- noqa: Should accept a string instead of a integer + {{ get_csv_sql(create_table_sql, sql) }}; + {% endcall %} + + {% set target_relation = this.incorporate(type='table') %} + + {% set should_revoke = should_revoke(old_relation, full_refresh_mode) %} + {% do apply_grants(target_relation, grant_config, should_revoke=should_revoke) %} + + {% do persist_docs(target_relation, model) %} + + {% if full_refresh_mode or not exists_as_table %} + {% do create_indexes(target_relation) %} + {% endif %} + + {{ run_hooks(post_hooks, inside_transaction=True) }} + + -- `COMMIT` happens here + {{ adapter.commit() }} + + {{ run_hooks(post_hooks, inside_transaction=False) }} + + {{ return({'relations': [target_relation]}) }} + +{% endmaterialization %} diff --git a/dbt_internal_packages/dbt-adapters/macros/materializations/snapshots/helpers.sql b/dbt_internal_packages/dbt-adapters/macros/materializations/snapshots/helpers.sql new file mode 100644 index 0000000..f57a5b7 --- /dev/null +++ b/dbt_internal_packages/dbt-adapters/macros/materializations/snapshots/helpers.sql @@ -0,0 +1,320 @@ +{# + Add new columns to the table if applicable +#} +-- funcsign: (relation, list[base_column]) -> string +{% macro create_columns(relation, columns) %} + {{ adapter.dispatch('create_columns', 'dbt')(relation, columns) }} +{% endmacro %} + +-- funcsign: (relation, list[base_column]) -> string +{% macro default__create_columns(relation, columns) %} + {% for column in columns %} + {% call statement() %} + alter table {{ relation.render() }} add column "{{ column.name }}" {{ column.data_type }}; + {% endcall %} + {% endfor %} +{% endmacro %} + +-- funcsign: (relation) -> string +{% macro post_snapshot(staging_relation) %} + {{ adapter.dispatch('post_snapshot', 'dbt')(staging_relation) }} +{% endmacro %} + +-- funcsign: (relation) -> string +{% macro default__post_snapshot(staging_relation) %} + {# no-op #} +{% endmacro %} + +-- funcsign: () -> string +{% macro get_true_sql() %} + {{ adapter.dispatch('get_true_sql', 'dbt')() }} +{% endmacro %} + +-- funcsign: () -> string +{% macro default__get_true_sql() %} + {{ return('TRUE') }} +{% endmacro %} + +-- funcsign: (strategy, string, relation) -> string +{% macro snapshot_staging_table(strategy, source_sql, target_relation) -%} + {{ adapter.dispatch('snapshot_staging_table', 'dbt')(strategy, source_sql, target_relation) }} +{% endmacro %} + +-- funcsign: () -> struct{dbt_valid_to: string, dbt_valid_from: string, dbt_scd_id: string, dbt_updated_at: string, dbt_is_deleted: string} +{% macro get_snapshot_table_column_names() %} + {{ return({'dbt_valid_to': 'dbt_valid_to', 'dbt_valid_from': 'dbt_valid_from', 'dbt_scd_id': 'dbt_scd_id', 'dbt_updated_at': 'dbt_updated_at', 'dbt_is_deleted': 'dbt_is_deleted'}) }} +{% endmacro %} + +-- funcsign: (strategy, string, relation) -> string +{% macro default__snapshot_staging_table(strategy, source_sql, target_relation) -%} + {% set columns = config.get('snapshot_table_column_names') or get_snapshot_table_column_names() %} + {% if strategy.hard_deletes == 'new_record' %} + {% set new_scd_id = snapshot_hash_arguments([columns.dbt_scd_id, snapshot_get_time()]) %} + {% endif %} + with snapshot_query as ( + + {{ source_sql }} + + ), + + snapshotted_data as ( + + select *, {{ unique_key_fields(strategy.unique_key) }} + from {{ target_relation }} + where + {% if config.get('dbt_valid_to_current') %} + {% set source_unique_key = columns.dbt_valid_to | trim %} + {% set target_unique_key = config.get('dbt_valid_to_current') | trim %} -- noqa: optional[string] does not support trim + + {# The exact equals semantics between NULL values depends on the current behavior flag set. Also, update records if the source field is null #} + ( {{ equals(source_unique_key, target_unique_key) }} or {{ source_unique_key }} is null ) + {% else %} + {{ columns.dbt_valid_to }} is null + {% endif %} + + ), + + insertions_source_data as ( + + select *, {{ unique_key_fields(strategy.unique_key) }}, + {{ strategy.updated_at }} as {{ columns.dbt_updated_at }}, + {{ strategy.updated_at }} as {{ columns.dbt_valid_from }}, + {{ get_dbt_valid_to_current(strategy, columns) }}, + {{ strategy.scd_id }} as {{ columns.dbt_scd_id }} + + from snapshot_query + ), + + updates_source_data as ( + + select *, {{ unique_key_fields(strategy.unique_key) }}, + {{ strategy.updated_at }} as {{ columns.dbt_updated_at }}, + {{ strategy.updated_at }} as {{ columns.dbt_valid_from }}, + {{ strategy.updated_at }} as {{ columns.dbt_valid_to }} + + from snapshot_query + ), + + {%- if strategy.hard_deletes == 'invalidate' or strategy.hard_deletes == 'new_record' %} + + deletes_source_data as ( + + select *, {{ unique_key_fields(strategy.unique_key) }} + from snapshot_query + ), + {% endif %} + + insertions as ( + + select + 'insert' as dbt_change_type, + source_data.* + {%- if strategy.hard_deletes == 'new_record' -%} + ,'False' as {{ columns.dbt_is_deleted }} + {%- endif %} + + from insertions_source_data as source_data + left outer join snapshotted_data + on {{ unique_key_join_on(strategy.unique_key, "snapshotted_data", "source_data") }} + where {{ unique_key_is_null(strategy.unique_key, "snapshotted_data") }} + or ({{ unique_key_is_not_null(strategy.unique_key, "snapshotted_data") }} and ({{ strategy.row_changed }}) + + ) + + ), + + updates as ( + + select + 'update' as dbt_change_type, + source_data.*, + snapshotted_data.{{ columns.dbt_scd_id }} + {%- if strategy.hard_deletes == 'new_record' -%} + , snapshotted_data.{{ columns.dbt_is_deleted }} + {%- endif %} + + from updates_source_data as source_data + join snapshotted_data + on {{ unique_key_join_on(strategy.unique_key, "snapshotted_data", "source_data") }} + where ( + {{ strategy.row_changed }} + ) + ) + + {%- if strategy.hard_deletes == 'invalidate' or strategy.hard_deletes == 'new_record' %} + , + deletes as ( + + select + 'delete' as dbt_change_type, + source_data.*, + {{ snapshot_get_time() }} as {{ columns.dbt_valid_from }}, + {{ snapshot_get_time() }} as {{ columns.dbt_updated_at }}, + {{ snapshot_get_time() }} as {{ columns.dbt_valid_to }}, + snapshotted_data.{{ columns.dbt_scd_id }} + {%- if strategy.hard_deletes == 'new_record' -%} + , snapshotted_data.{{ columns.dbt_is_deleted }} + {%- endif %} + from snapshotted_data + left join deletes_source_data as source_data + on {{ unique_key_join_on(strategy.unique_key, "snapshotted_data", "source_data") }} + where {{ unique_key_is_null(strategy.unique_key, "source_data") }} + ) + {%- endif %} + + {%- if strategy.hard_deletes == 'new_record' %} + {% set source_sql_cols = get_column_schema_from_query(source_sql) %} + , + deletion_records as ( + + select + 'insert' as dbt_change_type, + {%- for col in source_sql_cols -%} + snapshotted_data.{{ adapter.quote(col.column) }}, + {% endfor -%} + {%- if strategy.unique_key | is_list -%} + {%- for key in strategy.unique_key -%} + snapshotted_data.{{ key }} as dbt_unique_key_{{ loop.index }}, + {% endfor -%} + {%- else -%} + snapshotted_data.dbt_unique_key as dbt_unique_key, + {% endif -%} + {{ snapshot_get_time() }} as {{ columns.dbt_valid_from }}, + {{ snapshot_get_time() }} as {{ columns.dbt_updated_at }}, + snapshotted_data.{{ columns.dbt_valid_to }} as {{ columns.dbt_valid_to }}, + {{ new_scd_id }} as {{ columns.dbt_scd_id }}, + 'True' as {{ columns.dbt_is_deleted }} + from snapshotted_data + left join deletes_source_data as source_data + on {{ unique_key_join_on(strategy.unique_key, "snapshotted_data", "source_data") }} + where {{ unique_key_is_null(strategy.unique_key, "source_data") }} + + ) + {%- endif %} + + select * from insertions + union all + select * from updates + {%- if strategy.hard_deletes == 'invalidate' or strategy.hard_deletes == 'new_record' %} + union all + select * from deletes + {%- endif %} + {%- if strategy.hard_deletes == 'new_record' %} + union all + select * from deletion_records + {%- endif %} + + +{%- endmacro %} + +-- funcsign: (strategy, string) -> string +{% macro build_snapshot_table(strategy, sql) -%} + {{ adapter.dispatch('build_snapshot_table', 'dbt')(strategy, sql) }} +{% endmacro %} + +-- funcsign: (strategy, string) -> string +{% macro default__build_snapshot_table(strategy, sql) %} + {% set columns = config.get('snapshot_table_column_names') or get_snapshot_table_column_names() %} + + select *, + {{ strategy.scd_id }} as {{ columns.dbt_scd_id }}, + {{ strategy.updated_at }} as {{ columns.dbt_updated_at }}, + {{ strategy.updated_at }} as {{ columns.dbt_valid_from }}, + {{ get_dbt_valid_to_current(strategy, columns) }} + {%- if strategy.hard_deletes == 'new_record' -%} + , 'False' as {{ columns.dbt_is_deleted }} + {% endif -%} + from ( + {{ sql }} + ) sbq + +{% endmacro %} + +-- funcsign: (strategy, string, relation) -> relation +{% macro build_snapshot_staging_table(strategy, sql, target_relation) %} + {% set temp_relation = make_temp_relation(target_relation) %} + + {% set select = snapshot_staging_table(strategy, sql, target_relation) %} + + {% call statement('build_snapshot_staging_relation') %} + {{ create_table_as(True, temp_relation, select) }} + {% endcall %} + + {% do return(temp_relation) %} +{% endmacro %} + +-- funcsign: (string) -> string +{% macro get_updated_at_column_data_type(snapshot_sql) %} + {% set snapshot_sql_column_schema = get_column_schema_from_query(snapshot_sql) %} + {% set dbt_updated_at_data_type = null %} + {% set ns = namespace() -%} {#-- handle for-loop scoping with a namespace --#} + {% set ns.dbt_updated_at_data_type = null -%} + {% for column in snapshot_sql_column_schema %} + {% if ((column.column == 'dbt_updated_at') or (column.column == 'DBT_UPDATED_AT')) %} + {% set ns.dbt_updated_at_data_type = column.dtype %} + {% endif %} + {% endfor %} + {{ return(ns.dbt_updated_at_data_type or none) }} +{% endmacro %} + +-- funcsign: (string) -> string +{% macro check_time_data_types(sql) %} + {% set dbt_updated_at_data_type = get_updated_at_column_data_type(sql) %} + {% set snapshot_get_time_data_type = get_snapshot_get_time_data_type() %} + {% if snapshot_get_time_data_type is not none and dbt_updated_at_data_type is not none and snapshot_get_time_data_type != dbt_updated_at_data_type %} + {% if exceptions.warn_snapshot_timestamp_data_types %} + {{ exceptions.warn_snapshot_timestamp_data_types(snapshot_get_time_data_type, dbt_updated_at_data_type) }} + {% endif %} + {% endif %} +{% endmacro %} + +-- funcsign: (strategy, list[base_column]) -> string +{% macro get_dbt_valid_to_current(strategy, columns) %} + {% set dbt_valid_to_current = config.get('dbt_valid_to_current') or "null" %} + coalesce(nullif({{ strategy.updated_at }}, {{ strategy.updated_at }}), {{dbt_valid_to_current}}) + as {{ columns.dbt_valid_to }} +{% endmacro %} + +-- funcsign: (string|list[string]|none) -> string +{% macro unique_key_fields(unique_key) %} + {% if unique_key | is_list %} + {% for key in unique_key %} + {{ key }} as dbt_unique_key_{{ loop.index }} + {%- if not loop.last %} , {%- endif %} + {% endfor %} + {% else %} + {{ unique_key }} as dbt_unique_key + {% endif %} +{% endmacro %} + +-- funcsign: (string|list[string]|none, string, string) -> string +{% macro unique_key_join_on(unique_key, identifier, from_identifier) %} + {% if unique_key | is_list %} + {% for key in unique_key %} + {% set source_unique_key = (identifier ~ ".dbt_unique_key_" ~ loop.index) | trim %} + {% set target_unique_key = (from_identifier ~ ".dbt_unique_key_" ~ loop.index) | trim %} + {{ equals(source_unique_key, target_unique_key) }} + {%- if not loop.last %} and {%- endif %} + {% endfor %} + {% else %} + {{ identifier }}.dbt_unique_key = {{ from_identifier }}.dbt_unique_key + {% endif %} +{% endmacro %} + +-- funcsign: (string|list[string]|none, string) -> string +{% macro unique_key_is_null(unique_key, identifier) %} + {% if unique_key | is_list %} + {{ identifier }}.dbt_unique_key_1 is null + {% else %} + {{ identifier }}.dbt_unique_key is null + {% endif %} +{% endmacro %} + +-- funcsign: (string|list[string]|none, string) -> string +{% macro unique_key_is_not_null(unique_key, identifier) %} + {% if unique_key | is_list %} + {{ identifier }}.dbt_unique_key_1 is not null + {% else %} + {{ identifier }}.dbt_unique_key is not null + {% endif %} +{% endmacro %} diff --git a/dbt_internal_packages/dbt-adapters/macros/materializations/snapshots/snapshot.sql b/dbt_internal_packages/dbt-adapters/macros/materializations/snapshots/snapshot.sql new file mode 100644 index 0000000..60544ff --- /dev/null +++ b/dbt_internal_packages/dbt-adapters/macros/materializations/snapshots/snapshot.sql @@ -0,0 +1,111 @@ +{% materialization snapshot, default %} + + {%- set target_table = model.get('alias', model.get('name')) -%} + + {%- set strategy_name = config.get('strategy') -%} + {%- set unique_key = config.get('unique_key') %} + -- grab current tables grants config for comparision later on + {%- set grant_config = config.get('grants') -%} + + {% set target_relation_exists, target_relation = get_or_create_relation( + database=model.database, + schema=model.schema, + identifier=target_table, + type='table') -%} + + {%- if not target_relation.is_table -%} + {% do exceptions.relation_wrong_type(target_relation, 'table') %} + {%- endif -%} + + + {{ run_hooks(pre_hooks, inside_transaction=False) }} + + {{ run_hooks(pre_hooks, inside_transaction=True) }} + + {% set strategy_macro = strategy_dispatch(strategy_name) %} + + + {# The model['config'] parameter below is no longer used, but passing anyway for compatibility #} + {# It was a dictionary of config, instead of the config object from the context #} + {% set strategy = strategy_macro(model, "snapshotted_data", "source_data", model['config'], target_relation_exists) %} + + {% if not target_relation_exists %} + + {% set build_sql = build_snapshot_table(strategy, model['compiled_code']) %} + {% set build_or_select_sql = build_sql %} + {% set final_sql = create_table_as(False, target_relation, build_sql) %} + + {% else %} + + {% set columns = config.get("snapshot_table_column_names") or get_snapshot_table_column_names() %} + + {{ adapter.assert_valid_snapshot_target_given_strategy(target_relation, columns, strategy) }} + + {% set build_or_select_sql = snapshot_staging_table(strategy, sql, target_relation) %} + {% set staging_table = build_snapshot_staging_table(strategy, sql, target_relation) %} + + -- this may no-op if the database does not require column expansion + {% do adapter.expand_target_column_types(from_relation=staging_table, + to_relation=target_relation) %} + + {% set remove_columns = ['dbt_change_type', 'DBT_CHANGE_TYPE', 'dbt_unique_key', 'DBT_UNIQUE_KEY'] %} + {% if unique_key | is_list %} + {% for key in strategy.unique_key %} + {{ remove_columns.append('dbt_unique_key_' + loop.index|string) }} + {{ remove_columns.append('DBT_UNIQUE_KEY_' + loop.index|string) }} + {% endfor %} + {% endif %} + + {% set missing_columns = adapter.get_missing_columns(staging_table, target_relation) + | rejectattr('name', 'in', remove_columns) + | list %} + + {% do create_columns(target_relation, missing_columns) %} + + {% set source_columns = adapter.get_columns_in_relation(staging_table) + | rejectattr('name', 'in', remove_columns) + | list %} + + {% set quoted_source_columns = [] %} + {% for column in source_columns %} + {% do quoted_source_columns.append(adapter.quote(column.name)) %} + {% endfor %} + + {% set final_sql = snapshot_merge_sql( + target = target_relation, + source = staging_table, + insert_cols = quoted_source_columns + ) + %} + + {% endif %} + + + {{ check_time_data_types(build_or_select_sql) }} + + {% call statement('main') %} + {{ final_sql }} + {% endcall %} + + {% set should_revoke = should_revoke(target_relation_exists, full_refresh_mode=False) %} + {% do apply_grants(target_relation, grant_config, should_revoke=should_revoke) %} + + {% do persist_docs(target_relation, model) %} + + {% if not target_relation_exists %} + {% do create_indexes(target_relation) %} + {% endif %} + + {{ run_hooks(post_hooks, inside_transaction=True) }} + + {{ adapter.commit() }} + + {% if staging_table is defined %} + {% do post_snapshot(staging_table) %} + {% endif %} + + {{ run_hooks(post_hooks, inside_transaction=False) }} + + {{ return({'relations': [target_relation]}) }} + +{% endmaterialization %} diff --git a/dbt_internal_packages/dbt-adapters/macros/materializations/snapshots/snapshot_merge.sql b/dbt_internal_packages/dbt-adapters/macros/materializations/snapshots/snapshot_merge.sql new file mode 100644 index 0000000..f213003 --- /dev/null +++ b/dbt_internal_packages/dbt-adapters/macros/materializations/snapshots/snapshot_merge.sql @@ -0,0 +1,34 @@ +-- funcsign: (relation, string, list[string]) -> string +{% macro snapshot_merge_sql(target, source, insert_cols) -%} + {{ adapter.dispatch('snapshot_merge_sql', 'dbt')(target, source, insert_cols) }} +{%- endmacro %} + +-- funcsign: (relation, string, list[string]) -> string +{% macro default__snapshot_merge_sql(target, source, insert_cols) -%} + {%- set insert_cols_csv = insert_cols | join(', ') -%} + + {%- set columns = config.get("snapshot_table_column_names") or get_snapshot_table_column_names() -%} + + merge into {{ target.render() }} as DBT_INTERNAL_DEST + using {{ source }} as DBT_INTERNAL_SOURCE + on DBT_INTERNAL_SOURCE.{{ columns.dbt_scd_id }} = DBT_INTERNAL_DEST.{{ columns.dbt_scd_id }} + + when matched + {% if config.get("dbt_valid_to_current") %} + {% set source_unique_key = ("DBT_INTERNAL_DEST." ~ columns.dbt_valid_to) | trim %} + {% set target_unique_key = config.get('dbt_valid_to_current') | trim %} + and ({{ equals(source_unique_key, target_unique_key) }} or {{ source_unique_key }} is null) + + {% else %} + and DBT_INTERNAL_DEST.{{ columns.dbt_valid_to }} is null + {% endif %} + and DBT_INTERNAL_SOURCE.dbt_change_type in ('update', 'delete') + then update + set {{ columns.dbt_valid_to }} = DBT_INTERNAL_SOURCE.{{ columns.dbt_valid_to }} + + when not matched + and DBT_INTERNAL_SOURCE.dbt_change_type = 'insert' + then insert ({{ insert_cols_csv }}) + values ({{ insert_cols_csv }}) + +{% endmacro %} diff --git a/dbt_internal_packages/dbt-adapters/macros/materializations/snapshots/strategies.sql b/dbt_internal_packages/dbt-adapters/macros/materializations/snapshots/strategies.sql new file mode 100644 index 0000000..ac843cc --- /dev/null +++ b/dbt_internal_packages/dbt-adapters/macros/materializations/snapshots/strategies.sql @@ -0,0 +1,187 @@ +{# + Dispatch strategies by name, optionally qualified to a package +#} +-- funcsign: (string) -> (model, string, string, model.config, bool) -> strategy +{% macro strategy_dispatch(name) -%} +{% set original_name = name %} + {% if '.' in name %} + {% set package_name, name = name.split(".", 1) %} + {% else %} + {% set package_name = none %} + {% endif %} + + {% if package_name is none %} + {% set package_context = context %} + {% elif package_name in context %} + {% set package_context = context[package_name] %} + {% else %} + {% set error_msg %} + Could not find package '{{package_name}}', called with '{{original_name}}' + {% endset %} + {{ exceptions.raise_compiler_error(error_msg | trim) }} + {% endif %} + + {%- set search_name = 'snapshot_' ~ name ~ '_strategy' -%} + + {% if search_name not in package_context %} + {% set error_msg %} + The specified strategy macro '{{name}}' was not found in package '{{ package_name }}' + {% endset %} + {{ exceptions.raise_compiler_error(error_msg | trim) }} + {% endif %} + {{ return(package_context[search_name]) }} +{%- endmacro %} + + +{# + Create SCD Hash SQL fields cross-db +#} +-- funcsign: (list[string]) -> string +{% macro snapshot_hash_arguments(args) -%} + {{ adapter.dispatch('snapshot_hash_arguments', 'dbt')(args) }} +{%- endmacro %} + +-- funcsign: (list[string]) -> string +{% macro default__snapshot_hash_arguments(args) -%} + md5({%- for arg in args -%} + coalesce(cast({{ arg }} as varchar ), '') + {% if not loop.last %} || '|' || {% endif %} + {%- endfor -%}) +{%- endmacro %} + +{# + Core strategy definitions +#} + +{% macro snapshot_timestamp_strategy(node, snapshotted_rel, current_rel, model_config, target_exists) %} + {# The model_config parameter is no longer used, but is passed in anyway for compatibility. #} + {% set primary_key = config.get('unique_key') %} + {% set updated_at = config.get('updated_at') %} + {% set hard_deletes = adapter.get_hard_deletes_behavior(config) %} + {% set invalidate_hard_deletes = hard_deletes == 'invalidate' %} + {% set columns = config.get("snapshot_table_column_names") or get_snapshot_table_column_names() %} + + {#/* + The snapshot relation might not have an {{ updated_at }} value if the + snapshot strategy is changed from `check` to `timestamp`. We + should use a dbt-created column for the comparison in the snapshot + table instead of assuming that the user-supplied {{ updated_at }} + will be present in the historical data. + + See https://github.com/dbt-labs/dbt-core/issues/2350 + */ #} + {% set row_changed_expr -%} + ({{ snapshotted_rel }}.{{ columns.dbt_valid_from }} < {{ current_rel }}.{{ updated_at }}) + {%- endset %} + + {% set scd_args = api.Relation.scd_args(primary_key, updated_at) %} -- noqa: updated_at should be a string + {% set scd_id_expr = snapshot_hash_arguments(scd_args) %} + + {% do return({ + "unique_key": primary_key, + "updated_at": updated_at, + "row_changed": row_changed_expr, + "scd_id": scd_id_expr, + "invalidate_hard_deletes": invalidate_hard_deletes, + "hard_deletes": hard_deletes + }) %} +{% endmacro %} + +-- funcsign: (timestamp) -> string +{% macro snapshot_string_as_time(timestamp) -%} + {{ adapter.dispatch('snapshot_string_as_time', 'dbt')(timestamp) }} +{%- endmacro %} + +-- funcsign: (timestamp) -> string +{% macro default__snapshot_string_as_time(timestamp) %} + {% do exceptions.raise_not_implemented( + 'snapshot_string_as_time macro not implemented for adapter '+adapter.type() + ) %} +{% endmacro %} + +-- funcsign: (model, bool, list[string]|string) -> tuple[bool, list[base_column]] +{% macro snapshot_check_all_get_existing_columns(node, target_exists, check_cols_config) -%} + {%- if not target_exists -%} + {#-- no table yet -> return whatever the query does --#} + {{ return((false, query_columns)) }} + {%- endif -%} + + {#-- handle any schema changes --#} + {%- set target_relation = adapter.get_relation(database=node.database, schema=node.schema, identifier=node.alias) -%} + + {% if check_cols_config == 'all' %} + {%- set query_columns = get_columns_in_query(node['compiled_code']) -%} + + {% elif check_cols_config is iterable and (check_cols_config | length) > 0 %} + {#-- query for proper casing/quoting, to support comparison below --#} + {%- set select_check_cols_from_target -%} + {#-- N.B. The whitespace below is necessary to avoid edge case issue with comments --#} + {#-- See: https://github.com/dbt-labs/dbt-core/issues/6781 --#} + select {{ check_cols_config | join(', ') }} from ( + {{ node['compiled_code'] }} + ) subq + {%- endset -%} + {% set query_columns = get_columns_in_query(select_check_cols_from_target) %} + + {% else %} + {% do exceptions.raise_compiler_error("Invalid value for 'check_cols': " ~ check_cols_config) %} + {% endif %} + + {%- set existing_cols = adapter.get_columns_in_relation(target_relation) | map(attribute = 'name') | list -%} -- noqa: should be a relation instead of an optional[string] + {%- set ns = namespace() -%} {#-- handle for-loop scoping with a namespace --#} + {%- set ns.column_added = false -%} + + {%- set intersection = [] -%} + {%- for col in query_columns -%} + {%- if col in existing_cols -%} + {%- do intersection.append(adapter.quote(col)) -%} + {%- else -%} + {% set ns.column_added = true %} + {%- endif -%} + {%- endfor -%} + {{ return((ns.column_added, intersection)) }} +{%- endmacro %} +-- funcsign: (model, string, string, config, bool) -> struct{unique_key: list[string]|string|none, updated_at: string, row_changed: string, scd_id: string, invalidate_hard_deletes: bool, hard_deletes: string} +{% macro snapshot_check_strategy(node, snapshotted_rel, current_rel, model_config, target_exists) %} + {# The model_config parameter is no longer used, but is passed in anyway for compatibility. #} + {% set check_cols_config = config.get('check_cols') %} + {% set primary_key = config.get('unique_key') %} + {% set hard_deletes = adapter.get_hard_deletes_behavior(config) %} + {% set invalidate_hard_deletes = hard_deletes == 'invalidate' %} + {% set updated_at = config.get('updated_at') or snapshot_get_time() %} + + {% set column_added = false %} + + {% set column_added, check_cols = snapshot_check_all_get_existing_columns(node, target_exists, check_cols_config) %} -- noqa: check_cols_config is a string|list[string]|none + + {%- set row_changed_expr -%} + ( + {%- if column_added -%} + {{ get_true_sql() }} + {%- else -%} + {%- for col in check_cols -%} + {{ snapshotted_rel }}.{{ col }} != {{ current_rel }}.{{ col }} + or + ( + (({{ snapshotted_rel }}.{{ col }} is null) and not ({{ current_rel }}.{{ col }} is null)) + or + ((not {{ snapshotted_rel }}.{{ col }} is null) and ({{ current_rel }}.{{ col }} is null)) + ) + {%- if not loop.last %} or {% endif -%} + {%- endfor -%} + {%- endif -%} + ) + {%- endset %} + + {% set scd_args = api.Relation.scd_args(primary_key, updated_at) %} -- noqa: primary_key is a string|list[string]|none + {% set scd_id_expr = snapshot_hash_arguments(scd_args) %} + + {% do return({ + "unique_key": primary_key, + "updated_at": updated_at, + "row_changed": row_changed_expr, + "scd_id": scd_id_expr, + "invalidate_hard_deletes": invalidate_hard_deletes, + "hard_deletes": hard_deletes + }) %} +{% endmacro %} diff --git a/dbt_internal_packages/dbt-adapters/macros/materializations/tests/helpers.sql b/dbt_internal_packages/dbt-adapters/macros/materializations/tests/helpers.sql new file mode 100644 index 0000000..788a190 --- /dev/null +++ b/dbt_internal_packages/dbt-adapters/macros/materializations/tests/helpers.sql @@ -0,0 +1,45 @@ +-- funcsign: (string, string, string, string, optional[integer]) -> string +{% macro get_test_sql(main_sql, fail_calc, warn_if, error_if, limit) -%} + {{ adapter.dispatch('get_test_sql', 'dbt')(main_sql, fail_calc, warn_if, error_if, limit) }} +{%- endmacro %} + +-- funcsign: (string, string, string, string, optional[integer]) -> string +{% macro default__get_test_sql(main_sql, fail_calc, warn_if, error_if, limit) -%} + select + {{ fail_calc }} as failures, + {{ fail_calc }} {{ warn_if }} as should_warn, + {{ fail_calc }} {{ error_if }} as should_error + from ( + {{ main_sql }} + {{ "limit " ~ limit if limit != none }} + ) dbt_internal_test +{%- endmacro %} + +-- funcsign: (string, string, list[string]) -> string +{% macro get_unit_test_sql(main_sql, expected_fixture_sql, expected_column_names) -%} + {{ adapter.dispatch('get_unit_test_sql', 'dbt')(main_sql, expected_fixture_sql, expected_column_names) }} +{%- endmacro %} + +-- funcsign: (string, string, list[string]) -> string +{% macro default__get_unit_test_sql(main_sql, expected_fixture_sql, expected_column_names) -%} +-- Build actual result given inputs +with dbt_internal_unit_test_actual as ( + select + {% for expected_column_name in expected_column_names %}{{expected_column_name}}{% if not loop.last -%},{% endif %}{%- endfor -%}, {{ dbt.string_literal("actual") }} as {{ adapter.quote("actual_or_expected") }} + from ( + {{ main_sql }} + ) _dbt_internal_unit_test_actual +), +-- Build expected result +dbt_internal_unit_test_expected as ( + select + {% for expected_column_name in expected_column_names %}{{expected_column_name}}{% if not loop.last -%}, {% endif %}{%- endfor -%}, {{ dbt.string_literal("expected") }} as {{ adapter.quote("actual_or_expected") }} + from ( + {{ expected_fixture_sql }} + ) _dbt_internal_unit_test_expected +) +-- Union actual and expected results +select * from dbt_internal_unit_test_actual +union all +select * from dbt_internal_unit_test_expected +{%- endmacro %} diff --git a/dbt_internal_packages/dbt-adapters/macros/materializations/tests/test.sql b/dbt_internal_packages/dbt-adapters/macros/materializations/tests/test.sql new file mode 100644 index 0000000..26e9126 --- /dev/null +++ b/dbt_internal_packages/dbt-adapters/macros/materializations/tests/test.sql @@ -0,0 +1,66 @@ +{%- materialization test, default -%} + + {% set relations = [] %} + {% set limit = config.get('limit') %} + + {% set sql_with_limit %} + {{ get_limit_subquery_sql(sql, limit) }} + {% endset %} + + {% if should_store_failures() %} + + {% set identifier = model['alias'] %} + {% set old_relation = adapter.get_relation(database=database, schema=schema, identifier=identifier) %} + + {% set store_failures_as = config.get('store_failures_as') %} + -- if `--store-failures` is invoked via command line and `store_failures_as` is not set, + -- config.get('store_failures_as', 'table') returns None, not 'table' + {% if store_failures_as == none %}{% set store_failures_as = 'table' %}{% endif %} + {% if store_failures_as not in ['table', 'view'] %} + {{ exceptions.raise_compiler_error( + "'" ~ store_failures_as ~ "' is not a valid value for `store_failures_as`. " + "Accepted values are: ['ephemeral', 'table', 'view']" + ) }} + {% endif %} + + {% set target_relation = api.Relation.create( + identifier=identifier, schema=schema, database=database, type=store_failures_as) -%} %} + + {% if old_relation %} + {% do adapter.drop_relation(old_relation) %} + {% endif %} + + {% call statement(auto_begin=True) %} + {{ get_create_sql(target_relation, sql_with_limit) }} + {% endcall %} + + {% do relations.append(target_relation) %} + + {# Since the test failures have already been saved to the database, reuse that result rather than querying again #} + {% set main_sql %} + select * + from {{ target_relation }} + {% endset %} + + {{ adapter.commit() }} + + {% else %} + + {% set main_sql = sql_with_limit %} + + {% endif %} + + {% set fail_calc = config.get('fail_calc') %} + {% set warn_if = config.get('warn_if') %} + {% set error_if = config.get('error_if') %} + + {% call statement('main', fetch_result=True) -%} + + {# The limit has already been included above, and we do not want to duplicate it again. We also want to be safe for macro overrides treating `limit` as a required parameter. #} + {{ get_test_sql(main_sql, fail_calc, warn_if, error_if, limit=none)}} + + {%- endcall %} + + {{ return({'relations': relations}) }} + +{%- endmaterialization -%} diff --git a/dbt_internal_packages/dbt-adapters/macros/materializations/tests/unit.sql b/dbt_internal_packages/dbt-adapters/macros/materializations/tests/unit.sql new file mode 100644 index 0000000..11a6b4c --- /dev/null +++ b/dbt_internal_packages/dbt-adapters/macros/materializations/tests/unit.sql @@ -0,0 +1,33 @@ +{%- materialization unit, default -%} + + {% set relations = [] %} + + {% set expected_rows = config.get('expected_rows') %} + {% set expected_sql = config.get('expected_sql') %} + {% set tested_expected_column_names = expected_rows[0].keys() if (expected_rows | length ) > 0 else get_columns_in_query(sql) %} %} -- noqa: operations may fail as expected_rows may be none + + {%- set target_relation = this.incorporate(type='table') -%} + {%- set temp_relation = make_temp_relation(target_relation)-%} + {% do run_query(get_create_table_as_sql(True, temp_relation, get_empty_subquery_sql(sql))) %} + {%- set columns_in_relation = adapter.get_columns_in_relation(temp_relation) -%} + {%- set column_name_to_data_types = {} -%} + {%- for column in columns_in_relation -%} + {%- do column_name_to_data_types.update({column.name|lower: column.data_type}) -%} + {%- endfor -%} + + {% if not expected_sql %} + {% set expected_sql = get_expected_sql(expected_rows, column_name_to_data_types) %} + {% endif %} + {% set unit_test_sql = get_unit_test_sql(sql, expected_sql, tested_expected_column_names) %} + + {% call statement('main', fetch_result=True) -%} + + {{ unit_test_sql }} + + {%- endcall %} + + {% do adapter.drop_relation(temp_relation) %} + + {{ return({'relations': relations}) }} + +{%- endmaterialization -%} diff --git a/dbt_internal_packages/dbt-adapters/macros/materializations/tests/where_subquery.sql b/dbt_internal_packages/dbt-adapters/macros/materializations/tests/where_subquery.sql new file mode 100644 index 0000000..8e15c86 --- /dev/null +++ b/dbt_internal_packages/dbt-adapters/macros/materializations/tests/where_subquery.sql @@ -0,0 +1,16 @@ +{% macro get_where_subquery(relation) -%} + {% do return(adapter.dispatch('get_where_subquery', 'dbt')(relation)) %} +{%- endmacro %} + +-- funcsign: (relation) -> string +{% macro default__get_where_subquery(relation) -%} + {% set where = config.get('where', '') %} + {% if where %} + {%- set filtered -%} + (select * from {{ relation }} where {{ where }}) dbt_subquery + {%- endset -%} + {% do return(filtered) %} + {%- else -%} + {% do return(relation) %} + {%- endif -%} +{%- endmacro %} diff --git a/dbt_internal_packages/dbt-adapters/macros/python_model/python.sql b/dbt_internal_packages/dbt-adapters/macros/python_model/python.sql new file mode 100644 index 0000000..afecaf1 --- /dev/null +++ b/dbt_internal_packages/dbt-adapters/macros/python_model/python.sql @@ -0,0 +1,117 @@ +-- funcsign: (string) -> string +{% macro resolve_model_name(input_model_name) %} + {{ return(adapter.dispatch('resolve_model_name', 'dbt')(input_model_name)) }} +{% endmacro %} + +-- funcsign: (string) -> string +{%- macro default__resolve_model_name(input_model_name) -%} + {{ input_model_name | string | replace('"', '\"') }} +{%- endmacro -%} + +-- funcsign: (model) -> string +{% macro build_ref_function(model) %} + + {%- set ref_dict = {} -%} + {%- for _ref in model.refs -%} + {% set _ref_args = [_ref.get('package'), _ref['name']] if _ref.get('package') else [_ref['name'],] %} + {%- set resolved = ref(*_ref_args, v=_ref.get('version')) -%} + + {# + We want to get the string of the returned relation by calling .render() in order to skip sample/empty + mode rendering logic. However, people override the default ref macro, and often return a string instead + of a relation (like the ref macro does by default). Thus, to make sure we dont blow things up, we have + to ensure the resolved relation has a .render() method. + #} + {%- if resolved.render is defined and resolved.render is callable -%} + {%- set resolved = resolved.render() -%} + {%- endif -%} + + {%- if _ref.get('version') -%} + {% do _ref_args.extend(["v" ~ _ref['version']]) %} + {%- endif -%} + {%- do ref_dict.update({_ref_args | join('.'): resolve_model_name(resolved)}) -%} + {%- endfor -%} + +def ref(*args, **kwargs): + refs = {{ ref_dict | tojson }} + key = '.'.join(args) + version = kwargs.get("v") or kwargs.get("version") + if version: + key += f".v{version}" + dbt_load_df_function = kwargs.get("dbt_load_df_function") + return dbt_load_df_function(refs[key]) + +{% endmacro %} + +{% macro build_source_function(model) %} + + {%- set source_dict = {} -%} + {%- for _source in model.sources -%} + {%- set resolved = source(*_source) -%} + {%- do source_dict.update({_source | join('.'): resolve_model_name(resolved)}) -%} + {%- endfor -%} + +def source(*args, dbt_load_df_function): + sources = {{ source_dict | tojson }} + key = '.'.join(args) + return dbt_load_df_function(sources[key]) + +{% endmacro %} + +{% macro build_config_dict(model) %} + {%- set config_dict = {} -%} + {% set config_dbt_used = zip(model.config.config_keys_used, model.config.config_keys_defaults) | list %} + {%- for key, default in config_dbt_used -%} + {# weird type testing with enum, would be much easier to write this logic in Python! #} + {%- if key == "language" -%} + {%- set value = "python" -%} + {%- endif -%} + {%- set value = model.config.get(key, default) -%} + {%- do config_dict.update({key: value}) -%} + {%- endfor -%} +config_dict = {{ config_dict }} +{% endmacro %} + +{% macro py_script_postfix(model) %} +# This part is user provided model code +# you will need to copy the next section to run the code +# COMMAND ---------- +# this part is dbt logic for get ref work, do not modify + +{{ build_ref_function(model ) }} +{{ build_source_function(model ) }} +{{ build_config_dict(model) }} + +class config: + def __init__(self, *args, **kwargs): + pass + + @staticmethod + def get(key, default=None): + return config_dict.get(key, default) + +class this: + """dbt.this() or dbt.this.identifier""" + database = "{{ this.database }}" + schema = "{{ this.schema }}" + identifier = "{{ this.identifier }}" + {% set this_relation_name = resolve_model_name(this) %} + def __repr__(self): + return '{{ this_relation_name }}' + + +class dbtObj: + def __init__(self, load_df_function) -> string: + self.source = lambda *args: source(*args, dbt_load_df_function=load_df_function) + self.ref = lambda *args, **kwargs: ref(*args, **kwargs, dbt_load_df_function=load_df_function) + self.config = config + self.this = this() + self.is_incremental = {{ is_incremental() }} + +# COMMAND ---------- +{{py_script_comment()}} +{% endmacro %} + +{#-- entry point for add instuctions for running compiled_code --#} +{%macro py_script_comment()%} +{%endmacro%} diff --git a/dbt_internal_packages/dbt-adapters/macros/relations/column/columns_spec_ddl.sql b/dbt_internal_packages/dbt-adapters/macros/relations/column/columns_spec_ddl.sql new file mode 100644 index 0000000..df08619 --- /dev/null +++ b/dbt_internal_packages/dbt-adapters/macros/relations/column/columns_spec_ddl.sql @@ -0,0 +1,97 @@ +-- funcsign: () -> string +{%- macro get_table_columns_and_constraints() -%} + {{ adapter.dispatch('get_table_columns_and_constraints', 'dbt')() }} +{%- endmacro -%} + +-- funcsign: () -> string +{% macro default__get_table_columns_and_constraints() -%} + {{ return(table_columns_and_constraints()) }} +{%- endmacro %} + +-- funcsign: () -> string +{% macro table_columns_and_constraints() %} + {# loop through user_provided_columns to create DDL with data types and constraints #} + {%- set raw_column_constraints = adapter.render_raw_columns_constraints(raw_columns=model['columns']) -%} + {%- set raw_model_constraints = adapter.render_raw_model_constraints(raw_constraints=model['constraints']) -%} + ( + {% for c in raw_column_constraints -%} + {{ c }}{{ "," if not loop.last or raw_model_constraints }} + {% endfor %} + {% for c in raw_model_constraints -%} + {{ c }}{{ "," if not loop.last }} + {% endfor -%} + ) +{% endmacro %} + +-- funcsign: (string) -> string +{%- macro get_assert_columns_equivalent(sql) -%} + {{ adapter.dispatch('get_assert_columns_equivalent', 'dbt')(sql) }} +{%- endmacro -%} + +-- funcsign: (string) -> string +{% macro default__get_assert_columns_equivalent(sql) -%} + {{ return(assert_columns_equivalent(sql)) }} +{%- endmacro %} + +{# + Compares the column schema provided by a model's sql file to the column schema provided by a model's schema file. + If any differences in name, data_type or number of columns exist between the two schemas, raises a compiler error +#} +-- funcsign: (string) -> string +{% macro assert_columns_equivalent(sql) %} + + {#-- First ensure the user has defined 'columns' in yaml specification --#} + {%- set user_defined_columns = model['columns'] -%} + {%- if not user_defined_columns -%} + {{ exceptions.raise_contract_error([], []) }} + {%- endif -%} + + {#-- Obtain the column schema provided by sql file. #} + {%- set sql_file_provided_columns = get_column_schema_from_query(sql, config.get('sql_header', none)) -%} + {#--Obtain the column schema provided by the schema file by generating an 'empty schema' query from the model's columns. #} + {%- set schema_file_provided_columns = get_column_schema_from_query(get_empty_schema_sql(user_defined_columns)) -%} + + {#-- create dictionaries with name and formatted data type and strings for exception #} + {%- set sql_columns = format_columns(sql_file_provided_columns) -%} + {%- set yaml_columns = format_columns(schema_file_provided_columns) -%} + + {%- if sql_columns|length != yaml_columns|length -%} + {%- do exceptions.raise_contract_error(yaml_columns, sql_columns) -%} + {%- endif -%} + + {%- for sql_col in sql_columns -%} + {%- set yaml_col = [] -%} + {%- for this_col in yaml_columns -%} + {%- if this_col['name'] == sql_col['name'] -%} + {%- do yaml_col.append(this_col) -%} + {%- break -%} + {%- endif -%} + {%- endfor -%} + {%- if not yaml_col -%} + {#-- Column with name not found in yaml #} + {%- do exceptions.raise_contract_error(yaml_columns, sql_columns) -%} + {%- endif -%} + {%- if sql_col['formatted'] != yaml_col[0]['formatted'] -%} + {#-- Column data types don't match #} + {%- do exceptions.raise_contract_error(yaml_columns, sql_columns) -%} + {%- endif -%} + {%- endfor -%} + +{% endmacro %} + +-- funcsign: (list[ANY]) -> list[ANY] +{% macro format_columns(columns) %} + {% set formatted_columns = [] %} + {% for column in columns %} + {%- set formatted_column = adapter.dispatch('format_column', 'dbt')(column) -%} + {%- do formatted_columns.append(formatted_column) -%} + {% endfor %} + {{ return(formatted_columns) }} +{% endmacro %} + +-- funcsign: (base_column) -> struct {name: string, data_type: string, formatted: string} +{% macro default__format_column(column) -%} + {% set data_type = column.dtype %} + {% set formatted = column.column.lower() ~ " " ~ data_type %} + {{ return({'name': column.name, 'data_type': data_type, 'formatted': formatted}) }} +{%- endmacro -%} diff --git a/dbt_internal_packages/dbt-adapters/macros/relations/create.sql b/dbt_internal_packages/dbt-adapters/macros/relations/create.sql new file mode 100644 index 0000000..80f89a4 --- /dev/null +++ b/dbt_internal_packages/dbt-adapters/macros/relations/create.sql @@ -0,0 +1,24 @@ +-- funcsign: (relation, string) -> string +{%- macro get_create_sql(relation, sql) -%} + {{- log('Applying CREATE to: ' ~ relation) -}} + {{- adapter.dispatch('get_create_sql', 'dbt')(relation, sql) -}} +{%- endmacro -%} + +-- funcsign: (relation, string) -> string +{%- macro default__get_create_sql(relation, sql) -%} + + {%- if relation.is_view -%} + {{ get_create_view_as_sql(relation, sql) }} + + {%- elif relation.is_table -%} + {{ get_create_table_as_sql(False, relation, sql) }} + + {%- elif relation.is_materialized_view -%} + {{ get_create_materialized_view_as_sql(relation, sql) }} + + {%- else -%} + {{- exceptions.raise_compiler_error("`get_create_sql` has not been implemented for: " ~ relation.type ) -}} + + {%- endif -%} + +{%- endmacro -%} diff --git a/dbt_internal_packages/dbt-adapters/macros/relations/create_backup.sql b/dbt_internal_packages/dbt-adapters/macros/relations/create_backup.sql new file mode 100644 index 0000000..ab7484f --- /dev/null +++ b/dbt_internal_packages/dbt-adapters/macros/relations/create_backup.sql @@ -0,0 +1,18 @@ +-- funcsign: (relation) -> string +{%- macro get_create_backup_sql(relation) -%} + {{- log('Applying CREATE BACKUP to: ' ~ relation) -}} + {{- adapter.dispatch('get_create_backup_sql', 'dbt')(relation) -}} +{%- endmacro -%} + +-- funcsign: (relation) -> string +{%- macro default__get_create_backup_sql(relation) -%} + + -- get the standard backup name + {% set backup_relation = make_backup_relation(relation, relation.type) %} + + -- drop any pre-existing backup + {{ get_drop_sql(backup_relation) }}; + + {{ get_rename_sql(relation, backup_relation.identifier) }} + +{%- endmacro -%} diff --git a/dbt_internal_packages/dbt-adapters/macros/relations/create_intermediate.sql b/dbt_internal_packages/dbt-adapters/macros/relations/create_intermediate.sql new file mode 100644 index 0000000..98fb223 --- /dev/null +++ b/dbt_internal_packages/dbt-adapters/macros/relations/create_intermediate.sql @@ -0,0 +1,18 @@ +-- funcsign: (relation, string) -> string +{%- macro get_create_intermediate_sql(relation, sql) -%} + {{- log('Applying CREATE INTERMEDIATE to: ' ~ relation) -}} + {{- adapter.dispatch('get_create_intermediate_sql', 'dbt')(relation, sql) -}} +{%- endmacro -%} + +-- funcsign: (relation, string) -> string +{%- macro default__get_create_intermediate_sql(relation, sql) -%} + + -- get the standard intermediate name + {% set intermediate_relation = make_intermediate_relation(relation) %} + + -- drop any pre-existing intermediate + {{ get_drop_sql(intermediate_relation) }}; + + {{ get_create_sql(intermediate_relation, sql) }} + +{%- endmacro -%} diff --git a/dbt_internal_packages/dbt-adapters/macros/relations/drop.sql b/dbt_internal_packages/dbt-adapters/macros/relations/drop.sql new file mode 100644 index 0000000..5d6e431 --- /dev/null +++ b/dbt_internal_packages/dbt-adapters/macros/relations/drop.sql @@ -0,0 +1,43 @@ +-- funcsign: (relation) -> string +{%- macro get_drop_sql(relation) -%} + {{- log('Applying DROP to: ' ~ relation) -}} + {{- adapter.dispatch('get_drop_sql', 'dbt')(relation) -}} +{%- endmacro -%} + +-- funcsign: (relation) -> string +{%- macro default__get_drop_sql(relation) -%} + + {%- if relation.is_view -%} + {{ drop_view(relation) }} + + {%- elif relation.is_table -%} + {{ drop_table(relation) }} + + {%- elif relation.is_materialized_view -%} + {{ drop_materialized_view(relation) }} + + {%- else -%} + drop {{ relation.type }} if exists {{ relation.render() }} cascade + + {%- endif -%} + +{%- endmacro -%} + +-- funcsign: (relation) -> string +{% macro drop_relation(relation) -%} + {{ return(adapter.dispatch('drop_relation', 'dbt')(relation)) }} +{% endmacro %} + +-- funcsign: (relation) -> string +{% macro default__drop_relation(relation) -%} + {% call statement('drop_relation', auto_begin=False) -%} + {{ get_drop_sql(relation) }} + {%- endcall %} +{% endmacro %} + +-- funcsign: (optional[relation]) -> string +{% macro drop_relation_if_exists(relation) %} + {% if relation is not none %} + {{ adapter.drop_relation(relation) }} + {% endif %} +{% endmacro %} diff --git a/dbt_internal_packages/dbt-adapters/macros/relations/drop_backup.sql b/dbt_internal_packages/dbt-adapters/macros/relations/drop_backup.sql new file mode 100644 index 0000000..76349c6 --- /dev/null +++ b/dbt_internal_packages/dbt-adapters/macros/relations/drop_backup.sql @@ -0,0 +1,15 @@ +-- funcsign: (relation) -> string +{%- macro get_drop_backup_sql(relation) -%} + {{- log('Applying DROP BACKUP to: ' ~ relation) -}} + {{- adapter.dispatch('get_drop_backup_sql', 'dbt')(relation) -}} +{%- endmacro -%} + +-- funcsign: (relation) -> string +{%- macro default__get_drop_backup_sql(relation) -%} + + -- get the standard backup name + {% set backup_relation = make_backup_relation(relation, relation.type) %} + + {{ get_drop_sql(backup_relation) }} + +{%- endmacro -%} diff --git a/dbt_internal_packages/dbt-adapters/macros/relations/get_relations_by_pattern.sql b/dbt_internal_packages/dbt-adapters/macros/relations/get_relations_by_pattern.sql new file mode 100644 index 0000000..e2b8f8a --- /dev/null +++ b/dbt_internal_packages/dbt-adapters/macros/relations/get_relations_by_pattern.sql @@ -0,0 +1,43 @@ +-- This is ported from the override version of get_relations_by_pattern in tests/data/internal-analytics +-- with var('single-tenant-exclusions') usage being extracted to a parameter - excluded_schemas +-- funcsign: (string, string, string, string, boolean, list[ANY]) -> list[ANY] +{% macro get_relations_by_pattern_internal(schema_pattern, table_pattern, exclude='', database=target.database, quote_table=False, excluded_schemas=[]) %} + {{ return(adapter.dispatch('get_relations_by_pattern_internal')(schema_pattern, table_pattern, exclude, database, quote_table, excluded_schemas)) }} +{% endmacro %} + +-- funcsign: (string, string, string, string, boolean, list[ANY]) -> list[ANY] +{% macro default__get_relations_by_pattern_internal(schema_pattern, table_pattern, exclude='', database=target.database, quote_table=False, excluded_schemas=[]) %} + + {%- call statement('get_tables', fetch_result=True) %} + + {{ adapter.dispatch('get_tables_by_pattern_sql')(schema_pattern, table_pattern, exclude, database) }} + + {%- endcall -%} + + {%- set table_list = load_result('get_tables') -%} + + {%- if table_list and table_list['table'] -%} + {%- set tbl_relations = [] -%} + {%- for row in table_list['table'] -%} + {% if row.table_schema not in excluded_schemas %} + {% if quote_table %} + {% set table_name = '"' ~ row.table_name ~ '"' %} + {% else %} + {% set table_name = row.table_name %} + {% endif %} + {%- set tbl_relation = api.Relation.create( + database=database, + schema=row.table_schema, + identifier=table_name, + type=row.table_type + ) -%} + {%- do tbl_relations.append(tbl_relation) -%} + {% endif %} + {%- endfor -%} + + {{ return(tbl_relations) }} + {%- else -%} + {{ return([]) }} + {%- endif -%} + +{% endmacro %} \ No newline at end of file diff --git a/dbt_internal_packages/dbt-adapters/macros/relations/get_relations_by_pattern_sql.sql b/dbt_internal_packages/dbt-adapters/macros/relations/get_relations_by_pattern_sql.sql new file mode 100644 index 0000000..ef098f0 --- /dev/null +++ b/dbt_internal_packages/dbt-adapters/macros/relations/get_relations_by_pattern_sql.sql @@ -0,0 +1,99 @@ +-- funcsign: (string, string, string, string) -> string +{% macro get_tables_by_pattern_sql(schema_pattern, table_pattern, exclude='', database=target.database) %} + {{ return(adapter.dispatch('get_tables_by_pattern_sql') + (schema_pattern, table_pattern, exclude, database)) }} +{% endmacro %} + +-- funcsign: (string, string, string, string) -> string +{% macro default__get_tables_by_pattern_sql(schema_pattern, table_pattern, exclude='', database=target.database) %} + + select distinct + table_schema as {{ adapter.quote('table_schema') }}, + table_name as {{ adapter.quote('table_name') }}, + {{ adapter.dispatch('get_table_types_sql')() }} + from {{ database }}.information_schema.tables + where table_schema ilike '{{ schema_pattern }}' + and table_name ilike '{{ table_pattern }}' + and table_name not ilike '{{ exclude }}' + +{% endmacro %} + +-- funcsign: (string, string, string, string) -> string +{% macro redshift__get_tables_by_pattern_sql(schema_pattern, table_pattern, exclude='', database=target.database) %} + + {% set sql %} + select distinct + table_schema as {{ adapter.quote('table_schema') }}, + table_name as {{ adapter.quote('table_name') }}, + {{ adapter.dispatch('get_table_types_sql')() }} + from "{{ database }}"."information_schema"."tables" + where table_schema ilike '{{ schema_pattern }}' + and table_name ilike '{{ table_pattern }}' + and table_name not ilike '{{ exclude }}' + union all + select distinct + schemaname as {{ adapter.quote('table_schema') }}, + tablename as {{ adapter.quote('table_name') }}, + 'external' as {{ adapter.quote('table_type') }} + from svv_external_tables + where redshift_database_name = '{{ database }}' + and schemaname ilike '{{ schema_pattern }}' + and table_name ilike '{{ table_pattern }}' + and table_name not ilike '{{ exclude }}' + {% endset %} + + {{ return(sql) }} +{% endmacro %} + +-- funcsign: (string, string, string, string) -> string +{% macro bigquery__get_tables_by_pattern_sql(schema_pattern, table_pattern, exclude='', database=target.database) %} + + {% if '%' in schema_pattern %} + {% set schemata=_bigquery__get_matching_schemata(schema_pattern, database) %} + {% else %} + {% set schemata=[schema_pattern] %} + {% endif %} + + {% set sql %} + {% for schema in schemata %} + select distinct + table_schema, + table_name, + {{ adapter.dispatch('get_table_types_sql')() }} + + from {{ adapter.quote(database) }}.{{ schema }}.INFORMATION_SCHEMA.TABLES + where lower(table_name) like lower ('{{ table_pattern }}') + and lower(table_name) not like lower ('{{ exclude }}') + + {% if not loop.last %} union all {% endif %} + + {% endfor %} + {% endset %} + + {{ return(sql) }} + +{% endmacro %} + +-- funcsign: (string, string) -> string +{% macro _bigquery__get_matching_schemata(schema_pattern, database) %} + {% if execute %} + + {% set sql %} + select schema_name from {{ adapter.quote(database) }}.INFORMATION_SCHEMA.SCHEMATA + where lower(schema_name) like lower('{{ schema_pattern }}') + {% endset %} + + {% set results=run_query(sql) %} + + {% set schemata=results.columns['schema_name'].values() %} + + {{ return(schemata) }} + + {% else %} + + {{ return([]) }} + + {% endif %} + + +{% endmacro %} diff --git a/dbt_internal_packages/dbt-adapters/macros/relations/get_table_types_sql.sql b/dbt_internal_packages/dbt-adapters/macros/relations/get_table_types_sql.sql new file mode 100644 index 0000000..8528175 --- /dev/null +++ b/dbt_internal_packages/dbt-adapters/macros/relations/get_table_types_sql.sql @@ -0,0 +1,34 @@ +-- funcsign: () -> string +{%- macro get_table_types_sql() -%} + {{ return(adapter.dispatch('get_table_types_sql')()) }} +{%- endmacro -%} + +-- funcsign: () -> string +{% macro default__get_table_types_sql() %} + case table_type + when 'BASE TABLE' then 'table' + when 'EXTERNAL TABLE' then 'external' + when 'MATERIALIZED VIEW' then 'materializedview' + else lower(table_type) + end as {{ adapter.quote('table_type') }} +{% endmacro %} + +-- funcsign: () -> string +{% macro postgres__get_table_types_sql() %} + case table_type + when 'BASE TABLE' then 'table' + when 'FOREIGN' then 'external' + when 'MATERIALIZED VIEW' then 'materializedview' + else lower(table_type) + end as {{ adapter.quote('table_type') }} +{% endmacro %} + +-- funcsign: () -> string +{% macro databricks__get_table_types_sql() %} + case table_type + when 'MANAGED' then 'table' + when 'BASE TABLE' then 'table' + when 'MATERIALIZED VIEW' then 'materializedview' + else lower(table_type) + end as {{ adapter.quote('table_type') }} +{% endmacro %} diff --git a/dbt_internal_packages/dbt-adapters/macros/relations/materialized_view/alter.sql b/dbt_internal_packages/dbt-adapters/macros/relations/materialized_view/alter.sql new file mode 100644 index 0000000..4ce7104 --- /dev/null +++ b/dbt_internal_packages/dbt-adapters/macros/relations/materialized_view/alter.sql @@ -0,0 +1,58 @@ +-- funcsign: (relation, string, string, relation, relation, relation) -> string +{% macro get_alter_materialized_view_as_sql( + relation, + configuration_changes, + sql, + existing_relation, + backup_relation, + intermediate_relation +) %} + {{- log('Applying ALTER to: ' ~ relation) -}} + {{- adapter.dispatch('get_alter_materialized_view_as_sql', 'dbt')( + relation, + configuration_changes, + sql, + existing_relation, + backup_relation, + intermediate_relation + ) -}} +{% endmacro %} + +-- funcsign: (relation, string, string, relation, relation, relation) -> string +{% macro default__get_alter_materialized_view_as_sql( + relation, + configuration_changes, + sql, + existing_relation, + backup_relation, + intermediate_relation +) %} + {{ exceptions.raise_compiler_error("Materialized views have not been implemented for this adapter.") }} +{% endmacro %} + + +-- funcsign: (relation, config) -> string +{% macro get_materialized_view_configuration_changes(existing_relation, new_config) %} + /* {# + It's recommended that configuration changes be formatted as follows: + {"": [{"action": "", "context": ...}]} + + For example: + { + "indexes": [ + {"action": "drop", "context": "index_abc"}, + {"action": "create", "context": {"columns": ["column_1", "column_2"], "type": "hash", "unique": True}}, + ], + } + + Either way, `get_materialized_view_configuration_changes` needs to align with `get_alter_materialized_view_as_sql`. + #} */ + {{- log('Determining configuration changes on: ' ~ existing_relation) -}} + {%- do return(adapter.dispatch('get_materialized_view_configuration_changes', 'dbt')(existing_relation, new_config)) -%} +{% endmacro %} + + +-- funcsign: (relation, config) -> string +{% macro default__get_materialized_view_configuration_changes(existing_relation, new_config) %} + {{ exceptions.raise_compiler_error("Materialized views have not been implemented for this adapter.") }} +{% endmacro %} diff --git a/dbt_internal_packages/dbt-adapters/macros/relations/materialized_view/create.sql b/dbt_internal_packages/dbt-adapters/macros/relations/materialized_view/create.sql new file mode 100644 index 0000000..2eb8950 --- /dev/null +++ b/dbt_internal_packages/dbt-adapters/macros/relations/materialized_view/create.sql @@ -0,0 +1,11 @@ +-- funcsign: (optional[relation], string) -> string +{% macro get_create_materialized_view_as_sql(relation, sql) -%} + {{- adapter.dispatch('get_create_materialized_view_as_sql', 'dbt')(relation, sql) -}} +{%- endmacro %} + +-- funcsign: (optional[relation], string) -> string +{% macro default__get_create_materialized_view_as_sql(relation, sql) -%} + {{ exceptions.raise_compiler_error( + "`get_create_materialized_view_as_sql` has not been implemented for this adapter." + ) }} +{% endmacro %} diff --git a/dbt_internal_packages/dbt-adapters/macros/relations/materialized_view/drop.sql b/dbt_internal_packages/dbt-adapters/macros/relations/materialized_view/drop.sql new file mode 100644 index 0000000..d96e181 --- /dev/null +++ b/dbt_internal_packages/dbt-adapters/macros/relations/materialized_view/drop.sql @@ -0,0 +1,14 @@ +{# /* +This was already implemented. Instead of creating a new macro that aligns with the standard, +this was reused and the default was maintained. This gets called by `drop_relation`, which +actually executes the drop, and `get_drop_sql`, which returns the template. +*/ #} +-- funcsign: (relation) -> string +{% macro drop_materialized_view(relation) -%} + {{- adapter.dispatch('drop_materialized_view', 'dbt')(relation) -}} +{%- endmacro %} + +-- funcsign: (relation) -> string +{% macro default__drop_materialized_view(relation) -%} + drop materialized view if exists {{ relation.render() }} cascade +{%- endmacro %} diff --git a/dbt_internal_packages/dbt-adapters/macros/relations/materialized_view/refresh.sql b/dbt_internal_packages/dbt-adapters/macros/relations/materialized_view/refresh.sql new file mode 100644 index 0000000..c77f119 --- /dev/null +++ b/dbt_internal_packages/dbt-adapters/macros/relations/materialized_view/refresh.sql @@ -0,0 +1,10 @@ +-- funcsign: (relation) -> string +{% macro refresh_materialized_view(relation) %} + {{- log('Applying REFRESH to: ' ~ relation) -}} + {{- adapter.dispatch('refresh_materialized_view', 'dbt')(relation) -}} +{% endmacro %} + +-- funcsign: (relation) -> string +{% macro default__refresh_materialized_view(relation) %} + {{ exceptions.raise_compiler_error("`refresh_materialized_view` has not been implemented for this adapter.") }} +{% endmacro %} diff --git a/dbt_internal_packages/dbt-adapters/macros/relations/materialized_view/rename.sql b/dbt_internal_packages/dbt-adapters/macros/relations/materialized_view/rename.sql new file mode 100644 index 0000000..abd5bab --- /dev/null +++ b/dbt_internal_packages/dbt-adapters/macros/relations/materialized_view/rename.sql @@ -0,0 +1,10 @@ +{% macro get_rename_materialized_view_sql(relation, new_name) %} + {{- adapter.dispatch('get_rename_materialized_view_sql', 'dbt')(relation, new_name) -}} +{% endmacro %} + + +{% macro default__get_rename_materialized_view_sql(relation, new_name) %} + {{ exceptions.raise_compiler_error( + "`get_rename_materialized_view_sql` has not been implemented for this adapter." + ) }} +{% endmacro %} diff --git a/dbt_internal_packages/dbt-adapters/macros/relations/materialized_view/replace.sql b/dbt_internal_packages/dbt-adapters/macros/relations/materialized_view/replace.sql new file mode 100644 index 0000000..0660f86 --- /dev/null +++ b/dbt_internal_packages/dbt-adapters/macros/relations/materialized_view/replace.sql @@ -0,0 +1,10 @@ +{% macro get_replace_materialized_view_sql(relation, sql) %} + {{- adapter.dispatch('get_replace_materialized_view_sql', 'dbt')(relation, sql) -}} +{% endmacro %} + + +{% macro default__get_replace_materialized_view_sql(relation, sql) %} + {{ exceptions.raise_compiler_error( + "`get_replace_materialized_view_sql` has not been implemented for this adapter." + ) }} +{% endmacro %} diff --git a/dbt_internal_packages/dbt-adapters/macros/relations/rename.sql b/dbt_internal_packages/dbt-adapters/macros/relations/rename.sql new file mode 100644 index 0000000..fbe7a47 --- /dev/null +++ b/dbt_internal_packages/dbt-adapters/macros/relations/rename.sql @@ -0,0 +1,37 @@ +-- funcsign: (relation, relation) -> string +{%- macro get_rename_sql(relation, new_name) -%} + {{- log('Applying RENAME to: ' ~ relation) -}} + {{- adapter.dispatch('get_rename_sql', 'dbt')(relation, new_name) -}} +{%- endmacro -%} + +-- funcsign: (relation, relation) -> string +{%- macro default__get_rename_sql(relation, new_name) -%} + + {%- if relation.is_view -%} + {{ get_rename_view_sql(relation, new_name) }} + + {%- elif relation.is_table -%} + {{ get_rename_table_sql(relation, new_name) }} + + {%- elif relation.is_materialized_view -%} + {{ get_rename_materialized_view_sql(relation, new_name) }} + + {%- else -%} + {{- exceptions.raise_compiler_error("`get_rename_sql` has not been implemented for: " ~ relation.type ) -}} + + {%- endif -%} + +{%- endmacro -%} + +-- funcsign: (relation, relation) -> string +{% macro rename_relation(from_relation, to_relation) -%} + {{ return(adapter.dispatch('rename_relation', 'dbt')(from_relation, to_relation)) }} +{% endmacro %} + +-- funcsign: (relation, relation) -> string +{% macro default__rename_relation(from_relation, to_relation) -%} + {% set target_name = adapter.quote_as_configured(to_relation.identifier, 'identifier') %} + {% call statement('rename_relation') -%} + alter table {{ from_relation.render() }} rename to {{ target_name }} + {%- endcall %} +{% endmacro %} diff --git a/dbt_internal_packages/dbt-adapters/macros/relations/rename_intermediate.sql b/dbt_internal_packages/dbt-adapters/macros/relations/rename_intermediate.sql new file mode 100644 index 0000000..294a5d3 --- /dev/null +++ b/dbt_internal_packages/dbt-adapters/macros/relations/rename_intermediate.sql @@ -0,0 +1,15 @@ +-- funcsign: (relation) -> string +{%- macro get_rename_intermediate_sql(relation) -%} + {{- log('Applying RENAME INTERMEDIATE to: ' ~ relation) -}} + {{- adapter.dispatch('get_rename_intermediate_sql', 'dbt')(relation) -}} +{%- endmacro -%} + +-- funcsign: (relation) -> string +{%- macro default__get_rename_intermediate_sql(relation) -%} + + -- get the standard intermediate name + {% set intermediate_relation = make_intermediate_relation(relation) %} + + {{ get_rename_sql(intermediate_relation, relation.identifier) }} + +{%- endmacro -%} diff --git a/dbt_internal_packages/dbt-adapters/macros/relations/replace.sql b/dbt_internal_packages/dbt-adapters/macros/relations/replace.sql new file mode 100644 index 0000000..c31136b --- /dev/null +++ b/dbt_internal_packages/dbt-adapters/macros/relations/replace.sql @@ -0,0 +1,51 @@ +-- funcsign: (relation, relation, string) -> string +{% macro get_replace_sql(existing_relation, target_relation, sql) %} + {{- log('Applying REPLACE to: ' ~ existing_relation) -}} + {{- adapter.dispatch('get_replace_sql', 'dbt')(existing_relation, target_relation, sql) -}} +{% endmacro %} + +-- funcsign: (relation, relation, string) -> string +{% macro default__get_replace_sql(existing_relation, target_relation, sql) %} + + {# /* use a create or replace statement if possible */ #} + + {% set is_replaceable = existing_relation.type == target_relation.type and existing_relation.can_be_replaced %} + + {% if is_replaceable and existing_relation.is_view %} + {{ get_replace_view_sql(target_relation, sql) }} + + {% elif is_replaceable and existing_relation.is_table %} + {{ get_replace_table_sql(target_relation, sql) }} + + {% elif is_replaceable and existing_relation.is_materialized_view %} + {{ get_replace_materialized_view_sql(target_relation, sql) }} + + {# /* a create or replace statement is not possible, so try to stage and/or backup to be safe */ #} + + {# /* create target_relation as an intermediate relation, then swap it out with the existing one using a backup */ #} + {%- elif target_relation.can_be_renamed and existing_relation.can_be_renamed -%} + {{ get_create_intermediate_sql(target_relation, sql) }}; + {{ get_create_backup_sql(existing_relation) }}; + {{ get_rename_intermediate_sql(target_relation) }}; + {{ get_drop_backup_sql(existing_relation) }} + + {# /* create target_relation as an intermediate relation, then swap it out with the existing one without using a backup */ #} + {%- elif target_relation.can_be_renamed -%} + {{ get_create_intermediate_sql(target_relation, sql) }}; + {{ get_drop_sql(existing_relation) }}; + {{ get_rename_intermediate_sql(target_relation) }} + + {# /* create target_relation in place by first backing up the existing relation */ #} + {%- elif existing_relation.can_be_renamed -%} + {{ get_create_backup_sql(existing_relation) }}; + {{ get_create_sql(target_relation, sql) }}; + {{ get_drop_backup_sql(existing_relation) }} + + {# /* no renaming is allowed, so just drop and create */ #} + {%- else -%} + {{ get_drop_sql(existing_relation) }}; + {{ get_create_sql(target_relation, sql) }} + + {%- endif -%} + +{% endmacro %} diff --git a/dbt_internal_packages/dbt-adapters/macros/relations/schema.sql b/dbt_internal_packages/dbt-adapters/macros/relations/schema.sql new file mode 100644 index 0000000..2288d1d --- /dev/null +++ b/dbt_internal_packages/dbt-adapters/macros/relations/schema.sql @@ -0,0 +1,10 @@ +-- funcsign: (string) -> string +{% macro drop_schema_named(schema_name) %} + {{ return(adapter.dispatch('drop_schema_named', 'dbt') (schema_name)) }} +{% endmacro %} + +-- funcsign: (string) -> string +{% macro default__drop_schema_named(schema_name) %} + {% set schema_relation = api.Relation.create(schema=schema_name) %} + {{ adapter.drop_schema(schema_relation) }} +{% endmacro %} diff --git a/dbt_internal_packages/dbt-adapters/macros/relations/table/create.sql b/dbt_internal_packages/dbt-adapters/macros/relations/table/create.sql new file mode 100644 index 0000000..0fe96fb --- /dev/null +++ b/dbt_internal_packages/dbt-adapters/macros/relations/table/create.sql @@ -0,0 +1,65 @@ +-- funcsign: (bool, relation, string) -> string +{% macro get_create_table_as_sql(temporary, relation, sql) -%} + {{ adapter.dispatch('get_create_table_as_sql', 'dbt')(temporary, relation, sql) }} +{%- endmacro %} + +-- funcsign: (bool, relation, string) -> string +{% macro default__get_create_table_as_sql(temporary, relation, sql) -%} + {{ return(create_table_as(temporary, relation, sql)) }} +{% endmacro %} + + +/* {# keep logic under old macro name for backwards compatibility #} */ +-- funcsign: (bool, relation, string, optional[string]) -> string +{% macro create_table_as(temporary, relation, compiled_code, language='sql') -%} + {# backward compatibility for create_table_as that does not support language #} + {% if language == "sql" %} + {{ adapter.dispatch('create_table_as', 'dbt')(temporary, relation, compiled_code)}} + {% else %} + {{ adapter.dispatch('create_table_as', 'dbt')(temporary, relation, compiled_code, language) }} + {% endif %} + +{%- endmacro %} + +-- funcsign: (bool, relation, string) -> string +{% macro default__create_table_as(temporary, relation, sql) -%} + {%- set sql_header = config.get('sql_header', none) -%} + + {{ sql_header if sql_header is not none }} + + create {% if temporary: -%}temporary{%- endif %} table + {{ relation.include(database=(not temporary), schema=(not temporary)) }} + {% set contract_config = config.get('contract') %} + {% if contract_config.enforced and (not temporary) %} + {{ get_assert_columns_equivalent(sql) }} + {{ get_table_columns_and_constraints() }} + {%- set sql = get_select_subquery(sql) %} + {% endif %} + as ( + {{ sql }} + ); +{%- endmacro %} + +-- funcsign: () -> string +{% macro default__get_column_names() %} + {#- loop through user_provided_columns to get column names -#} + {%- set user_provided_columns = model['columns'] -%} + {%- for i in user_provided_columns %} + {%- set col = user_provided_columns[i] -%} + {%- set col_name = adapter.quote(col['name']) if col.get('quote') else col['name'] -%} + {{ col_name }}{{ ", " if not loop.last }} + {%- endfor -%} +{% endmacro %} + +-- funcsign: (string) -> string +{% macro get_select_subquery(sql) %} + {{ return(adapter.dispatch('get_select_subquery', 'dbt')(sql)) }} +{% endmacro %} + +-- funcsign: (string) -> string +{% macro default__get_select_subquery(sql) %} + select {{ adapter.dispatch('get_column_names', 'dbt')() }} + from ( + {{ sql }} + ) as model_subq +{%- endmacro %} diff --git a/dbt_internal_packages/dbt-adapters/macros/relations/table/drop.sql b/dbt_internal_packages/dbt-adapters/macros/relations/table/drop.sql new file mode 100644 index 0000000..a0b9617 --- /dev/null +++ b/dbt_internal_packages/dbt-adapters/macros/relations/table/drop.sql @@ -0,0 +1,15 @@ +{# /* +This was already implemented. Instead of creating a new macro that aligns with the standard, +this was reused and the default was maintained. This gets called by `drop_relation`, which +actually executes the drop, and `get_drop_sql`, which returns the template. +*/ #} + +-- funcsign: (relation) -> string +{% macro drop_table(relation) -%} + {{- adapter.dispatch('drop_table', 'dbt')(relation) -}} +{%- endmacro %} + +-- funcsign: (relation) -> string +{% macro default__drop_table(relation) -%} + drop table if exists {{ relation.render() }} cascade +{%- endmacro %} diff --git a/dbt_internal_packages/dbt-adapters/macros/relations/table/rename.sql b/dbt_internal_packages/dbt-adapters/macros/relations/table/rename.sql new file mode 100644 index 0000000..814cbf7 --- /dev/null +++ b/dbt_internal_packages/dbt-adapters/macros/relations/table/rename.sql @@ -0,0 +1,11 @@ +-- funcsign: (relation, string) -> string +{% macro get_rename_table_sql(relation, new_name) %} + {{- adapter.dispatch('get_rename_table_sql', 'dbt')(relation, new_name) -}} +{% endmacro %} + +-- funcsign: (relation, string) -> string +{% macro default__get_rename_table_sql(relation, new_name) %} + {{ exceptions.raise_compiler_error( + "`get_rename_table_sql` has not been implemented for this adapter." + ) }} +{% endmacro %} diff --git a/dbt_internal_packages/dbt-adapters/macros/relations/table/replace.sql b/dbt_internal_packages/dbt-adapters/macros/relations/table/replace.sql new file mode 100644 index 0000000..69bfa2d --- /dev/null +++ b/dbt_internal_packages/dbt-adapters/macros/relations/table/replace.sql @@ -0,0 +1,10 @@ +{% macro get_replace_table_sql(relation, sql) %} + {{- adapter.dispatch('get_replace_table_sql', 'dbt')(relation, sql) -}} +{% endmacro %} + + +{% macro default__get_replace_table_sql(relation, sql) %} + {{ exceptions.raise_compiler_error( + "`get_replace_table_sql` has not been implemented for this adapter." + ) }} +{% endmacro %} diff --git a/dbt_internal_packages/dbt-adapters/macros/relations/view/create.sql b/dbt_internal_packages/dbt-adapters/macros/relations/view/create.sql new file mode 100644 index 0000000..fe5e942 --- /dev/null +++ b/dbt_internal_packages/dbt-adapters/macros/relations/view/create.sql @@ -0,0 +1,31 @@ +-- funcsign: (relation, string) -> string +{% macro get_create_view_as_sql(relation, sql) -%} + {{ adapter.dispatch('get_create_view_as_sql', 'dbt')(relation, sql) }} +{%- endmacro %} + +-- funcsign: (relation, string) -> string +{% macro default__get_create_view_as_sql(relation, sql) -%} + {{ return(create_view_as(relation, sql)) }} +{% endmacro %} + + +/* {# keep logic under old name for backwards compatibility #} */ +-- funcsign: (relation, string) -> string +{% macro create_view_as(relation, sql) -%} + {{ adapter.dispatch('create_view_as', 'dbt')(relation, sql) }} +{%- endmacro %} + +-- funcsign: (relation, string) -> string +{% macro default__create_view_as(relation, sql) -%} + {%- set sql_header = config.get('sql_header', none) -%} + + {{ sql_header if sql_header is not none }} + create view {{ relation.render() }} + {% set contract_config = config.get('contract') %} + {% if contract_config.enforced %} + {{ get_assert_columns_equivalent(sql) }} + {%- endif %} + as ( + {{ sql }} + ); +{%- endmacro %} diff --git a/dbt_internal_packages/dbt-adapters/macros/relations/view/drop.sql b/dbt_internal_packages/dbt-adapters/macros/relations/view/drop.sql new file mode 100644 index 0000000..2ea6ae5 --- /dev/null +++ b/dbt_internal_packages/dbt-adapters/macros/relations/view/drop.sql @@ -0,0 +1,14 @@ +{# /* +This was already implemented. Instead of creating a new macro that aligns with the standard, +this was reused and the default was maintained. This gets called by `drop_relation`, which +actually executes the drop, and `get_drop_sql`, which returns the template. +*/ #} +-- funcsign: (relation) -> string +{% macro drop_view(relation) -%} + {{- adapter.dispatch('drop_view', 'dbt')(relation) -}} +{%- endmacro %} + +-- funcsign: (relation) -> string +{% macro default__drop_view(relation) -%} + drop view if exists {{ relation.render() }} cascade +{%- endmacro %} diff --git a/dbt_internal_packages/dbt-adapters/macros/relations/view/rename.sql b/dbt_internal_packages/dbt-adapters/macros/relations/view/rename.sql new file mode 100644 index 0000000..32917c8 --- /dev/null +++ b/dbt_internal_packages/dbt-adapters/macros/relations/view/rename.sql @@ -0,0 +1,11 @@ +-- funcsign: (relation, string) -> string +{% macro get_rename_view_sql(relation, new_name) %} + {{- adapter.dispatch('get_rename_view_sql', 'dbt')(relation, new_name) -}} +{% endmacro %} + +-- funcsign: (relation, string) -> string +{% macro default__get_rename_view_sql(relation, new_name) %} + {{ exceptions.raise_compiler_error( + "`get_rename_view_sql` has not been implemented for this adapter." + ) }} +{% endmacro %} diff --git a/dbt_internal_packages/dbt-adapters/macros/relations/view/replace.sql b/dbt_internal_packages/dbt-adapters/macros/relations/view/replace.sql new file mode 100644 index 0000000..a0f0dc7 --- /dev/null +++ b/dbt_internal_packages/dbt-adapters/macros/relations/view/replace.sql @@ -0,0 +1,66 @@ +{% macro get_replace_view_sql(relation, sql) %} + {{- adapter.dispatch('get_replace_view_sql', 'dbt')(relation, sql) -}} +{% endmacro %} + + +{% macro default__get_replace_view_sql(relation, sql) %} + {{ exceptions.raise_compiler_error( + "`get_replace_view_sql` has not been implemented for this adapter." + ) }} +{% endmacro %} + + +/* {# + Core materialization implementation. BigQuery and Snowflake are similar + because both can use `create or replace view` where the resulting view's columns + are not necessarily the same as those of the existing view. On Redshift, this would + result in: ERROR: cannot change number of columns in view + + This implementation is superior to the create_temp, swap_with_existing, drop_old + paradigm because transactions don't run DDL queries atomically on Snowflake. By using + `create or replace view`, the materialization becomes atomic in nature. +#} */ + +{% macro create_or_replace_view() %} + {%- set identifier = model['alias'] -%} + + {%- set old_relation = adapter.get_relation(database=database, schema=schema, identifier=identifier) -%} + {%- set exists_as_view = (old_relation is not none and old_relation.is_view) -%} + + {%- set target_relation = api.Relation.create( + identifier=identifier, schema=schema, database=database, + type='view') -%} + {% set grant_config = config.get('grants') %} + + {{ run_hooks(pre_hooks) }} + + -- If there's a table with the same name and we weren't told to full refresh, + -- that's an error. If we were told to full refresh, drop it. This behavior differs + -- for Snowflake and BigQuery, so multiple dispatch is used. + {%- if old_relation is not none and old_relation.is_table -%} + {{ handle_existing_table(should_full_refresh(), old_relation) }} + {%- endif -%} + + -- build model + {% call statement('main') -%} + {{ get_create_view_as_sql(target_relation, sql) }} + {%- endcall %} + + {% set should_revoke = should_revoke(exists_as_view, full_refresh_mode=True) %} + {% do apply_grants(target_relation, grant_config, should_revoke=should_revoke) %} + + {{ run_hooks(post_hooks) }} + + {{ return({'relations': [target_relation]}) }} + +{% endmacro %} + + +{% macro handle_existing_table(full_refresh, old_relation) %} + {{ adapter.dispatch('handle_existing_table', 'dbt')(full_refresh, old_relation) }} +{% endmacro %} + +{% macro default__handle_existing_table(full_refresh, old_relation) %} + {{ log("Dropping relation " ~ old_relation.render() ~ " because it is of type " ~ old_relation.type) }} + {{ adapter.drop_relation(old_relation) }} +{% endmacro %} diff --git a/dbt_internal_packages/dbt-adapters/macros/unit_test_sql/get_fixture_sql.sql b/dbt_internal_packages/dbt-adapters/macros/unit_test_sql/get_fixture_sql.sql new file mode 100644 index 0000000..7e2a7e6 --- /dev/null +++ b/dbt_internal_packages/dbt-adapters/macros/unit_test_sql/get_fixture_sql.sql @@ -0,0 +1,108 @@ +-- funcsign: (list[ANY], optional[dict[string, string]]) -> string +{% macro get_fixture_sql(rows, column_name_to_data_types) %} +-- Fixture for {{ model.name }} +{% set default_row = {} %} + +{%- if not column_name_to_data_types -%} +{#-- Use defer_relation IFF it is available in the manifest and 'this' is missing from the database --#} +{%- set this_or_defer_relation = defer_relation if (defer_relation and not load_relation(this)) else this -%} +{%- set columns_in_relation = adapter.get_columns_in_relation(this_or_defer_relation) -%} + +{%- set column_name_to_data_types = {} -%} +{%- for column in columns_in_relation -%} +{#-- This needs to be a case-insensitive comparison --#} +{%- do column_name_to_data_types.update({column.name|lower: column.data_type}) -%} +{%- endfor -%} +{%- endif -%} + +{%- if not column_name_to_data_types -%} + {{ exceptions.raise_compiler_error("Not able to get columns for unit test '" ~ model.name ~ "' from relation " ~ this ~ " because the relation doesn't exist") }} +{%- endif -%} + +{%- for column_name, column_type in column_name_to_data_types.items() -%} + {%- do default_row.update({column_name: (safe_cast("null", column_type) | trim )}) -%} +{%- endfor -%} + +{{ validate_fixture_rows(rows, row_number) }} + +{%- for row in rows -%} +{%- set formatted_row = format_row(row, column_name_to_data_types) -%} +{%- set default_row_copy = default_row.copy() -%} +{%- do default_row_copy.update(formatted_row) -%} +select +{%- for column_name, column_value in default_row_copy.items() %} {{ column_value }} as {{ column_name }}{% if not loop.last -%}, {%- endif %} +{%- endfor %} +{%- if not loop.last %} +union all +{% endif %} +{%- endfor -%} + +{%- if (rows | length) == 0 -%} + select + {%- for column_name, column_value in default_row.items() %} {{ column_value }} as {{ column_name }}{% if not loop.last -%},{%- endif %} + {%- endfor %} + limit 0 +{%- endif -%} +{% endmacro %} + +-- funcsign: (list[ANY], dict[string, string]) -> string +{% macro get_expected_sql(rows, column_name_to_data_types) %} + +{%- if (rows | length) == 0 -%} + select * from dbt_internal_unit_test_actual + limit 0 +{%- else -%} +{%- for row in rows -%} +{%- set formatted_row = format_row(row, column_name_to_data_types) -%} +select +{%- for column_name, column_value in formatted_row.items() %} {{ column_value }} as {{ column_name }}{% if not loop.last -%}, {%- endif %} +{%- endfor %} +{%- if not loop.last %} +union all +{% endif %} +{%- endfor -%} +{%- endif -%} + +{% endmacro %} + +-- funcsign: (dict[string, string], dict[string, string]) -> dict[string, string] +{%- macro format_row(row, column_name_to_data_types) -%} + {#-- generate case-insensitive formatted row --#} + {% set formatted_row = {} %} + {%- for column_name, column_value in row.items() -%} + {% set column_name = column_name|lower %} + + {%- if column_name not in column_name_to_data_types %} + {#-- if user-provided row contains column name that relation does not contain, raise an error --#} + {% set fixture_name = "expected output" if model.resource_type == 'unit_test' else ("'" ~ model.name ~ "'") %} + {{ exceptions.raise_compiler_error( + "Invalid column name: '" ~ column_name ~ "' in unit test fixture for " ~ fixture_name ~ "." + "\nAccepted columns for " ~ fixture_name ~ " are: " ~ (column_name_to_data_types.keys()|list) + ) }} + {%- endif -%} + + {%- set column_type = column_name_to_data_types[column_name] %} + + {#-- sanitize column_value: wrap yaml strings in quotes, apply cast --#} + {%- set column_value_clean = column_value -%} + {%- if column_value is string -%} + {%- set column_value_clean = dbt.string_literal(dbt.escape_single_quotes(column_value)) -%} + {%- elif column_value is none -%} + {%- set column_value_clean = 'null' -%} + {%- endif -%} + + {%- set row_update = {column_name: safe_cast(column_value_clean, column_type) } -%} + {%- do formatted_row.update(row_update) -%} + {%- endfor -%} + {{ return(formatted_row) }} +{%- endmacro -%} + +-- funcsign: (optional[list[ANY]], integer) -> string +{%- macro validate_fixture_rows(rows, row_number) -%} + {{ return(adapter.dispatch('validate_fixture_rows', 'dbt')(rows, row_number)) }} +{%- endmacro -%} + +-- funcsign: (optional[list[ANY]], integer) -> string +{%- macro default__validate_fixture_rows(rows, row_number) -%} + {# This is an abstract method for adapter overrides as needed #} +{%- endmacro -%} diff --git a/dbt_internal_packages/dbt-adapters/macros/utils/any_value.sql b/dbt_internal_packages/dbt-adapters/macros/utils/any_value.sql new file mode 100644 index 0000000..0f1ab4b --- /dev/null +++ b/dbt_internal_packages/dbt-adapters/macros/utils/any_value.sql @@ -0,0 +1,11 @@ +-- funcsign: (string) -> string +{% macro any_value(expression) -%} + {{ return(adapter.dispatch('any_value', 'dbt') (expression)) }} +{% endmacro %} + +-- funcsign: (string) -> string +{% macro default__any_value(expression) -%} + + any_value({{ expression }}) + +{%- endmacro %} diff --git a/dbt_internal_packages/dbt-adapters/macros/utils/array_append.sql b/dbt_internal_packages/dbt-adapters/macros/utils/array_append.sql new file mode 100644 index 0000000..c141e68 --- /dev/null +++ b/dbt_internal_packages/dbt-adapters/macros/utils/array_append.sql @@ -0,0 +1,10 @@ +-- funcsign: (list[any], any) -> list[any] +{% macro array_append(array, new_element) -%} + {{ return(adapter.dispatch('array_append', 'dbt')(array, new_element)) }} +{%- endmacro %} + +{# new_element must be the same data type as elements in array to match postgres functionality #} +-- funcsign: (list[any], any) -> list[any] +{% macro default__array_append(array, new_element) -%} + array_append({{ array }}, {{ new_element }}) +{%- endmacro %} diff --git a/dbt_internal_packages/dbt-adapters/macros/utils/array_concat.sql b/dbt_internal_packages/dbt-adapters/macros/utils/array_concat.sql new file mode 100644 index 0000000..d729137 --- /dev/null +++ b/dbt_internal_packages/dbt-adapters/macros/utils/array_concat.sql @@ -0,0 +1,9 @@ +-- funcsign: (list[any], list[any]) -> list[any] +{% macro array_concat(array_1, array_2) -%} + {{ return(adapter.dispatch('array_concat', 'dbt')(array_1, array_2)) }} +{%- endmacro %} + +-- funcsign: (list[any], list[any]) -> list[any] +{% macro default__array_concat(array_1, array_2) -%} + array_cat({{ array_1 }}, {{ array_2 }}) +{%- endmacro %} diff --git a/dbt_internal_packages/dbt-adapters/macros/utils/array_construct.sql b/dbt_internal_packages/dbt-adapters/macros/utils/array_construct.sql new file mode 100644 index 0000000..73e1592 --- /dev/null +++ b/dbt_internal_packages/dbt-adapters/macros/utils/array_construct.sql @@ -0,0 +1,14 @@ +-- funcsign: (list[any], string) -> list[any] +{% macro array_construct(inputs=[], data_type=api.Column.translate_type('integer')) -%} + {{ return(adapter.dispatch('array_construct', 'dbt')(inputs, data_type)) }} +{%- endmacro %} + +{# all inputs must be the same data type to match postgres functionality #} +-- funcsign: (list[any], string) -> list[any] +{% macro default__array_construct(inputs, data_type) -%} + {% if inputs|length > 0 %} + array[ {{ inputs|join(' , ') }} ] + {% else %} + array[]::{{data_type}}[] + {% endif %} +{%- endmacro %} diff --git a/dbt_internal_packages/dbt-adapters/macros/utils/bool_or.sql b/dbt_internal_packages/dbt-adapters/macros/utils/bool_or.sql new file mode 100644 index 0000000..34e59d9 --- /dev/null +++ b/dbt_internal_packages/dbt-adapters/macros/utils/bool_or.sql @@ -0,0 +1,9 @@ +{% macro bool_or(expression) -%} + {{ return(adapter.dispatch('bool_or', 'dbt') (expression)) }} +{% endmacro %} + +{% macro default__bool_or(expression) -%} + + bool_or({{ expression }}) + +{%- endmacro %} diff --git a/dbt_internal_packages/dbt-adapters/macros/utils/cast.sql b/dbt_internal_packages/dbt-adapters/macros/utils/cast.sql new file mode 100644 index 0000000..7d24c0e --- /dev/null +++ b/dbt_internal_packages/dbt-adapters/macros/utils/cast.sql @@ -0,0 +1,9 @@ +-- funcsign: (string, string) -> string +{% macro cast(field, type) %} + {{ return(adapter.dispatch('cast', 'dbt') (field, type)) }} +{% endmacro %} + +-- funcsign: (string, string) -> string +{% macro default__cast(field, type) %} + cast({{field}} as {{type}}) +{% endmacro %} diff --git a/dbt_internal_packages/dbt-adapters/macros/utils/cast_bool_to_text.sql b/dbt_internal_packages/dbt-adapters/macros/utils/cast_bool_to_text.sql new file mode 100644 index 0000000..5f5c033 --- /dev/null +++ b/dbt_internal_packages/dbt-adapters/macros/utils/cast_bool_to_text.sql @@ -0,0 +1,7 @@ +{% macro cast_bool_to_text(field) %} + {{ adapter.dispatch('cast_bool_to_text', 'dbt') (field) }} +{% endmacro %} + +{% macro default__cast_bool_to_text(field) %} + cast({{ field }} as {{ api.Column.translate_type('string') }}) +{% endmacro %} diff --git a/dbt_internal_packages/dbt-adapters/macros/utils/concat.sql b/dbt_internal_packages/dbt-adapters/macros/utils/concat.sql new file mode 100644 index 0000000..27bf3c9 --- /dev/null +++ b/dbt_internal_packages/dbt-adapters/macros/utils/concat.sql @@ -0,0 +1,7 @@ +{% macro concat(fields) -%} + {{ return(adapter.dispatch('concat', 'dbt')(fields)) }} +{%- endmacro %} + +{% macro default__concat(fields) -%} + {{ fields|join(' || ') }} +{%- endmacro %} diff --git a/dbt_internal_packages/dbt-adapters/macros/utils/data_types.sql b/dbt_internal_packages/dbt-adapters/macros/utils/data_types.sql new file mode 100644 index 0000000..f762ffa --- /dev/null +++ b/dbt_internal_packages/dbt-adapters/macros/utils/data_types.sql @@ -0,0 +1,133 @@ +{# string ------------------------------------------------- #} + +-- funcsign: () -> string +{%- macro type_string() -%} + {{ return(adapter.dispatch('type_string', 'dbt')()) }} +{%- endmacro -%} + +-- funcsign: () -> string +{% macro default__type_string() %} + {{ return(api.Column.translate_type("string")) }} +{% endmacro %} + +-- This will return 'text' by default +-- On Postgres + Snowflake, that's equivalent to varchar (no size) +-- Redshift will treat that as varchar(256) + + +{# timestamp ------------------------------------------------- #} + +-- funcsign: () -> string +{%- macro type_timestamp() -%} + {{ return(adapter.dispatch('type_timestamp', 'dbt')()) }} +{%- endmacro -%} + +-- funcsign: () -> string +{% macro default__type_timestamp() %} + {{ return(api.Column.translate_type("timestamp")) }} +{% endmacro %} + +/* +POSTGRES +https://www.postgresql.org/docs/current/datatype-datetime.html: +The SQL standard requires that writing just `timestamp` +be equivalent to `timestamp without time zone`, and +PostgreSQL honors that behavior. +`timestamptz` is accepted as an abbreviation for `timestamp with time zone`; +this is a PostgreSQL extension. + +SNOWFLAKE +https://docs.snowflake.com/en/sql-reference/data-types-datetime.html#timestamp +The TIMESTAMP_* variation associated with TIMESTAMP is specified by the +TIMESTAMP_TYPE_MAPPING session parameter. The default is TIMESTAMP_NTZ. + +BIGQUERY +TIMESTAMP means 'timestamp with time zone' +DATETIME means 'timestamp without time zone' +TODO: shouldn't this return DATETIME instead of TIMESTAMP, for consistency with other databases? +e.g. dateadd returns a DATETIME + +/* Snowflake: +https://docs.snowflake.com/en/sql-reference/data-types-datetime.html#timestamp +The TIMESTAMP_* variation associated with TIMESTAMP is specified by the TIMESTAMP_TYPE_MAPPING session parameter. The default is TIMESTAMP_NTZ. +*/ + + +{# float ------------------------------------------------- #} + +{%- macro type_float() -%} + {{ return(adapter.dispatch('type_float', 'dbt')()) }} +{%- endmacro -%} + +{% macro default__type_float() %} + {{ return(api.Column.translate_type("float")) }} +{% endmacro %} + +{# numeric ------------------------------------------------- #} + +{%- macro type_numeric() -%} + {{ return(adapter.dispatch('type_numeric', 'dbt')()) }} +{%- endmacro -%} + +/* +This one can't be just translate_type, since precision/scale make it a bit more complicated. + +On most databases, the default (precision, scale) is something like: + Redshift: (18, 0) + Snowflake: (38, 0) + Postgres: (<=131072, 0) + +https://www.postgresql.org/docs/current/datatype-numeric.html: +Specifying NUMERIC without any precision or scale creates an “unconstrained numeric” +column in which numeric values of any length can be stored, up to the implementation limits. +A column of this kind will not coerce input values to any particular scale, +whereas numeric columns with a declared scale will coerce input values to that scale. +(The SQL standard requires a default scale of 0, i.e., coercion to integer precision. +We find this a bit useless. If you're concerned about portability, always specify +the precision and scale explicitly.) +*/ + +{% macro default__type_numeric() %} + {{ return(api.Column.numeric_type("numeric", 28, 6)) }} -- noqa: this warning is in the comment +{% endmacro %} + + +{# bigint ------------------------------------------------- #} + +{%- macro type_bigint() -%} + {{ return(adapter.dispatch('type_bigint', 'dbt')()) }} +{%- endmacro -%} + +-- We don't have a conversion type for 'bigint' in TYPE_LABELS, +-- so this actually just returns the string 'bigint' + +{% macro default__type_bigint() %} + {{ return(api.Column.translate_type("bigint")) }} +{% endmacro %} + +-- Good news: BigQuery now supports 'bigint' (and 'int') as an alias for 'int64' + +{# int ------------------------------------------------- #} + +{%- macro type_int() -%} + {{ return(adapter.dispatch('type_int', 'dbt')()) }} +{%- endmacro -%} + +{%- macro default__type_int() -%} + {{ return(api.Column.translate_type("integer")) }} +{%- endmacro -%} + +-- returns 'int' everywhere, except BigQuery, where it returns 'int64' +-- (but BigQuery also now accepts 'int' as a valid alias for 'int64') + +{# bool ------------------------------------------------- #} + +{%- macro type_boolean() -%} + {{ return(adapter.dispatch('type_boolean', 'dbt')()) }} +{%- endmacro -%} + +{%- macro default__type_boolean() -%} + {{ return(api.Column.translate_type("boolean")) }} +{%- endmacro -%} + +-- returns 'boolean' everywhere. BigQuery accepts 'boolean' as a valid alias for 'bool' diff --git a/dbt_internal_packages/dbt-adapters/macros/utils/date.sql b/dbt_internal_packages/dbt-adapters/macros/utils/date.sql new file mode 100644 index 0000000..d41b443 --- /dev/null +++ b/dbt_internal_packages/dbt-adapters/macros/utils/date.sql @@ -0,0 +1,10 @@ +{% macro date(year, month, day) %} + {{ return(adapter.dispatch('date', 'dbt') (year, month, day)) }} +{% endmacro %} + + +{% macro default__date(year, month, day) -%} + {%- set dt = modules.datetime.date(year, month, day) -%} + {%- set iso_8601_formatted_date = dt.strftime('%Y-%m-%d') -%} + to_date('{{ iso_8601_formatted_date }}', 'YYYY-MM-DD') +{%- endmacro %} diff --git a/dbt_internal_packages/dbt-adapters/macros/utils/date_spine.sql b/dbt_internal_packages/dbt-adapters/macros/utils/date_spine.sql new file mode 100644 index 0000000..833fbcc --- /dev/null +++ b/dbt_internal_packages/dbt-adapters/macros/utils/date_spine.sql @@ -0,0 +1,75 @@ +{% macro get_intervals_between(start_date, end_date, datepart) -%} + {{ return(adapter.dispatch('get_intervals_between', 'dbt')(start_date, end_date, datepart)) }} +{%- endmacro %} + +{% macro default__get_intervals_between(start_date, end_date, datepart) -%} + {%- call statement('get_intervals_between', fetch_result=True) %} + + select {{ dbt.datediff(start_date, end_date, datepart) }} + + {%- endcall -%} + + {%- set value_list = load_result('get_intervals_between') -%} + + {%- if value_list and value_list['data'] -%} + {%- set values = value_list['data'] | map(attribute=0) | list %} + {{ return(values[0]) }} + {%- else -%} + {{ return(1) }} + {%- endif -%} + +{%- endmacro %} + + + + +{% macro date_spine(datepart, start_date, end_date) %} + {{ return(adapter.dispatch('date_spine', 'dbt')(datepart, start_date, end_date)) }} +{%- endmacro %} + +{% macro default__date_spine(datepart, start_date, end_date) %} + + + {# call as follows: + + date_spine( + "day", + "to_date('01/01/2016', 'mm/dd/yyyy')", + "dbt.dateadd(week, 1, current_date)" + ) #} + + + with rawdata as ( + + {{dbt.generate_series( + dbt.get_intervals_between(start_date, end_date, datepart) + )}} + + ), + + all_periods as ( + + select ( + {{ + dbt.dateadd( + datepart, + "row_number() over (order by 1) - 1", + start_date + ) + }} + ) as date_{{datepart}} + from rawdata + + ), + + filtered as ( + + select * + from all_periods + where date_{{datepart}} <= {{ end_date }} + + ) + + select * from filtered + +{% endmacro %} diff --git a/dbt_internal_packages/dbt-adapters/macros/utils/date_trunc.sql b/dbt_internal_packages/dbt-adapters/macros/utils/date_trunc.sql new file mode 100644 index 0000000..deadc40 --- /dev/null +++ b/dbt_internal_packages/dbt-adapters/macros/utils/date_trunc.sql @@ -0,0 +1,7 @@ +{% macro date_trunc(datepart, date) -%} + {{ return(adapter.dispatch('date_trunc', 'dbt') (datepart, date)) }} +{%- endmacro %} + +{% macro default__date_trunc(datepart, date) -%} + date_trunc('{{datepart}}', {{date}}) +{%- endmacro %} diff --git a/dbt_internal_packages/dbt-adapters/macros/utils/dateadd.sql b/dbt_internal_packages/dbt-adapters/macros/utils/dateadd.sql new file mode 100644 index 0000000..2e24609 --- /dev/null +++ b/dbt_internal_packages/dbt-adapters/macros/utils/dateadd.sql @@ -0,0 +1,14 @@ +{% macro dateadd(datepart, interval, from_date_or_timestamp) %} + {{ return(adapter.dispatch('dateadd', 'dbt')(datepart, interval, from_date_or_timestamp)) }} +{% endmacro %} + + +{% macro default__dateadd(datepart, interval, from_date_or_timestamp) %} + + dateadd( + {{ datepart }}, + {{ interval }}, + {{ from_date_or_timestamp }} + ) + +{% endmacro %} diff --git a/dbt_internal_packages/dbt-adapters/macros/utils/datediff.sql b/dbt_internal_packages/dbt-adapters/macros/utils/datediff.sql new file mode 100644 index 0000000..7d70d33 --- /dev/null +++ b/dbt_internal_packages/dbt-adapters/macros/utils/datediff.sql @@ -0,0 +1,14 @@ +{% macro datediff(first_date, second_date, datepart) %} + {{ return(adapter.dispatch('datediff', 'dbt')(first_date, second_date, datepart)) }} +{% endmacro %} + + +{% macro default__datediff(first_date, second_date, datepart) -%} + + datediff( + {{ datepart }}, + {{ first_date }}, + {{ second_date }} + ) + +{%- endmacro %} diff --git a/dbt_internal_packages/dbt-adapters/macros/utils/equals.sql b/dbt_internal_packages/dbt-adapters/macros/utils/equals.sql new file mode 100644 index 0000000..8efdc0f --- /dev/null +++ b/dbt_internal_packages/dbt-adapters/macros/utils/equals.sql @@ -0,0 +1,18 @@ +-- ai +-- funcsign: (string, string) -> string +{% macro equals(expr1, expr2) %} + {{ return(adapter.dispatch('equals', 'dbt') (expr1, expr2)) }} +{%- endmacro %} + +-- ai +-- funcsign: (string, string) -> string +{% macro default__equals(expr1, expr2) -%} +{%- if adapter.behavior.enable_truthy_nulls_equals_macro.no_warn %} + case when (({{ expr1 }} = {{ expr2 }}) or ({{ expr1 }} is null and {{ expr2 }} is null)) + then 0 + else 1 + end = 0 +{%- else -%} + ({{ expr1 }} = {{ expr2 }}) +{%- endif %} +{% endmacro %} diff --git a/dbt_internal_packages/dbt-adapters/macros/utils/escape_single_quotes.sql b/dbt_internal_packages/dbt-adapters/macros/utils/escape_single_quotes.sql new file mode 100644 index 0000000..ed816e9 --- /dev/null +++ b/dbt_internal_packages/dbt-adapters/macros/utils/escape_single_quotes.sql @@ -0,0 +1,10 @@ +-- funcsign: (string) -> string +{% macro escape_single_quotes(expression) %} + {{ return(adapter.dispatch('escape_single_quotes', 'dbt') (expression)) }} +{% endmacro %} + +{# /*Default to replacing a single apostrophe with two apostrophes: they're -> they''re*/ #} +-- funcsign: (string) -> string +{% macro default__escape_single_quotes(expression) -%} +{{ expression | replace("'","''") }} +{%- endmacro %} diff --git a/dbt_internal_packages/dbt-adapters/macros/utils/except.sql b/dbt_internal_packages/dbt-adapters/macros/utils/except.sql new file mode 100644 index 0000000..91d5401 --- /dev/null +++ b/dbt_internal_packages/dbt-adapters/macros/utils/except.sql @@ -0,0 +1,9 @@ +{% macro except() %} + {{ return(adapter.dispatch('except', 'dbt')()) }} +{% endmacro %} + +{% macro default__except() %} + + except + +{% endmacro %} diff --git a/dbt_internal_packages/dbt-adapters/macros/utils/generate_series.sql b/dbt_internal_packages/dbt-adapters/macros/utils/generate_series.sql new file mode 100644 index 0000000..f6a0960 --- /dev/null +++ b/dbt_internal_packages/dbt-adapters/macros/utils/generate_series.sql @@ -0,0 +1,53 @@ +{% macro get_powers_of_two(upper_bound) %} + {{ return(adapter.dispatch('get_powers_of_two', 'dbt')(upper_bound)) }} +{% endmacro %} + +{% macro default__get_powers_of_two(upper_bound) %} + + {% if upper_bound <= 0 %} + {{ exceptions.raise_compiler_error("upper bound must be positive") }} + {% endif %} + + {% for _ in range(1, 100) %} + {% if upper_bound <= 2 ** loop.index %}{{ return(loop.index) }}{% endif %} + {% endfor %} + +{% endmacro %} + + +{% macro generate_series(upper_bound) %} + {{ return(adapter.dispatch('generate_series', 'dbt')(upper_bound)) }} +{% endmacro %} + +{% macro default__generate_series(upper_bound) %} + + {% set n = dbt.get_powers_of_two(upper_bound) %} + + with p as ( + select 0 as generated_number union all select 1 + ), unioned as ( + + select + + {% for i in range(n) %} + p{{i}}.generated_number * power(2, {{i}}) + {% if not loop.last %} + {% endif %} + {% endfor %} + + 1 + as generated_number + + from + + {% for i in range(n) %} + p as p{{i}} + {% if not loop.last %} cross join {% endif %} + {% endfor %} + + ) + + select * + from unioned + where generated_number <= {{upper_bound}} + order by generated_number + +{% endmacro %} diff --git a/dbt_internal_packages/dbt-adapters/macros/utils/hash.sql b/dbt_internal_packages/dbt-adapters/macros/utils/hash.sql new file mode 100644 index 0000000..efa12db --- /dev/null +++ b/dbt_internal_packages/dbt-adapters/macros/utils/hash.sql @@ -0,0 +1,7 @@ +{% macro hash(field) -%} + {{ return(adapter.dispatch('hash', 'dbt') (field)) }} +{%- endmacro %} + +{% macro default__hash(field) -%} + md5(cast({{ field }} as {{ api.Column.translate_type('string') }})) +{%- endmacro %} diff --git a/dbt_internal_packages/dbt-adapters/macros/utils/intersect.sql b/dbt_internal_packages/dbt-adapters/macros/utils/intersect.sql new file mode 100644 index 0000000..6e8ede0 --- /dev/null +++ b/dbt_internal_packages/dbt-adapters/macros/utils/intersect.sql @@ -0,0 +1,9 @@ +{% macro intersect() %} + {{ return(adapter.dispatch('intersect', 'dbt')()) }} +{% endmacro %} + +{% macro default__intersect() %} + + intersect + +{% endmacro %} diff --git a/dbt_internal_packages/dbt-adapters/macros/utils/last_day.sql b/dbt_internal_packages/dbt-adapters/macros/utils/last_day.sql new file mode 100644 index 0000000..6a1aa99 --- /dev/null +++ b/dbt_internal_packages/dbt-adapters/macros/utils/last_day.sql @@ -0,0 +1,15 @@ +{% macro last_day(date, datepart) %} + {{ return(adapter.dispatch('last_day', 'dbt') (date, datepart)) }} +{% endmacro %} + +{%- macro default_last_day(date, datepart) -%} + cast( + {{dbt.dateadd('day', '-1', + dbt.dateadd(datepart, '1', dbt.date_trunc(datepart, date)) + )}} + as date) +{%- endmacro -%} + +{% macro default__last_day(date, datepart) -%} + {{dbt.default_last_day(date, datepart)}} +{%- endmacro %} diff --git a/dbt_internal_packages/dbt-adapters/macros/utils/length.sql b/dbt_internal_packages/dbt-adapters/macros/utils/length.sql new file mode 100644 index 0000000..1b2fd55 --- /dev/null +++ b/dbt_internal_packages/dbt-adapters/macros/utils/length.sql @@ -0,0 +1,11 @@ +{% macro length(expression) -%} + {{ return(adapter.dispatch('length', 'dbt') (expression)) }} +{% endmacro %} + +{% macro default__length(expression) %} + + length( + {{ expression }} + ) + +{%- endmacro -%} diff --git a/dbt_internal_packages/dbt-adapters/macros/utils/listagg.sql b/dbt_internal_packages/dbt-adapters/macros/utils/listagg.sql new file mode 100644 index 0000000..f785ca1 --- /dev/null +++ b/dbt_internal_packages/dbt-adapters/macros/utils/listagg.sql @@ -0,0 +1,30 @@ +{% macro listagg(measure, delimiter_text="','", order_by_clause=none, limit_num=none) -%} + {{ return(adapter.dispatch('listagg', 'dbt') (measure, delimiter_text, order_by_clause, limit_num)) }} +{%- endmacro %} + +{% macro default__listagg(measure, delimiter_text, order_by_clause, limit_num) -%} + + {% if limit_num -%} + array_to_string( + array_slice( + array_agg( + {{ measure }} + ){% if order_by_clause -%} + within group ({{ order_by_clause }}) + {%- endif %} + ,0 + ,{{ limit_num }} + ), + {{ delimiter_text }} + ) + {%- else %} + listagg( + {{ measure }}, + {{ delimiter_text }} + ) + {% if order_by_clause -%} + within group ({{ order_by_clause }}) + {%- endif %} + {%- endif %} + +{%- endmacro %} diff --git a/dbt_internal_packages/dbt-adapters/macros/utils/literal.sql b/dbt_internal_packages/dbt-adapters/macros/utils/literal.sql new file mode 100644 index 0000000..8382312 --- /dev/null +++ b/dbt_internal_packages/dbt-adapters/macros/utils/literal.sql @@ -0,0 +1,9 @@ +-- funcsign: (string) -> string +{%- macro string_literal(value) -%} + {{ return(adapter.dispatch('string_literal', 'dbt') (value)) }} +{%- endmacro -%} + +-- funcsign: (string) -> string +{% macro default__string_literal(value) -%} + '{{ value }}' +{%- endmacro %} diff --git a/dbt_internal_packages/dbt-adapters/macros/utils/position.sql b/dbt_internal_packages/dbt-adapters/macros/utils/position.sql new file mode 100644 index 0000000..dde3ee2 --- /dev/null +++ b/dbt_internal_packages/dbt-adapters/macros/utils/position.sql @@ -0,0 +1,11 @@ +{% macro position(substring_text, string_text) -%} + {{ return(adapter.dispatch('position', 'dbt') (substring_text, string_text)) }} +{% endmacro %} + +{% macro default__position(substring_text, string_text) %} + + position( + {{ substring_text }} in {{ string_text }} + ) + +{%- endmacro -%} diff --git a/dbt_internal_packages/dbt-adapters/macros/utils/replace.sql b/dbt_internal_packages/dbt-adapters/macros/utils/replace.sql new file mode 100644 index 0000000..478809f --- /dev/null +++ b/dbt_internal_packages/dbt-adapters/macros/utils/replace.sql @@ -0,0 +1,14 @@ +{% macro replace(field, old_chars, new_chars) -%} + {{ return(adapter.dispatch('replace', 'dbt') (field, old_chars, new_chars)) }} +{% endmacro %} + +{% macro default__replace(field, old_chars, new_chars) %} + + replace( + {{ field }}, + {{ old_chars }}, + {{ new_chars }} + ) + + +{% endmacro %} diff --git a/dbt_internal_packages/dbt-adapters/macros/utils/right.sql b/dbt_internal_packages/dbt-adapters/macros/utils/right.sql new file mode 100644 index 0000000..5782a25 --- /dev/null +++ b/dbt_internal_packages/dbt-adapters/macros/utils/right.sql @@ -0,0 +1,12 @@ +{% macro right(string_text, length_expression) -%} + {{ return(adapter.dispatch('right', 'dbt') (string_text, length_expression)) }} +{% endmacro %} + +{% macro default__right(string_text, length_expression) %} + + right( + {{ string_text }}, + {{ length_expression }} + ) + +{%- endmacro -%} diff --git a/dbt_internal_packages/dbt-adapters/macros/utils/safe_cast.sql b/dbt_internal_packages/dbt-adapters/macros/utils/safe_cast.sql new file mode 100644 index 0000000..c3bf197 --- /dev/null +++ b/dbt_internal_packages/dbt-adapters/macros/utils/safe_cast.sql @@ -0,0 +1,11 @@ +-- funcsign: (string, string) -> string +{% macro safe_cast(field, type) %} + {{ return(adapter.dispatch('safe_cast', 'dbt') (field, type)) }} +{% endmacro %} + +-- funcsign: (string, string) -> string +{% macro default__safe_cast(field, type) %} + {# most databases don't support this function yet + so we just need to use cast #} + cast({{field}} as {{type}}) +{% endmacro %} diff --git a/dbt_internal_packages/dbt-adapters/macros/utils/split_part.sql b/dbt_internal_packages/dbt-adapters/macros/utils/split_part.sql new file mode 100644 index 0000000..13da017 --- /dev/null +++ b/dbt_internal_packages/dbt-adapters/macros/utils/split_part.sql @@ -0,0 +1,29 @@ +-- funcsign: (string, string, int) -> string +{% macro split_part(string_text, delimiter_text, part_number) %} + {{ return(adapter.dispatch('split_part', 'dbt') (string_text, delimiter_text, part_number)) }} +{% endmacro %} + +-- funcsign: (string, string, int) -> string +{% macro default__split_part(string_text, delimiter_text, part_number) %} + + split_part( + {{ string_text }}, + {{ delimiter_text }}, + {{ part_number }} + ) + +{% endmacro %} + +-- funcsign: (string, string, int) -> string +{% macro _split_part_negative(string_text, delimiter_text, part_number) %} + + split_part( + {{ string_text }}, + {{ delimiter_text }}, + length({{ string_text }}) + - length( + replace({{ string_text }}, {{ delimiter_text }}, '') + ) + 2 + {{ part_number }} + ) + +{% endmacro %} diff --git a/dbt_internal_packages/dbt-adapters/tests/generic/builtin.sql b/dbt_internal_packages/dbt-adapters/tests/generic/builtin.sql new file mode 100644 index 0000000..23a7507 --- /dev/null +++ b/dbt_internal_packages/dbt-adapters/tests/generic/builtin.sql @@ -0,0 +1,30 @@ +/* {# + Generic tests can be defined in `macros/` or in `tests/generic`. + These four tests are built into the dbt-core global project. + To support extensibility to other adapters and SQL dialects, + they call 'dispatched' macros. By default, they will use + the SQL defined in `global_project/macros/generic_test_sql` +#} */ + +{% test unique(model, column_name) %} + {% set macro = adapter.dispatch('test_unique', 'dbt') %} + {{ macro(model, column_name) }} +{% endtest %} + + +{% test not_null(model, column_name) %} + {% set macro = adapter.dispatch('test_not_null', 'dbt') %} + {{ macro(model, column_name) }} +{% endtest %} + + +{% test accepted_values(model, column_name, values, quote=True) %} + {% set macro = adapter.dispatch('test_accepted_values', 'dbt') %} + {{ macro(model, column_name, values, quote) }} +{% endtest %} + + +{% test relationships(model, column_name, to, field) %} + {% set macro = adapter.dispatch('test_relationships', 'dbt') %} + {{ macro(model, column_name, to, field) }} +{% endtest %} diff --git a/dbt_internal_packages/dbt-postgres/dbt_project.yml b/dbt_internal_packages/dbt-postgres/dbt_project.yml new file mode 100644 index 0000000..88fcd55 --- /dev/null +++ b/dbt_internal_packages/dbt-postgres/dbt_project.yml @@ -0,0 +1 @@ +name: dbt_postgres diff --git a/dbt_internal_packages/dbt-postgres/macros/adapters.sql b/dbt_internal_packages/dbt-postgres/macros/adapters.sql new file mode 100644 index 0000000..9117539 --- /dev/null +++ b/dbt_internal_packages/dbt-postgres/macros/adapters.sql @@ -0,0 +1,257 @@ +{% macro postgres__create_table_as(temporary, relation, sql) -%} + {%- set unlogged = config.get('unlogged', default=false) -%} + {%- set sql_header = config.get('sql_header', none) -%} + + {{ sql_header if sql_header is not none }} + + create {% if temporary -%} + temporary + {%- elif unlogged -%} + unlogged + {%- endif %} table {{ relation }} + {% set contract_config = config.get('contract') %} + {% if contract_config.enforced %} + {{ get_assert_columns_equivalent(sql) }} + {% endif -%} + {% if contract_config.enforced and (not temporary) -%} + {{ get_table_columns_and_constraints() }} ; + insert into {{ relation }} ( + {{ adapter.dispatch('get_column_names', 'dbt')() }} + ) + {%- set sql = get_select_subquery(sql) %} + {% else %} + as + {% endif %} + ( + {{ sql }} + ); +{%- endmacro %} + +{% macro postgres__get_create_index_sql(relation, index_dict) -%} + {%- set index_config = adapter.parse_index(index_dict) -%} + {%- set comma_separated_columns = ", ".join(index_config.columns) -%} + {%- set index_name = index_config.render(relation) -%} + + create {% if index_config.unique -%} + unique + {%- endif %} index if not exists + "{{ index_name }}" + on {{ relation }} {% if index_config.type -%} + using {{ index_config.type }} + {%- endif %} + ({{ comma_separated_columns }}) +{%- endmacro %} + +{% macro postgres__create_schema(relation) -%} + {% if relation.database -%} + {{ adapter.verify_database(relation.database) }} + {%- endif -%} + {%- call statement('create_schema') -%} + create schema if not exists {{ relation.without_identifier().include(database=False) }} + {%- endcall -%} +{% endmacro %} + +{% macro postgres__drop_schema(relation) -%} + {% if relation.database -%} + {{ adapter.verify_database(relation.database) }} + {%- endif -%} + {%- call statement('drop_schema') -%} + drop schema if exists {{ relation.without_identifier().include(database=False) }} cascade + {%- endcall -%} +{% endmacro %} + +{% macro postgres__get_columns_in_relation(relation) -%} + {% call statement('get_columns_in_relation', fetch_result=True) %} + select + column_name, + data_type, + character_maximum_length, + numeric_precision, + numeric_scale + + from {{ relation.information_schema('columns') }} + where table_name = '{{ relation.identifier }}' + {% if relation.schema %} + and table_schema = '{{ relation.schema }}' + {% endif %} + order by ordinal_position + + {% endcall %} + {% set table = load_result('get_columns_in_relation').table %} + {{ return(sql_convert_columns_in_relation(table)) }} +{% endmacro %} + + +{% macro postgres__list_relations_without_caching(schema_relation) %} + {% call statement('list_relations_without_caching', fetch_result=True) -%} + select + '{{ schema_relation.database }}' as database, + tablename as name, + schemaname as schema, + 'table' as type + from pg_tables + where schemaname ilike '{{ schema_relation.schema }}' + union all + select + '{{ schema_relation.database }}' as database, + viewname as name, + schemaname as schema, + 'view' as type + from pg_views + where schemaname ilike '{{ schema_relation.schema }}' + union all + select + '{{ schema_relation.database }}' as database, + matviewname as name, + schemaname as schema, + 'materialized_view' as type + from pg_matviews + where schemaname ilike '{{ schema_relation.schema }}' + {% endcall %} + {{ return(load_result('list_relations_without_caching').table) }} +{% endmacro %} + +{% macro postgres__information_schema_name(database) -%} + {% if database_name -%} + {{ adapter.verify_database(database_name) }} + {%- endif -%} + information_schema +{%- endmacro %} + +{% macro postgres__list_schemas(database) %} + {% if database -%} + {{ adapter.verify_database(database) }} + {%- endif -%} + {% call statement('list_schemas', fetch_result=True, auto_begin=False) %} + select distinct nspname from pg_namespace + {% endcall %} + {{ return(load_result('list_schemas').table) }} +{% endmacro %} + +{% macro postgres__check_schema_exists(information_schema, schema) -%} + {% if information_schema.database -%} + {{ adapter.verify_database(information_schema.database) }} + {%- endif -%} + {% call statement('check_schema_exists', fetch_result=True, auto_begin=False) %} + select count(*) from pg_namespace where nspname = '{{ schema }}' + {% endcall %} + {{ return(load_result('check_schema_exists').table) }} +{% endmacro %} + +{# + Postgres tables have a maximum length of 63 characters, anything longer is silently truncated. + Temp and backup relations add a lot of extra characters to the end of table names to ensure uniqueness. + To prevent this going over the character limit, the base_relation name is truncated to ensure + that name + suffix + uniquestring is < 63 characters. +#} + +{% macro postgres__make_relation_with_suffix(base_relation, suffix, dstring) %} + {% if dstring %} + {% set dt = modules.datetime.datetime.now() %} + {% set dtstring = dt.strftime("%H%M%S%f") %} + {% set suffix = suffix ~ dtstring %} + {% endif %} + {% set suffix_length = suffix|length %} + {% set relation_max_name_length = base_relation.relation_max_name_length() %} + {% if suffix_length > relation_max_name_length %} + {% do exceptions.raise_compiler_error('Relation suffix is too long (' ~ suffix_length ~ ' characters). Maximum length is ' ~ relation_max_name_length ~ ' characters.') %} + {% endif %} + {% set identifier = base_relation.identifier[:relation_max_name_length - suffix_length] ~ suffix %} + + {{ return(base_relation.incorporate(path={"identifier": identifier })) }} +{% endmacro %} + +{% macro postgres__make_intermediate_relation(base_relation, suffix) %} + {{ return(postgres__make_relation_with_suffix(base_relation, suffix, dstring=False)) }} +{% endmacro %} + +{% macro postgres__make_temp_relation(base_relation, suffix) %} + {% set temp_relation = postgres__make_relation_with_suffix(base_relation, suffix, dstring=True) %} + {{ return(temp_relation.incorporate(path={"schema": none, + "database": none})) }} +{% endmacro %} + +{% macro postgres__make_backup_relation(base_relation, backup_relation_type, suffix) %} + {% set backup_relation = postgres__make_relation_with_suffix(base_relation, suffix, dstring=False) %} + {{ return(backup_relation.incorporate(type=backup_relation_type)) }} +{% endmacro %} + +{# + By using dollar-quoting like this, users can embed anything they want into their comments + (including nested dollar-quoting), as long as they do not use this exact dollar-quoting + label. It would be nice to just pick a new one but eventually you do have to give up. +#} +{% macro postgres_escape_comment(comment) -%} + {% if comment is not string %} + {% do exceptions.raise_compiler_error('cannot escape a non-string: ' ~ comment) %} + {% endif %} + {%- set magic = '$dbt_comment_literal_block$' -%} + {%- if magic in comment -%} + {%- do exceptions.raise_compiler_error('The string ' ~ magic ~ ' is not allowed in comments.') -%} + {%- endif -%} + {{ magic }}{{ comment }}{{ magic }} +{%- endmacro %} + + +{% macro postgres__alter_relation_comment(relation, comment) %} + {% set escaped_comment = postgres_escape_comment(comment) %} + {% if relation.type == 'materialized_view' -%} + {% set relation_type = "materialized view" %} + {%- else -%} + {%- set relation_type = relation.type -%} + {%- endif -%} + comment on {{ relation_type }} {{ relation }} is {{ escaped_comment }}; +{% endmacro %} + + +{% macro postgres__alter_column_comment(relation, column_dict) %} + {% set existing_columns = adapter.get_columns_in_relation(relation) | map(attribute="name") | list %} + {% for column_name in column_dict if (column_name in existing_columns) %} + {% set comment = column_dict[column_name]['description'] %} + {% set escaped_comment = postgres_escape_comment(comment) %} + comment on column {{ relation }}.{{ adapter.quote(column_name) if column_dict[column_name]['quote'] else column_name }} is {{ escaped_comment }}; + {% endfor %} +{% endmacro %} + +{%- macro postgres__get_show_grant_sql(relation) -%} + select grantee, privilege_type + from {{ relation.information_schema('role_table_grants') }} + where grantor = current_role + and grantee != current_role + and table_schema = '{{ relation.schema }}' + and table_name = '{{ relation.identifier }}' +{%- endmacro -%} + +{% macro postgres__copy_grants() %} + {{ return(False) }} +{% endmacro %} + + +{% macro postgres__get_show_indexes_sql(relation) %} + select + i.relname as name, + m.amname as method, + ix.indisunique as "unique", + array_to_string(array_agg(a.attname), ',') as column_names + from pg_index ix + join pg_class i + on i.oid = ix.indexrelid + join pg_am m + on m.oid=i.relam + join pg_class t + on t.oid = ix.indrelid + join pg_namespace n + on n.oid = t.relnamespace + join pg_attribute a + on a.attrelid = t.oid + and a.attnum = ANY(ix.indkey) + where t.relname = '{{ relation.identifier }}' + and n.nspname = '{{ relation.schema }}' + and t.relkind in ('r', 'm') + group by 1, 2, 3 + order by 1, 2, 3 +{% endmacro %} + +{%- macro postgres__get_drop_index_sql(relation, index_name) -%} + drop index if exists "{{ relation.schema }}"."{{ index_name }}" +{%- endmacro -%} diff --git a/dbt_internal_packages/dbt-postgres/macros/catalog.sql b/dbt_internal_packages/dbt-postgres/macros/catalog.sql new file mode 100644 index 0000000..42c3ada --- /dev/null +++ b/dbt_internal_packages/dbt-postgres/macros/catalog.sql @@ -0,0 +1,67 @@ + +{% macro postgres__get_catalog_relations(dbschema, relations) -%} + {%- call statement('catalog', fetch_result=True) -%} + + {# + If the user has multiple databases set and the first one is wrong, this will fail. + But we won't fail in the case where there are multiple quoting-difference-only dbs, which is better. + #} + + {{ adapter.verify_database(database) }} + + select + '{{ dbschema.database }}' as table_database, + sch.nspname as table_schema, + tbl.relname as table_name, + case tbl.relkind + when 'v' then 'VIEW' + when 'm' then 'MATERIALIZED VIEW' + else 'BASE TABLE' + end as table_type, + tbl_desc.description as table_comment, + col.attname as column_name, + col.attnum as column_index, + pg_catalog.format_type(col.atttypid, col.atttypmod) as column_type, + col_desc.description as column_comment, + pg_get_userbyid(tbl.relowner) as table_owner + + from pg_catalog.pg_namespace sch + join pg_catalog.pg_class tbl on tbl.relnamespace = sch.oid + join pg_catalog.pg_attribute col on col.attrelid = tbl.oid + left outer join pg_catalog.pg_description tbl_desc on (tbl_desc.objoid = tbl.oid and tbl_desc.objsubid = 0) + left outer join pg_catalog.pg_description col_desc on (col_desc.objoid = tbl.oid and col_desc.objsubid = col.attnum) + where ( + {%- for relation in relations -%} + {%- if relation.identifier -%} + (upper(sch.nspname) = upper('{{ relation.schema }}') and + upper(tbl.relname) = upper('{{ relation.identifier }}')) + {%- else-%} + upper(sch.nspname) = upper('{{ relation.schema }}') + {%- endif -%} + {%- if not loop.last %} or {% endif -%} + {%- endfor -%} + ) + and not pg_is_other_temp_schema(sch.oid) -- not a temporary schema belonging to another session + and tbl.relpersistence in ('p', 'u') -- [p]ermanent table or [u]nlogged table. Exclude [t]emporary tables + and tbl.relkind in ('r', 'v', 'f', 'p', 'm') -- o[r]dinary table, [v]iew, [f]oreign table, [p]artitioned table, [m]aterialized view. Other values are [i]ndex, [S]equence, [c]omposite type, [t]OAST table + and col.attnum > 0 -- negative numbers are used for system columns such as oid + and not col.attisdropped -- column as not been dropped + + order by + sch.nspname, + tbl.relname, + col.attnum + + {%- endcall -%} + + {{ return(load_result('catalog').table) }} +{%- endmacro %} + + +{% macro postgres__get_catalog(dbschema, schemas) -%} + {%- set relations = [] -%} + {%- for schema in schemas -%} + {%- set dummy = relations.append({'schema': schema}) -%} + {%- endfor -%} + {{ return(postgres__get_catalog_relations(dbschema, relations)) }} +{%- endmacro %} diff --git a/dbt_internal_packages/dbt-postgres/macros/materializations/incremental_strategies.sql b/dbt_internal_packages/dbt-postgres/macros/materializations/incremental_strategies.sql new file mode 100644 index 0000000..1d37366 --- /dev/null +++ b/dbt_internal_packages/dbt-postgres/macros/materializations/incremental_strategies.sql @@ -0,0 +1,20 @@ +{% macro postgres__get_incremental_default_sql(arg_dict) %} + + {% if arg_dict["unique_key"] %} + {% do return(get_incremental_delete_insert_sql(arg_dict)) %} + {% else %} + {% do return(get_incremental_append_sql(arg_dict)) %} + {% endif %} + +{% endmacro %} + + +{% macro postgres__get_incremental_microbatch_sql(arg_dict) %} + + {% if arg_dict["unique_key"] %} + {% do return(adapter.dispatch('get_incremental_merge_sql', 'dbt')(arg_dict)) %} + {% else %} + {{ exceptions.raise_compiler_error("dbt-postgres 'microbatch' requires a `unique_key` config") }} + {% endif %} + +{% endmacro %} diff --git a/dbt_internal_packages/dbt-postgres/macros/materializations/snapshot_merge.sql b/dbt_internal_packages/dbt-postgres/macros/materializations/snapshot_merge.sql new file mode 100644 index 0000000..0b4deb1 --- /dev/null +++ b/dbt_internal_packages/dbt-postgres/macros/materializations/snapshot_merge.sql @@ -0,0 +1,25 @@ + +{% macro postgres__snapshot_merge_sql(target, source, insert_cols) -%} + {%- set insert_cols_csv = insert_cols | join(', ') -%} + + {%- set columns = config.get("snapshot_table_column_names") or get_snapshot_table_column_names() -%} + + update {{ target }} + set {{ columns.dbt_valid_to }} = DBT_INTERNAL_SOURCE.{{ columns.dbt_valid_to }} + from {{ source }} as DBT_INTERNAL_SOURCE + where DBT_INTERNAL_SOURCE.{{ columns.dbt_scd_id }}::text = {{ target }}.{{ columns.dbt_scd_id }}::text + and DBT_INTERNAL_SOURCE.dbt_change_type::text in ('update'::text, 'delete'::text) + {% if config.get("dbt_valid_to_current") %} + and ({{ target }}.{{ columns.dbt_valid_to }} = {{ config.get('dbt_valid_to_current') }} or {{ target }}.{{ columns.dbt_valid_to }} is null); + {% else %} + and {{ target }}.{{ columns.dbt_valid_to }} is null; + {% endif %} + + + insert into {{ target }} ({{ insert_cols_csv }}) + select {% for column in insert_cols -%} + DBT_INTERNAL_SOURCE.{{ column }} {%- if not loop.last %}, {%- endif %} + {%- endfor %} + from {{ source }} as DBT_INTERNAL_SOURCE + where DBT_INTERNAL_SOURCE.dbt_change_type::text = 'insert'::text; +{% endmacro %} diff --git a/dbt_internal_packages/dbt-postgres/macros/relations.sql b/dbt_internal_packages/dbt-postgres/macros/relations.sql new file mode 100644 index 0000000..dd50cf0 --- /dev/null +++ b/dbt_internal_packages/dbt-postgres/macros/relations.sql @@ -0,0 +1,80 @@ +{% macro postgres__get_relations() -%} + + {# + -- in pg_depend, objid is the dependent, refobjid is the referenced object + -- > a pg_depend entry indicates that the referenced object cannot be + -- > dropped without also dropping the dependent object. + #} + + {%- call statement('relations', fetch_result=True) -%} + with relation as ( + select + pg_rewrite.ev_class as class, + pg_rewrite.oid as id + from pg_rewrite + ), + class as ( + select + oid as id, + relname as name, + relnamespace as schema, + relkind as kind + from pg_class + ), + dependency as ( + select distinct + pg_depend.objid as id, + pg_depend.refobjid as ref + from pg_depend + ), + schema as ( + select + pg_namespace.oid as id, + pg_namespace.nspname as name + from pg_namespace + where nspname != 'information_schema' and nspname not like 'pg\_%' + ), + referenced as ( + select + relation.id AS id, + referenced_class.name , + referenced_class.schema , + referenced_class.kind + from relation + join class as referenced_class on relation.class=referenced_class.id + where referenced_class.kind in ('r', 'v', 'm') + ), + relationships as ( + select + referenced.name as referenced_name, + referenced.schema as referenced_schema_id, + dependent_class.name as dependent_name, + dependent_class.schema as dependent_schema_id, + referenced.kind as kind + from referenced + join dependency on referenced.id=dependency.id + join class as dependent_class on dependency.ref=dependent_class.id + where + (referenced.name != dependent_class.name or + referenced.schema != dependent_class.schema) + ) + + select + referenced_schema.name as referenced_schema, + relationships.referenced_name as referenced_name, + dependent_schema.name as dependent_schema, + relationships.dependent_name as dependent_name + from relationships + join schema as dependent_schema on relationships.dependent_schema_id=dependent_schema.id + join schema as referenced_schema on relationships.referenced_schema_id=referenced_schema.id + group by referenced_schema, referenced_name, dependent_schema, dependent_name + order by referenced_schema, referenced_name, dependent_schema, dependent_name; + + {%- endcall -%} + + {{ return(load_result('relations').table) }} +{% endmacro %} + +{% macro postgres_get_relations() %} + {{ return(postgres__get_relations()) }} +{% endmacro %} diff --git a/dbt_internal_packages/dbt-postgres/macros/relations/materialized_view/alter.sql b/dbt_internal_packages/dbt-postgres/macros/relations/materialized_view/alter.sql new file mode 100644 index 0000000..429b7e5 --- /dev/null +++ b/dbt_internal_packages/dbt-postgres/macros/relations/materialized_view/alter.sql @@ -0,0 +1,51 @@ +{% macro postgres__get_alter_materialized_view_as_sql( + relation, + configuration_changes, + sql, + existing_relation, + backup_relation, + intermediate_relation +) %} + + -- apply a full refresh immediately if needed + {% if configuration_changes.requires_full_refresh %} + + {{ get_replace_sql(existing_relation, relation, sql) }} + + -- otherwise apply individual changes as needed + {% else %} + + {{ postgres__update_indexes_on_materialized_view(relation, configuration_changes.indexes) }} + + {%- endif -%} + +{% endmacro %} + + +{%- macro postgres__update_indexes_on_materialized_view(relation, index_changes) -%} + {{- log("Applying UPDATE INDEXES to: " ~ relation) -}} + + {%- for _index_change in index_changes -%} + {%- set _index = _index_change.context -%} + + {%- if _index_change.action == "drop" -%} + + {{ postgres__get_drop_index_sql(relation, _index.name) }} + + {%- elif _index_change.action == "create" -%} + + {{ postgres__get_create_index_sql(relation, _index.as_node_config) }} + + {%- endif -%} + {{ ';' if not loop.last else "" }} + + {%- endfor -%} + +{%- endmacro -%} + + +{% macro postgres__get_materialized_view_configuration_changes(existing_relation, new_config) %} + {% set _existing_materialized_view = postgres__describe_materialized_view(existing_relation) %} + {% set _configuration_changes = existing_relation.get_materialized_view_config_change_collection(_existing_materialized_view, new_config.model) %} + {% do return(_configuration_changes) %} +{% endmacro %} diff --git a/dbt_internal_packages/dbt-postgres/macros/relations/materialized_view/create.sql b/dbt_internal_packages/dbt-postgres/macros/relations/materialized_view/create.sql new file mode 100644 index 0000000..89c1823 --- /dev/null +++ b/dbt_internal_packages/dbt-postgres/macros/relations/materialized_view/create.sql @@ -0,0 +1,8 @@ +{% macro postgres__get_create_materialized_view_as_sql(relation, sql) %} + create materialized view if not exists {{ relation }} as {{ sql }}; + + {% for _index_dict in config.get('indexes', []) -%} + {{- get_create_index_sql(relation, _index_dict) -}}{{ ';' if not loop.last else "" }} + {%- endfor -%} + +{% endmacro %} diff --git a/dbt_internal_packages/dbt-postgres/macros/relations/materialized_view/describe.sql b/dbt_internal_packages/dbt-postgres/macros/relations/materialized_view/describe.sql new file mode 100644 index 0000000..cb133b6 --- /dev/null +++ b/dbt_internal_packages/dbt-postgres/macros/relations/materialized_view/describe.sql @@ -0,0 +1,5 @@ +{% macro postgres__describe_materialized_view(relation) %} + -- for now just get the indexes, we don't need the name or the query yet + {% set _indexes = run_query(get_show_indexes_sql(relation)) %} + {% do return({'indexes': _indexes}) %} +{% endmacro %} diff --git a/dbt_internal_packages/dbt-postgres/macros/relations/materialized_view/drop.sql b/dbt_internal_packages/dbt-postgres/macros/relations/materialized_view/drop.sql new file mode 100644 index 0000000..2263bb6 --- /dev/null +++ b/dbt_internal_packages/dbt-postgres/macros/relations/materialized_view/drop.sql @@ -0,0 +1,3 @@ +{% macro postgres__drop_materialized_view(relation) -%} + drop materialized view if exists {{ relation }} cascade +{%- endmacro %} diff --git a/dbt_internal_packages/dbt-postgres/macros/relations/materialized_view/refresh.sql b/dbt_internal_packages/dbt-postgres/macros/relations/materialized_view/refresh.sql new file mode 100644 index 0000000..48b863e --- /dev/null +++ b/dbt_internal_packages/dbt-postgres/macros/relations/materialized_view/refresh.sql @@ -0,0 +1,3 @@ +{% macro postgres__refresh_materialized_view(relation) %} + refresh materialized view {{ relation }} +{% endmacro %} diff --git a/dbt_internal_packages/dbt-postgres/macros/relations/materialized_view/rename.sql b/dbt_internal_packages/dbt-postgres/macros/relations/materialized_view/rename.sql new file mode 100644 index 0000000..293ec9d --- /dev/null +++ b/dbt_internal_packages/dbt-postgres/macros/relations/materialized_view/rename.sql @@ -0,0 +1,3 @@ +{% macro postgres__get_rename_materialized_view_sql(relation, new_name) %} + alter materialized view {{ relation }} rename to {{ new_name }} +{% endmacro %} diff --git a/dbt_internal_packages/dbt-postgres/macros/relations/table/drop.sql b/dbt_internal_packages/dbt-postgres/macros/relations/table/drop.sql new file mode 100644 index 0000000..146cfc8 --- /dev/null +++ b/dbt_internal_packages/dbt-postgres/macros/relations/table/drop.sql @@ -0,0 +1,3 @@ +{% macro postgres__drop_table(relation) -%} + drop table if exists {{ relation }} cascade +{%- endmacro %} diff --git a/dbt_internal_packages/dbt-postgres/macros/relations/table/rename.sql b/dbt_internal_packages/dbt-postgres/macros/relations/table/rename.sql new file mode 100644 index 0000000..1693c01 --- /dev/null +++ b/dbt_internal_packages/dbt-postgres/macros/relations/table/rename.sql @@ -0,0 +1,4 @@ +-- funcsign: (relation, string) -> string +{% macro postgres__get_rename_table_sql(relation, new_name) %} + alter table {{ relation }} rename to {{ new_name }} +{% endmacro %} diff --git a/dbt_internal_packages/dbt-postgres/macros/relations/table/replace.sql b/dbt_internal_packages/dbt-postgres/macros/relations/table/replace.sql new file mode 100644 index 0000000..3750edf --- /dev/null +++ b/dbt_internal_packages/dbt-postgres/macros/relations/table/replace.sql @@ -0,0 +1,17 @@ +{% macro postgres__get_replace_table_sql(relation, sql) -%} + + {%- set sql_header = config.get('sql_header', none) -%} + {{ sql_header if sql_header is not none }} + + create or replace table {{ relation }} + {% set contract_config = config.get('contract') %} + {% if contract_config.enforced %} + {{ get_assert_columns_equivalent(sql) }} + {{ get_table_columns_and_constraints() }} + {%- set sql = get_select_subquery(sql) %} + {% endif %} + as ( + {{ sql }} + ); + +{%- endmacro %} diff --git a/dbt_internal_packages/dbt-postgres/macros/relations/view/drop.sql b/dbt_internal_packages/dbt-postgres/macros/relations/view/drop.sql new file mode 100644 index 0000000..46bd5a0 --- /dev/null +++ b/dbt_internal_packages/dbt-postgres/macros/relations/view/drop.sql @@ -0,0 +1,3 @@ +{% macro postgres__drop_view(relation) -%} + drop view if exists {{ relation }} cascade +{%- endmacro %} diff --git a/dbt_internal_packages/dbt-postgres/macros/relations/view/rename.sql b/dbt_internal_packages/dbt-postgres/macros/relations/view/rename.sql new file mode 100644 index 0000000..3c890a5 --- /dev/null +++ b/dbt_internal_packages/dbt-postgres/macros/relations/view/rename.sql @@ -0,0 +1,3 @@ +{% macro postgres__get_rename_view_sql(relation, new_name) %} + alter view {{ relation }} rename to {{ new_name }} +{% endmacro %} diff --git a/dbt_internal_packages/dbt-postgres/macros/relations/view/replace.sql b/dbt_internal_packages/dbt-postgres/macros/relations/view/replace.sql new file mode 100644 index 0000000..e2724c3 --- /dev/null +++ b/dbt_internal_packages/dbt-postgres/macros/relations/view/replace.sql @@ -0,0 +1,15 @@ +{% macro postgres__get_replace_view_sql(relation, sql) -%} + + {%- set sql_header = config.get('sql_header', none) -%} + {{ sql_header if sql_header is not none }} + + create or replace view {{ relation }} + {% set contract_config = config.get('contract') %} + {% if contract_config.enforced %} + {{ get_assert_columns_equivalent(sql) }} + {%- endif %} + as ( + {{ sql }} + ); + +{%- endmacro %} diff --git a/dbt_internal_packages/dbt-postgres/macros/timestamps.sql b/dbt_internal_packages/dbt-postgres/macros/timestamps.sql new file mode 100644 index 0000000..7233571 --- /dev/null +++ b/dbt_internal_packages/dbt-postgres/macros/timestamps.sql @@ -0,0 +1,20 @@ +{% macro postgres__current_timestamp() -%} + now() +{%- endmacro %} + +{% macro postgres__snapshot_string_as_time(timestamp) -%} + {%- set result = "'" ~ timestamp ~ "'::timestamp without time zone" -%} + {{ return(result) }} +{%- endmacro %} + +{% macro postgres__snapshot_get_time() -%} + {{ current_timestamp() }}::timestamp without time zone +{%- endmacro %} + +{% macro postgres__current_timestamp_backcompat() %} + current_timestamp::{{ type_timestamp() }} +{% endmacro %} + +{% macro postgres__current_timestamp_in_utc_backcompat() %} + (current_timestamp at time zone 'utc')::{{ type_timestamp() }} +{% endmacro %} diff --git a/dbt_internal_packages/dbt-postgres/macros/utils/any_value.sql b/dbt_internal_packages/dbt-postgres/macros/utils/any_value.sql new file mode 100644 index 0000000..6fcb4ee --- /dev/null +++ b/dbt_internal_packages/dbt-postgres/macros/utils/any_value.sql @@ -0,0 +1,7 @@ +{#- /*Postgres doesn't support any_value, so we're using min() to get the same result*/ -#} + +{% macro postgres__any_value(expression) -%} + + min({{ expression }}) + +{%- endmacro %} diff --git a/dbt_internal_packages/dbt-postgres/macros/utils/dateadd.sql b/dbt_internal_packages/dbt-postgres/macros/utils/dateadd.sql new file mode 100644 index 0000000..97009cc --- /dev/null +++ b/dbt_internal_packages/dbt-postgres/macros/utils/dateadd.sql @@ -0,0 +1,5 @@ +{% macro postgres__dateadd(datepart, interval, from_date_or_timestamp) %} + + {{ from_date_or_timestamp }} + ((interval '1 {{ datepart }}') * ({{ interval }})) + +{% endmacro %} diff --git a/dbt_internal_packages/dbt-postgres/macros/utils/datediff.sql b/dbt_internal_packages/dbt-postgres/macros/utils/datediff.sql new file mode 100644 index 0000000..b452529 --- /dev/null +++ b/dbt_internal_packages/dbt-postgres/macros/utils/datediff.sql @@ -0,0 +1,32 @@ +{% macro postgres__datediff(first_date, second_date, datepart) -%} + + {% if datepart == 'year' %} + (date_part('year', ({{second_date}})::date) - date_part('year', ({{first_date}})::date)) + {% elif datepart == 'quarter' %} + ({{ datediff(first_date, second_date, 'year') }} * 4 + date_part('quarter', ({{second_date}})::date) - date_part('quarter', ({{first_date}})::date)) + {% elif datepart == 'month' %} + ({{ datediff(first_date, second_date, 'year') }} * 12 + date_part('month', ({{second_date}})::date) - date_part('month', ({{first_date}})::date)) + {% elif datepart == 'day' %} + (({{second_date}})::date - ({{first_date}})::date) + {% elif datepart == 'week' %} + ({{ datediff(first_date, second_date, 'day') }} / 7 + case + when date_part('dow', ({{first_date}})::timestamp) <= date_part('dow', ({{second_date}})::timestamp) then + case when {{first_date}} <= {{second_date}} then 0 else -1 end + else + case when {{first_date}} <= {{second_date}} then 1 else 0 end + end) + {% elif datepart == 'hour' %} + ({{ datediff(first_date, second_date, 'day') }} * 24 + date_part('hour', ({{second_date}})::timestamp) - date_part('hour', ({{first_date}})::timestamp)) + {% elif datepart == 'minute' %} + ({{ datediff(first_date, second_date, 'hour') }} * 60 + date_part('minute', ({{second_date}})::timestamp) - date_part('minute', ({{first_date}})::timestamp)) + {% elif datepart == 'second' %} + ({{ datediff(first_date, second_date, 'minute') }} * 60 + floor(date_part('second', ({{second_date}})::timestamp)) - floor(date_part('second', ({{first_date}})::timestamp))) + {% elif datepart == 'millisecond' %} + ({{ datediff(first_date, second_date, 'minute') }} * 60000 + floor(date_part('millisecond', ({{second_date}})::timestamp)) - floor(date_part('millisecond', ({{first_date}})::timestamp))) + {% elif datepart == 'microsecond' %} + ({{ datediff(first_date, second_date, 'minute') }} * 60000000 + floor(date_part('microsecond', ({{second_date}})::timestamp)) - floor(date_part('microsecond', ({{first_date}})::timestamp))) + {% else %} + {{ exceptions.raise_compiler_error("Unsupported datepart for macro datediff in postgres: {!r}".format(datepart)) }} + {% endif %} + +{%- endmacro %} diff --git a/dbt_internal_packages/dbt-postgres/macros/utils/last_day.sql b/dbt_internal_packages/dbt-postgres/macros/utils/last_day.sql new file mode 100644 index 0000000..1699530 --- /dev/null +++ b/dbt_internal_packages/dbt-postgres/macros/utils/last_day.sql @@ -0,0 +1,14 @@ +{% macro postgres__last_day(date, datepart) -%} + + {%- if datepart == 'quarter' -%} + -- postgres dateadd does not support quarter interval. + cast( + {{dbt.dateadd('day', '-1', + dbt.dateadd('month', '3', dbt.date_trunc(datepart, date)) + )}} + as date) + {%- else -%} + {{dbt.default_last_day(date, datepart)}} + {%- endif -%} + +{%- endmacro %} diff --git a/dbt_internal_packages/dbt-postgres/macros/utils/listagg.sql b/dbt_internal_packages/dbt-postgres/macros/utils/listagg.sql new file mode 100644 index 0000000..f3e1942 --- /dev/null +++ b/dbt_internal_packages/dbt-postgres/macros/utils/listagg.sql @@ -0,0 +1,23 @@ +{% macro postgres__listagg(measure, delimiter_text, order_by_clause, limit_num) -%} + + {% if limit_num -%} + array_to_string( + (array_agg( + {{ measure }} + {% if order_by_clause -%} + {{ order_by_clause }} + {%- endif %} + ))[1:{{ limit_num }}], + {{ delimiter_text }} + ) + {%- else %} + string_agg( + {{ measure }}, + {{ delimiter_text }} + {% if order_by_clause -%} + {{ order_by_clause }} + {%- endif %} + ) + {%- endif %} + +{%- endmacro %} diff --git a/dbt_internal_packages/dbt-postgres/macros/utils/split_part.sql b/dbt_internal_packages/dbt-postgres/macros/utils/split_part.sql new file mode 100644 index 0000000..e4174d2 --- /dev/null +++ b/dbt_internal_packages/dbt-postgres/macros/utils/split_part.sql @@ -0,0 +1,9 @@ +{% macro postgres__split_part(string_text, delimiter_text, part_number) %} + + {% if part_number >= 0 %} + {{ dbt.default__split_part(string_text, delimiter_text, part_number) }} + {% else %} + {{ dbt._split_part_negative(string_text, delimiter_text, part_number) }} + {% endif %} + +{% endmacro %} diff --git a/docker-compose.yml b/docker-compose.yml index 0aee0d3..44597ae 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -16,4 +16,4 @@ services: environment: POSTGRES_PASSWORD: postgres ports: - - 5432:5432 + - 55432:5432 diff --git a/models/business_vault/customer_pit.sql b/models/business_vault/customer_pit.sql index a3392a6..b20a07f 100644 --- a/models/business_vault/customer_pit.sql +++ b/models/business_vault/customer_pit.sql @@ -3,21 +3,21 @@ {%- set yaml_metadata -%} source_model: hub_customer src_pk: CUSTOMER_PK -as_of_dates_table: AS_OF_DATE +as_of_dates_table: as_of_date satellites: - SAT_CUSTOMER: + sat_customer: pk: PK: CUSTOMER_PK ldts: LDTS: LOAD_DATE - SAT_CUSTOMER_CRM: + sat_customer_crm: pk: PK: CUSTOMER_PK ldts: LDTS: LOAD_DATE stage_tables: - STG_CUSTOMERS: LOAD_DATE - STG_CUSTOMERS_CRM: LOAD_DATE + stg_customers: LOAD_DATE + stg_customers_crm: LOAD_DATE src_ldts: LOAD_DATE {%- endset -%} @@ -27,7 +27,7 @@ src_ldts: LOAD_DATE {% set src_pk = metadata_dict['src_pk'] %} {% set as_of_dates_table = metadata_dict['as_of_dates_table'] %} {% set satellites = metadata_dict['satellites'] %} -{% set stage_tables_ldts = metadata_dict['stage_tables_ldts'] %} +{% set stage_tables_ldts = metadata_dict['stage_tables'] %} {% set src_ldts = metadata_dict['src_ldts'] %} {{ automate_dv.pit(source_model=source_model, src_pk=src_pk, diff --git a/models/business_vault/our_customer_pit.sql b/models/business_vault/our_customer_pit.sql index d5954f6..8b23a73 100644 --- a/models/business_vault/our_customer_pit.sql +++ b/models/business_vault/our_customer_pit.sql @@ -1,4 +1,4 @@ -with all_history ( +with all_history as ( select hc.customer_pk, sc.last_name, sc.first_name, @@ -11,7 +11,6 @@ with all_history ( LEFT JOIN {{ref('sat_customer')}} sc ON sc.customer_pk = hc.customer_pk LEFT JOIN {{ref('sat_customer_crm')}} scc ON scc.customer_pk = hc.customer_pk ) - -SELECT customer_pk, last_name, first_name, age, sc_effective_from, scc_effective_from +select customer_pk, last_name, first_name, age, sc_effective_from, scc_effective_from from all_history where 1=1 \ No newline at end of file diff --git a/models/customers_rating.sql b/models/customers_rating.sql new file mode 100644 index 0000000..64ca657 --- /dev/null +++ b/models/customers_rating.sql @@ -0,0 +1,19 @@ +with ord_stat as( +select + hc.customer_key , count(distinct ho.order_key) as order_count +from + {{ref('link_customer_order')}} lco + inner join {{ref('hub_order')}} ho on ho.order_pk = lco.order_pk + inner join {{ref('sat_order')}} so on so.order_pk = lco.order_pk + inner join {{ref('hub_customer')}} hc on hc.customer_pk =lco.customer_pk + where so.status ='completed' + group by hc.customer_key + ) +select + hc.customer_key, + coalesce(o.order_count,0) as order_count +from + {{ref('hub_customer')}} hc + left join ord_stat o on o.customer_key = hc.customer_key +order by + coalesce(o.order_count,0) desc \ No newline at end of file diff --git a/models/weekly_orders_stat.sql b/models/weekly_orders_stat.sql new file mode 100644 index 0000000..1d62227 --- /dev/null +++ b/models/weekly_orders_stat.sql @@ -0,0 +1,9 @@ +WITH orders_weekly AS ( + SELECT + DATE_TRUNC('week', order_date) AS calendar_week, + COUNT(DISTINCT id) AS orders_count + from {{ ref('stg_orders') }} so + GROUP BY calendar_week +) +SELECT * FROM orders_weekly +ORDER BY calendar_week \ No newline at end of file diff --git a/seeds/source_orders.csv b/seeds/source_orders.csv index c487062..0022b15 100644 --- a/seeds/source_orders.csv +++ b/seeds/source_orders.csv @@ -98,3 +98,4 @@ id,user_id,order_date,status 97,89,2018-04-07,placed 98,41,2018-04-07,placed 99,85,2018-04-09,placed +1000,85,2025-11-24,completed \ No newline at end of file