From 408f866be72d95bc924895e592d0dffb04343a02 Mon Sep 17 00:00:00 2001 From: Alice Naghshineh Date: Tue, 27 Dec 2022 17:19:08 -0700 Subject: [PATCH 1/6] feat: First pass at implementing UDF materialization --- dbt/adapters/bigquery/impl.py | 22 +++++++++--- dbt/include/bigquery/macros/adapters.sql | 35 +++++++++++++++++++ .../bigquery/macros/materializations/udf.sql | 16 +++++++++ 3 files changed, 68 insertions(+), 5 deletions(-) create mode 100644 dbt/include/bigquery/macros/materializations/udf.sql diff --git a/dbt/adapters/bigquery/impl.py b/dbt/adapters/bigquery/impl.py index ec749c316..8e9d6849b 100644 --- a/dbt/adapters/bigquery/impl.py +++ b/dbt/adapters/bigquery/impl.py @@ -747,20 +747,26 @@ def _get_catalog_schemas(self, manifest: Manifest) -> SchemaSearchMap: ) return result + @staticmethod + def _get_description_option(self, config: Dict[str, Any], node: Dict[str, Any]) -> Dict[str, str]: + opts = {} + + if config.persist_relation_docs() and "description" in node: # type: ignore[attr-defined] + description = sql_escape(node["description"]) + opts["description"] = '"""{}"""'.format(description) + + return opts + @available.parse(lambda *a, **k: {}) def get_common_options( self, config: Dict[str, Any], node: Dict[str, Any], temporary: bool = False ) -> Dict[str, Any]: - opts = {} + opts = self._get_description_option(config, node) if (config.get("hours_to_expiration") is not None) and (not temporary): expiration = f'TIMESTAMP_ADD(CURRENT_TIMESTAMP(), INTERVAL {config.get("hours_to_expiration")} hour)' opts["expiration_timestamp"] = expiration - if config.persist_relation_docs() and "description" in node: # type: ignore[attr-defined] - description = sql_escape(node["description"]) - opts["description"] = '"""{}"""'.format(description) - if config.get("labels"): labels = config.get("labels", {}) opts["labels"] = list(labels.items()) # type: ignore[assignment] @@ -797,6 +803,12 @@ def get_view_options(self, config: Dict[str, Any], node: Dict[str, Any]) -> Dict opts = self.get_common_options(config, node) return opts + @available.parse(lambda *a, **k: {}) + def get_udf_options(self, config: Dict[str, Any], node: Dict[str, Any]) -> Dict[str, str]: + opts = self._get_description_option(config, node) + + return opts + @available.parse_none def grant_access_to(self, entity, entity_type, role, grant_target_dict): """ diff --git a/dbt/include/bigquery/macros/adapters.sql b/dbt/include/bigquery/macros/adapters.sql index 07cf3c3e5..a98db7df9 100644 --- a/dbt/include/bigquery/macros/adapters.sql +++ b/dbt/include/bigquery/macros/adapters.sql @@ -89,6 +89,41 @@ {% endmacro %} +{% macro bigquery_udf_options(config, node, temporary) %} + {% set opts = adapter.get_udf_options(config, node, temporary) %} + {%- do return(bigquery_options(opts)) -%} +{%- endmacro -%} + +{% macro bigquery_udf_returns(udf_returns_type) %} + {% set udf_returns -%} + RETURNS {{ udf_returns_type }} + {%- endset %} + {%- do return(udf_returns) -%} +{%- endmacro -%} + +{% macro bigquery_udf_args(udf_args_dict) %} + {% set udf_args -%} + {% for arg_key, arg_val in udf_args_dict.items() %} + {{ arg_key }} {{ arg_value }}{% if not loop.last %},{% endif %} + {% endfor %}) + {%- endset %} + {%- do return(udf_args) -%} +{%- endmacro -%} + +{% macro bigquery__get_create_udf_as_sql(relation, sql) -%} + {%- set udf_returns_type = config.get('returns', none) -%} + {%- set udf_args_dict = config.get('args', none) -%} + {%- set sql_header = config.get('sql_header', none) -%} + + {{ sql_header if sql_header is not none }} + + create or replace function {{ relation }} ( + {{ bigquery_udf_args(udf_args_dict) }} + ) {% if udf_returns_type -%}{{ bigquery_udf_returns(udf_returns_type) }}{% endif %} as ( + {{ sql }} + ) {{ bigquery_udf_options(config, model) }}; +{%- endmacro -%} + {% macro bigquery__drop_schema(relation) -%} {{ adapter.drop_schema(relation) }} {% endmacro %} diff --git a/dbt/include/bigquery/macros/materializations/udf.sql b/dbt/include/bigquery/macros/materializations/udf.sql new file mode 100644 index 000000000..22bd2472c --- /dev/null +++ b/dbt/include/bigquery/macros/materializations/udf.sql @@ -0,0 +1,16 @@ +{% materialization udf, adapter='bigquery' %} + {%- set target_relation = this %} + + {{ run_hooks(pre_hooks) }} + + -- create the UDF + {%- call statement('main') -%} + {{ get_create_udf_as_sql(target_relation, sql) }} + {%- endcall -%} + + {{ run_hooks(post_hooks) }} + + -- TODO - grants + + {{ return({'relations': [target_relation]}) }} +{% endmaterialization %} From 6adda75fdafde3956fb727779b41941f774212d5 Mon Sep 17 00:00:00 2001 From: Alice Naghshineh Date: Tue, 27 Dec 2022 17:41:42 -0700 Subject: [PATCH 2/6] docs: Add some inline docs around UDF docs & grants --- dbt/include/bigquery/macros/materializations/udf.sql | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/dbt/include/bigquery/macros/materializations/udf.sql b/dbt/include/bigquery/macros/materializations/udf.sql index 22bd2472c..079d51157 100644 --- a/dbt/include/bigquery/macros/materializations/udf.sql +++ b/dbt/include/bigquery/macros/materializations/udf.sql @@ -10,7 +10,14 @@ {{ run_hooks(post_hooks) }} - -- TODO - grants + -- We do not need to explicitly call persist_docs as in other + -- materializations because UDF documentation is handled in the + -- get_create_udf_as_sql macro. There is no concept of column-level + -- documentation for UDFs. + + -- Not calling apply_grants because dataset-level grants not + -- yet supported in dbt, and BigQuery does not support permissions + -- at the level of individual UDFs {{ return({'relations': [target_relation]}) }} {% endmaterialization %} From ed0864f06cab57752e2ebe9f427516a310508f8e Mon Sep 17 00:00:00 2001 From: Alice Naghshineh Date: Thu, 29 Dec 2022 13:59:31 -0700 Subject: [PATCH 3/6] fix: Fix some issues in bigquery_udf_args and bigquery__get_create_udf_as_sql --- dbt/adapters/bigquery/impl.py | 7 ++++++ dbt/include/bigquery/macros/adapters.sql | 22 ++++++++++--------- .../bigquery/macros/materializations/udf.sql | 2 +- 3 files changed, 20 insertions(+), 11 deletions(-) diff --git a/dbt/adapters/bigquery/impl.py b/dbt/adapters/bigquery/impl.py index 8e9d6849b..ab2f33261 100644 --- a/dbt/adapters/bigquery/impl.py +++ b/dbt/adapters/bigquery/impl.py @@ -749,6 +749,9 @@ def _get_catalog_schemas(self, manifest: Manifest) -> SchemaSearchMap: @staticmethod def _get_description_option(self, config: Dict[str, Any], node: Dict[str, Any]) -> Dict[str, str]: + """ + Sets the description, if applicable, as part of the relation options + """ opts = {} if config.persist_relation_docs() and "description" in node: # type: ignore[attr-defined] @@ -805,6 +808,10 @@ def get_view_options(self, config: Dict[str, Any], node: Dict[str, Any]) -> Dict @available.parse(lambda *a, **k: {}) def get_udf_options(self, config: Dict[str, Any], node: Dict[str, Any]) -> Dict[str, str]: + """ + Constructs a dictionary of options for creating a UDF. + Currently, only supports a description option. + """ opts = self._get_description_option(config, node) return opts diff --git a/dbt/include/bigquery/macros/adapters.sql b/dbt/include/bigquery/macros/adapters.sql index a98db7df9..dbed19d63 100644 --- a/dbt/include/bigquery/macros/adapters.sql +++ b/dbt/include/bigquery/macros/adapters.sql @@ -95,31 +95,33 @@ {%- endmacro -%} {% macro bigquery_udf_returns(udf_returns_type) %} - {% set udf_returns -%} - RETURNS {{ udf_returns_type }} - {%- endset %} + {% if udf_returns_type -%} + {% set udf_returns -%} + RETURNS {{ udf_returns_type }} + {%- endset %} + {% endif %} {%- do return(udf_returns) -%} {%- endmacro -%} -{% macro bigquery_udf_args(udf_args_dict) %} +{% macro bigquery_udf_args(udf_args_array) %} {% set udf_args -%} - {% for arg_key, arg_val in udf_args_dict.items() %} - {{ arg_key }} {{ arg_value }}{% if not loop.last %},{% endif %} - {% endfor %}) + {%- for udf_arg in udf_args_array -%} + {{ udf_arg.name }} {{ udf_arg.type }}{% if not loop.last %},{% endif %} + {% endfor %} {%- endset %} {%- do return(udf_args) -%} {%- endmacro -%} {% macro bigquery__get_create_udf_as_sql(relation, sql) -%} {%- set udf_returns_type = config.get('returns', none) -%} - {%- set udf_args_dict = config.get('args', none) -%} + {%- set udf_args_array = config.get('args', []) -%} {%- set sql_header = config.get('sql_header', none) -%} {{ sql_header if sql_header is not none }} create or replace function {{ relation }} ( - {{ bigquery_udf_args(udf_args_dict) }} - ) {% if udf_returns_type -%}{{ bigquery_udf_returns(udf_returns_type) }}{% endif %} as ( + {{ bigquery_udf_args(udf_args_array) }} + ) {{ bigquery_udf_returns(udf_returns_type) }} as ( {{ sql }} ) {{ bigquery_udf_options(config, model) }}; {%- endmacro -%} diff --git a/dbt/include/bigquery/macros/materializations/udf.sql b/dbt/include/bigquery/macros/materializations/udf.sql index 079d51157..394988ed7 100644 --- a/dbt/include/bigquery/macros/materializations/udf.sql +++ b/dbt/include/bigquery/macros/materializations/udf.sql @@ -5,7 +5,7 @@ -- create the UDF {%- call statement('main') -%} - {{ get_create_udf_as_sql(target_relation, sql) }} + {{ bigquery__get_create_udf_as_sql(target_relation, sql) }} {%- endcall -%} {{ run_hooks(post_hooks) }} From 969a97735f7daee79ad0007b5bbb77d8d698b063 Mon Sep 17 00:00:00 2001 From: Alice Naghshineh Date: Sat, 31 Dec 2022 18:01:34 -0700 Subject: [PATCH 4/6] test: Add integration tests for UDf materialization --- dbt/adapters/bigquery/connections.py | 13 +++ dbt/include/bigquery/macros/adapters.sql | 10 +- .../bigquery/macros/materializations/udf.sql | 6 +- .../test_bigquery_udf_materialization.py | 93 +++++++++++++++++++ .../bigquery_test/udf-models/udf_no_args.sql | 9 ++ .../udf-models/udf_with_args.sql | 14 +++ .../udf-models/udf_with_description.sql | 9 ++ .../bigquery_test/udf-models/udfs.yml | 8 ++ 8 files changed, 154 insertions(+), 8 deletions(-) create mode 100644 tests/integration/bigquery_test/test_bigquery_udf_materialization.py create mode 100644 tests/integration/bigquery_test/udf-models/udf_no_args.sql create mode 100644 tests/integration/bigquery_test/udf-models/udf_with_args.sql create mode 100644 tests/integration/bigquery_test/udf-models/udf_with_description.sql create mode 100644 tests/integration/bigquery_test/udf-models/udfs.yml diff --git a/dbt/adapters/bigquery/connections.py b/dbt/adapters/bigquery/connections.py index e4643f660..c85672e7c 100644 --- a/dbt/adapters/bigquery/connections.py +++ b/dbt/adapters/bigquery/connections.py @@ -587,6 +587,19 @@ def table_ref(database, schema, table_name): dataset_ref = google.cloud.bigquery.DatasetReference(database, schema) return google.cloud.bigquery.TableReference(dataset_ref, table_name) + @staticmethod + def routine_ref(database: str, schema: str, routine_name: str) -> google.cloud.bigquery.RoutineReference: + return google.cloud.bigquery.RoutineReference.from_string(f"{database}.{schema}.{routine_name}") + + def get_bq_routine(self, database: str, schema: str, identifier: str) -> google.cloud.bigquery.Routine: + """Get a BigQuery UDF for a schema/model.""" + conn = self.get_thread_connection() + # backwards compatibility: fill in with defaults if not specified + database = database or conn.credentials.database + schema = schema or conn.credentials.schema + routine_ref = self.routine_ref(database, schema, identifier) + return conn.handle.get_routine(routine_ref) + def get_bq_table(self, database, schema, identifier): """Get a bigquery table for a schema/model.""" conn = self.get_thread_connection() diff --git a/dbt/include/bigquery/macros/adapters.sql b/dbt/include/bigquery/macros/adapters.sql index dbed19d63..a814e5f4b 100644 --- a/dbt/include/bigquery/macros/adapters.sql +++ b/dbt/include/bigquery/macros/adapters.sql @@ -94,10 +94,10 @@ {%- do return(bigquery_options(opts)) -%} {%- endmacro -%} -{% macro bigquery_udf_returns(udf_returns_type) %} - {% if udf_returns_type -%} +{% macro bigquery_udf_returns(udf_return_type) %} + {% if udf_return_type -%} {% set udf_returns -%} - RETURNS {{ udf_returns_type }} + RETURNS {{ udf_return_type }} {%- endset %} {% endif %} {%- do return(udf_returns) -%} @@ -113,7 +113,7 @@ {%- endmacro -%} {% macro bigquery__get_create_udf_as_sql(relation, sql) -%} - {%- set udf_returns_type = config.get('returns', none) -%} + {%- set udf_return_type = config.get('return_type', none) -%} {%- set udf_args_array = config.get('args', []) -%} {%- set sql_header = config.get('sql_header', none) -%} @@ -121,7 +121,7 @@ create or replace function {{ relation }} ( {{ bigquery_udf_args(udf_args_array) }} - ) {{ bigquery_udf_returns(udf_returns_type) }} as ( + ) {{ bigquery_udf_returns(udf_return_type) }} as ( {{ sql }} ) {{ bigquery_udf_options(config, model) }}; {%- endmacro -%} diff --git a/dbt/include/bigquery/macros/materializations/udf.sql b/dbt/include/bigquery/macros/materializations/udf.sql index 394988ed7..6fcb5603e 100644 --- a/dbt/include/bigquery/macros/materializations/udf.sql +++ b/dbt/include/bigquery/macros/materializations/udf.sql @@ -3,7 +3,7 @@ {{ run_hooks(pre_hooks) }} - -- create the UDF + -- Create the UDF {%- call statement('main') -%} {{ bigquery__get_create_udf_as_sql(target_relation, sql) }} {%- endcall -%} @@ -16,8 +16,8 @@ -- documentation for UDFs. -- Not calling apply_grants because dataset-level grants not - -- yet supported in dbt, and BigQuery does not support permissions - -- at the level of individual UDFs + -- yet natively supported in dbt, and BigQuery does not support + -- permissions at the level of individual UDFs {{ return({'relations': [target_relation]}) }} {% endmaterialization %} diff --git a/tests/integration/bigquery_test/test_bigquery_udf_materialization.py b/tests/integration/bigquery_test/test_bigquery_udf_materialization.py new file mode 100644 index 000000000..aad6d8129 --- /dev/null +++ b/tests/integration/bigquery_test/test_bigquery_udf_materialization.py @@ -0,0 +1,93 @@ +from tests.integration.base import DBTIntegrationTest, use_profile + +class TestBigQueryUDFMaterialization(DBTIntegrationTest): + + @property + def schema(self): + return "bigquery_test" + + @property + def models(self): + return "udf-models" + + @property + def project_config(self): + return { + 'config-version': 2, + 'vars': { + 'udf_description': self.udf_description + } + } + + @property + def udf_description(self): + return 'this is a UDF' + + @use_profile('bigquery') + def test__bigquery_materialize_udf_no_args(self): + results = self.run_dbt(['run', '--select', 'udf_no_args']) + self.assertEqual(len(results), 1) + + with self.get_connection() as conn: + client = conn.handle + + udf = client.get_routine( + self.adapter.connections.get_bq_routine( + self.default_database, self.unique_schema(), 'udf_no_args' + ) + ) + + self.assertEqual(len(udf.arguments), 0) + self.assertEqual(udf.return_type, 'INT64') + # No description specified, so should be None + self.assertIsNone(udf.description) + + @use_profile('bigquery') + def test__bigquery_materialize_udf_with_args(self): + results = self.run_dbt(['run', '--select', 'udf_with_args']) + self.assertEqual(len(results), 1) + + with self.get_connection() as conn: + client = conn.handle + + udf = client.get_routine( + self.adapter.connections.get_bq_routine( + self.default_database, self.unique_schema(), 'udf_no_args' + ) + ) + + # Make sure the UDF has two args as specified in model config + self.assertEqual(len(udf.arguments), 2) + + # Check the name & type of the first arg + self.assertEqual(udf.arguments[0].name, 'color') + self.assertEqual(udf.arguments[0].data_type.type_kind.name, 'STRING') + + # Check the name & type of the second arg + self.assertEqual(udf.arguments[1].name, 'is_pretty') + self.assertEqual(udf.arguments[1].data_type.type_kind.name, 'BOOL') + + # Model config did not specify return_type, so should be None + self.assertIsNone(udf.return_type) + # No description specified, so should be None + self.assertIsNone(udf.description) + + + @use_profile('bigquery') + def test__bigquery_materialize_udf_with_description(self): + results = self.run_dbt(['run', '--select', 'udf_with_description']) + self.assertEqual(len(results), 1) + + with self.get_connection() as conn: + client = conn.handle + + udf = client.get_routine( + self.adapter.connections.get_bq_routine( + self.default_database, self.unique_schema(), 'udf_with_description' + ) + ) + # Check that the description persisted + self.assertEqual(udf.description, self.udf_description) + + self.assertEqual(len(udf.arguments), 0) + self.assertIsNone(udf.return_type) diff --git a/tests/integration/bigquery_test/udf-models/udf_no_args.sql b/tests/integration/bigquery_test/udf-models/udf_no_args.sql new file mode 100644 index 000000000..2db998551 --- /dev/null +++ b/tests/integration/bigquery_test/udf-models/udf_no_args.sql @@ -0,0 +1,9 @@ +{{ + config( + materialized='udf', + return_type='INT64', + persist_docs={ 'relation': false } + ) +}} + +ifnull(null,0,1) diff --git a/tests/integration/bigquery_test/udf-models/udf_with_args.sql b/tests/integration/bigquery_test/udf-models/udf_with_args.sql new file mode 100644 index 000000000..41418e915 --- /dev/null +++ b/tests/integration/bigquery_test/udf-models/udf_with_args.sql @@ -0,0 +1,14 @@ +{{ + config( + materialized='udf', + args=[ + {'name': 'color', 'type': 'STRING'}, + {'name': 'is_pretty', 'type': 'BOOL'} + ] + ) +}} + +case + when is_pretty then concat('pretty', ' ', color) + else concat('not pretty', ' ', color) +end diff --git a/tests/integration/bigquery_test/udf-models/udf_with_description.sql b/tests/integration/bigquery_test/udf-models/udf_with_description.sql new file mode 100644 index 000000000..16fce8337 --- /dev/null +++ b/tests/integration/bigquery_test/udf-models/udf_with_description.sql @@ -0,0 +1,9 @@ +{{ + config( + materialized='udf', + persist_docs={ 'relation': true } + ) +}} + +-- Dummy UDF that returns the following string +'UDF with description' diff --git a/tests/integration/bigquery_test/udf-models/udfs.yml b/tests/integration/bigquery_test/udf-models/udfs.yml new file mode 100644 index 000000000..d83e4e0b4 --- /dev/null +++ b/tests/integration/bigquery_test/udf-models/udfs.yml @@ -0,0 +1,8 @@ +version: 2 + +models: +- name: udf_no_args + description: 'Should not persist' + +- name: udf_with_description + description: '{{ var("udf_description") }}' From 400eea0d7a17b34d043b40f2133a3ba71d421658 Mon Sep 17 00:00:00 2001 From: Alice Naghshineh Date: Tue, 3 Jan 2023 16:51:07 -0700 Subject: [PATCH 5/6] docs: Add inline docs --- dbt/adapters/bigquery/connections.py | 3 ++- dbt/adapters/bigquery/impl.py | 13 ++++++++++++- dbt/include/bigquery/macros/adapters.sql | 9 +++++++++ .../test_bigquery_udf_materialization.py | 17 +++++++++++++++++ 4 files changed, 40 insertions(+), 2 deletions(-) diff --git a/dbt/adapters/bigquery/connections.py b/dbt/adapters/bigquery/connections.py index c85672e7c..81010e7c9 100644 --- a/dbt/adapters/bigquery/connections.py +++ b/dbt/adapters/bigquery/connections.py @@ -589,10 +589,11 @@ def table_ref(database, schema, table_name): @staticmethod def routine_ref(database: str, schema: str, routine_name: str) -> google.cloud.bigquery.RoutineReference: + """Returns a RoutineReference object referencing a BigQuery routine.""" return google.cloud.bigquery.RoutineReference.from_string(f"{database}.{schema}.{routine_name}") def get_bq_routine(self, database: str, schema: str, identifier: str) -> google.cloud.bigquery.Routine: - """Get a BigQuery UDF for a schema/model.""" + """Get a BigQuery routine (UDF) for a schema/model.""" conn = self.get_thread_connection() # backwards compatibility: fill in with defaults if not specified database = database or conn.credentials.database diff --git a/dbt/adapters/bigquery/impl.py b/dbt/adapters/bigquery/impl.py index ab2f33261..329501e06 100644 --- a/dbt/adapters/bigquery/impl.py +++ b/dbt/adapters/bigquery/impl.py @@ -764,6 +764,11 @@ def _get_description_option(self, config: Dict[str, Any], node: Dict[str, Any]) def get_common_options( self, config: Dict[str, Any], node: Dict[str, Any], temporary: bool = False ) -> Dict[str, Any]: + """ + Constructs a dictionary of common options for configuring Bigquery + tables and views. Supports config options for setting a description, + the number of hours to expiration, and labels. + """ opts = self._get_description_option(config, node) if (config.get("hours_to_expiration") is not None) and (not temporary): @@ -780,6 +785,9 @@ def get_common_options( def get_table_options( self, config: Dict[str, Any], node: Dict[str, Any], temporary: bool ) -> Dict[str, Any]: + """ + Constructs a dictionary of options for configuring a table. + """ opts = self.get_common_options(config, node, temporary) if config.get("kms_key_name") is not None: @@ -803,13 +811,16 @@ def get_table_options( @available.parse(lambda *a, **k: {}) def get_view_options(self, config: Dict[str, Any], node: Dict[str, Any]) -> Dict[str, Any]: + """ + Constructs a dictionary of options for configuring a view. + """ opts = self.get_common_options(config, node) return opts @available.parse(lambda *a, **k: {}) def get_udf_options(self, config: Dict[str, Any], node: Dict[str, Any]) -> Dict[str, str]: """ - Constructs a dictionary of options for creating a UDF. + Constructs a dictionary of options for configuring a UDF. Currently, only supports a description option. """ opts = self._get_description_option(config, node) diff --git a/dbt/include/bigquery/macros/adapters.sql b/dbt/include/bigquery/macros/adapters.sql index a814e5f4b..1728684cc 100644 --- a/dbt/include/bigquery/macros/adapters.sql +++ b/dbt/include/bigquery/macros/adapters.sql @@ -89,11 +89,15 @@ {% endmacro %} +-- Define the UDF options. Currently, only UDF description supported. +-- E.g., OPTIONS(description="description here") {% macro bigquery_udf_options(config, node, temporary) %} {% set opts = adapter.get_udf_options(config, node, temporary) %} {%- do return(bigquery_options(opts)) -%} {%- endmacro -%} +-- Define the return type for the UDF, if applicable. +-- E.g., RETURNS STRING {% macro bigquery_udf_returns(udf_return_type) %} {% if udf_return_type -%} {% set udf_returns -%} @@ -103,6 +107,8 @@ {%- do return(udf_returns) -%} {%- endmacro -%} +-- Create the comma-separated list of UDF args. +-- E.g., arg1 STRING, arg2 INT64 {% macro bigquery_udf_args(udf_args_array) %} {% set udf_args -%} {%- for udf_arg in udf_args_array -%} @@ -112,6 +118,9 @@ {%- do return(udf_args) -%} {%- endmacro -%} +-- Construct the SQL DDL for creating the UDF. +-- Sets the appropriate UDF args, return type, and options +-- based on the UDF model configuration {% macro bigquery__get_create_udf_as_sql(relation, sql) -%} {%- set udf_return_type = config.get('return_type', none) -%} {%- set udf_args_array = config.get('args', []) -%} diff --git a/tests/integration/bigquery_test/test_bigquery_udf_materialization.py b/tests/integration/bigquery_test/test_bigquery_udf_materialization.py index aad6d8129..e4c346260 100644 --- a/tests/integration/bigquery_test/test_bigquery_udf_materialization.py +++ b/tests/integration/bigquery_test/test_bigquery_udf_materialization.py @@ -25,6 +25,12 @@ def udf_description(self): @use_profile('bigquery') def test__bigquery_materialize_udf_no_args(self): + """ + Tests the materialization of a UDF model without arguments. + Asserts successful creation without args. + Also asserts that the UDF has the expected return type and + no description, based on the UDF model's config. + """ results = self.run_dbt(['run', '--select', 'udf_no_args']) self.assertEqual(len(results), 1) @@ -44,6 +50,12 @@ def test__bigquery_materialize_udf_no_args(self): @use_profile('bigquery') def test__bigquery_materialize_udf_with_args(self): + """ + Tests the materialization of a UDF model with arguments. + Asserts successful creation with the expected args. + Also asserts that the UDF has neither a return type nor + a description, based on the UDF model's config. + """ results = self.run_dbt(['run', '--select', 'udf_with_args']) self.assertEqual(len(results), 1) @@ -75,6 +87,11 @@ def test__bigquery_materialize_udf_with_args(self): @use_profile('bigquery') def test__bigquery_materialize_udf_with_description(self): + """ + Tests the materialization of a UDF model with a description. + Asserts successful creation. Also asserts the UDF's description + matches the one provided in the model properties file. + """ results = self.run_dbt(['run', '--select', 'udf_with_description']) self.assertEqual(len(results), 1) From dfaedcdd7d63338b73759676941491a11d4a33d2 Mon Sep 17 00:00:00 2001 From: Alice Naghshineh Date: Tue, 3 Jan 2023 17:08:53 -0700 Subject: [PATCH 6/6] chore: Add CHANGELOG entry --- .changes/unreleased/Features-20230103-170813.yaml | 7 +++++++ 1 file changed, 7 insertions(+) create mode 100644 .changes/unreleased/Features-20230103-170813.yaml diff --git a/.changes/unreleased/Features-20230103-170813.yaml b/.changes/unreleased/Features-20230103-170813.yaml new file mode 100644 index 000000000..e06cf9396 --- /dev/null +++ b/.changes/unreleased/Features-20230103-170813.yaml @@ -0,0 +1,7 @@ +kind: Features +body: Add materialization for BQ persistent SQL UDF +time: 2023-01-03T17:08:13.915345-07:00 +custom: + Author: anaghshineh + Issue: "451" + PR: "454"