diff --git a/dbt-adapters/.changes/unreleased/Features-20251210-203204.yaml b/dbt-adapters/.changes/unreleased/Features-20251210-203204.yaml new file mode 100644 index 000000000..ece24e586 --- /dev/null +++ b/dbt-adapters/.changes/unreleased/Features-20251210-203204.yaml @@ -0,0 +1,6 @@ +kind: Features +body: Add scaffolding for resolving function relations from data warehouses +time: 2025-12-10T20:32:04.674335-06:00 +custom: + Author: QMalcolm + Issue: "1488" diff --git a/dbt-adapters/src/dbt/adapters/sql/impl.py b/dbt-adapters/src/dbt/adapters/sql/impl.py index e39c47b18..b554c81c4 100644 --- a/dbt-adapters/src/dbt/adapters/sql/impl.py +++ b/dbt-adapters/src/dbt/adapters/sql/impl.py @@ -12,6 +12,7 @@ from dbt.adapters.sql.connections import SQLConnectionManager LIST_RELATIONS_MACRO_NAME = "list_relations_without_caching" +LIST_FUNCTION_RELATIONS_MACRO_NAME = "list_function_relations_without_caching" GET_COLUMNS_IN_RELATION_MACRO_NAME = "get_columns_in_relation" LIST_SCHEMAS_MACRO_NAME = "list_schemas" CHECK_SCHEMA_EXISTS_MACRO_NAME = "check_schema_exists" diff --git a/dbt-adapters/src/dbt/include/global_project/macros/adapters/metadata.sql b/dbt-adapters/src/dbt/include/global_project/macros/adapters/metadata.sql index 0aa7aabb4..93a0079de 100644 --- a/dbt-adapters/src/dbt/include/global_project/macros/adapters/metadata.sql +++ b/dbt-adapters/src/dbt/include/global_project/macros/adapters/metadata.sql @@ -77,6 +77,15 @@ 'list_relations_without_caching macro not implemented for adapter '+adapter.type()) }} {% endmacro %} +{% macro list_function_relations_without_caching(schema_relation) %} + {{ return(adapter.dispatch('list_function_relations_without_caching', 'dbt')(schema_relation)) }} +{% endmacro %} + +{% macro default__list_function_relations_without_caching(schema_relation) %} + {{ exceptions.raise_not_implemented( + 'list_function_relations_without_caching macro not implemented for adapter '+adapter.type()) }} +{% endmacro %} + {% macro get_catalog_for_single_relation(relation) %} {{ return(adapter.dispatch('get_catalog_for_single_relation', 'dbt')(relation)) }} {% endmacro %} diff --git a/dbt-postgres/.changes/unreleased/Fixes-20251203-130720.yaml b/dbt-postgres/.changes/unreleased/Fixes-20251203-130720.yaml new file mode 100644 index 000000000..92e570e5c --- /dev/null +++ b/dbt-postgres/.changes/unreleased/Fixes-20251203-130720.yaml @@ -0,0 +1,6 @@ +kind: Fixes +body: Ensure function relations can be found +time: 2025-12-03T13:07:20.569073-08:00 +custom: + Author: QMalcolm + Issue: "1488" diff --git a/dbt-postgres/src/dbt/include/postgres/macros/adapters.sql b/dbt-postgres/src/dbt/include/postgres/macros/adapters.sql index 1d20e6b3f..26c412608 100644 --- a/dbt-postgres/src/dbt/include/postgres/macros/adapters.sql +++ b/dbt-postgres/src/dbt/include/postgres/macros/adapters.sql @@ -107,6 +107,14 @@ 'materialized_view' as type from pg_matviews where schemaname ilike '{{ schema_relation.schema }}' + union all + select + '{{ schema_relation.database }}' as database, + proname as name, + ns.nspname as schema, + 'function' as type + from pg_proc + join pg_namespace as ns on pronamespace = ns.oid {% endcall %} {{ return(load_result('list_relations_without_caching').table) }} {% endmacro %} diff --git a/dbt-postgres/tests/functional/functions/test_udfs.py b/dbt-postgres/tests/functional/functions/test_udfs.py index 7c314d9c3..7e52c2957 100644 --- a/dbt-postgres/tests/functional/functions/test_udfs.py +++ b/dbt-postgres/tests/functional/functions/test_udfs.py @@ -6,6 +6,7 @@ ErrorForUnsupportedType, PythonUDFNotSupported, SqlUDFDefaultArgSupport, + CanFindScalarFunctionRelation, ) @@ -35,3 +36,7 @@ class TestPostgresPythonUDFNotSupported(PythonUDFNotSupported): class TestPostgresDefaultArgsSupportSQLUDFs(SqlUDFDefaultArgSupport): expect_default_arg_support = True + + +class TestPostgresCanFindScalarFunctionRelation(CanFindScalarFunctionRelation): + pass diff --git a/dbt-snowflake/.changes/unreleased/Fixes-20251210-203334.yaml b/dbt-snowflake/.changes/unreleased/Fixes-20251210-203334.yaml new file mode 100644 index 000000000..f25b0d7a9 --- /dev/null +++ b/dbt-snowflake/.changes/unreleased/Fixes-20251210-203334.yaml @@ -0,0 +1,6 @@ +kind: Fixes +body: Fix lookup of function relations in warehouse +time: 2025-12-10T20:33:34.080424-06:00 +custom: + Author: QMalcolm + Issue: "1488" diff --git a/dbt-snowflake/src/dbt/adapters/snowflake/impl.py b/dbt-snowflake/src/dbt/adapters/snowflake/impl.py index 789621c2a..a24f6dcf9 100644 --- a/dbt-snowflake/src/dbt/adapters/snowflake/impl.py +++ b/dbt-snowflake/src/dbt/adapters/snowflake/impl.py @@ -11,6 +11,7 @@ from dbt.adapters.sql.impl import ( LIST_SCHEMAS_MACRO_NAME, LIST_RELATIONS_MACRO_NAME, + LIST_FUNCTION_RELATIONS_MACRO_NAME, ) from dbt_common.contracts.constraints import ConstraintType from dbt_common.contracts.metadata import ( @@ -285,6 +286,9 @@ def list_relations_without_caching( try: schema_objects = self.execute_macro(LIST_RELATIONS_MACRO_NAME, kwargs=kwargs) + schema_functions = self.execute_macro( + LIST_FUNCTION_RELATIONS_MACRO_NAME, kwargs=kwargs + ) except DbtDatabaseError as exc: # if the schema doesn't exist, we just want to return. # Alternatively, we could query the list of schemas before we start @@ -295,11 +299,29 @@ def list_relations_without_caching( return [] raise - columns = ["database_name", "schema_name", "name", "kind", "is_dynamic", "is_iceberg"] + object_columns = [ + "database_name", + "schema_name", + "name", + "kind", + "is_dynamic", + "is_iceberg", + ] + function_columns = ["catalog_name", "schema_name", "name"] schema_objects = schema_objects.rename( column_names=[col.lower() for col in schema_objects.column_names] ) - return [self._parse_list_relations_result(obj) for obj in schema_objects.select(columns)] + schema_functions = schema_functions.rename( + column_names=[col.lower() for col in schema_functions.column_names] + ) + object_relations = [ + self._parse_list_relations_result(obj) for obj in schema_objects.select(object_columns) + ] + function_relations = [ + self._parse_list_function_relations_result(obj) + for obj in schema_functions.select(function_columns) + ] + return object_relations + function_relations def _parse_list_relations_result(self, result: "agate.Row") -> SnowflakeRelation: database, schema, identifier, relation_type, is_dynamic, is_iceberg = result @@ -329,6 +351,15 @@ def _parse_list_relations_result(self, result: "agate.Row") -> SnowflakeRelation quote_policy=quote_policy, ) + def _parse_list_function_relations_result(self, result: "agate.Row") -> SnowflakeRelation: + database, schema, identifier = result + return self.Relation.create( + database=database, + schema=schema, + identifier=identifier, + type=self.Relation.Function, + ) + def quote_seed_column(self, column: str, quote_config: Optional[bool]) -> str: quote_columns: bool = False if isinstance(quote_config, bool): diff --git a/dbt-snowflake/src/dbt/include/snowflake/macros/metadata/list_relations_without_caching.sql b/dbt-snowflake/src/dbt/include/snowflake/macros/metadata/list_relations_without_caching.sql index 8bf6c1bbf..6c33b47a1 100644 --- a/dbt-snowflake/src/dbt/include/snowflake/macros/metadata/list_relations_without_caching.sql +++ b/dbt-snowflake/src/dbt/include/snowflake/macros/metadata/list_relations_without_caching.sql @@ -63,3 +63,70 @@ show objects in {{ schema }} {%- do return(_sql) -%} {% endmacro %} + + +{% macro snowflake__list_function_relations_without_caching(schema_relation, max_iter=10000, max_results_per_iter=10000) %} + + {%- if schema_relation is string -%} + {%- set schema = schema_relation -%} + {%- else -%} + {%- set schema = schema_relation.include(identifier=False) -%} + {%- endif -%} + + {%- set max_results_per_iter = adapter.config.flags.get('list_relations_per_page', max_results_per_iter) -%} + {%- set max_iter = adapter.config.flags.get('list_relations_page_limit', max_iter) -%} + {%- set too_many_relations_msg -%} + dbt is currently configured to list a maximum of {{ max_results_per_iter * max_iter }} objects per schema. + {{ schema }} exceeds this limit. If this is expected, you may configure this limit + by setting list_relations_per_page and list_relations_page_limit in your project flags. + It is recommended to start by increasing list_relations_page_limit. + {%- endset -%} + + {%- set paginated_state = namespace(paginated_results=[], watermark=none) -%} + + {#- + loop an extra time to catch the breach of max iterations + Note: while range is 0-based, loop.index starts at 1 + -#} + {%- for _ in range(max_iter + 1) -%} + + {#- + raise a warning and break if we still didn't exit and we're beyond the max iterations limit + Note: while range is 0-based, loop.index starts at 1 + -#} + {%- if loop.index == max_iter + 1 -%} + {%- do exceptions.warn(too_many_relations_msg) -%} + {%- break -%} + {%- endif -%} + + {%- set show_functions_sql = snowflake__show_functions_sql(schema, max_results_per_iter, paginated_state.watermark) -%} + {%- set paginated_result = run_query(show_functions_sql) -%} + {%- do paginated_state.paginated_results.append(paginated_result) -%} + {%- set paginated_state.watermark = paginated_result.columns.get('name').values()[-1] -%} + + {#- we got less results than the max_results_per_iter (includes 0), meaning we reached the end -#} + {%- if (paginated_result | length) < max_results_per_iter -%} + {%- break -%} + {%- endif -%} + + {%- endfor -%} + + {#- grab the first table in the paginated results to access the `merge` method -#} + {%- set agate_table = paginated_state.paginated_results[0] -%} + {%- do return(agate_table.merge(paginated_state.paginated_results)) -%} + +{% endmacro %} + + +{% macro snowflake__show_functions_sql(schema, max_results_per_iter=10000, watermark=none) %} + +{%- set _sql -%} +show functions in {{ schema }} + limit {{ max_results_per_iter }} + {% if watermark is not none -%} from '{{ watermark }}' {%- endif %} +; +{%- endset -%} + +{%- do return(_sql) -%} + +{% endmacro %} diff --git a/dbt-snowflake/tests/functional/functions/test_udfs.py b/dbt-snowflake/tests/functional/functions/test_udfs.py index d8f4a04d4..19282c7bf 100644 --- a/dbt-snowflake/tests/functional/functions/test_udfs.py +++ b/dbt-snowflake/tests/functional/functions/test_udfs.py @@ -14,6 +14,7 @@ PythonUDFEntryPointRequired, SqlUDFDefaultArgSupport, PythonUDFDefaultArgSupport, + CanFindScalarFunctionRelation, ) from dbt.tests.util import run_dbt from dbt_common.events.event_catcher import EventCatcher @@ -105,3 +106,12 @@ def functions(self): class TestSnowflakeDefaultArgsSupportPythonUDFs(PythonUDFDefaultArgSupport): expect_default_arg_support = True + + +class TestSnowflakeCanFindScalarFunctionRelation(CanFindScalarFunctionRelation): + @pytest.fixture(scope="class") + def functions(self): + return { + "price_for_xlarge.sql": MY_UDF_SQL, + "price_for_xlarge.yml": MY_UDF_YML, + } diff --git a/dbt-tests-adapter/.changes/unreleased/Features-20251203-130528.yaml b/dbt-tests-adapter/.changes/unreleased/Features-20251203-130528.yaml new file mode 100644 index 000000000..4ab8d9634 --- /dev/null +++ b/dbt-tests-adapter/.changes/unreleased/Features-20251203-130528.yaml @@ -0,0 +1,6 @@ +kind: Features +body: Add test for checking that function relations can be found +time: 2025-12-03T13:05:28.838376-08:00 +custom: + Author: QMalcolm + Issue: "1488" diff --git a/dbt-tests-adapter/src/dbt/tests/adapter/functions/test_udfs.py b/dbt-tests-adapter/src/dbt/tests/adapter/functions/test_udfs.py index 635b83e06..68a5bf59b 100644 --- a/dbt-tests-adapter/src/dbt/tests/adapter/functions/test_udfs.py +++ b/dbt-tests-adapter/src/dbt/tests/adapter/functions/test_udfs.py @@ -36,7 +36,7 @@ def check_function_volatility(self, sql: str): assert "STABLE" not in sql assert "IMMUTABLE" not in sql - def test_udfs(self, project, sql_event_catcher): + def test_udfs(self, project, adapter, sql_event_catcher): result = run_dbt(["build", "--debug"], callbacks=[sql_event_catcher.catch]) assert len(result.results) == 1 @@ -240,3 +240,19 @@ def functions(self): "price_for_xlarge.py": files.MY_UDF_PYTHON, "price_for_xlarge.yml": files.MY_UDF_PYTHON_WITH_DEFAULT_ARG_YML, } + + +class CanFindScalarFunctionRelation(UDFsBasic): + def test_udfs(self, project): + result = run_dbt(["build", "--debug"]) + + assert len(result.results) == 1 + node = result.results[0].node + assert isinstance(node, FunctionNode) + + with project.adapter.connection_named("_test_scalar_function_relation"): + relation = project.adapter.get_relation( + database=node.database, schema=node.schema, identifier=node.name + ) + + assert relation is not None