From 90b447b5b12affe406258a3ffe81f9e490711f45 Mon Sep 17 00:00:00 2001 From: Valerii Mironchenko Date: Mon, 9 Sep 2024 18:53:32 +0300 Subject: [PATCH] Fix Snowflake adapter warnings + add more debug logs (#107) * Fix pydantic dependency warning with snowflake-connector-python * Fix pydantic warning according to using attr name that corresponds to parent class method * Update snowflake tests according to client and adapter changes + pipes * Linting * Add debugging logs * Modify properties to ttl cached methods due to avoid duplication of nesting logging * Add invocation of Snowflake adapter tests in workflow * Bump generic collector version --------- Co-authored-by: Valerii Mironchenko --- .github/workflows/run-tests.yaml | 7 +- odd-collector/odd_collector/__version__.py | 2 +- .../adapters/snowflake/adapter.py | 42 +- .../adapters/snowflake/domain/pipe.py | 2 +- .../adapters/snowflake/mappers/column.py | 5 + .../adapters/snowflake/mappers/database.py | 3 + .../adapters/snowflake/mappers/pipe.py | 9 +- .../snowflake/mappers/relationships/mapper.py | 6 + .../adapters/snowflake/mappers/schema.py | 5 +- .../adapters/snowflake/mappers/table.py | 4 + .../adapters/snowflake/mappers/view.py | 7 +- odd-collector/poetry.lock | 172 +++---- odd-collector/pyproject.toml | 6 +- .../tests/adapters/snowflake/test_adapter.py | 478 ++++++++++-------- 14 files changed, 418 insertions(+), 330 deletions(-) diff --git a/.github/workflows/run-tests.yaml b/.github/workflows/run-tests.yaml index 6a386a7..b28a6e9 100644 --- a/.github/workflows/run-tests.yaml +++ b/.github/workflows/run-tests.yaml @@ -35,7 +35,10 @@ jobs: unixodbc-dev openssl libsasl2-dev poetry install - # for now, only tests for PostgreSQL adapter are being invoked, others need to be checked and updated for future use + # for now, only tests for PostgreSQL and Snowflake adapters are being invoked, + # others need to be checked and updated for future use - name: Run tests working-directory: odd-collector - run: poetry run pytest tests/integration/test_postgres.py -v + run: | + poetry run pytest tests/integration/test_postgres.py -v + poetry run pytest tests/adapters/snowflake -v diff --git a/odd-collector/odd_collector/__version__.py b/odd-collector/odd_collector/__version__.py index 2404d47..1fc8518 100644 --- a/odd-collector/odd_collector/__version__.py +++ b/odd-collector/odd_collector/__version__.py @@ -1 +1 @@ -VERSION = "0.1.64" +VERSION = "0.1.65" diff --git a/odd-collector/odd_collector/adapters/snowflake/adapter.py b/odd-collector/odd_collector/adapters/snowflake/adapter.py index 11f9f95..a80e6d3 100644 --- a/odd-collector/odd_collector/adapters/snowflake/adapter.py +++ b/odd-collector/odd_collector/adapters/snowflake/adapter.py @@ -70,14 +70,14 @@ def _fk_constraints(self) -> list[ForeignKeyConstraint]: def _unique_constraints(self) -> list[UniqueConstraint]: return self._get_metadata()["unique_constraints"] - @property - def _pipe_entities(self) -> list[tuple[Pipe, DataEntity]]: + @ttl_cache(ttl=CACHE_TTL) + def get_pipe_entities(self) -> list[tuple[Pipe, DataEntity]]: pipes: list[Pipe] = [] for raw_pipe in self._raw_pipes: pipes.extend( Pipe( catalog=raw_pipe.pipe_catalog, - schema=raw_pipe.pipe_schema, + schema_name=raw_pipe.pipe_schema, name=raw_pipe.pipe_name, definition=raw_pipe.definition, stage_url=raw_stage.stage_url, @@ -89,8 +89,8 @@ def _pipe_entities(self) -> list[tuple[Pipe, DataEntity]]: ) return [(pipe, map_pipe(pipe, self.generator)) for pipe in pipes] - @property - def _table_entities(self) -> list[tuple[Table, DataEntity]]: + @ttl_cache(ttl=CACHE_TTL) + def get_table_entities(self) -> list[tuple[Table, DataEntity]]: result = [] for table in self._tables: @@ -100,32 +100,36 @@ def _table_entities(self) -> list[tuple[Table, DataEntity]]: result.append((table, map_table(table, self.generator))) return result - @property - def _relationship_entities(self) -> list[DataEntity]: + @ttl_cache(ttl=CACHE_TTL) + def get_relationship_entities(self) -> list[DataEntity]: return DataEntityRelationshipsMapper( oddrn_generator=self.generator, unique_constraints=self._unique_constraints, - table_entities_pair=self._table_entities, + table_entities_pair=self.get_table_entities(), ).map(self._fk_constraints) - @property - def _schema_entities(self) -> list[DataEntity]: - return map_schemas(self._table_entities, self._pipe_entities, self.generator) + @ttl_cache(ttl=CACHE_TTL) + def get_schema_entities(self) -> list[DataEntity]: + return map_schemas( + self.get_table_entities(), self.get_pipe_entities(), self.generator + ) - @property - def _database_entity(self) -> DataEntity: - return map_database(self._database_name, self._schema_entities, self.generator) + @ttl_cache(ttl=CACHE_TTL) + def get_database_entity(self) -> DataEntity: + return map_database( + self._database_name, self.get_schema_entities(), self.generator + ) def get_data_entity_list(self) -> DataEntityList: try: return DataEntityList( data_source_oddrn=self.get_data_source_oddrn(), items=[ - *[te[1] for te in self._table_entities], - *self._schema_entities, - self._database_entity, - *[pe[1] for pe in self._pipe_entities], - *self._relationship_entities, + *[te[1] for te in self.get_table_entities()], + *self.get_schema_entities(), + self.get_database_entity(), + *[pe[1] for pe in self.get_pipe_entities()], + *self.get_relationship_entities(), ], ) except Exception as e: diff --git a/odd-collector/odd_collector/adapters/snowflake/domain/pipe.py b/odd-collector/odd_collector/adapters/snowflake/domain/pipe.py index 097f644..4f805a6 100644 --- a/odd-collector/odd_collector/adapters/snowflake/domain/pipe.py +++ b/odd-collector/odd_collector/adapters/snowflake/domain/pipe.py @@ -77,7 +77,7 @@ def stage_full_name(self) -> str: class Pipe(Connectable): catalog: str - schema: str + schema_name: str name: str definition: str stage_url: Optional[str] = None diff --git a/odd-collector/odd_collector/adapters/snowflake/mappers/column.py b/odd-collector/odd_collector/adapters/snowflake/mappers/column.py index 886e7c1..9beb745 100644 --- a/odd-collector/odd_collector/adapters/snowflake/mappers/column.py +++ b/odd-collector/odd_collector/adapters/snowflake/mappers/column.py @@ -2,6 +2,7 @@ from typing import Dict, List from odd_collector.adapters.snowflake.domain import Column +from odd_collector.adapters.snowflake.logger import logger from odd_models.models import DataSetField, DataSetFieldType, Type from oddrn_generator import SnowflakeGenerator @@ -52,6 +53,10 @@ def map_columns( result: List[DataSetField] = [] for column in columns: + logger.debug( + f"Mapping column from table {column.table_schema}.{column.table_name}: {column.column_name}" + ) + column_oddrn_key = f"{parent_path.value}_columns" generator_params = {column_oddrn_key: column.column_name} generator.set_oddrn_paths(**generator_params) diff --git a/odd-collector/odd_collector/adapters/snowflake/mappers/database.py b/odd-collector/odd_collector/adapters/snowflake/mappers/database.py index 8c28aea..5b1368e 100644 --- a/odd-collector/odd_collector/adapters/snowflake/mappers/database.py +++ b/odd-collector/odd_collector/adapters/snowflake/mappers/database.py @@ -2,6 +2,7 @@ from typing import List from funcy import lpluck_attr +from odd_collector.adapters.snowflake.logger import logger from odd_models.models import DataEntity, DataEntityGroup, DataEntityType from oddrn_generator import SnowflakeGenerator @@ -11,6 +12,8 @@ def map_database( schemas_entities: List[DataEntity], generator: SnowflakeGenerator, ) -> DataEntity: + logger.debug(f"Mapping database: {database_name}") + generator = deepcopy(generator) oddrn = generator.get_oddrn_by_path("databases", database_name) return DataEntity( diff --git a/odd-collector/odd_collector/adapters/snowflake/mappers/pipe.py b/odd-collector/odd_collector/adapters/snowflake/mappers/pipe.py index 62a22be..43d33cd 100644 --- a/odd-collector/odd_collector/adapters/snowflake/mappers/pipe.py +++ b/odd-collector/odd_collector/adapters/snowflake/mappers/pipe.py @@ -1,22 +1,23 @@ -import logging from abc import abstractmethod from copy import deepcopy from typing import List import sqlparse from odd_collector.adapters.snowflake.domain import Pipe +from odd_collector.adapters.snowflake.logger import logger from odd_models.models import DataEntity, DataEntityType, DataTransformer from oddrn_generator import SnowflakeGenerator from oddrn_generator.generators import S3Generator from .view import _map_connection -logger = logging.getLogger("Snowpipe") - def map_pipe(pipe: Pipe, generator: SnowflakeGenerator) -> DataEntity: + full_pipe_name = f"{pipe.schema_name}.{pipe.name}" + logger.debug(f"Mapping pipe: {full_pipe_name}") + generator = deepcopy(generator) - generator.set_oddrn_paths(schemas=pipe.schema, pipes=pipe.name) + generator.set_oddrn_paths(schemas=pipe.schema_name, pipes=pipe.name) return DataEntity( oddrn=generator.get_oddrn_by_path("pipes"), diff --git a/odd-collector/odd_collector/adapters/snowflake/mappers/relationships/mapper.py b/odd-collector/odd_collector/adapters/snowflake/mappers/relationships/mapper.py index 781f134..418fb67 100644 --- a/odd-collector/odd_collector/adapters/snowflake/mappers/relationships/mapper.py +++ b/odd-collector/odd_collector/adapters/snowflake/mappers/relationships/mapper.py @@ -3,6 +3,7 @@ Table, UniqueConstraint, ) +from odd_collector.adapters.snowflake.logger import logger from odd_collector.adapters.snowflake.mappers.relationships.relationship_mapper import ( RelationshipMapper, ) @@ -35,6 +36,11 @@ def _map_data_entity_relationship( referenced_schema_name = fk_cons.referenced_schema_name referenced_table_name = fk_cons.referenced_table_name + logger.debug( + f"Mapping relationship referencing to the table {referenced_schema_name}.{referenced_table_name}: " + f"{fk_cons.constraint_name}" + ) + self.oddrn_generator.set_oddrn_paths( schemas=schema_name, tables=table_name, diff --git a/odd-collector/odd_collector/adapters/snowflake/mappers/schema.py b/odd-collector/odd_collector/adapters/snowflake/mappers/schema.py index 98cc2a4..f82c223 100644 --- a/odd-collector/odd_collector/adapters/snowflake/mappers/schema.py +++ b/odd-collector/odd_collector/adapters/snowflake/mappers/schema.py @@ -1,6 +1,7 @@ from collections import defaultdict from copy import deepcopy +from odd_collector.adapters.snowflake.logger import logger from odd_models.models import DataEntity, DataEntityGroup, DataEntityType from oddrn_generator import SnowflakeGenerator @@ -20,11 +21,13 @@ def map_schemas( grouped[table.table_catalog][table.table_schema].add(entity.oddrn) for pipe, entity in pipe_entities: - grouped[pipe.catalog][pipe.schema].add(entity.oddrn) + grouped[pipe.catalog][pipe.schema_name].add(entity.oddrn) entities = [] for catalog, schemas in grouped.items(): for schema, oddrns in schemas.items(): + logger.debug(f"Mapping schema: {schema}") + generator.set_oddrn_paths(databases=catalog, schemas=schema) oddrn = generator.get_oddrn_by_path("schemas") entity = DataEntity( diff --git a/odd-collector/odd_collector/adapters/snowflake/mappers/table.py b/odd-collector/odd_collector/adapters/snowflake/mappers/table.py index b561f94..bbd4018 100644 --- a/odd-collector/odd_collector/adapters/snowflake/mappers/table.py +++ b/odd-collector/odd_collector/adapters/snowflake/mappers/table.py @@ -1,6 +1,7 @@ from copy import deepcopy from odd_collector.adapters.snowflake.domain import Table +from odd_collector.adapters.snowflake.logger import logger from odd_models.models import DataEntity, DataEntityType, DataSet from oddrn_generator import SnowflakeGenerator @@ -11,6 +12,9 @@ def map_table(table: Table, generator: SnowflakeGenerator) -> DataEntity: + full_table_name = f"{table.table_schema}.{table.table_name}" + logger.debug(f"Mapping table: {full_table_name}") + generator = deepcopy(generator) generator.set_oddrn_paths(schemas=table.table_schema, tables=table.table_name) diff --git a/odd-collector/odd_collector/adapters/snowflake/mappers/view.py b/odd-collector/odd_collector/adapters/snowflake/mappers/view.py index 46282d3..78cf17d 100644 --- a/odd-collector/odd_collector/adapters/snowflake/mappers/view.py +++ b/odd-collector/odd_collector/adapters/snowflake/mappers/view.py @@ -15,6 +15,9 @@ def map_view(view: View, generator: SnowflakeGenerator) -> DataEntity: + full_view_name = f"{view.table_schema}.{view.table_name}" + logger.debug(f"Mapping view: {full_view_name}") + generator = deepcopy(generator) generator.set_oddrn_paths(schemas=view.table_schema, views=view.table_name) @@ -22,7 +25,9 @@ def map_view(view: View, generator: SnowflakeGenerator) -> DataEntity: try: sql = sqlparse.format(view.view_definition) except Exception: - logger.warning(f"Couldn't parse view definition {view.view_definition}") + logger.warning( + f"Couldn't parse {full_view_name} view definition: {view.view_definition}" + ) return DataEntity( oddrn=generator.get_oddrn_by_path("views"), diff --git a/odd-collector/poetry.lock b/odd-collector/poetry.lock index ef9165e..0780db6 100644 --- a/odd-collector/poetry.lock +++ b/odd-collector/poetry.lock @@ -2977,13 +2977,13 @@ tqdm = ">=4.64.1,<5.0.0" [[package]] name = "odd-models" -version = "2.0.47" +version = "2.0.50" description = "Open Data Discovery Models" optional = false python-versions = "<4.0,>=3.9" files = [ - {file = "odd_models-2.0.47-py3-none-any.whl", hash = "sha256:fe93cfc1ddd99b9f6307040df5b9f1af862bde643efa43a6146e6f65cba11f5d"}, - {file = "odd_models-2.0.47.tar.gz", hash = "sha256:c22ebaa216e9591e1cd1b1bb0f9d701f5dc36088b97f537cfef50a4853ae6060"}, + {file = "odd_models-2.0.50-py3-none-any.whl", hash = "sha256:cd6a17c561b043e45c4c08bd6eadce0e069124245823677641a254774fcd745c"}, + {file = "odd_models-2.0.50.tar.gz", hash = "sha256:c9645f417278288ac22ab1a18f3c07f63f3c5f3df6ed4b1492c0d1380fce4be6"}, ] [package.dependencies] @@ -3077,20 +3077,6 @@ files = [ [package.dependencies] cryptography = ">=3.2.1" -[[package]] -name = "oscrypto" -version = "1.3.0" -description = "TLS (SSL) sockets, key generation, encryption, decryption, signing, verification and KDFs using the OS crypto libraries. Does not require a compiler, and relies on the OS for patching. Works on Windows, OS X and Linux/BSD." -optional = false -python-versions = "*" -files = [ - {file = "oscrypto-1.3.0-py2.py3-none-any.whl", hash = "sha256:2b2f1d2d42ec152ca90ccb5682f3e051fb55986e1b170ebde472b133713e7085"}, - {file = "oscrypto-1.3.0.tar.gz", hash = "sha256:6f5fef59cb5b3708321db7cca56aed8ad7e662853351e7991fcf60ec606d47a4"}, -] - -[package.dependencies] -asn1crypto = ">=1.5.1" - [[package]] name = "packaging" version = "23.2" @@ -3259,6 +3245,22 @@ files = [ docs = ["furo", "olefile", "sphinx (>=2.4)", "sphinx-copybutton", "sphinx-inline-tabs", "sphinx-removed-in", "sphinxext-opengraph"] tests = ["check-manifest", "coverage", "defusedxml", "markdown2", "olefile", "packaging", "pyroma", "pytest", "pytest-cov", "pytest-timeout"] +[[package]] +name = "platformdirs" +version = "4.2.2" +description = "A small Python package for determining appropriate platform-specific dirs, e.g. a `user data dir`." +optional = false +python-versions = ">=3.8" +files = [ + {file = "platformdirs-4.2.2-py3-none-any.whl", hash = "sha256:2d7a1657e36a80ea911db832a8a6ece5ee53d8de21edd5cc5879af6530b1bfee"}, + {file = "platformdirs-4.2.2.tar.gz", hash = "sha256:38b7b51f512eed9e84a22788b4bce1de17c0adb134d6becb09836e37d8654cd3"}, +] + +[package.extras] +docs = ["furo (>=2023.9.10)", "proselint (>=0.13)", "sphinx (>=7.2.6)", "sphinx-autodoc-typehints (>=1.25.2)"] +test = ["appdirs (==1.4.4)", "covdefaults (>=2.3)", "pytest (>=7.4.3)", "pytest-cov (>=4.1)", "pytest-mock (>=3.12)"] +type = ["mypy (>=1.8)"] + [[package]] name = "pluggy" version = "1.5.0" @@ -3491,47 +3493,6 @@ files = [ {file = "pycparser-2.21.tar.gz", hash = "sha256:e644fdec12f7872f86c58ff790da456218b10f863970249516d60a5eaca77206"}, ] -[[package]] -name = "pycryptodomex" -version = "3.19.0" -description = "Cryptographic library for Python" -optional = false -python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" -files = [ - {file = "pycryptodomex-3.19.0-cp27-cp27m-macosx_10_9_x86_64.whl", hash = "sha256:ff64fd720def623bf64d8776f8d0deada1cc1bf1ec3c1f9d6f5bb5bd098d034f"}, - {file = "pycryptodomex-3.19.0-cp27-cp27m-manylinux2010_i686.whl", hash = "sha256:61056a1fd3254f6f863de94c233b30dd33bc02f8c935b2000269705f1eeeffa4"}, - {file = "pycryptodomex-3.19.0-cp27-cp27m-manylinux2010_x86_64.whl", hash = "sha256:258c4233a3fe5a6341780306a36c6fb072ef38ce676a6d41eec3e591347919e8"}, - {file = "pycryptodomex-3.19.0-cp27-cp27m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6e45bb4635b3c4e0a00ca9df75ef6295838c85c2ac44ad882410cb631ed1eeaa"}, - {file = "pycryptodomex-3.19.0-cp27-cp27m-musllinux_1_1_aarch64.whl", hash = "sha256:a12144d785518f6491ad334c75ccdc6ad52ea49230b4237f319dbb7cef26f464"}, - {file = "pycryptodomex-3.19.0-cp27-cp27m-win32.whl", hash = "sha256:1789d89f61f70a4cd5483d4dfa8df7032efab1118f8b9894faae03c967707865"}, - {file = "pycryptodomex-3.19.0-cp27-cp27m-win_amd64.whl", hash = "sha256:eb2fc0ec241bf5e5ef56c8fbec4a2634d631e4c4f616a59b567947a0f35ad83c"}, - {file = "pycryptodomex-3.19.0-cp27-cp27mu-manylinux2010_i686.whl", hash = "sha256:c9a68a2f7bd091ccea54ad3be3e9d65eded813e6d79fdf4cc3604e26cdd6384f"}, - {file = "pycryptodomex-3.19.0-cp27-cp27mu-manylinux2010_x86_64.whl", hash = "sha256:8df69e41f7e7015a90b94d1096ec3d8e0182e73449487306709ec27379fff761"}, - {file = "pycryptodomex-3.19.0-cp27-cp27mu-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:917033016ecc23c8933205585a0ab73e20020fdf671b7cd1be788a5c4039840b"}, - {file = "pycryptodomex-3.19.0-cp27-cp27mu-musllinux_1_1_aarch64.whl", hash = "sha256:e8e5ecbd4da4157889fce8ba49da74764dd86c891410bfd6b24969fa46edda51"}, - {file = "pycryptodomex-3.19.0-cp35-abi3-macosx_10_9_universal2.whl", hash = "sha256:a77b79852175064c822b047fee7cf5a1f434f06ad075cc9986aa1c19a0c53eb0"}, - {file = "pycryptodomex-3.19.0-cp35-abi3-macosx_10_9_x86_64.whl", hash = "sha256:5b883e1439ab63af976656446fb4839d566bb096f15fc3c06b5a99cde4927188"}, - {file = "pycryptodomex-3.19.0-cp35-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a3866d68e2fc345162b1b9b83ef80686acfe5cec0d134337f3b03950a0a8bf56"}, - {file = "pycryptodomex-3.19.0-cp35-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c74eb1f73f788facece7979ce91594dc177e1a9b5d5e3e64697dd58299e5cb4d"}, - {file = "pycryptodomex-3.19.0-cp35-abi3-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:7cb51096a6a8d400724104db8a7e4f2206041a1f23e58924aa3d8d96bcb48338"}, - {file = "pycryptodomex-3.19.0-cp35-abi3-musllinux_1_1_aarch64.whl", hash = "sha256:a588a1cb7781da9d5e1c84affd98c32aff9c89771eac8eaa659d2760666f7139"}, - {file = "pycryptodomex-3.19.0-cp35-abi3-musllinux_1_1_i686.whl", hash = "sha256:d4dd3b381ff5a5907a3eb98f5f6d32c64d319a840278ceea1dcfcc65063856f3"}, - {file = "pycryptodomex-3.19.0-cp35-abi3-musllinux_1_1_x86_64.whl", hash = "sha256:263de9a96d2fcbc9f5bd3a279f14ea0d5f072adb68ebd324987576ec25da084d"}, - {file = "pycryptodomex-3.19.0-cp35-abi3-win32.whl", hash = "sha256:67c8eb79ab33d0fbcb56842992298ddb56eb6505a72369c20f60bc1d2b6fb002"}, - {file = "pycryptodomex-3.19.0-cp35-abi3-win_amd64.whl", hash = "sha256:09c9401dc06fb3d94cb1ec23b4ea067a25d1f4c6b7b118ff5631d0b5daaab3cc"}, - {file = "pycryptodomex-3.19.0-pp27-pypy_73-manylinux2010_x86_64.whl", hash = "sha256:edbe083c299835de7e02c8aa0885cb904a75087d35e7bab75ebe5ed336e8c3e2"}, - {file = "pycryptodomex-3.19.0-pp27-pypy_73-win32.whl", hash = "sha256:136b284e9246b4ccf4f752d435c80f2c44fc2321c198505de1d43a95a3453b3c"}, - {file = "pycryptodomex-3.19.0-pp310-pypy310_pp73-macosx_10_9_x86_64.whl", hash = "sha256:5d73e9fa3fe830e7b6b42afc49d8329b07a049a47d12e0ef9225f2fd220f19b2"}, - {file = "pycryptodomex-3.19.0-pp310-pypy310_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0b2f1982c5bc311f0aab8c293524b861b485d76f7c9ab2c3ac9a25b6f7655975"}, - {file = "pycryptodomex-3.19.0-pp310-pypy310_pp73-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:bfb040b5dda1dff1e197d2ef71927bd6b8bfcb9793bc4dfe0bb6df1e691eaacb"}, - {file = "pycryptodomex-3.19.0-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:800a2b05cfb83654df80266692f7092eeefe2a314fa7901dcefab255934faeec"}, - {file = "pycryptodomex-3.19.0-pp39-pypy39_pp73-macosx_10_9_x86_64.whl", hash = "sha256:c01678aee8ac0c1a461cbc38ad496f953f9efcb1fa19f5637cbeba7544792a53"}, - {file = "pycryptodomex-3.19.0-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2126bc54beccbede6eade00e647106b4f4c21e5201d2b0a73e9e816a01c50905"}, - {file = "pycryptodomex-3.19.0-pp39-pypy39_pp73-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:b801216c48c0886742abf286a9a6b117e248ca144d8ceec1f931ce2dd0c9cb40"}, - {file = "pycryptodomex-3.19.0-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:50cb18d4dd87571006fd2447ccec85e6cec0136632a550aa29226ba075c80644"}, - {file = "pycryptodomex-3.19.0.tar.gz", hash = "sha256:af83a554b3f077564229865c45af0791be008ac6469ef0098152139e6bd4b5b6"}, -] - [[package]] name = "pydantic" version = "2.7.1" @@ -4593,55 +4554,73 @@ files = [ [[package]] name = "snowflake-connector-python" -version = "2.9.0" +version = "3.12.1" description = "Snowflake Connector for Python" optional = false -python-versions = ">=3.7" +python-versions = ">=3.8" files = [ - {file = "snowflake-connector-python-2.9.0.tar.gz", hash = "sha256:7551b2404b26850fb12c6232d015ba5d10adb13b091e379fe8b8366492f624b8"}, - {file = "snowflake_connector_python-2.9.0-cp310-cp310-macosx_10_14_x86_64.whl", hash = "sha256:a66f90e76232db02754c34bdc1a0cda90f698658d38e1411cc6ddbe14e40f2a8"}, - {file = "snowflake_connector_python-2.9.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:649dc6764c1f2cf2a8aeac3cbb880c141cc38a38b72683325f87dc053539efcb"}, - {file = "snowflake_connector_python-2.9.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:db5152f24a60544f1efee4c4022d7a07428d5bab78858bb824fd966895e4078a"}, - {file = "snowflake_connector_python-2.9.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8ad9af11a59a8bb258f514c951f9d849747c64a860e77f512f5f113ef7067617"}, - {file = "snowflake_connector_python-2.9.0-cp310-cp310-win_amd64.whl", hash = "sha256:e720b0e24cb34caa5ef1ff099e3d41dcc4f95c04ab70fafff390cc8045dfdb56"}, - {file = "snowflake_connector_python-2.9.0-cp37-cp37m-macosx_10_14_x86_64.whl", hash = "sha256:308e7332d9f045b6f2d615a0199839ebddf6edcc614d58939d6c01926dc0fa4e"}, - {file = "snowflake_connector_python-2.9.0-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7d341672d8fbf711e8cf96a61d91ab0f065d4c212a3a2384f08dc6de75d211bd"}, - {file = "snowflake_connector_python-2.9.0-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2796eb05c264fd37ba2d922dcc27f04e2e4f7b938b7cae7f79b2ea7ed7d0cd63"}, - {file = "snowflake_connector_python-2.9.0-cp37-cp37m-win_amd64.whl", hash = "sha256:99d93e7ceef0aed159aee7ac36d032e8e8753be61a506f91b89b5e696d25bb62"}, - {file = "snowflake_connector_python-2.9.0-cp38-cp38-macosx_10_14_x86_64.whl", hash = "sha256:f27ef8988147688d3f9cbe798f72919b454cc295e386c21926391b4d3487d88c"}, - {file = "snowflake_connector_python-2.9.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:301affd1567dda268a1e57c0241721e4aa3e2bb103b4a3d2bf08e42768cf610a"}, - {file = "snowflake_connector_python-2.9.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e805a4e7631ef68bc373173fcd74057dfecf1003424766e6ed32948aa18130af"}, - {file = "snowflake_connector_python-2.9.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:014f2aa6544f1af2a190c161ad11880dc0e23cf0936b43cd90c29d6dc2bca66f"}, - {file = "snowflake_connector_python-2.9.0-cp38-cp38-win_amd64.whl", hash = "sha256:9654a7af2c407daf8525416933356eeacb7265a85a42b8e3ade2c1b912fa3295"}, - {file = "snowflake_connector_python-2.9.0-cp39-cp39-macosx_10_14_x86_64.whl", hash = "sha256:6f46c1fb9f9cc5053203b654595d4395b9a68f2cd8dccdeb805e3a823d0b5e5b"}, - {file = "snowflake_connector_python-2.9.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:3bf763bc4f8fdc215ed2236bc709c19d2019b41b16a5df64d366fd139ca1edf0"}, - {file = "snowflake_connector_python-2.9.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:fb10e761554f3e26c586f3c10d6bdc8df502016424d210978dfa175c16ecd70f"}, - {file = "snowflake_connector_python-2.9.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:23401da4ac80867f1c65c7275938fadd4a796b500748b39b26f7d3e36a10405e"}, - {file = "snowflake_connector_python-2.9.0-cp39-cp39-win_amd64.whl", hash = "sha256:6994dca106e4fb2c26135e7d3d05fdbd927cee4e63f8d3f8167b43d5b3d2c9d3"}, + {file = "snowflake_connector_python-3.12.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:c0979324bd96019f500f6c987d4720c9e4d7176df54b1b5aa96875be8c8ff57b"}, + {file = "snowflake_connector_python-3.12.1-cp310-cp310-macosx_11_0_x86_64.whl", hash = "sha256:c889a85966ec6a3384799e594e97301a4be0705d7763a5177104866b75383d8c"}, + {file = "snowflake_connector_python-3.12.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4bfb5fe8db051771480059ffddd5127653f4ac1168c76293655da33c2a2904d7"}, + {file = "snowflake_connector_python-3.12.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d1061af4a3a3e66b0c99ab0f8bae5eda28e6324618143b3f5b2d81d1649b8557"}, + {file = "snowflake_connector_python-3.12.1-cp310-cp310-win_amd64.whl", hash = "sha256:3edcf3591b6071ddb02413a0000dea42ee6fe811693d176915edb8687b03ce89"}, + {file = "snowflake_connector_python-3.12.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:226a714eb68bbae328fe49b705ecb304fbd44ea6a7afbb329ba3c389ac9111bc"}, + {file = "snowflake_connector_python-3.12.1-cp311-cp311-macosx_11_0_x86_64.whl", hash = "sha256:7319f63c09efed853d7652cbb38ecc23068e86dbce8340444056787993a854d9"}, + {file = "snowflake_connector_python-3.12.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f86b42a076e14900dc6af2f096343ccf4314d324e7e1153b667d6ee53c60334b"}, + {file = "snowflake_connector_python-3.12.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d231f0d5fb8d7a96b9ab5e9500035bd9f259c80d4b3c482163d156928fb0e546"}, + {file = "snowflake_connector_python-3.12.1-cp311-cp311-win_amd64.whl", hash = "sha256:d9f1bc6b35344b170e2fb30314aa64709b28539084be88e95aacf094e13259eb"}, + {file = "snowflake_connector_python-3.12.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:0114370c274ed64fe4aee2333b01e9ff88272837bdaa65fb3a3ee4820dca61b4"}, + {file = "snowflake_connector_python-3.12.1-cp312-cp312-macosx_11_0_x86_64.whl", hash = "sha256:dadd262196cce0132ca7e766f055e00c00497a88fdf83fd48143eb4a469a4527"}, + {file = "snowflake_connector_python-3.12.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:473642c0e628b8b9f264cbf31c7f4de44974373db43052b6542a66e751159caf"}, + {file = "snowflake_connector_python-3.12.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bddc4cdcd991f9538726a7c293d2637bb5aed43db68246e06c92c49a6df2b692"}, + {file = "snowflake_connector_python-3.12.1-cp312-cp312-win_amd64.whl", hash = "sha256:b06c63ec0381df1f4da6c4326330a1a40c8fc21fd3dcc2f58df4de395d676893"}, + {file = "snowflake_connector_python-3.12.1-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:3c24119ad64c20a8a691760c81e7d846feea4a6103ba84470116c60f7f31a1b8"}, + {file = "snowflake_connector_python-3.12.1-cp38-cp38-macosx_11_0_x86_64.whl", hash = "sha256:a8ba32c91ebf4de6d3f981cfd6324fb4b833696b639c350f5e5984371957e6f9"}, + {file = "snowflake_connector_python-3.12.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:cde5643d8237fc109fed68c6a806297ebe3adeb56ac6865430a78fcaba27f2ef"}, + {file = "snowflake_connector_python-3.12.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4a4bc4212db73feab5a79ad28b1d03743cbe48df1e346d219747afde5425c35d"}, + {file = "snowflake_connector_python-3.12.1-cp38-cp38-win_amd64.whl", hash = "sha256:7e5d7a0f1b827304b3ba250fa98c25385a7158ea5333e7857cda2ea91433a354"}, + {file = "snowflake_connector_python-3.12.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:a56f9df9db2b03caf9bc7a45f51d7cdfe307b5e2cde7edaa93b67c2d81789db6"}, + {file = "snowflake_connector_python-3.12.1-cp39-cp39-macosx_11_0_x86_64.whl", hash = "sha256:a1ead374d96cf21cb249bf91fe814ab1e1baaa3c3f2391116ccefab8bfa36374"}, + {file = "snowflake_connector_python-3.12.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:38698260175321ddef5504170ac1f9e5e92b897844d55ac2fc77bf0783435299"}, + {file = "snowflake_connector_python-3.12.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d7f8699ff60924105253e465a54ad150469ddf65082ce029387d65ca404a46cc"}, + {file = "snowflake_connector_python-3.12.1-cp39-cp39-win_amd64.whl", hash = "sha256:93e79497ae0f0be1a10cf2649900db0011e391ede47cbef2803814c32e1d63d6"}, + {file = "snowflake_connector_python-3.12.1.tar.gz", hash = "sha256:e43b7d4b4488ecd97b5bf62539cc502d7e84d8215c547eaeb4dd928c0b7212b9"}, ] [package.dependencies] asn1crypto = ">0.24.0,<2.0.0" certifi = ">=2017.4.17" cffi = ">=1.9,<2.0.0" -charset-normalizer = ">=2,<3" -cryptography = ">=3.1.0,<41.0.0" +charset-normalizer = ">=2,<4" +cryptography = ">=3.1.0" filelock = ">=3.5,<4" idna = ">=2.5,<4" -oscrypto = "<2.0.0" -pycryptodomex = ">=3.2,<3.5.0 || >3.5.0,<4.0.0" +packaging = "*" +platformdirs = ">=2.6.0,<5.0.0" pyjwt = "<3.0.0" -pyOpenSSL = ">=16.2.0,<23.0.0" +pyOpenSSL = ">=16.2.0,<25.0.0" pytz = "*" requests = "<3.0.0" -setuptools = ">34.0.0" +sortedcontainers = ">=2.4.0" +tomlkit = "*" typing-extensions = ">=4.3,<5" -urllib3 = ">=1.21.1,<1.27" +urllib3 = {version = ">=1.21.1,<2.0.0", markers = "python_version < \"3.10\""} [package.extras] -development = ["Cython", "coverage", "more-itertools", "numpy (<1.24.0)", "pendulum (!=2.1.1)", "pexpect", "pytest (<7.3.0)", "pytest-cov", "pytest-rerunfailures", "pytest-timeout", "pytest-xdist", "pytzdata"] -pandas = ["pandas (>=1.0.0,<1.6.0)", "pyarrow (>=8.0.0,<8.1.0)"] -secure-local-storage = ["keyring (!=16.1.0,<24.0.0)"] +development = ["Cython", "coverage", "more-itertools", "numpy (<1.27.0)", "pendulum (!=2.1.1)", "pexpect", "pytest (<7.5.0)", "pytest-cov", "pytest-rerunfailures", "pytest-timeout", "pytest-xdist", "pytzdata"] +pandas = ["pandas (>=1.0.0,<3.0.0)", "pyarrow"] +secure-local-storage = ["keyring (>=23.1.0,<26.0.0)"] + +[[package]] +name = "sortedcontainers" +version = "2.4.0" +description = "Sorted Containers -- Sorted List, Sorted Dict, Sorted Set" +optional = false +python-versions = "*" +files = [ + {file = "sortedcontainers-2.4.0-py2.py3-none-any.whl", hash = "sha256:a163dcaede0f1c021485e957a39245190e74249897e2ae4b2aa38595db237ee0"}, + {file = "sortedcontainers-2.4.0.tar.gz", hash = "sha256:25caa5a06cc30b6b83d11423433f65d1f9d76c4c6a0c90e3379eaa43b9bfdb88"}, +] [[package]] name = "sql-metadata" @@ -5005,6 +4984,17 @@ files = [ {file = "tomli-2.0.1.tar.gz", hash = "sha256:de526c12914f0c550d15924c62d72abc48d6fe7364aa87328337a31007fe8a4f"}, ] +[[package]] +name = "tomlkit" +version = "0.13.2" +description = "Style preserving TOML library" +optional = false +python-versions = ">=3.8" +files = [ + {file = "tomlkit-0.13.2-py3-none-any.whl", hash = "sha256:7a974427f6e119197f670fbbbeae7bef749a6c14e793db934baefc1b5f03efde"}, + {file = "tomlkit-0.13.2.tar.gz", hash = "sha256:fff5fe59a87295b278abd31bec92c15d9bc4a06885ab12bcea52c71119392e79"}, +] + [[package]] name = "toolz" version = "0.12.0" @@ -5726,4 +5716,4 @@ cffi = ["cffi (>=1.11)"] [metadata] lock-version = "2.0" python-versions = "^3.9" -content-hash = "f010b6341fd00d233b02fa2e88d8bed0fb9db4c16fb10107ebafde0f4f973675" +content-hash = "c2d9f975bd41b0d578248081bf2672c208caa4361670380e96e175a4cd30639d" diff --git a/odd-collector/pyproject.toml b/odd-collector/pyproject.toml index 7dd572e..5eb7c1e 100644 --- a/odd-collector/pyproject.toml +++ b/odd-collector/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "odd-collector" -version = "0.1.64" +version = "0.1.65" description = "ODD Collector" authors = ["Open Data Discovery "] keywords = [ @@ -35,7 +35,7 @@ pyodbc = "4.0.35" pymongo = { extras = ["srv"], version = "4.0.2" } presto-python-client = "0.8.2" pymssql = "2.2.10" -snowflake-connector-python = "^2.9" +snowflake-connector-python = "^3.12.1" sqlalchemy = "^2.0.29" tableauserverclient = "0.19.0" tarantool = "0.8.0" @@ -50,7 +50,7 @@ mlflow = "^2.12.1" sql-metadata = "^2.9.0" odd-collector-sdk = "^0.3.59" clickhouse-connect = "^0.5.14" -odd-models = "^2.0.47" +odd-models = "^2.0.50" couchbase = "^4.1.3" pyhive = { version = "^0.6.5", extras = ["hive"] } duckdb = "^0.10.2" diff --git a/odd-collector/tests/adapters/snowflake/test_adapter.py b/odd-collector/tests/adapters/snowflake/test_adapter.py index f8602be..d7890ac 100644 --- a/odd-collector/tests/adapters/snowflake/test_adapter.py +++ b/odd-collector/tests/adapters/snowflake/test_adapter.py @@ -1,12 +1,14 @@ import datetime -from typing import List, Optional, Type +from typing import Optional +from unittest.mock import MagicMock, patch import pytest from funcy import filter, first, lfilter from odd_collector.adapters.snowflake.adapter import Adapter -from odd_collector.adapters.snowflake.client import SnowflakeClientBase from odd_collector.adapters.snowflake.domain import Column, Connection, Table, View +from odd_collector.adapters.snowflake.domain.fk_constraint import ForeignKeyConstraint from odd_collector.adapters.snowflake.domain.pipe import RawPipe, RawStage +from odd_collector.adapters.snowflake.domain.unique_constraint import UniqueConstraint from odd_collector.domain.plugin import SnowflakePlugin from odd_models.models import DataEntity, DataEntityType from pydantic import SecretStr @@ -16,196 +18,9 @@ TABLE_NAME = "TEST_TABLE" FIRST_VIEW = "FIRST_VIEW" SECOND_VIEW = "SECOND_VIEW" - - -class TestClient(SnowflakeClientBase): - def get_raw_pipes(self) -> List[RawPipe]: - return [] - - def get_raw_stages(self) -> List[RawStage]: - return [] - - def get_tables(self) -> List[Table]: - tables = [ - Table( - upstream=[], - downstream=[ - Connection( - table_catalog=DATABASE_NAME, - table_name=FIRST_VIEW, - table_schema=SCHEMA, - domain="VIEW", - ) - ], - table_catalog=DATABASE_NAME, - table_schema=SCHEMA, - table_name=TABLE_NAME, - table_owner="ACCOUNTADMIN", - table_type="BASE TABLE", - is_transient="NO", - clustering_key=None, - row_count=0, - retention_time=1, - created=datetime.datetime.now(), - last_altered=datetime.datetime.now(), - table_comment=None, - self_referencing_column_name=None, - reference_generation=None, - user_defined_type_catalog=None, - user_defined_type_schema=None, - user_defined_type_name=None, - is_insertable_into="YES", - is_typed="YES", - columns=[ - Column( - table_catalog=DATABASE_NAME, - table_schema=SCHEMA, - table_name=TABLE_NAME, - column_name="NAME", - ordinal_position=1, - column_default=None, - is_nullable="YES", - data_type="TEXT", - character_maximum_length=16777216, - character_octet_length=16777216, - numeric_precision=None, - numeric_precision_radix=None, - numeric_scale=None, - collation_name=None, - is_identity="NO", - identity_generation=None, - identity_start=None, - identity_increment=None, - identity_cycle=None, - comment=None, - ) - ], - ), - View( - upstream=[ - Connection( - table_catalog=DATABASE_NAME, - table_name=TABLE_NAME, - table_schema=SCHEMA, - domain="TABLE", - ) - ], - downstream=[ - Connection( - table_catalog=DATABASE_NAME, - table_name=SECOND_VIEW, - table_schema=SCHEMA, - domain="VIEW", - ) - ], - table_catalog=DATABASE_NAME, - table_schema=SCHEMA, - table_name=FIRST_VIEW, - table_owner="ACCOUNTADMIN", - table_type="VIEW", - is_transient=None, - clustering_key=None, - row_count=None, - retention_time=None, - created=datetime.datetime.now(), - last_altered=datetime.datetime.now(), - table_comment=None, - self_referencing_column_name=None, - reference_generation=None, - user_defined_type_catalog=None, - user_defined_type_schema=None, - user_defined_type_name=None, - is_insertable_into="YES", - is_typed="YES", - columns=[ - Column( - table_catalog=DATABASE_NAME, - table_schema=SCHEMA, - table_name=FIRST_VIEW, - column_name="NAME", - ordinal_position=1, - column_default=None, - is_nullable="YES", - data_type="TEXT", - character_maximum_length=16777216, - character_octet_length=16777216, - numeric_precision=None, - numeric_precision_radix=None, - numeric_scale=None, - collation_name=None, - is_identity="NO", - identity_generation=None, - identity_start=None, - identity_increment=None, - identity_cycle=None, - comment=None, - ) - ], - view_definition="create view test_view as\n -- comment = ''\n select * from test;", - is_updatable="NO", - is_secure="NO", - view_comment=None, - ), - View( - upstream=[ - Connection( - table_catalog=DATABASE_NAME, - table_name=FIRST_VIEW, - table_schema=SCHEMA, - domain="VIEW", - ) - ], - downstream=[], - table_catalog=DATABASE_NAME, - table_schema=SCHEMA, - table_name=SECOND_VIEW, - table_owner="ACCOUNTADMIN", - table_type="VIEW", - is_transient=None, - clustering_key=None, - row_count=None, - retention_time=None, - created=datetime.datetime.now(), - last_altered=datetime.datetime.now(), - table_comment=None, - self_referencing_column_name=None, - reference_generation=None, - user_defined_type_catalog=None, - user_defined_type_schema=None, - user_defined_type_name=None, - is_insertable_into="YES", - is_typed="YES", - columns=[ - Column( - table_catalog=DATABASE_NAME, - table_schema=SCHEMA, - table_name=SECOND_VIEW, - column_name="NAME", - ordinal_position=1, - column_default=None, - is_nullable="YES", - data_type="TEXT", - character_maximum_length=16777216, - character_octet_length=16777216, - numeric_precision=None, - numeric_precision_radix=None, - numeric_scale=None, - collation_name=None, - is_identity="NO", - identity_generation=None, - identity_start=None, - identity_increment=None, - identity_cycle=None, - comment=None, - ) - ], - view_definition="create or replace view TEST_1.PUBLIC.TEST_VIEW_2(\n\tNAME\n) as\n -- comment = ''\n select * from TEST_VIEW;", - is_updatable="NO", - is_secure="NO", - view_comment=None, - ), - ] - return tables +FIRST_PIPE = "MY_PIPE_1" +SECOND_PIPE = "MY_PIPE_2" +STAGE_NAME = "MY_INTERNAL_STAGE" @pytest.fixture() @@ -224,42 +39,291 @@ def config() -> SnowflakePlugin: @pytest.fixture() -def client(config: SnowflakePlugin) -> Type[SnowflakeClientBase]: - return TestClient +def raw_pipes() -> list[RawPipe]: + return [ + RawPipe( + pipe_catalog=DATABASE_NAME, + pipe_schema=SCHEMA, + pipe_name=FIRST_PIPE, + definition=f"COPY INTO {TABLE_NAME}\nFROM @my_internal_stage\nFILE_FORMAT = (TYPE = 'CSV')", + ), + RawPipe( + pipe_catalog=DATABASE_NAME, + pipe_schema=SCHEMA, + pipe_name=SECOND_PIPE, + definition=f"COPY INTO {TABLE_NAME}\nFROM @my_internal_stage\nFILE_FORMAT = (TYPE = 'JSON')\nMATCH_BY_COLUMN_NAME = CASE_INSENSITIVE", + ), + ] -def _find_database(seq: List[DataEntity]) -> Optional[DataEntity]: +@pytest.fixture() +def raw_stages() -> list[RawStage]: + return [ + RawStage( + stage_name=STAGE_NAME, + stage_catalog=DATABASE_NAME, + stage_schema=SCHEMA, + stage_url=None, + stage_type="Internal Named", + ) + ] + + +@pytest.fixture() +def fk_constraints() -> list[ForeignKeyConstraint]: + return [] + + +@pytest.fixture() +def unique_constraints() -> list[UniqueConstraint]: + return [] + + +@pytest.fixture() +def tables() -> list[Table]: + return [ + Table( + upstream=[], + downstream=[ + Connection( + table_catalog=DATABASE_NAME, + table_name=FIRST_VIEW, + table_schema=SCHEMA, + domain="VIEW", + ) + ], + table_catalog=DATABASE_NAME, + table_schema=SCHEMA, + table_name=TABLE_NAME, + table_owner="ACCOUNTADMIN", + table_type="BASE TABLE", + is_transient="NO", + clustering_key=None, + row_count=0, + retention_time=1, + created=datetime.datetime.now(), + last_altered=datetime.datetime.now(), + table_comment=None, + self_referencing_column_name=None, + reference_generation=None, + user_defined_type_catalog=None, + user_defined_type_schema=None, + user_defined_type_name=None, + is_insertable_into="YES", + is_typed="YES", + columns=[ + Column( + table_catalog=DATABASE_NAME, + table_schema=SCHEMA, + table_name=TABLE_NAME, + column_name="NAME", + ordinal_position=1, + column_default=None, + is_nullable="YES", + data_type="TEXT", + character_maximum_length=16777216, + character_octet_length=16777216, + numeric_precision=None, + numeric_precision_radix=None, + numeric_scale=None, + collation_name=None, + is_identity="NO", + identity_generation=None, + identity_start=None, + identity_increment=None, + identity_cycle=None, + comment=None, + ) + ], + ), + View( + upstream=[ + Connection( + table_catalog=DATABASE_NAME, + table_name=TABLE_NAME, + table_schema=SCHEMA, + domain="TABLE", + ) + ], + downstream=[ + Connection( + table_catalog=DATABASE_NAME, + table_name=SECOND_VIEW, + table_schema=SCHEMA, + domain="VIEW", + ) + ], + table_catalog=DATABASE_NAME, + table_schema=SCHEMA, + table_name=FIRST_VIEW, + table_owner="ACCOUNTADMIN", + table_type="VIEW", + is_transient=None, + clustering_key=None, + row_count=None, + retention_time=None, + created=datetime.datetime.now(), + last_altered=datetime.datetime.now(), + table_comment=None, + self_referencing_column_name=None, + reference_generation=None, + user_defined_type_catalog=None, + user_defined_type_schema=None, + user_defined_type_name=None, + is_insertable_into="YES", + is_typed="YES", + columns=[ + Column( + table_catalog=DATABASE_NAME, + table_schema=SCHEMA, + table_name=FIRST_VIEW, + column_name="NAME", + ordinal_position=1, + column_default=None, + is_nullable="YES", + data_type="TEXT", + character_maximum_length=16777216, + character_octet_length=16777216, + numeric_precision=None, + numeric_precision_radix=None, + numeric_scale=None, + collation_name=None, + is_identity="NO", + identity_generation=None, + identity_start=None, + identity_increment=None, + identity_cycle=None, + comment=None, + ) + ], + view_definition="create view test_view as\n -- comment = ''\n select * from test;", + is_updatable="NO", + is_secure="NO", + view_comment=None, + ), + View( + upstream=[ + Connection( + table_catalog=DATABASE_NAME, + table_name=FIRST_VIEW, + table_schema=SCHEMA, + domain="VIEW", + ) + ], + downstream=[], + table_catalog=DATABASE_NAME, + table_schema=SCHEMA, + table_name=SECOND_VIEW, + table_owner="ACCOUNTADMIN", + table_type="VIEW", + is_transient=None, + clustering_key=None, + row_count=None, + retention_time=None, + created=datetime.datetime.now(), + last_altered=datetime.datetime.now(), + table_comment=None, + self_referencing_column_name=None, + reference_generation=None, + user_defined_type_catalog=None, + user_defined_type_schema=None, + user_defined_type_name=None, + is_insertable_into="YES", + is_typed="YES", + columns=[ + Column( + table_catalog=DATABASE_NAME, + table_schema=SCHEMA, + table_name=SECOND_VIEW, + column_name="NAME", + ordinal_position=1, + column_default=None, + is_nullable="YES", + data_type="TEXT", + character_maximum_length=16777216, + character_octet_length=16777216, + numeric_precision=None, + numeric_precision_radix=None, + numeric_scale=None, + collation_name=None, + is_identity="NO", + identity_generation=None, + identity_start=None, + identity_increment=None, + identity_cycle=None, + comment=None, + ) + ], + view_definition="create or replace view TEST_1.PUBLIC.TEST_VIEW_2(\n\tNAME\n) as\n -- comment = ''\n select * from TEST_VIEW;", + is_updatable="NO", + is_secure="NO", + view_comment=None, + ), + ] + + +def _find_database(seq: list[DataEntity]) -> Optional[DataEntity]: return first(filter(lambda entity: entity.name == DATABASE_NAME, seq)) -def _find_schema(seq: List[DataEntity]) -> Optional[DataEntity]: +def _find_schema(seq: list[DataEntity]) -> Optional[DataEntity]: return first(filter(lambda entity: entity.name == SCHEMA, seq)) -def _find_tables(seq: List[DataEntity]) -> List[DataEntity]: +def _find_tables(seq: list[DataEntity]) -> list[DataEntity]: return lfilter( lambda entity: entity.type in {DataEntityType.TABLE, DataEntityType.VIEW}, seq ) -def test_adapter(config: SnowflakePlugin, client: Type[SnowflakeClientBase]): - adapter = Adapter(config, client) +def _find_pipes(seq: list[DataEntity]) -> list[DataEntity]: + return lfilter(lambda entity: entity.name in (FIRST_PIPE, SECOND_PIPE), seq) + + +@patch("odd_collector.adapters.snowflake.adapter.SnowflakeClient") +def test_adapter( + mock_snowflake_client, + config: SnowflakePlugin, + tables: list[Table], + raw_pipes: list[RawPipe], + raw_stages: list[RawStage], + fk_constraints: list[ForeignKeyConstraint], + unique_constraints: list[UniqueConstraint], +): + # Create a mock instance of the SnowflakeClient + mock_client_instance = MagicMock() + + # Mock the return values for the client methods + mock_client_instance.get_tables.return_value = tables + mock_client_instance.get_raw_pipes.return_value = raw_pipes + mock_client_instance.get_raw_stages.return_value = raw_stages + mock_client_instance.get_fk_constraints.return_value = fk_constraints + mock_client_instance.get_unique_constraints.return_value = unique_constraints + + # Set the mock client instance as the return value of the context manager + mock_snowflake_client.return_value.__enter__.return_value = mock_client_instance + + # Create the adapter with the mocked client + adapter = Adapter(config) data_entity_list = adapter.get_data_entity_list() data_entities = data_entity_list.items - assert len(data_entities) == 5 # 3 Tables(Views) 1 Schema 1 Database + assert len(data_entities) == 7 # 1 Database; 1 Schema; 3 Tables + Views; 2 Pipes database_entity: DataEntity = _find_database(data_entities) schema_entity: DataEntity = _find_schema(data_entities) - tables_entities: List[DataEntity] = _find_tables(data_entities) + table_entities: list[DataEntity] = _find_tables(data_entities) + pipe_entities: list[DataEntity] = _find_pipes(data_entities) assert database_entity is not None - assert schema_entity.oddrn in database_entity.data_entity_group.entities_list - assert schema_entity is not None - for table_entity in tables_entities: - assert table_entity.oddrn in schema_entity.data_entity_group.entities_list + assert table_entities is not None + assert pipe_entities is not None - assert tables_entities is not None - assert len(tables_entities) == 3 + assert len(table_entities) == 3 + assert len(pipe_entities) == 2 + + assert schema_entity.oddrn in database_entity.data_entity_group.entities_list + for table_entity in table_entities: + assert table_entity.oddrn in schema_entity.data_entity_group.entities_list