fix(ingest/mssql): Add UNIQUEIDENTIFIER data type as String (#8642)

Co-authored-by: Andrew Sikowitz <andrew.sikowitz@acryl.io>
datahub-project · Sep 12, 2023 · a021053 · a021053
1 parent aee1e68
commit a021053
Show file tree

Hide file tree

Showing 6 changed files with 68 additions and 14 deletions.
diff --git a/metadata-ingestion/src/datahub/ingestion/source/sql/mssql/source.py b/metadata-ingestion/src/datahub/ingestion/source/sql/mssql/source.py
@@ -46,12 +46,17 @@
     BasicSQLAlchemyConfig,
     make_sqlalchemy_uri,
 )
-from datahub.metadata.schema_classes import BooleanTypeClass, UnionTypeClass
+from datahub.metadata.schema_classes import (
+    BooleanTypeClass,
+    StringTypeClass,
+    UnionTypeClass,
+)
 
 logger: logging.Logger = logging.getLogger(__name__)
 
 register_custom_type(sqlalchemy.dialects.mssql.BIT, BooleanTypeClass)
 register_custom_type(sqlalchemy.dialects.mssql.SQL_VARIANT, UnionTypeClass)
+register_custom_type(sqlalchemy.dialects.mssql.UNIQUEIDENTIFIER, StringTypeClass)
 
 
 class SQLServerConfig(BasicSQLAlchemyConfig):

diff --git a/...-ingestion/tests/integration/sql_server/golden_files/golden_mces_mssql_no_db_to_file.json b/...-ingestion/tests/integration/sql_server/golden_files/golden_mces_mssql_no_db_to_file.json
@@ -1752,6 +1752,18 @@
                                 "recursive": false,
                                 "isPartOfKey": true
                             },
+                            {
+                                "fieldPath": "SomeId",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "UNIQUEIDENTIFIER()",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
                             {
                                 "fieldPath": "Name",
                                 "nullable": true,

diff --git a/...estion/tests/integration/sql_server/golden_files/golden_mces_mssql_no_db_with_filter.json b/...estion/tests/integration/sql_server/golden_files/golden_mces_mssql_no_db_with_filter.json
@@ -1752,6 +1752,18 @@
                                 "recursive": false,
                                 "isPartOfKey": true
                             },
+                            {
+                                "fieldPath": "SomeId",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "UNIQUEIDENTIFIER()",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
                             {
                                 "fieldPath": "Name",
                                 "nullable": true,

diff --git a/metadata-ingestion/tests/integration/sql_server/golden_files/golden_mces_mssql_to_file.json b/metadata-ingestion/tests/integration/sql_server/golden_files/golden_mces_mssql_to_file.json
@@ -1752,6 +1752,18 @@
                                 "recursive": false,
                                 "isPartOfKey": true
                             },
+                            {
+                                "fieldPath": "SomeId",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "UNIQUEIDENTIFIER()",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
                             {
                                 "fieldPath": "Name",
                                 "nullable": true,

diff --git a/...tion/tests/integration/sql_server/golden_files/golden_mces_mssql_with_lower_case_urn.json b/...tion/tests/integration/sql_server/golden_files/golden_mces_mssql_with_lower_case_urn.json
@@ -1752,6 +1752,18 @@
                                 "recursive": false,
                                 "isPartOfKey": true
                             },
+                            {
+                                "fieldPath": "SomeId",
+                                "nullable": false,
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.StringType": {}
+                                    }
+                                },
+                                "nativeDataType": "UNIQUEIDENTIFIER()",
+                                "recursive": false,
+                                "isPartOfKey": false
+                            },
                             {
                                 "fieldPath": "Name",
                                 "nullable": true,

diff --git a/metadata-ingestion/tests/integration/sql_server/setup/setup.sql b/metadata-ingestion/tests/integration/sql_server/setup/setup.sql
@@ -34,7 +34,8 @@ CREATE TABLE Foo.Persons (
 GO
 CREATE TABLE Foo.SalesReason 
    (
-      TempID int NOT NULL, 
+      TempID int NOT NULL,
+      SomeId UNIQUEIDENTIFIER NOT NULL DEFAULT NEWID(),
       Name nvarchar(50)
       , CONSTRAINT PK_TempSales PRIMARY KEY NONCLUSTERED (TempID)
       , CONSTRAINT FK_TempSales_SalesReason FOREIGN KEY (TempID)
@@ -49,20 +50,20 @@ AS
     SELECT @ID AS ThatDB;
 GO
 
-GO  
-EXEC sys.sp_addextendedproperty   
-@name = N'MS_Description',   
-@value = N'Description for table Items of schema Foo.',   
-@level0type = N'SCHEMA', @level0name = 'Foo',  
-@level1type = N'TABLE',  @level1name = 'Items';  
+GO
+EXEC sys.sp_addextendedproperty
+@name = N'MS_Description',
+@value = N'Description for table Items of schema Foo.',
+@level0type = N'SCHEMA', @level0name = 'Foo',
+@level1type = N'TABLE',  @level1name = 'Items';
 GO
 
-GO  
-EXEC sys.sp_addextendedproperty   
-@name = N'MS_Description',   
-@value = N'Description for column LastName of table Persons of schema Foo.',  
-@level0type = N'SCHEMA', @level0name = 'Foo',  
-@level1type = N'TABLE', @level1name = 'Persons',   
+GO
+EXEC sys.sp_addextendedproperty
+@name = N'MS_Description',
+@value = N'Description for column LastName of table Persons of schema Foo.',
+@level0type = N'SCHEMA', @level0name = 'Foo',
+@level1type = N'TABLE', @level1name = 'Persons',
 @level2type = N'COLUMN',@level2name = 'LastName';
 GO
 USE msdb ;