diff --git a/.github/scripts/check_python_package.py b/.github/scripts/check_python_package.py index f1f3005691700..1b23d8e621ef0 100644 --- a/.github/scripts/check_python_package.py +++ b/.github/scripts/check_python_package.py @@ -1,18 +1,33 @@ import setuptools +import os folders = ["./smoke-test/tests"] for folder in folders: print(f"Checking folder {folder}") - a = [i for i in setuptools.find_packages(folder) if "cypress" not in i] - b = [i for i in setuptools.find_namespace_packages(folder) if "cypress" not in i] + packages = [i for i in setuptools.find_packages(folder) if "cypress" not in i] + namespace_packages = [ + i for i in setuptools.find_namespace_packages(folder) if "cypress" not in i + ] - in_a_not_b = set(a) - set(b) - in_b_not_a = set(b) - set(a) + print("Packages found:", packages) + print("Namespace packages found:", namespace_packages) + + in_packages_not_namespace = set(packages) - set(namespace_packages) + in_namespace_not_packages = set(namespace_packages) - set(packages) + + if in_packages_not_namespace: + print(f"Packages not in namespace packages: {in_packages_not_namespace}") + if in_namespace_not_packages: + print(f"Namespace packages not in packages: {in_namespace_not_packages}") + for pkg in in_namespace_not_packages: + pkg_path = os.path.join(folder, pkg.replace(".", os.path.sep)) + print(f"Contents of {pkg_path}:") + print(os.listdir(pkg_path)) assert ( - len(in_a_not_b) == 0 - ), f"Found packages in {folder} that are not in namespace packages: {in_a_not_b}" + len(in_packages_not_namespace) == 0 + ), f"Found packages in {folder} that are not in namespace packages: {in_packages_not_namespace}" assert ( - len(in_b_not_a) == 0 - ), f"Found namespace packages in {folder} that are not in packages: {in_b_not_a}" + len(in_namespace_not_packages) == 0 + ), f"Found namespace packages in {folder} that are not in packages: {in_namespace_not_packages}" diff --git a/.github/workflows/airflow-plugin.yml b/.github/workflows/airflow-plugin.yml index ab5b3eb48da7f..d4f0a1369da25 100644 --- a/.github/workflows/airflow-plugin.yml +++ b/.github/workflows/airflow-plugin.yml @@ -52,7 +52,7 @@ jobs: extra_pip_requirements: 'apache-airflow~=2.8.1 -c https://raw.githubusercontent.com/apache/airflow/constraints-2.8.1/constraints-3.10.txt' extra_pip_extras: plugin-v2 - python-version: "3.11" - extra_pip_requirements: 'apache-airflow~=2.9.3 -c https://raw.githubusercontent.com/apache/airflow/constraints-2.9.3/constraints-3.10.txt' + extra_pip_requirements: 'apache-airflow~=2.9.3 -c https://raw.githubusercontent.com/apache/airflow/constraints-2.9.3/constraints-3.11.txt' extra_pip_extras: plugin-v2 fail-fast: false steps: diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/chart/mappers/InputFieldsMapper.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/chart/mappers/InputFieldsMapper.java index 49c2d17ce0958..269fb7d4ddf79 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/chart/mappers/InputFieldsMapper.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/chart/mappers/InputFieldsMapper.java @@ -5,10 +5,13 @@ import com.linkedin.datahub.graphql.QueryContext; import com.linkedin.datahub.graphql.generated.InputField; import com.linkedin.datahub.graphql.types.dataset.mappers.SchemaFieldMapper; +import java.net.URISyntaxException; import java.util.stream.Collectors; import javax.annotation.Nonnull; import javax.annotation.Nullable; +import lombok.extern.slf4j.Slf4j; +@Slf4j public class InputFieldsMapper { public static final InputFieldsMapper INSTANCE = new InputFieldsMapper(); @@ -31,13 +34,24 @@ public com.linkedin.datahub.graphql.generated.InputFields apply( .map( field -> { InputField fieldResult = new InputField(); + Urn parentUrn = entityUrn; - if (field.hasSchemaField()) { - fieldResult.setSchemaField( - SchemaFieldMapper.map(context, field.getSchemaField(), entityUrn)); - } if (field.hasSchemaFieldUrn()) { fieldResult.setSchemaFieldUrn(field.getSchemaFieldUrn().toString()); + try { + parentUrn = + Urn.createFromString(field.getSchemaFieldUrn().getEntityKey().get(0)); + } catch (URISyntaxException e) { + log.error( + "Field urn resolution: failed to extract parentUrn successfully from {}. Falling back to {}", + field.getSchemaFieldUrn(), + entityUrn, + e); + } + } + if (field.hasSchemaField()) { + fieldResult.setSchemaField( + SchemaFieldMapper.map(context, field.getSchemaField(), parentUrn)); } return fieldResult; }) diff --git a/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/common/steps/ClearSystemMetadataServiceStep.java b/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/common/steps/ClearSystemMetadataServiceStep.java new file mode 100644 index 0000000000000..889d2f0a79edf --- /dev/null +++ b/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/common/steps/ClearSystemMetadataServiceStep.java @@ -0,0 +1,56 @@ +package com.linkedin.datahub.upgrade.common.steps; + +import com.linkedin.datahub.upgrade.UpgradeContext; +import com.linkedin.datahub.upgrade.UpgradeStep; +import com.linkedin.datahub.upgrade.UpgradeStepResult; +import com.linkedin.datahub.upgrade.impl.DefaultUpgradeStepResult; +import com.linkedin.datahub.upgrade.nocode.NoCodeUpgrade; +import com.linkedin.metadata.systemmetadata.SystemMetadataService; +import java.util.function.Function; + +public class ClearSystemMetadataServiceStep implements UpgradeStep { + + private final SystemMetadataService _systemMetadataService; + private final boolean _alwaysRun; + + public ClearSystemMetadataServiceStep( + final SystemMetadataService systemMetadataService, final boolean alwaysRun) { + _systemMetadataService = systemMetadataService; + _alwaysRun = alwaysRun; + } + + @Override + public String id() { + return "ClearSystemMetadataServiceStep"; + } + + @Override + public boolean skip(UpgradeContext context) { + if (_alwaysRun) { + return false; + } + if (context.parsedArgs().containsKey(NoCodeUpgrade.CLEAN_ARG_NAME)) { + return false; + } + context.report().addLine("Cleanup has not been requested."); + return true; + } + + @Override + public int retryCount() { + return 1; + } + + @Override + public Function executable() { + return (context) -> { + try { + _systemMetadataService.clear(); + } catch (Exception e) { + context.report().addLine("Failed to clear system metadata service", e); + return new DefaultUpgradeStepResult(id(), UpgradeStepResult.Result.FAILED); + } + return new DefaultUpgradeStepResult(id(), UpgradeStepResult.Result.SUCCEEDED); + }; + } +} diff --git a/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/common/steps/GMSDisableWriteModeStep.java b/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/common/steps/GMSDisableWriteModeStep.java index 42b3c6b3ccc49..a80adabc60e74 100644 --- a/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/common/steps/GMSDisableWriteModeStep.java +++ b/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/common/steps/GMSDisableWriteModeStep.java @@ -13,7 +13,7 @@ @RequiredArgsConstructor public class GMSDisableWriteModeStep implements UpgradeStep { - private final SystemEntityClient entityClient; + private final SystemEntityClient systemEntityClient; @Override public String id() { @@ -29,7 +29,7 @@ public int retryCount() { public Function executable() { return (context) -> { try { - entityClient.setWritable(context.opContext(), false); + systemEntityClient.setWritable(context.opContext(), false); } catch (Exception e) { log.error("Failed to turn write mode off in GMS", e); context.report().addLine("Failed to turn write mode off in GMS"); diff --git a/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/common/steps/GMSEnableWriteModeStep.java b/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/common/steps/GMSEnableWriteModeStep.java index c43644fda7ddf..a6b2942fabffd 100644 --- a/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/common/steps/GMSEnableWriteModeStep.java +++ b/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/common/steps/GMSEnableWriteModeStep.java @@ -12,7 +12,7 @@ @Slf4j @RequiredArgsConstructor public class GMSEnableWriteModeStep implements UpgradeStep { - private final SystemEntityClient entityClient; + private final SystemEntityClient systemEntityClient; @Override public String id() { @@ -28,7 +28,7 @@ public int retryCount() { public Function executable() { return (context) -> { try { - entityClient.setWritable(context.opContext(), true); + systemEntityClient.setWritable(context.opContext(), true); } catch (Exception e) { log.error("Failed to turn write mode back on in GMS", e); context.report().addLine("Failed to turn write mode back on in GMS"); diff --git a/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/config/RestoreBackupConfig.java b/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/config/RestoreBackupConfig.java index ec6e5a4a8f04d..b4eafb4ad3d24 100644 --- a/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/config/RestoreBackupConfig.java +++ b/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/config/RestoreBackupConfig.java @@ -6,6 +6,7 @@ import com.linkedin.metadata.graph.GraphService; import com.linkedin.metadata.models.registry.EntityRegistry; import com.linkedin.metadata.search.EntitySearchService; +import com.linkedin.metadata.systemmetadata.SystemMetadataService; import io.ebean.Database; import javax.annotation.Nonnull; import lombok.extern.slf4j.Slf4j; @@ -26,8 +27,9 @@ public class RestoreBackupConfig { "ebeanServer", "entityService", "systemEntityClient", - "graphService", + "systemMetadataService", "searchService", + "graphService", "entityRegistry" }) @ConditionalOnProperty(name = "entityService.impl", havingValue = "ebean", matchIfMissing = true) @@ -35,13 +37,23 @@ public class RestoreBackupConfig { public RestoreBackup createInstance() { final Database ebeanServer = applicationContext.getBean(Database.class); final EntityService entityService = applicationContext.getBean(EntityService.class); - final SystemEntityClient entityClient = applicationContext.getBean(SystemEntityClient.class); - final GraphService graphClient = applicationContext.getBean(GraphService.class); - final EntitySearchService searchClient = applicationContext.getBean(EntitySearchService.class); + final SystemEntityClient systemEntityClient = + applicationContext.getBean(SystemEntityClient.class); + final SystemMetadataService systemMetadataService = + applicationContext.getBean(SystemMetadataService.class); + final EntitySearchService entitySearchService = + applicationContext.getBean(EntitySearchService.class); + final GraphService graphService = applicationContext.getBean(GraphService.class); final EntityRegistry entityRegistry = applicationContext.getBean(EntityRegistry.class); return new RestoreBackup( - ebeanServer, entityService, entityRegistry, entityClient, graphClient, searchClient); + ebeanServer, + entityService, + entityRegistry, + systemEntityClient, + systemMetadataService, + entitySearchService, + graphService); } @Bean(name = "restoreBackup") @@ -49,6 +61,6 @@ public RestoreBackup createInstance() { @Nonnull public RestoreBackup createNotImplInstance() { log.warn("restoreIndices is not supported for cassandra!"); - return new RestoreBackup(null, null, null, null, null, null); + return new RestoreBackup(null, null, null, null, null, null, null); } } diff --git a/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/config/RestoreIndicesConfig.java b/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/config/RestoreIndicesConfig.java index 008bdf5cfac38..949b75edaa6ba 100644 --- a/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/config/RestoreIndicesConfig.java +++ b/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/config/RestoreIndicesConfig.java @@ -4,6 +4,7 @@ import com.linkedin.metadata.entity.EntityService; import com.linkedin.metadata.graph.GraphService; import com.linkedin.metadata.search.EntitySearchService; +import com.linkedin.metadata.systemmetadata.SystemMetadataService; import io.ebean.Database; import javax.annotation.Nonnull; import lombok.extern.slf4j.Slf4j; @@ -20,17 +21,26 @@ public class RestoreIndicesConfig { @Autowired ApplicationContext applicationContext; @Bean(name = "restoreIndices") - @DependsOn({"ebeanServer", "entityService", "searchService", "graphService"}) + @DependsOn({ + "ebeanServer", + "entityService", + "systemMetadataService", + "searchService", + "graphService" + }) @ConditionalOnProperty(name = "entityService.impl", havingValue = "ebean", matchIfMissing = true) @Nonnull public RestoreIndices createInstance() { final Database ebeanServer = applicationContext.getBean(Database.class); final EntityService entityService = applicationContext.getBean(EntityService.class); + final SystemMetadataService systemMetadataService = + applicationContext.getBean(SystemMetadataService.class); final EntitySearchService entitySearchService = applicationContext.getBean(EntitySearchService.class); final GraphService graphService = applicationContext.getBean(GraphService.class); - return new RestoreIndices(ebeanServer, entityService, entitySearchService, graphService); + return new RestoreIndices( + ebeanServer, entityService, systemMetadataService, entitySearchService, graphService); } @Bean(name = "restoreIndices") @@ -38,6 +48,6 @@ public RestoreIndices createInstance() { @Nonnull public RestoreIndices createNotImplInstance() { log.warn("restoreIndices is not supported for cassandra!"); - return new RestoreIndices(null, null, null, null); + return new RestoreIndices(null, null, null, null, null); } } diff --git a/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/restorebackup/RestoreBackup.java b/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/restorebackup/RestoreBackup.java index bcaeaa34e8936..7496655e581b0 100644 --- a/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/restorebackup/RestoreBackup.java +++ b/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/restorebackup/RestoreBackup.java @@ -6,6 +6,7 @@ import com.linkedin.datahub.upgrade.UpgradeStep; import com.linkedin.datahub.upgrade.common.steps.ClearGraphServiceStep; import com.linkedin.datahub.upgrade.common.steps.ClearSearchServiceStep; +import com.linkedin.datahub.upgrade.common.steps.ClearSystemMetadataServiceStep; import com.linkedin.datahub.upgrade.common.steps.GMSDisableWriteModeStep; import com.linkedin.datahub.upgrade.common.steps.GMSEnableWriteModeStep; import com.linkedin.entity.client.SystemEntityClient; @@ -13,6 +14,7 @@ import com.linkedin.metadata.graph.GraphService; import com.linkedin.metadata.models.registry.EntityRegistry; import com.linkedin.metadata.search.EntitySearchService; +import com.linkedin.metadata.systemmetadata.SystemMetadataService; import io.ebean.Database; import java.util.ArrayList; import java.util.List; @@ -26,13 +28,20 @@ public RestoreBackup( @Nullable final Database server, final EntityService entityService, final EntityRegistry entityRegistry, - final SystemEntityClient entityClient, - final GraphService graphClient, - final EntitySearchService searchClient) { + final SystemEntityClient systemEntityClient, + final SystemMetadataService systemMetadataService, + final EntitySearchService entitySearchService, + final GraphService graphClient) { if (server != null) { _steps = buildSteps( - server, entityService, entityRegistry, entityClient, graphClient, searchClient); + server, + entityService, + entityRegistry, + systemEntityClient, + systemMetadataService, + entitySearchService, + graphClient); } else { _steps = List.of(); } @@ -52,16 +61,18 @@ private List buildSteps( final Database server, final EntityService entityService, final EntityRegistry entityRegistry, - final SystemEntityClient entityClient, - final GraphService graphClient, - final EntitySearchService searchClient) { + final SystemEntityClient systemEntityClient, + final SystemMetadataService systemMetadataService, + final EntitySearchService entitySearchService, + final GraphService graphClient) { final List steps = new ArrayList<>(); - steps.add(new GMSDisableWriteModeStep(entityClient)); - steps.add(new ClearSearchServiceStep(searchClient, true)); + steps.add(new GMSDisableWriteModeStep(systemEntityClient)); + steps.add(new ClearSystemMetadataServiceStep(systemMetadataService, true)); + steps.add(new ClearSearchServiceStep(entitySearchService, true)); steps.add(new ClearGraphServiceStep(graphClient, true)); steps.add(new ClearAspectV2TableStep(server)); steps.add(new RestoreStorageStep(entityService, entityRegistry)); - steps.add(new GMSEnableWriteModeStep(entityClient)); + steps.add(new GMSEnableWriteModeStep(systemEntityClient)); return steps; } diff --git a/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/restoreindices/RestoreIndices.java b/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/restoreindices/RestoreIndices.java index 9bc42e23a9974..9d239a5622486 100644 --- a/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/restoreindices/RestoreIndices.java +++ b/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/restoreindices/RestoreIndices.java @@ -6,9 +6,11 @@ import com.linkedin.datahub.upgrade.UpgradeStep; import com.linkedin.datahub.upgrade.common.steps.ClearGraphServiceStep; import com.linkedin.datahub.upgrade.common.steps.ClearSearchServiceStep; +import com.linkedin.datahub.upgrade.common.steps.ClearSystemMetadataServiceStep; import com.linkedin.metadata.entity.EntityService; import com.linkedin.metadata.graph.GraphService; import com.linkedin.metadata.search.EntitySearchService; +import com.linkedin.metadata.systemmetadata.SystemMetadataService; import io.ebean.Database; import java.util.ArrayList; import java.util.List; @@ -32,10 +34,13 @@ public class RestoreIndices implements Upgrade { public RestoreIndices( @Nullable final Database server, final EntityService entityService, + final SystemMetadataService systemMetadataService, final EntitySearchService entitySearchService, final GraphService graphService) { if (server != null) { - _steps = buildSteps(server, entityService, entitySearchService, graphService); + _steps = + buildSteps( + server, entityService, systemMetadataService, entitySearchService, graphService); } else { _steps = List.of(); } @@ -54,9 +59,11 @@ public List steps() { private List buildSteps( final Database server, final EntityService entityService, + final SystemMetadataService systemMetadataService, final EntitySearchService entitySearchService, final GraphService graphService) { final List steps = new ArrayList<>(); + steps.add(new ClearSystemMetadataServiceStep(systemMetadataService, false)); steps.add(new ClearSearchServiceStep(entitySearchService, false)); steps.add(new ClearGraphServiceStep(graphService, false)); steps.add(new SendMAEStep(server, entityService)); diff --git a/datahub-web-react/src/app/settings/SettingsPage.tsx b/datahub-web-react/src/app/settings/SettingsPage.tsx index e0a15c73a626d..24bcd17ca7f9c 100644 --- a/datahub-web-react/src/app/settings/SettingsPage.tsx +++ b/datahub-web-react/src/app/settings/SettingsPage.tsx @@ -8,6 +8,7 @@ import { FilterOutlined, TeamOutlined, PushpinOutlined, + ControlOutlined, } from '@ant-design/icons'; import { Redirect, Route, useHistory, useLocation, useRouteMatch, Switch } from 'react-router'; import styled from 'styled-components'; @@ -17,11 +18,17 @@ import { ManagePermissions } from '../permissions/ManagePermissions'; import { useAppConfig } from '../useAppConfig'; import { AccessTokens } from './AccessTokens'; import { Preferences } from './Preferences'; +import { Features } from './features/Features'; import { ManageViews } from '../entity/view/ManageViews'; import { useUserContext } from '../context/useUserContext'; import { ManageOwnership } from '../entity/ownership/ManageOwnership'; import ManagePosts from './posts/ManagePosts'; +const MenuItem = styled(Menu.Item)` + display: flex; + align-items: center; +`; + const PageContainer = styled.div` display: flex; overflow: auto; @@ -59,6 +66,17 @@ const ItemTitle = styled.span` const menuStyle = { width: 256, 'margin-top': 8, overflow: 'hidden auto' }; +const NewTag = styled.span` + padding: 4px 8px; + margin-left: 8px; + + border-radius: 24px; + background: #f1fbfe; + + color: #09739a; + font-size: 12px; +`; + /** * URL Paths for each settings page. */ @@ -70,6 +88,7 @@ const PATHS = [ { path: 'views', content: }, { path: 'ownership', content: }, { path: 'posts', content: }, + { path: 'features', content: }, ]; /** @@ -80,6 +99,7 @@ const DEFAULT_PATH = PATHS[0]; export const SettingsPage = () => { const { path, url } = useRouteMatch(); const { pathname } = useLocation(); + const history = useHistory(); const subRoutes = PATHS.map((p) => p.path.replace('/', '')); const currPathName = pathname.replace(path, ''); @@ -101,6 +121,7 @@ export const SettingsPage = () => { const showViews = isViewsEnabled || false; const showOwnershipTypes = me && me?.platformPrivileges?.manageOwnershipTypes; const showHomePagePosts = me && me?.platformPrivileges?.manageGlobalAnnouncements && !readOnlyModeEnabled; + const showFeatures = true; // TODO: Add feature flag for this return ( @@ -143,6 +164,13 @@ export const SettingsPage = () => { )} {(showViews || showOwnershipTypes || showHomePagePosts) && ( + {showFeatures && ( + + + Features + New! + + )} {showViews && ( My Views diff --git a/datahub-web-react/src/app/settings/features/Feature.tsx b/datahub-web-react/src/app/settings/features/Feature.tsx new file mode 100644 index 0000000000000..2c090aae696f8 --- /dev/null +++ b/datahub-web-react/src/app/settings/features/Feature.tsx @@ -0,0 +1,179 @@ +import React from 'react'; + +import styled from 'styled-components'; + +import { Divider, Typography, Switch, Card, Button, Tooltip } from 'antd'; +import { ArrowRightOutlined } from '@ant-design/icons'; +import { ANTD_GRAY } from '../../entity/shared/constants'; + +const Title = styled(Typography.Title)` + && { + margin-bottom: 8px; + } +`; + +const FeatureRow = styled.div` + display: flex; + align-items: flex-start; + justify-content: space-between; +`; + +const FeatureOptionRow = styled.div` + display: flex; + justify-content: space-between; + + &:not(:last-child) { + margin-bottom: 8px; + } +`; + +const SettingsOptionRow = styled.div` + display: flex; + justify-content: space-between; + align-items: center; + padding: 0 16px; + + &:not(:last-child) { + margin-bottom: 8px; + } +`; + +const DescriptionText = styled(Typography.Text)` + color: ${ANTD_GRAY[7]}; + font-size: 11px; +`; + +const SettingTitle = styled.div` + display: flex; + align-items: center; + gap: 8px; + font-size: 14px; + margin-bottom: 4px; +`; + +const OptionTitle = styled(Typography.Text)` + display: flex; + align-items: center; + gap: 8px; + font-size: 12px; +`; + +const learnMoreLinkStyle = { + flex: 1, + display: 'flex', + alignItems: 'center', + gap: '8px', + color: '#1890FF', + fontSize: '12px', + cursor: 'pointer', +}; + +const NewTag = styled.div` + padding: 4px 8px; + + border-radius: 24px; + background: #f1fbfe; + + color: #09739a; + font-size: 12px; +`; + +const DataHubOnlyTag = styled.div` + padding: 2px 8px; + + border-radius: 24px; + background: #c9fff2; + + color: #50a494; + font-size: 12px; +`; + +export interface FeatureType { + key: string; + title: string; + description: string; + settings: Array<{ + key: string; + title: string; + isAvailable: boolean; + buttonText: string; + onClick?: () => void; + }>; + options: Array<{ + key: string; + title: string; + description: string; + isAvailable: boolean; + checked: boolean; + onChange?: (checked: boolean) => void; + }>; + isNew: boolean; + learnMoreLink?: string; +} + +export const Feature = ({ key, title, description, settings, options, isNew, learnMoreLink }: FeatureType) => ( + + +
+ + + {title} + + {isNew && New!} + +
+ {description} +
+
+
+ {learnMoreLink && ( + + Learn more + + )} +
+ + + {settings.map((option) => ( + <> + + + + {option.title} + + + + + + + + ))} + + {options.map((option, index) => ( + <> + + + + {option.title} + {!option.isAvailable && ( + Only available on DataHub Cloud + )} + +
+ {option.description} +
+ + (option.onChange ? option.onChange(checked) : null)} + disabled={!option.isAvailable} + /> + + {index !== options.length - 1 && } + + ))} + + +); diff --git a/datahub-web-react/src/app/settings/features/Features.tsx b/datahub-web-react/src/app/settings/features/Features.tsx new file mode 100644 index 0000000000000..ee8d7c628c1ef --- /dev/null +++ b/datahub-web-react/src/app/settings/features/Features.tsx @@ -0,0 +1,110 @@ +import React from 'react'; + +import styled from 'styled-components'; + +import { Divider, Typography } from 'antd'; +import { v4 as uuidv4 } from 'uuid'; + +import { Feature, FeatureType } from './Feature'; + +import { useGetDocPropagationSettings, useUpdateDocPropagationSettings } from './useDocPropagationSettings'; + +const Page = styled.div` + width: 100%; + display: flex; + justify-content: center; +`; + +const SourceContainer = styled.div` + width: 80%; + padding-top: 20px; + padding-right: 40px; + padding-left: 40px; +`; +const Container = styled.div` + padding-top: 0px; +`; + +const Title = styled(Typography.Title)` + && { + margin-bottom: 8px; + } +`; + +export const Features = () => { + /* + * Note: When adding new features, make sure to update the features array below + * and create a hook file for the new feature in the same directory + */ + + // Hooks to get and update the document propagation settings + const { isColPropagateChecked, setIsColPropagateChecked } = useGetDocPropagationSettings(); + const { updateDocPropagation } = useUpdateDocPropagationSettings(); + + // Features to display + const features: FeatureType[] = [ + { + key: uuidv4(), + title: 'Documentation Propagation', + description: 'Automatically propagate documentation from upstream to downstream columns and assets.', + settings: [ + { + key: uuidv4(), + title: 'Rollback Propagation Changes', + isAvailable: false, + buttonText: 'Rollback', + }, + { + key: uuidv4(), + title: 'Backfill existing documentation from upstream to downstream columns/assets', + isAvailable: false, + buttonText: 'Initialize', + }, + ], + options: [ + { + key: uuidv4(), + title: 'Column Level Propagation', + description: + 'Propagate new documentation from upstream to downstream columns based on column-level lineage relationships.', + isAvailable: true, + checked: isColPropagateChecked, + onChange: (checked: boolean) => { + setIsColPropagateChecked(checked); + updateDocPropagation(checked); + }, + }, + { + key: uuidv4(), + title: 'Asset Level Propagation', + description: + 'Propagate new documentation from upstream to downstream assets based on data lineage relationships.', + isAvailable: false, + checked: false, + }, + ], + isNew: true, + learnMoreLink: 'https://datahubproject.io/docs/automations/doc-propagation', + }, + ]; + + // Render + return ( + + + +
+ Features + + Explore and configure specific features + +
+ + + {features.map((feature) => ( + + ))} + + + ); +}; diff --git a/datahub-web-react/src/app/settings/features/useDocPropagationSettings.ts b/datahub-web-react/src/app/settings/features/useDocPropagationSettings.ts new file mode 100644 index 0000000000000..c93b610cff9d1 --- /dev/null +++ b/datahub-web-react/src/app/settings/features/useDocPropagationSettings.ts @@ -0,0 +1,50 @@ +import { useEffect, useState } from 'react'; + +import { message } from 'antd'; + +import { + useGetDocPropagationSettingsQuery, + useUpdateDocPropagationSettingsMutation, +} from '../../../graphql/app.generated'; + +// Hook to get the document propagation settings & manage state +export const useGetDocPropagationSettings = () => { + const { data, refetch } = useGetDocPropagationSettingsQuery(); + const [isColPropagateChecked, setIsColPropagateChecked] = useState(false); + + useEffect(() => { + const docPropSetting = data?.docPropagationSettings?.docColumnPropagation; + if (docPropSetting !== undefined) setIsColPropagateChecked(!!docPropSetting); + }, [data]); + + return { + isColPropagateChecked, + setIsColPropagateChecked, + refetch, + }; +}; + +// Hook to update the document propagation settings +export const useUpdateDocPropagationSettings = () => { + const [updateDocPropagationSettings] = useUpdateDocPropagationSettingsMutation(); + const { refetch } = useGetDocPropagationSettingsQuery(); + + const updateDocPropagation = async (checked: boolean) => { + try { + await updateDocPropagationSettings({ + variables: { + input: { + docColumnPropagation: checked, + }, + }, + }); + refetch(); + message.success('Successfully updated documentation propagation settings'); + } catch (e) { + message.error('Failed to update documentation propagation settings'); + refetch(); + } + }; + + return { updateDocPropagation }; +}; diff --git a/datahub-web-react/src/graphql/app.graphql b/datahub-web-react/src/graphql/app.graphql index bfca27a4ad106..e058a6fbb58e0 100644 --- a/datahub-web-react/src/graphql/app.graphql +++ b/datahub-web-react/src/graphql/app.graphql @@ -89,6 +89,16 @@ query getGlobalViewsSettings { } } +query getDocPropagationSettings { + docPropagationSettings { + docColumnPropagation + } +} + mutation updateGlobalViewsSettings($input: UpdateGlobalViewsSettingsInput!) { updateGlobalViewsSettings(input: $input) } + +mutation updateDocPropagationSettings($input: UpdateDocPropagationSettingsInput!) { + updateDocPropagationSettings(input: $input) +} diff --git a/docs-website/docusaurus.config.js b/docs-website/docusaurus.config.js index e3336d4970eed..ab033f7b04e4b 100644 --- a/docs-website/docusaurus.config.js +++ b/docs-website/docusaurus.config.js @@ -76,6 +76,12 @@ module.exports = { label: "Docs", position: "right", }, + { + to: "/cloud", + activeBasePath: "cloud", + label: "Cloud", + position: "right", + }, { to: "/learn", activeBasePath: "learn", diff --git a/docs-website/src/learn/_components/LearnListPage/index.jsx b/docs-website/src/learn/_components/LearnListPage/index.jsx index 4df87a340f21e..1ceec9afa1e8a 100644 --- a/docs-website/src/learn/_components/LearnListPage/index.jsx +++ b/docs-website/src/learn/_components/LearnListPage/index.jsx @@ -58,8 +58,9 @@ function BlogListPageContent(props) { For: {audiences.map((audience) => ( diff --git a/docs-website/src/learn/_components/LearnListPage/styles.module.scss b/docs-website/src/learn/_components/LearnListPage/styles.module.scss index d08b48a011de0..ce86e124afdb8 100644 --- a/docs-website/src/learn/_components/LearnListPage/styles.module.scss +++ b/docs-website/src/learn/_components/LearnListPage/styles.module.scss @@ -4,4 +4,10 @@ align-items: center; gap: 10px; flex-wrap: wrap; -} \ No newline at end of file + + .buttonActive { + background-color: var(--ifm-color-primary); + border: 1px solid var(--ifm-color-primary); + color: #ffffff; + } +} diff --git a/docs-website/src/learn/business-metric.md b/docs-website/src/learn/business-metric.md index ff8677b998ead..1378168f42195 100644 --- a/docs-website/src/learn/business-metric.md +++ b/docs-website/src/learn/business-metric.md @@ -72,13 +72,14 @@ DataHub Cloud offers comprehensive features designed to tackle the challenges of

- **[Approval Flows](https://datahubproject.io/docs/managed-datahub/approval-workflows):** Structured workflows for approving changes to metric definitions, maintaining accuracy and reliability. - - -![Untitled](https://prod-files-secure.s3.us-west-2.amazonaws.com/f818df0d-1067-44ab-99e1-8cf45d930c01/33ebd070-32a1-4875-b220-c31373f5eedf/Untitled.png) +

+ +
+ Lineage Tracking +

- **[Lineage Tracking](https://datahubproject.io/docs/generated/lineage/lineage-feature-guide):** Tools to track the origin and transformations of metrics, ensuring they align with standardized definitions. - - -![Screenshot 2024-07-10 at 12.07.28 PM.png](https://prod-files-secure.s3.us-west-2.amazonaws.com/f818df0d-1067-44ab-99e1-8cf45d930c01/39503957-ad64-4d2d-a5b2-b140abfc1f6c/Screenshot_2024-07-10_at_12.07.28_PM.png) By implementing these solutions, you can ensure that your business metrics are consistently defined and accurately used across all teams, supporting reliable analysis and decision-making. diff --git a/docs-website/src/pages/cloud/CompanyLogos/customersData.json b/docs-website/src/pages/cloud/CompanyLogos/customersData.json new file mode 100644 index 0000000000000..09943e8ef172f --- /dev/null +++ b/docs-website/src/pages/cloud/CompanyLogos/customersData.json @@ -0,0 +1,88 @@ +{ + "customers": [ + { + "link": { + "href": "https://robinhood.com", + "blank": true + }, + "logo": { + "asset": { + "_ref": "/img/logos/scrollingCompanies/acertus.webp" + }, + "alt": "Robinhood" + } + }, + { + "link": { + "href": "https://www.dpgmediagroup.com/", + "blank": true + }, + "logo": { + "asset": { + "_ref": "/img/logos/scrollingCompanies/autoscout24.webp" + }, + "alt": "DPG Media" + } + }, + { + "link": { + "href": "https://www.twilio.com", + "blank": true + }, + "logo": { + "asset": { + "_ref": "/img/logos/scrollingCompanies/betterup.webp" + }, + "alt": "Twilio" + } + }, + { + "link": { + "href": "https://myob.com", + "blank": true + }, + "logo": { + "asset": { + "_ref": "/img/logos/scrollingCompanies/depop.webp" + }, + "alt": "Myob" + } + }, + { + "link": { + "href": "https://regeneron.com", + "blank": true + }, + "logo": { + "asset": { + "_ref": "/img/logos/scrollingCompanies/dpg_media.webp" + }, + "alt": "Regeneron" + } + }, + { + "link": { + "href": "https://riskified.com", + "blank": true + }, + "logo": { + "asset": { + "_ref": "/img/logos/scrollingCompanies/myob.webp" + }, + "alt": "Riskified" + } + }, + { + "link": { + "href": "https://xero.com", + "blank": true + }, + "logo": { + "asset": { + "_ref": "/img/logos/scrollingCompanies/notion.webp" + }, + "alt": "Xero" + } + } + ] +} diff --git a/docs-website/src/pages/cloud/CompanyLogos/index.js b/docs-website/src/pages/cloud/CompanyLogos/index.js new file mode 100644 index 0000000000000..cd3a79f277893 --- /dev/null +++ b/docs-website/src/pages/cloud/CompanyLogos/index.js @@ -0,0 +1,49 @@ +import React, { useEffect, useRef } from 'react'; +import Link from '@docusaurus/Link'; +import styles from './logos.module.scss'; +import customersData from './customersData.json'; +import clsx from 'clsx'; + +const ScrollingCustomers = ({ noOverlay = true, spacing, ...rest }) => { + const customers = customersData.customers; + const duplicatedLogos = [...customers, ...customers, ...customers]; + const scrollingRef = useRef(null); + + useEffect(() => { + if (scrollingRef.current) { + scrollingRef.current.style.setProperty('--customers-length', customers.length); + } + }, [customers.length]); + + return ( +
+
+ {duplicatedLogos.map((customer, index) => ( + + {customer.logo.alt} + + ))} +
+
+ ); +}; + +export default ScrollingCustomers; diff --git a/docs-website/src/pages/cloud/CompanyLogos/logos.module.scss b/docs-website/src/pages/cloud/CompanyLogos/logos.module.scss new file mode 100644 index 0000000000000..bc76e6c072f1e --- /dev/null +++ b/docs-website/src/pages/cloud/CompanyLogos/logos.module.scss @@ -0,0 +1,56 @@ +@keyframes scrollingCustomerAnimate { + 0% { + transform: translateX(0); + } + 100% { + transform: translateX(calc(var(--customers-length) * -220px)); + } +} + +@media (max-width: 767px) { + @keyframes scrollingCustomerAnimate { + 0% { + transform: translateX(0); + } + 100% { + transform: translateX(calc(var(--customers-length) * -166px)); + } + } +} + +.scrollingCustomers { + position: relative; + overflow: hidden; +} + + +.scrollingCustomers__inner { + display: flex; + padding: 1.25rem 0; + position: relative; + align-items: center; + animation: scrollingCustomerAnimate 15s linear infinite; +} + +.scrollingCustomers__inner img { + max-width: 100px!important; + min-width: unset; + filter: invert(1) brightness(0) contrast(100); // make image black +} + +@media (max-width: 767px) { + .scrollingCustomers__inner img { + width: 126px !important; + } +} + +.animateScrollingCustomers { + display: flex; + animation: scrollingCustomerAnimate 15s linear infinite; +} + +@media (max-width: 767px) { + .animateScrollingCustomers { + width: calc(var(--customers-length) * 126px); + } +} diff --git a/docs-website/src/pages/cloud/Enterprise/index.js b/docs-website/src/pages/cloud/Enterprise/index.js new file mode 100644 index 0000000000000..03cef99058ea2 --- /dev/null +++ b/docs-website/src/pages/cloud/Enterprise/index.js @@ -0,0 +1,48 @@ +import React from "react"; +import clsx from "clsx"; +import Link from "@docusaurus/Link"; +import styles from "./styles.module.scss"; + + +const featuresContent = [ + { + title: "99.5% Uptime SLA", + description: "We’ll focus on keeping DataHub running, so you can focus on the things that really matter." + }, + { + title: "In-VPC Ingestion and
Data Quality evaluation", + description: "So your actual data never leaves your network." + }, + { + title: "SOC-2 Compliant", + description: "An incredibly high level of security you can trust." + }, +]; + +const Feature = ({ title, description }) => { + return ( +
+

+

{description}

+
+ ); +}; + +const Features = () => + featuresContent?.length > 0 ? ( +
+
+
Enterprise Ready
+
+ {featuresContent.map((props, idx) => ( + + ))} +
+ + +4 benefits + +
+
+ ) : null; + +export default Features; diff --git a/docs-website/src/pages/cloud/Enterprise/styles.module.scss b/docs-website/src/pages/cloud/Enterprise/styles.module.scss new file mode 100644 index 0000000000000..9d95dcc3990b6 --- /dev/null +++ b/docs-website/src/pages/cloud/Enterprise/styles.module.scss @@ -0,0 +1,40 @@ + +.container { + padding: 2vh 6vh; + padding-bottom: 8vh; + display: flex; + justify-content: center; + align-items: center; +} +.wrapper { + display: flex; + justify-content: center; + flex-direction: column; + width: 90%; + max-width: 1200px; + +} + +.title { + font-size: 2.8rem; + font-weight: 600; + margin-bottom: 2rem; +} + +.moreBenefits { + font-weight: 400; + color: var(--ifm-color-primary); + + &:hover { + text-decoration: none; + opacity: 0.8; + } + +} + +@media (max-width: 767px) { + + .container { + padding: 0px 36px; + } +} \ No newline at end of file diff --git a/docs-website/src/pages/cloud/FeatureCards/index.js b/docs-website/src/pages/cloud/FeatureCards/index.js new file mode 100644 index 0000000000000..4a45cbcbe1717 --- /dev/null +++ b/docs-website/src/pages/cloud/FeatureCards/index.js @@ -0,0 +1,130 @@ +import React from "react"; +import clsx from "clsx"; +import styles from "./styles.module.scss"; +import Link from "@docusaurus/Link"; +import { CheckCircleOutlined } from "@ant-design/icons"; + +const data = { + sections: [ + { + title: "Data Discovery", + icon: "/img/assets/data-discovery.svg", + cloudPageLink: "https://www.acryldata.io/acryl-datahub", + cloudBenefits: [ + { text: "Enhanced search ranking", link: "" }, // → + { text: "Personalization for every persona", link: "" }, // → + { text: "A browser extension for BI Tools", link: "" }, // → + { text: "AI-Powered Documentation", link: "" }, // → + { text: "No-Code Automations", link: "" }, // → + ], + coreBenefits: [ + { text: "Integrations for 50+ data sources"}, + { text: "Lineage for tables, columns, and jobs"}, + ], + hiddenCoreBenefits: "+4 benefits", + hiddenCoreBenefitsLink: '/docs/managed-datahub/managed-datahub-overview#search-and-discovery', + }, + { + title: "Data Observability", + icon: "/img/assets/data-ob.svg", + cloudPageLink: "https://www.acryldata.io/observe", + cloudBenefits: [ + { text: "Continuous data quality monitors", link: "" }, // → + { text: "End-to-end data incident tracking & management", link: "" }, // → + { text: "AI-Driven anomaly detection", link: "" }, // → + { text: "Executive Data Health Dashboard", link: "" }, // → + { text: "On-demand data quality evaluation via APIs & UI", link: "" }, // → + ], + coreBenefits: [ + { text: "Surface data quality results"}, + { text: "Create and manage data contracts" }, + ], + hiddenCoreBenefits: "+1 benefit", + hiddenCoreBenefitsLink: '/docs/managed-datahub/managed-datahub-overview#data-observability', + }, + { + title: "Data Governance", + icon: "/img/assets/data-governance.svg", + cloudPageLink: "https://www.acryldata.io/acryl-datahub#governance", + cloudBenefits: [ + { text: "Human-assisted asset certification workflows", link: "" }, // → + { text: "Automations to enforce governance standards", link: "" }, // → + { text: "End-to-end glossary management workflows", link: "" }, // → + { text: "Ownership management workflows", link: "" }, // → + ], + coreBenefits: [ + { text: "Shift-left governance"}, + { text: "Business glossaries"}, + ], + hiddenCoreBenefits: "+1 benefit", + hiddenCoreBenefitsLink: '/docs/managed-datahub/managed-datahub-overview#data-governance', + }, + ], +}; + +const Features = () => ( +
+ {data.sections.map((section, sectionIndex) => ( +
+
+
+
+
+ {section.title} +
{section.title}
+
+
+
+
DataHub Cloud includes:
+
+ {section.cloudBenefits.map((benefit, index) => ( +
+ + +
{benefit.text}
+ +
+ ))} +
+ + Explore in DataHub Cloud + +
+
+
+
+

{`In addition to DataHub Core:`}

+
+ {section.coreBenefits.map((benefit, index) => ( +
+ +
{benefit.text}
+
+ ))} +
+ {section.hiddenCoreBenefits} +
+
+
+
+ ))} +
+); + +export default Features; diff --git a/docs-website/src/pages/cloud/FeatureCards/styles.module.scss b/docs-website/src/pages/cloud/FeatureCards/styles.module.scss new file mode 100644 index 0000000000000..cfba5e217374d --- /dev/null +++ b/docs-website/src/pages/cloud/FeatureCards/styles.module.scss @@ -0,0 +1,164 @@ +.container { + padding: 48px 20px; +} + +.rowItem { + justify-content: center; +} + +.cardGroup { + display: flex; + justify-content: space-between; + position: relative; + align-items: flex-start; + margin: 0rem 1.2rem 2rem 1.2rem; +} + +.cardGroupInverse { + flex-direction: row-reverse; + .coreBenefitCard { + margin-left: 15rem; + } +} + +.title { + display: flex; + align-items: center; + margin: 2rem; + + .icon { + width: 4rem; + height: 4rem; + margin-right: 1rem; + } + + .titleText { + font-size: 32px; + font-weight: 700; + } +} + +.card { + background: white; + padding: 40px; + margin: 0; +} + +.sectionTitle { + font-size: 24px; + font-weight: 600; + line-height: 33px; +} + +.benefitIcon { + width: 32px; + height: 32px; + align-items: center; +} + +.moreBenefits { + font-family: Manrope; + font-size: 20px; + font-weight: 400; + line-height: 30px; + text-align: left; + color: rgba(24, 144, 255, 1); + + &:hover { + text-decoration: none; + opacity: 0.8; + } +} + +.featureList { + display: grid; + margin-top: 1rem; + font-weight: 400; + font-size: 20px; + text-align: left; + line-height: 27px; +} + +.exploreButton { + margin-top: 24px; +} + +.cloudBenefitCard { + border-radius: 16px; + border: 2px solid rgba(24, 144, 255, 1); + box-shadow: 0px 4px 25px 0px rgba(24, 144, 255, 0.25); + background: linear-gradient(180deg, #ffffff 27%, #e4f7ff 87%); + min-width: 543px; + z-index: 2; + position: relative; +} + +.coreBenefitCard { + border-radius: 16px; + border: 1px solid rgba(227, 227, 227, 1); + min-width: 451px; + z-index: 1; + position: relative; + margin-top: 14rem; + margin-left: -48px; + padding-left: 72px; +} + +.reversedRow .coreBenefitCard { + margin-left: auto!important; + padding-left: auto!important; + margin-right: -48px; + padding-right: 72px; +} + +.coreBenefitCard h3 { + margin-bottom: 2rem; +} +.coreBenefit { + display: flex; + flex-direction: row; + margin-bottom: 1rem; +} + +.cloudBenefit { + color: rgba(24, 144, 255, 1); + margin-bottom: 1rem; +} +.cloudBenefitLink { + display: flex; + flex-direction: row; +} +.cloudBenefitLinkText { + display: block; +} + +.reversedRow { + flex-direction: row-reverse; +} + +.reversedRow .title { + justify-content: flex-end; +} + +@media (max-width: 768px) { + .cardGroup { + flex-direction: column; + align-items: center; + } + + .cardGroupInverse { + flex-direction: column; + } + + .cloudBenefitCard, .coreBenefitCard { + min-width: 100%; + margin-top: 20px; + } + + .coreBenefitCard { + margin-top: -40px; + margin-left: 0 !important; + margin-right: 0 !important; + padding-top: 80px; + } +} diff --git a/docs-website/src/pages/cloud/UnifiedTabs/index.js b/docs-website/src/pages/cloud/UnifiedTabs/index.js new file mode 100644 index 0000000000000..0c7ff50ce5447 --- /dev/null +++ b/docs-website/src/pages/cloud/UnifiedTabs/index.js @@ -0,0 +1,76 @@ +import React, { useState } from 'react'; +import { CaretUpFilled } from '@ant-design/icons'; +import styles from './styles.module.scss'; +import clsx from 'clsx'; + +const TabbedComponent = () => { + const [activeTab, setActiveTab] = useState(0); + + const tabs = [ + { + title: 'Data Discovery', + description: 'All the search and discovery features of DataHub Core you already love, enhanced.', + icon: "/img/assets/data-discovery.svg", + link: "https://www.acryldata.io/acryl-datahub", + image: 'https://raw.githubusercontent.com/datahub-project/static-assets/main/imgs/saas/demo/discovery.webm', + }, + { + title: 'Data Observability', + description: 'Detect, resolve, and prevent data quality issues before they impact your business. Unify data health signals from all your data quality tools, including dbt tests and more.', + icon: "/img/assets/data-ob.svg", + link: "https://www.acryldata.io/observe", + image: 'https://raw.githubusercontent.com/datahub-project/static-assets/main/imgs/saas/demo/observe.webm', + }, + { + title: 'Data Governance', + description: 'Powerful Automation, Reporting and Organizational tools to help you govern effectively.', + icon: "/img/assets/data-governance.svg", + link: "https://www.acryldata.io/acryl-datahub#governance", + image: 'https://raw.githubusercontent.com/datahub-project/static-assets/main/imgs/saas/demo/governance.webm', + }, + ]; + + return ( +
+
One platform to rule them all
+
+
+ {tabs.map((tab, index) => ( + +
+ + {activeTab === index && ( +
+

{tab.description}

+ Learn More → +
+ )} +
+ {activeTab === index && ( +
+
+
+
+ )} + + ))} +
+
+
+
+
+
+
+ ); +}; + +export default TabbedComponent; diff --git a/docs-website/src/pages/cloud/UnifiedTabs/styles.module.scss b/docs-website/src/pages/cloud/UnifiedTabs/styles.module.scss new file mode 100644 index 0000000000000..7549f74305355 --- /dev/null +++ b/docs-website/src/pages/cloud/UnifiedTabs/styles.module.scss @@ -0,0 +1,179 @@ +.tabbedComponent { + text-align: center; + padding: 20px; + padding-top: 48px; + padding-bottom: 48px; + display: flex; + flex-direction: column; + align-items: center; +} + +.title { + font-size: 2.5rem; + font-weight: 500; + margin-bottom: 2rem; + text-align: center; + position: relative; + padding: 0 2rem; + display: block; + width: 100%; + + &:before, &:after { + content: " "; + height: 2px; + width: calc((100vw - 468px - 120px)/2); + background: #EEE; + display: block; + position: absolute; + top: 50%; + } + + &:before { + left: 0; + } + + &:after { + right: 0; + } +} + +.container { + display: flex; + flex-direction: column; // Changed to column for mobile view + background: white; + border-radius: 1rem; + overflow: hidden; + box-shadow: 0 4px 4px rgba(0, 0, 0, 0.1); + margin: 0rem 5rem; + max-width: 1200px; + width: 100%; +} + +.tabs { + display: flex; + flex-direction: column; + justify-content: flex-start; +} + +.tab { + align-items: center; + margin: 0; + border-bottom: 1px solid #e0e0e0; + position: relative; + + &.activeTab { + border-bottom: none; + } +} + +.activeTab .tabButton { + border-bottom: 1px solid #f1f1f1; +} + +.tabButton { + padding: 1.5rem; + background: none; + border: none; + cursor: pointer; + display: flex; + align-items: center; + width: 100%; + justify-content: space-between; + + .tabButton:focus { + outline: none; + } + + .tabTitle { + font-size: 1.5rem; + font-weight: 700; + margin-left: 1rem; + flex-grow: 1; + text-align: left; + } + + .icon { + width: 2.3rem; + height: 2.3rem; + } +} + +.arrow { + margin-left: auto; + transition: transform 0.2s; + color: black; +} + +.upsideDown { + transform: rotate(180deg); +} + +.dropdown { + background-color: #ffffff; + margin-top: 5px; + text-align: left; + padding: 1rem 1.5rem; + + .learnMore { + font-weight: 600; + font-size: 15px; + line-height: 23px; + + &:hover { + text-decoration: none; + opacity: 0.8; + } + } +} + +.imageContainer { + display: none; + justify-content: center; + align-items: center; + background-color: transparent; + margin: 1rem 0; // Added margin for spacing + height: 380px; + border-radius: 24px; +} + +.tabImage { + width: 100%; + height: 100%; + display: flex; + padding: 4px; + align-items: center; + justify-content: center; +} +.tabImage video { + max-height: 100%; + max-width: 100%; + border-radius: 22px; + box-shadow: 0px 4px 12px rgba(0, 0, 0, 0.1); +} + +.mobileImageContainer { + @media (max-width: 767px) { + display: flex; + } +} + +.webImageContainer { + @media (min-width: 768px) { + display: flex; + flex-grow: 1; + } +} + +@media (min-width: 768px) { + .container { + flex-direction: row; // Change back to row for larger screens + padding: 40px 32px; + } + .tabs { + width: 440px; + } + + .imageContainer { + margin: 1rem; // Reset margin for larger screens + } +} diff --git a/docs-website/src/pages/cloud/index.js b/docs-website/src/pages/cloud/index.js new file mode 100644 index 0000000000000..48f6a58e23581 --- /dev/null +++ b/docs-website/src/pages/cloud/index.js @@ -0,0 +1,90 @@ +import React from "react"; +import Layout from "@theme/Layout"; +import Link from "@docusaurus/Link"; +import useDocusaurusContext from "@docusaurus/useDocusaurusContext"; +import Enterprise from "./Enterprise"; +import { Section } from "../_components/Section"; +import ScrollingCustomers from "./CompanyLogos"; +import clsx from "clsx"; +import styles from "./styles.module.scss"; +import UnifiedTabs from "./UnifiedTabs"; +import FeatureCards from "./FeatureCards"; + + +function Home() { + const context = useDocusaurusContext(); + const { siteConfig = {} } = context; + // const { colorMode } = useColorMode(); + + + if (siteConfig.customFields.isSaas) { + window.location.replace("/docs"); + } + + return !siteConfig.customFields.isSaas ? ( + +
+
+
+
+

Try DataHub Cloud

+
+ Introducing DataHub as a Managed Service +
with Data Observability and Data Governance built-in.
+ +
+ + Book Demo + + + Product Tour + +
+
+
+
+ +
+ +
+ +
+
+ +
+
+
+
+
+
+

Get your free trial.

+
Data Discovery, Data Quality and Data Governance unified.
+ + + Book Demo + + + Product Tour + +
+
+ An extension of the DataHub Core project.
+ View Cloud Docs. + +
+
+
+
+
+ + ) : null; +} + +export default Home; diff --git a/docs-website/src/pages/cloud/styles.module.scss b/docs-website/src/pages/cloud/styles.module.scss new file mode 100644 index 0000000000000..d1ac31f3ef8cc --- /dev/null +++ b/docs-website/src/pages/cloud/styles.module.scss @@ -0,0 +1,51 @@ +.link { + &:hover { + text-decoration: none; + opacity: 0.8; + } +} + +.bgSection { + background-color: #FAFAFA !important; + } + +.hero { + :global { + .button { + margin-right: 1rem; + } + } + .hero__title { + font-size: 4rem; + } + + .hero__secondtitle { + font-size: 2rem; + font-weight: 300; + margin-bottom: 2rem; + + } + + .hero__subtitle { + margin-bottom: 2rem; + font-size: 1.75rem; + line-height: 2.5rem; + } + + .buttonLightBlue { + color: #1990FF !important; + background: #EAF3FF; + border-color: #EAF3FF; + :hover { + background: #D6E7FF; + } + } + + .learnMore { + font-size: 1.25rem; + font-weight: 600; + margin-top: 0.5rem; + + } + } + diff --git a/docs-website/src/pages/docs/_components/CustomerCardSection/index.jsx b/docs-website/src/pages/docs/_components/CustomerCardSection/index.jsx index ca34d89df8701..27067cb3930eb 100644 --- a/docs-website/src/pages/docs/_components/CustomerCardSection/index.jsx +++ b/docs-website/src/pages/docs/_components/CustomerCardSection/index.jsx @@ -57,7 +57,7 @@ const customerCardContent = [ 3. Community-driven project which continually evolves with industry trends and best practices ), - to: "https://www.acryldata.io/blog/data-contracts-in-datahub-combining-verifiability-with-holistic-data-management?utm_source=datahub&utm_medium=referral&utm_content=blog", + to: "https://youtu.be/wsCFnElN_Wo?si=i-bNAQAsbHJq5O9-", }, { customer: "Airtel", @@ -75,7 +75,7 @@ const customerCardContent = [ DataHub to take their data mesh implementation to the next level. ), - to: "https://youtu.be/wsCFnElN_Wo?si=i-bNAQAsbHJq5O9-", + to: "https://www.youtube.com/watch?v=yr24mM91BN4", }, ]; diff --git a/docs-website/static/img/assets/data-discovery.svg b/docs-website/static/img/assets/data-discovery.svg new file mode 100644 index 0000000000000..1a6c6f36a231a --- /dev/null +++ b/docs-website/static/img/assets/data-discovery.svg @@ -0,0 +1,4 @@ + + + + diff --git a/docs-website/static/img/assets/data-governance.svg b/docs-website/static/img/assets/data-governance.svg new file mode 100644 index 0000000000000..b1db48e3e76bf --- /dev/null +++ b/docs-website/static/img/assets/data-governance.svg @@ -0,0 +1,6 @@ + + + + + + diff --git a/docs-website/static/img/assets/data-ob.svg b/docs-website/static/img/assets/data-ob.svg new file mode 100644 index 0000000000000..d630b0a2333a2 --- /dev/null +++ b/docs-website/static/img/assets/data-ob.svg @@ -0,0 +1,8 @@ + + + + + + + + diff --git a/docs-website/static/img/cloud-bg.png b/docs-website/static/img/cloud-bg.png new file mode 100644 index 0000000000000..392aec2ff936c Binary files /dev/null and b/docs-website/static/img/cloud-bg.png differ diff --git a/docs-website/static/img/logos/companies/mediamarkt-saturn.png b/docs-website/static/img/logos/companies/mediamarkt-saturn.png new file mode 100644 index 0000000000000..6e3a39e0ae34b Binary files /dev/null and b/docs-website/static/img/logos/companies/mediamarkt-saturn.png differ diff --git a/docs-website/static/img/logos/companies/netflix.png b/docs-website/static/img/logos/companies/netflix.png new file mode 100755 index 0000000000000..151775b3b17bc Binary files /dev/null and b/docs-website/static/img/logos/companies/netflix.png differ diff --git a/docs-website/static/img/logos/companies/visa.png b/docs-website/static/img/logos/companies/visa.png new file mode 100644 index 0000000000000..0a0198bfb76a2 Binary files /dev/null and b/docs-website/static/img/logos/companies/visa.png differ diff --git a/docs-website/static/img/logos/scrollingCompanies/acertus.webp b/docs-website/static/img/logos/scrollingCompanies/acertus.webp new file mode 100644 index 0000000000000..20ff1c6d7d554 Binary files /dev/null and b/docs-website/static/img/logos/scrollingCompanies/acertus.webp differ diff --git a/docs-website/static/img/logos/scrollingCompanies/autoscout24.webp b/docs-website/static/img/logos/scrollingCompanies/autoscout24.webp new file mode 100644 index 0000000000000..27b34c6f724de Binary files /dev/null and b/docs-website/static/img/logos/scrollingCompanies/autoscout24.webp differ diff --git a/docs-website/static/img/logos/scrollingCompanies/betterup.webp b/docs-website/static/img/logos/scrollingCompanies/betterup.webp new file mode 100644 index 0000000000000..268e019de8fd4 Binary files /dev/null and b/docs-website/static/img/logos/scrollingCompanies/betterup.webp differ diff --git a/docs-website/static/img/logos/scrollingCompanies/depop.webp b/docs-website/static/img/logos/scrollingCompanies/depop.webp new file mode 100644 index 0000000000000..7c006bb862060 Binary files /dev/null and b/docs-website/static/img/logos/scrollingCompanies/depop.webp differ diff --git a/docs-website/static/img/logos/scrollingCompanies/dpg-media.png b/docs-website/static/img/logos/scrollingCompanies/dpg-media.png new file mode 100644 index 0000000000000..40022ef0294d7 Binary files /dev/null and b/docs-website/static/img/logos/scrollingCompanies/dpg-media.png differ diff --git a/docs-website/static/img/logos/scrollingCompanies/dpg_media.webp b/docs-website/static/img/logos/scrollingCompanies/dpg_media.webp new file mode 100644 index 0000000000000..0d5c42847068a Binary files /dev/null and b/docs-website/static/img/logos/scrollingCompanies/dpg_media.webp differ diff --git a/docs-website/static/img/logos/scrollingCompanies/event-callout-img.png b/docs-website/static/img/logos/scrollingCompanies/event-callout-img.png new file mode 100644 index 0000000000000..032c34fb6a10e Binary files /dev/null and b/docs-website/static/img/logos/scrollingCompanies/event-callout-img.png differ diff --git a/docs-website/static/img/logos/scrollingCompanies/graham-stirling.png b/docs-website/static/img/logos/scrollingCompanies/graham-stirling.png new file mode 100644 index 0000000000000..14bb23dcef0d9 Binary files /dev/null and b/docs-website/static/img/logos/scrollingCompanies/graham-stirling.png differ diff --git a/docs-website/static/img/logos/scrollingCompanies/icon-check.svg b/docs-website/static/img/logos/scrollingCompanies/icon-check.svg new file mode 100644 index 0000000000000..83a4d0983ebe0 --- /dev/null +++ b/docs-website/static/img/logos/scrollingCompanies/icon-check.svg @@ -0,0 +1,14 @@ + + + + + + + + + + + + + + diff --git a/docs-website/static/img/logos/scrollingCompanies/log-saxo.svg b/docs-website/static/img/logos/scrollingCompanies/log-saxo.svg new file mode 100644 index 0000000000000..f6ab4ae071b10 --- /dev/null +++ b/docs-website/static/img/logos/scrollingCompanies/log-saxo.svg @@ -0,0 +1,16 @@ + + + + + + + + + + + + + + + + diff --git a/docs-website/static/img/logos/scrollingCompanies/myob.png b/docs-website/static/img/logos/scrollingCompanies/myob.png new file mode 100644 index 0000000000000..c161532e650ba Binary files /dev/null and b/docs-website/static/img/logos/scrollingCompanies/myob.png differ diff --git a/docs-website/static/img/logos/scrollingCompanies/myob.webp b/docs-website/static/img/logos/scrollingCompanies/myob.webp new file mode 100644 index 0000000000000..509e3fba0804c Binary files /dev/null and b/docs-website/static/img/logos/scrollingCompanies/myob.webp differ diff --git a/docs-website/static/img/logos/scrollingCompanies/notion-logo.png b/docs-website/static/img/logos/scrollingCompanies/notion-logo.png new file mode 100644 index 0000000000000..f65884f53ead8 Binary files /dev/null and b/docs-website/static/img/logos/scrollingCompanies/notion-logo.png differ diff --git a/docs-website/static/img/logos/scrollingCompanies/notion.webp b/docs-website/static/img/logos/scrollingCompanies/notion.webp new file mode 100644 index 0000000000000..393756855d938 Binary files /dev/null and b/docs-website/static/img/logos/scrollingCompanies/notion.webp differ diff --git a/docs-website/static/img/logos/scrollingCompanies/ovo_energy.webp b/docs-website/static/img/logos/scrollingCompanies/ovo_energy.webp new file mode 100644 index 0000000000000..3e08a80f5c5f4 Binary files /dev/null and b/docs-website/static/img/logos/scrollingCompanies/ovo_energy.webp differ diff --git a/docs-website/static/img/logos/scrollingCompanies/regeneron.png b/docs-website/static/img/logos/scrollingCompanies/regeneron.png new file mode 100644 index 0000000000000..0a660b95c6d50 Binary files /dev/null and b/docs-website/static/img/logos/scrollingCompanies/regeneron.png differ diff --git a/docs-website/static/img/logos/scrollingCompanies/regeneron.webp b/docs-website/static/img/logos/scrollingCompanies/regeneron.webp new file mode 100644 index 0000000000000..45bed965f3a01 Binary files /dev/null and b/docs-website/static/img/logos/scrollingCompanies/regeneron.webp differ diff --git a/docs-website/static/img/logos/scrollingCompanies/riskified.png b/docs-website/static/img/logos/scrollingCompanies/riskified.png new file mode 100644 index 0000000000000..69b94b43fd56e Binary files /dev/null and b/docs-website/static/img/logos/scrollingCompanies/riskified.png differ diff --git a/docs-website/static/img/logos/scrollingCompanies/riskified.webp b/docs-website/static/img/logos/scrollingCompanies/riskified.webp new file mode 100644 index 0000000000000..a2a2f96d5ea7b Binary files /dev/null and b/docs-website/static/img/logos/scrollingCompanies/riskified.webp differ diff --git a/docs-website/static/img/logos/scrollingCompanies/robinhood.png b/docs-website/static/img/logos/scrollingCompanies/robinhood.png new file mode 100644 index 0000000000000..e75535a383f32 Binary files /dev/null and b/docs-website/static/img/logos/scrollingCompanies/robinhood.png differ diff --git a/docs-website/static/img/logos/scrollingCompanies/robinhood.webp b/docs-website/static/img/logos/scrollingCompanies/robinhood.webp new file mode 100644 index 0000000000000..661c25c0dd830 Binary files /dev/null and b/docs-website/static/img/logos/scrollingCompanies/robinhood.webp differ diff --git a/docs-website/static/img/logos/scrollingCompanies/saxo_bank.webp b/docs-website/static/img/logos/scrollingCompanies/saxo_bank.webp new file mode 100644 index 0000000000000..a4c1aae73fe48 Binary files /dev/null and b/docs-website/static/img/logos/scrollingCompanies/saxo_bank.webp differ diff --git a/docs-website/static/img/logos/scrollingCompanies/screenshot.png b/docs-website/static/img/logos/scrollingCompanies/screenshot.png new file mode 100644 index 0000000000000..59d982c5aec6d Binary files /dev/null and b/docs-website/static/img/logos/scrollingCompanies/screenshot.png differ diff --git a/docs-website/static/img/logos/scrollingCompanies/twilio-bg.png b/docs-website/static/img/logos/scrollingCompanies/twilio-bg.png new file mode 100644 index 0000000000000..74dcbf88a3595 Binary files /dev/null and b/docs-website/static/img/logos/scrollingCompanies/twilio-bg.png differ diff --git a/docs-website/static/img/logos/scrollingCompanies/twilio.png b/docs-website/static/img/logos/scrollingCompanies/twilio.png new file mode 100644 index 0000000000000..f226674d0ffbc Binary files /dev/null and b/docs-website/static/img/logos/scrollingCompanies/twilio.png differ diff --git a/docs-website/static/img/logos/scrollingCompanies/twilio.webp b/docs-website/static/img/logos/scrollingCompanies/twilio.webp new file mode 100644 index 0000000000000..5ad47d5d5c87e Binary files /dev/null and b/docs-website/static/img/logos/scrollingCompanies/twilio.webp differ diff --git a/docs-website/static/img/logos/scrollingCompanies/xero.png b/docs-website/static/img/logos/scrollingCompanies/xero.png new file mode 100644 index 0000000000000..653ddfb2c7686 Binary files /dev/null and b/docs-website/static/img/logos/scrollingCompanies/xero.png differ diff --git a/docs-website/static/img/logos/scrollingCompanies/xero.webp b/docs-website/static/img/logos/scrollingCompanies/xero.webp new file mode 100644 index 0000000000000..9f2b4cc0cf0f9 Binary files /dev/null and b/docs-website/static/img/logos/scrollingCompanies/xero.webp differ diff --git a/docs-website/static/img/logos/scrollingCompanies/zendesk.webp b/docs-website/static/img/logos/scrollingCompanies/zendesk.webp new file mode 100644 index 0000000000000..e790fdc2af6ed Binary files /dev/null and b/docs-website/static/img/logos/scrollingCompanies/zendesk.webp differ diff --git a/docs/api/graphql/getting-started.md b/docs/api/graphql/getting-started.md index 98aeca196600d..dfa556051bd4d 100644 --- a/docs/api/graphql/getting-started.md +++ b/docs/api/graphql/getting-started.md @@ -27,6 +27,7 @@ For more information on, please refer to the following links." - [Querying for Domain of a Dataset](/docs/api/tutorials/domains.md#read-domains) - [Querying for Glossary Terms of a Dataset](/docs/api/tutorials/terms.md#read-terms) - [Querying for Deprecation of a dataset](/docs/api/tutorials/deprecation.md#read-deprecation) +- [Querying for all DataJobs that belong to a DataFlow](/docs/lineage/airflow.md#get-all-datajobs-associated-with-a-dataflow) ### Search diff --git a/docs/lineage/airflow.md b/docs/lineage/airflow.md index 8680e36e2baf3..9d838ef8a4404 100644 --- a/docs/lineage/airflow.md +++ b/docs/lineage/airflow.md @@ -266,6 +266,34 @@ with DAG( - ingest this DAG, and it will remove all the obsolete pipelines and tasks from the Datahub based on the `cluster` value set in the `airflow.cfg` +## Get all dataJobs associated with a dataFlow + +If you are looking to find all tasks (aka DataJobs) that belong to a specific pipeline (aka DataFlow), you can use the following GraphQL query: + +```graphql +query { + dataFlow(urn: "urn:li:dataFlow:(airflow,db_etl,prod)") { + childJobs: relationships( + input: { + types: ["IsPartOf"], + direction: INCOMING, + start: 0, + count: 100 + } + ) { + total + relationships { + entity { + ... on DataJob { + urn + } + } + } + } + } +} +``` + ## Emit Lineage Directly If you can't use the plugin or annotate inlets/outlets, you can also emit lineage using the `DatahubEmitterOperator`. diff --git a/metadata-ingestion/src/datahub/cli/cli_utils.py b/metadata-ingestion/src/datahub/cli/cli_utils.py index 06861065ca6f2..21841b173c23d 100644 --- a/metadata-ingestion/src/datahub/cli/cli_utils.py +++ b/metadata-ingestion/src/datahub/cli/cli_utils.py @@ -13,7 +13,7 @@ from datahub.emitter.aspect import ASPECT_MAP, TIMESERIES_ASPECT_MAP from datahub.emitter.mcp import MetadataChangeProposalWrapper from datahub.emitter.request_helper import make_curl_command -from datahub.emitter.serialization_helper import post_json_transform +from datahub.emitter.serialization_helper import post_json_transform, pre_json_transform from datahub.metadata.com.linkedin.pegasus2avro.mxe import ( MetadataChangeEvent, MetadataChangeProposal, @@ -153,10 +153,11 @@ def post_entity( aspect_value: Dict, cached_session_host: Optional[Tuple[Session, str]] = None, is_async: Optional[str] = "false", + system_metadata: Union[None, SystemMetadataClass] = None, ) -> int: endpoint: str = "/aspects/?action=ingestProposal" - proposal = { + proposal: Dict[str, Any] = { "proposal": { "entityType": entity_type, "entityUrn": urn, @@ -169,6 +170,12 @@ def post_entity( }, "async": is_async, } + + if system_metadata is not None: + proposal["proposal"]["systemMetadata"] = json.dumps( + pre_json_transform(system_metadata.to_obj()) + ) + payload = json.dumps(proposal) url = gms_host + endpoint curl_command = make_curl_command(session, "POST", url, payload) diff --git a/metadata-ingestion/src/datahub/cli/put_cli.py b/metadata-ingestion/src/datahub/cli/put_cli.py index 40af54c7c7e2e..989b1a6d02fd0 100644 --- a/metadata-ingestion/src/datahub/cli/put_cli.py +++ b/metadata-ingestion/src/datahub/cli/put_cli.py @@ -1,12 +1,12 @@ import logging -from typing import Optional +from typing import Optional, Union import click from click_default_group import DefaultGroup from datahub.cli.cli_utils import post_entity from datahub.configuration.config_loader import load_config_file -from datahub.emitter.mcp import MetadataChangeProposalWrapper +from datahub.emitter.mcp import MetadataChangeProposalWrapper, SystemMetadataClass from datahub.ingestion.graph.client import get_default_graph from datahub.metadata.schema_classes import ( DataPlatformInfoClass as DataPlatformInfo, @@ -36,9 +36,15 @@ def put() -> None: @click.option("--urn", required=True, type=str) @click.option("-a", "--aspect", required=True, type=str) @click.option("-d", "--aspect-data", required=True, type=str) +@click.option( + "--run-id", + type=str, + required=False, + help="Run ID into which we should log the aspect.", +) @upgrade.check_upgrade @telemetry.with_telemetry() -def aspect(urn: str, aspect: str, aspect_data: str) -> None: +def aspect(urn: str, aspect: str, aspect_data: str, run_id: Optional[str]) -> None: """Update a single aspect of an entity""" entity_type = guess_entity_type(urn) @@ -48,6 +54,10 @@ def aspect(urn: str, aspect: str, aspect_data: str) -> None: client = get_default_graph() + system_metadata: Union[None, SystemMetadataClass] = None + if run_id: + system_metadata = SystemMetadataClass(runId=run_id) + # TODO: Replace with client.emit, requires figuring out the correct subsclass of _Aspect to create from the data status = post_entity( client._session, @@ -56,6 +66,7 @@ def aspect(urn: str, aspect: str, aspect_data: str) -> None: aspect_name=aspect, entity_type=entity_type, aspect_value=aspect_obj, + system_metadata=system_metadata, ) click.secho(f"Update succeeded with status {status}", fg="green") @@ -82,8 +93,11 @@ def aspect(urn: str, aspect: str, aspect_data: str) -> None: help="Logo URL that must be reachable from the DataHub UI.", required=True, ) +@click.option( + "--run-id", type=str, help="Run ID into which we should log the platform." +) def platform( - ctx: click.Context, name: str, display_name: Optional[str], logo: str + ctx: click.Context, name: str, display_name: Optional[str], logo: str, run_id: str ) -> None: """ Create or update a dataplatform entity in DataHub @@ -104,11 +118,12 @@ def platform( logoUrl=logo, ) datahub_graph = get_default_graph() - datahub_graph.emit( - MetadataChangeProposalWrapper( - entityUrn=str(platform_urn), aspect=data_platform_info - ) + mcp = MetadataChangeProposalWrapper( + entityUrn=str(platform_urn), + aspect=data_platform_info, + systemMetadata=SystemMetadataClass(runId=run_id), ) + datahub_graph.emit(mcp) click.echo( f"✅ Successfully wrote data platform metadata for {platform_urn} to DataHub ({datahub_graph})" ) diff --git a/metadata-ingestion/src/datahub/ingestion/source/dbt/dbt_common.py b/metadata-ingestion/src/datahub/ingestion/source/dbt/dbt_common.py index 9972e9e46369d..ead86acc299ca 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/dbt/dbt_common.py +++ b/metadata-ingestion/src/datahub/ingestion/source/dbt/dbt_common.py @@ -366,6 +366,13 @@ class DBTCommonConfig( description="When enabled, includes the compiled code in the emitted metadata.", ) + prefer_sql_parser_lineage: bool = Field( + default=False, + description="Normally we use dbt's metadata to generate table lineage. When enabled, we prefer results from the SQL parser when generating lineage instead. " + "This can be useful when dbt models reference tables directly, instead of using the ref() macro. " + "This requires that `skip_sources_in_lineage` is enabled.", + ) + @validator("target_platform") def validate_target_platform_value(cls, target_platform: str) -> str: if target_platform.lower() == DBT_PLATFORM: @@ -447,6 +454,16 @@ def validate_skip_sources_in_lineage( return skip_sources_in_lineage + @validator("prefer_sql_parser_lineage") + def validate_prefer_sql_parser_lineage( + cls, prefer_sql_parser_lineage: bool, values: Dict + ) -> bool: + if prefer_sql_parser_lineage and not values.get("skip_sources_in_lineage"): + raise ValueError( + "`prefer_sql_parser_lineage` requires that `skip_sources_in_lineage` is enabled." + ) + return prefer_sql_parser_lineage + @dataclass class DBTColumn: @@ -516,6 +533,9 @@ class DBTNode: columns: List[DBTColumn] = field(default_factory=list) upstream_nodes: List[str] = field(default_factory=list) # list of upstream dbt_name upstream_cll: List[DBTColumnLineageInfo] = field(default_factory=list) + raw_sql_parsing_result: Optional[ + SqlParsingResult + ] = None # only set for nodes that don't depend on ephemeral models cll_debug_info: Optional[SqlParsingDebugInfo] = None meta: Dict[str, Any] = field(default_factory=dict) @@ -1130,6 +1150,7 @@ def _infer_schemas_and_update_cll( # noqa: C901 # Run sql parser to infer the schema + generate column lineage. sql_result = None + depends_on_ephemeral_models = False if node.node_type in {"source", "test", "seed"}: # For sources, we generate CLL as a 1:1 mapping. # We don't support CLL for tests (assertions) or seeds. @@ -1148,6 +1169,8 @@ def _infer_schemas_and_update_cll( # noqa: C901 upstream_node.name, schema_resolver.platform ) } + if cte_mapping: + depends_on_ephemeral_models = True sql_result = self._parse_cll(node, cte_mapping, schema_resolver) else: @@ -1155,8 +1178,12 @@ def _infer_schemas_and_update_cll( # noqa: C901 # Save the column lineage. if self.config.include_column_lineage and sql_result: - # We only save the debug info here. We'll report errors based on it later, after - # applying the configured node filters. + # We save the raw info here. We use this for supporting `prefer_sql_parser_lineage`. + if not depends_on_ephemeral_models: + node.raw_sql_parsing_result = sql_result + + # We use this for error reporting. However, we only want to report errors + # after node filters are applied. node.cll_debug_info = sql_result.debug_info if sql_result.column_lineage: @@ -1171,6 +1198,7 @@ def _infer_schemas_and_update_cll( # noqa: C901 for column_lineage_info in sql_result.column_lineage for upstream_column in column_lineage_info.upstreams # Only include the CLL if the table in in the upstream list. + # TODO: Add some telemetry around this - how frequently does it filter stuff out? if target_platform_urn_to_dbt_name.get(upstream_column.table) in node.upstream_nodes ] @@ -1813,33 +1841,76 @@ def _translate_dbt_name_to_upstream_urn(dbt_name: str) -> str: if node.cll_debug_info and node.cll_debug_info.error: self.report.report_warning( - node.dbt_name, - f"Error parsing SQL to generate column lineage: {node.cll_debug_info.error}", + "Error parsing SQL to generate column lineage", + context=node.dbt_name, + exc=node.cll_debug_info.error, ) - cll = [ - FineGrainedLineage( - upstreamType=FineGrainedLineageUpstreamType.FIELD_SET, - downstreamType=FineGrainedLineageDownstreamType.FIELD_SET, - upstreams=[ - mce_builder.make_schema_field_urn( - _translate_dbt_name_to_upstream_urn( - upstream_column.upstream_dbt_name - ), - upstream_column.upstream_col, + + cll = None + if self.config.prefer_sql_parser_lineage and node.raw_sql_parsing_result: + sql_parsing_result = node.raw_sql_parsing_result + if sql_parsing_result and not sql_parsing_result.debug_info.table_error: + # If we have some table lineage from SQL parsing, use that. + upstream_urns = sql_parsing_result.in_tables + + cll = [] + for column_lineage in sql_parsing_result.column_lineage or []: + cll.append( + FineGrainedLineage( + upstreamType=FineGrainedLineageUpstreamType.FIELD_SET, + downstreamType=FineGrainedLineageDownstreamType.FIELD, + upstreams=[ + mce_builder.make_schema_field_urn( + upstream.table, upstream.column + ) + for upstream in column_lineage.upstreams + ], + downstreams=[ + mce_builder.make_schema_field_urn( + node_urn, column_lineage.downstream.column + ) + ], + confidenceScore=sql_parsing_result.debug_info.confidence, + ) ) - for upstream_column in upstreams - ], - downstreams=[ - mce_builder.make_schema_field_urn(node_urn, downstream) - ], - confidenceScore=( - node.cll_debug_info.confidence if node.cll_debug_info else None - ), - ) - for downstream, upstreams in itertools.groupby( - node.upstream_cll, lambda x: x.downstream_col - ) - ] + + else: + if self.config.prefer_sql_parser_lineage: + if node.upstream_cll: + self.report.report_warning( + "SQL parser lineage is not available for this node, falling back to dbt-based column lineage.", + context=node.dbt_name, + ) + else: + # SQL parsing failed entirely, which is already reported above. + pass + + cll = [ + FineGrainedLineage( + upstreamType=FineGrainedLineageUpstreamType.FIELD_SET, + downstreamType=FineGrainedLineageDownstreamType.FIELD, + upstreams=[ + mce_builder.make_schema_field_urn( + _translate_dbt_name_to_upstream_urn( + upstream_column.upstream_dbt_name + ), + upstream_column.upstream_col, + ) + for upstream_column in upstreams + ], + downstreams=[ + mce_builder.make_schema_field_urn(node_urn, downstream) + ], + confidenceScore=( + node.cll_debug_info.confidence + if node.cll_debug_info + else None + ), + ) + for downstream, upstreams in itertools.groupby( + node.upstream_cll, lambda x: x.downstream_col + ) + ] if not upstream_urns: return None diff --git a/metadata-ingestion/src/datahub/ingestion/source/looker/looker_file_loader.py b/metadata-ingestion/src/datahub/ingestion/source/looker/looker_file_loader.py index bc069bd1e59ac..fd670c23ad9cb 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/looker/looker_file_loader.py +++ b/metadata-ingestion/src/datahub/ingestion/source/looker/looker_file_loader.py @@ -60,7 +60,12 @@ def _load_viewfile( with open(path) as file: raw_file_content = file.read() except Exception as e: - self.reporter.failure("Failed to read lkml file", path, exc=e) + self.reporter.failure( + title="LKML File Loading Error", + message="A lookml file is not present on local storage or GitHub", + context=f"file path: {path}", + exc=e, + ) self.viewfile_cache[path] = None return None try: @@ -86,8 +91,19 @@ def _load_viewfile( self.viewfile_cache[path] = looker_viewfile return looker_viewfile except Exception as e: - self.reporter.failure("Failed to parse lkml file", path, exc=e) + self.reporter.failure( + title="LKML File Parsing Error", + message="The input file is not lookml file", + context=f"file path: {path}", + exc=e, + ) + + logger.debug(f"Raw file content for path {path}") + + logger.debug(raw_file_content) + self.viewfile_cache[path] = None + return None def load_viewfile( diff --git a/metadata-ingestion/src/datahub/ingestion/source/sql/sql_common.py b/metadata-ingestion/src/datahub/ingestion/source/sql/sql_common.py index 1624203c4096b..b3a5f134c61d6 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/sql/sql_common.py +++ b/metadata-ingestion/src/datahub/ingestion/source/sql/sql_common.py @@ -713,18 +713,17 @@ def loop_tables( # noqa: C901 data_reader, ) except Exception as e: - self.warn( - logger, - f"{schema}.{table}", - f"Ingestion error: {e}", + self.report.warning( + "Error processing table", + context=f"{schema}.{table}", + exc=e, ) - logger.debug( - f"Error processing table {schema}.{table}: Error was: {e} Traceback:", - exc_info=e, - ) - except Exception as e: - self.error(logger, f"{schema}", f"Tables error: {e}") + self.report.failure( + "Error processing tables", + context=schema, + exc=e, + ) def add_information_for_schema(self, inspector: Inspector, schema: str) -> None: pass @@ -1047,13 +1046,17 @@ def loop_views( sql_config=sql_config, ) except Exception as e: - self.warn( - logger, - f"{schema}.{view}", - f"Ingestion error: {e} {traceback.format_exc()}", + self.report.warning( + "Error processing view", + context=f"{schema}.{view}", + exc=e, ) except Exception as e: - self.error(logger, f"{schema}", f"Views error: {e}") + self.report.failure( + "Error processing views", + context=schema, + exc=e, + ) def _process_view( self, diff --git a/metadata-ingestion/tests/integration/dbt/dbt_enabled_with_schemas_mces_golden.json b/metadata-ingestion/tests/integration/dbt/dbt_enabled_with_schemas_mces_golden.json index aacb7093e5b4e..dc8c400b29157 100644 --- a/metadata-ingestion/tests/integration/dbt/dbt_enabled_with_schemas_mces_golden.json +++ b/metadata-ingestion/tests/integration/dbt/dbt_enabled_with_schemas_mces_golden.json @@ -271,7 +271,7 @@ "upstreams": [ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.customer,PROD),customer_id)" ], - "downstreamType": "FIELD_SET", + "downstreamType": "FIELD", "downstreams": [ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.dbt_postgres.customer_details,PROD),customer_id)" ], @@ -283,7 +283,7 @@ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.customer,PROD),first_name)", "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.customer,PROD),last_name)" ], - "downstreamType": "FIELD_SET", + "downstreamType": "FIELD", "downstreams": [ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.dbt_postgres.customer_details,PROD),full_name)" ], @@ -294,7 +294,7 @@ "upstreams": [ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.customer,PROD),email)" ], - "downstreamType": "FIELD_SET", + "downstreamType": "FIELD", "downstreams": [ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.dbt_postgres.customer_details,PROD),email)" ], @@ -305,7 +305,7 @@ "upstreams": [ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.address,PROD),address)" ], - "downstreamType": "FIELD_SET", + "downstreamType": "FIELD", "downstreams": [ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.dbt_postgres.customer_details,PROD),address)" ], @@ -316,7 +316,7 @@ "upstreams": [ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.city,PROD),city)" ], - "downstreamType": "FIELD_SET", + "downstreamType": "FIELD", "downstreams": [ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.dbt_postgres.customer_details,PROD),city)" ], @@ -327,7 +327,7 @@ "upstreams": [ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.address,PROD),postal_code)" ], - "downstreamType": "FIELD_SET", + "downstreamType": "FIELD", "downstreams": [ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.dbt_postgres.customer_details,PROD),postal_code)" ], @@ -338,7 +338,7 @@ "upstreams": [ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.address,PROD),phone)" ], - "downstreamType": "FIELD_SET", + "downstreamType": "FIELD", "downstreams": [ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.dbt_postgres.customer_details,PROD),phone)" ], @@ -549,7 +549,7 @@ "upstreams": [ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.dbt_postgres.payments_by_customer_by_month,PROD),billing_month)" ], - "downstreamType": "FIELD_SET", + "downstreamType": "FIELD", "downstreams": [ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.dbt_postgres.an-aliased-view-for-monthly-billing,PROD),billing_month)" ], @@ -560,7 +560,7 @@ "upstreams": [ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.dbt_postgres.payments_by_customer_by_month,PROD),customer_id)" ], - "downstreamType": "FIELD_SET", + "downstreamType": "FIELD", "downstreams": [ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.dbt_postgres.an-aliased-view-for-monthly-billing,PROD),customer_id)" ], @@ -571,7 +571,7 @@ "upstreams": [ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.dbt_postgres.payments_by_customer_by_month,PROD),amount)" ], - "downstreamType": "FIELD_SET", + "downstreamType": "FIELD", "downstreams": [ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.dbt_postgres.an-aliased-view-for-monthly-billing,PROD),amount)" ], @@ -828,7 +828,7 @@ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payment_p2020_05,PROD),amount)", "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payment_p2020_06,PROD),amount)" ], - "downstreamType": "FIELD_SET", + "downstreamType": "FIELD", "downstreams": [ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.dbt_postgres.an-aliased-view-for-payments,PROD),amount)" ], @@ -844,7 +844,7 @@ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payment_p2020_05,PROD),customer_id)", "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payment_p2020_06,PROD),customer_id)" ], - "downstreamType": "FIELD_SET", + "downstreamType": "FIELD", "downstreams": [ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.dbt_postgres.an-aliased-view-for-payments,PROD),customer_id)" ], @@ -860,7 +860,7 @@ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payment_p2020_05,PROD),payment_date)", "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payment_p2020_06,PROD),payment_date)" ], - "downstreamType": "FIELD_SET", + "downstreamType": "FIELD", "downstreams": [ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.dbt_postgres.an-aliased-view-for-payments,PROD),payment_date)" ], @@ -876,7 +876,7 @@ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payment_p2020_05,PROD),payment_id)", "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payment_p2020_06,PROD),payment_id)" ], - "downstreamType": "FIELD_SET", + "downstreamType": "FIELD", "downstreams": [ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.dbt_postgres.an-aliased-view-for-payments,PROD),payment_id)" ], @@ -892,7 +892,7 @@ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payment_p2020_05,PROD),rental_id)", "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payment_p2020_06,PROD),rental_id)" ], - "downstreamType": "FIELD_SET", + "downstreamType": "FIELD", "downstreams": [ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.dbt_postgres.an-aliased-view-for-payments,PROD),rental_id)" ], @@ -908,7 +908,7 @@ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payment_p2020_05,PROD),staff_id)", "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payment_p2020_06,PROD),staff_id)" ], - "downstreamType": "FIELD_SET", + "downstreamType": "FIELD", "downstreams": [ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.dbt_postgres.an-aliased-view-for-payments,PROD),staff_id)" ], @@ -1076,7 +1076,7 @@ "upstreams": [ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.dbt_postgres.an-aliased-view-for-payments,PROD),payment_date)" ], - "downstreamType": "FIELD_SET", + "downstreamType": "FIELD", "downstreams": [ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.dbt_postgres.payments_by_customer_by_month,PROD),billing_month)" ], @@ -1087,7 +1087,7 @@ "upstreams": [ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.dbt_postgres.an-aliased-view-for-payments,PROD),customer_id)" ], - "downstreamType": "FIELD_SET", + "downstreamType": "FIELD", "downstreams": [ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.dbt_postgres.payments_by_customer_by_month,PROD),customer_id)" ], @@ -1098,7 +1098,7 @@ "upstreams": [ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.dbt_postgres.an-aliased-view-for-payments,PROD),amount)" ], - "downstreamType": "FIELD_SET", + "downstreamType": "FIELD", "downstreams": [ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.dbt_postgres.payments_by_customer_by_month,PROD),amount)" ], diff --git a/metadata-ingestion/tests/integration/dbt/dbt_test_column_meta_mapping_golden.json b/metadata-ingestion/tests/integration/dbt/dbt_test_column_meta_mapping_golden.json index 5a35b4763af06..60f5bf4fbca9a 100644 --- a/metadata-ingestion/tests/integration/dbt/dbt_test_column_meta_mapping_golden.json +++ b/metadata-ingestion/tests/integration/dbt/dbt_test_column_meta_mapping_golden.json @@ -224,7 +224,7 @@ "upstreams": [ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.customer,PROD),customer_id)" ], - "downstreamType": "FIELD_SET", + "downstreamType": "FIELD", "downstreams": [ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.customer_details,PROD),customer_id)" ], @@ -236,7 +236,7 @@ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.customer,PROD),first_name)", "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.customer,PROD),last_name)" ], - "downstreamType": "FIELD_SET", + "downstreamType": "FIELD", "downstreams": [ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.customer_details,PROD),full_name)" ], @@ -248,7 +248,7 @@ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.customer_snapshot,PROD),first_name)", "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.customer_snapshot,PROD),last_name)" ], - "downstreamType": "FIELD_SET", + "downstreamType": "FIELD", "downstreams": [ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.customer_details,PROD),initial_full_name)" ], @@ -259,7 +259,7 @@ "upstreams": [ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.customer,PROD),email)" ], - "downstreamType": "FIELD_SET", + "downstreamType": "FIELD", "downstreams": [ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.customer_details,PROD),email)" ], @@ -270,7 +270,7 @@ "upstreams": [ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.address,PROD),address)" ], - "downstreamType": "FIELD_SET", + "downstreamType": "FIELD", "downstreams": [ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.customer_details,PROD),address)" ], @@ -281,7 +281,7 @@ "upstreams": [ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.city,PROD),city)" ], - "downstreamType": "FIELD_SET", + "downstreamType": "FIELD", "downstreams": [ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.customer_details,PROD),city)" ], @@ -292,7 +292,7 @@ "upstreams": [ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.address,PROD),postal_code)" ], - "downstreamType": "FIELD_SET", + "downstreamType": "FIELD", "downstreams": [ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.customer_details,PROD),postal_code)" ], @@ -303,7 +303,7 @@ "upstreams": [ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.address,PROD),phone)" ], - "downstreamType": "FIELD_SET", + "downstreamType": "FIELD", "downstreams": [ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.customer_details,PROD),phone)" ], @@ -491,7 +491,7 @@ "upstreams": [ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.payments_by_customer_by_month,PROD),billing_month)" ], - "downstreamType": "FIELD_SET", + "downstreamType": "FIELD", "downstreams": [ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.an-aliased-view-for-monthly-billing,PROD),billing_month)" ], @@ -502,7 +502,7 @@ "upstreams": [ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.payments_by_customer_by_month,PROD),customer_id)" ], - "downstreamType": "FIELD_SET", + "downstreamType": "FIELD", "downstreams": [ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.an-aliased-view-for-monthly-billing,PROD),customer_id)" ], @@ -513,7 +513,7 @@ "upstreams": [ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.payments_by_customer_by_month,PROD),amount)" ], - "downstreamType": "FIELD_SET", + "downstreamType": "FIELD", "downstreams": [ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.an-aliased-view-for-monthly-billing,PROD),amount)" ], @@ -524,7 +524,7 @@ "upstreams": [ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.customer_details,PROD),email)" ], - "downstreamType": "FIELD_SET", + "downstreamType": "FIELD", "downstreams": [ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.an-aliased-view-for-monthly-billing,PROD),email)" ], @@ -773,7 +773,7 @@ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payment_p2020_05,PROD),amount)", "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payment_p2020_06,PROD),amount)" ], - "downstreamType": "FIELD_SET", + "downstreamType": "FIELD", "downstreams": [ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.an_aliased_view_for_payments,PROD),amount)" ], @@ -789,7 +789,7 @@ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payment_p2020_05,PROD),customer_id)", "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payment_p2020_06,PROD),customer_id)" ], - "downstreamType": "FIELD_SET", + "downstreamType": "FIELD", "downstreams": [ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.an_aliased_view_for_payments,PROD),customer_id)" ], @@ -805,7 +805,7 @@ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payment_p2020_05,PROD),payment_date)", "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payment_p2020_06,PROD),payment_date)" ], - "downstreamType": "FIELD_SET", + "downstreamType": "FIELD", "downstreams": [ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.an_aliased_view_for_payments,PROD),payment_date)" ], @@ -821,7 +821,7 @@ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payment_p2020_05,PROD),payment_id)", "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payment_p2020_06,PROD),payment_id)" ], - "downstreamType": "FIELD_SET", + "downstreamType": "FIELD", "downstreams": [ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.an_aliased_view_for_payments,PROD),payment_id)" ], @@ -837,7 +837,7 @@ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payment_p2020_05,PROD),rental_id)", "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payment_p2020_06,PROD),rental_id)" ], - "downstreamType": "FIELD_SET", + "downstreamType": "FIELD", "downstreams": [ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.an_aliased_view_for_payments,PROD),rental_id)" ], @@ -853,7 +853,7 @@ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payment_p2020_05,PROD),staff_id)", "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payment_p2020_06,PROD),staff_id)" ], - "downstreamType": "FIELD_SET", + "downstreamType": "FIELD", "downstreams": [ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.an_aliased_view_for_payments,PROD),staff_id)" ], @@ -1049,7 +1049,7 @@ "upstreams": [ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.an_aliased_view_for_payments,PROD),payment_date)" ], - "downstreamType": "FIELD_SET", + "downstreamType": "FIELD", "downstreams": [ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payments_by_customer_by_month,PROD),billing_month)" ], @@ -1060,7 +1060,7 @@ "upstreams": [ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.an_aliased_view_for_payments,PROD),customer_id)" ], - "downstreamType": "FIELD_SET", + "downstreamType": "FIELD", "downstreams": [ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payments_by_customer_by_month,PROD),customer_id)" ], @@ -1071,7 +1071,7 @@ "upstreams": [ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.an_aliased_view_for_payments,PROD),amount)" ], - "downstreamType": "FIELD_SET", + "downstreamType": "FIELD", "downstreams": [ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payments_by_customer_by_month,PROD),amount)" ], @@ -1371,7 +1371,7 @@ "upstreams": [ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.customer,PROD),active)" ], - "downstreamType": "FIELD_SET", + "downstreamType": "FIELD", "downstreams": [ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.customer_snapshot,PROD),active)" ], @@ -1382,7 +1382,7 @@ "upstreams": [ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.customer,PROD),activebool)" ], - "downstreamType": "FIELD_SET", + "downstreamType": "FIELD", "downstreams": [ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.customer_snapshot,PROD),activebool)" ], @@ -1393,7 +1393,7 @@ "upstreams": [ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.customer,PROD),address_id)" ], - "downstreamType": "FIELD_SET", + "downstreamType": "FIELD", "downstreams": [ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.customer_snapshot,PROD),address_id)" ], @@ -1404,7 +1404,7 @@ "upstreams": [ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.customer,PROD),create_date)" ], - "downstreamType": "FIELD_SET", + "downstreamType": "FIELD", "downstreams": [ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.customer_snapshot,PROD),create_date)" ], @@ -1415,7 +1415,7 @@ "upstreams": [ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.customer,PROD),customer_id)" ], - "downstreamType": "FIELD_SET", + "downstreamType": "FIELD", "downstreams": [ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.customer_snapshot,PROD),customer_id)" ], @@ -1426,7 +1426,7 @@ "upstreams": [ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.customer,PROD),email)" ], - "downstreamType": "FIELD_SET", + "downstreamType": "FIELD", "downstreams": [ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.customer_snapshot,PROD),email)" ], @@ -1437,7 +1437,7 @@ "upstreams": [ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.customer,PROD),first_name)" ], - "downstreamType": "FIELD_SET", + "downstreamType": "FIELD", "downstreams": [ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.customer_snapshot,PROD),first_name)" ], @@ -1448,7 +1448,7 @@ "upstreams": [ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.customer,PROD),last_name)" ], - "downstreamType": "FIELD_SET", + "downstreamType": "FIELD", "downstreams": [ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.customer_snapshot,PROD),last_name)" ], @@ -1459,7 +1459,7 @@ "upstreams": [ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.customer,PROD),last_update)" ], - "downstreamType": "FIELD_SET", + "downstreamType": "FIELD", "downstreams": [ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.customer_snapshot,PROD),last_update)" ], @@ -1470,7 +1470,7 @@ "upstreams": [ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.customer,PROD),store_id)" ], - "downstreamType": "FIELD_SET", + "downstreamType": "FIELD", "downstreams": [ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.customer_snapshot,PROD),store_id)" ], diff --git a/metadata-ingestion/tests/integration/dbt/dbt_test_prefer_sql_parser_lineage_golden.json b/metadata-ingestion/tests/integration/dbt/dbt_test_prefer_sql_parser_lineage_golden.json new file mode 100644 index 0000000000000..81754fd6cbcac --- /dev/null +++ b/metadata-ingestion/tests/integration/dbt/dbt_test_prefer_sql_parser_lineage_golden.json @@ -0,0 +1,3126 @@ +[ +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.customer_details,PROD)", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "Model" + ] + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "dbt-prefer-sql-parser-lineage", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.customer_details,PROD)", + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "json": { + "platform": "urn:li:dataPlatform:dbt" + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "dbt-prefer-sql-parser-lineage", + "lastRunId": "no-run-id-provided" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.customer_details,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": { + "node_type": "model", + "materialization": "ephemeral", + "dbt_file_path": "models/transform/customer_details.sql", + "language": "sql", + "dbt_unique_id": "model.sample_dbt.customer_details", + "dbt_package_name": "sample_dbt", + "manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v11.json", + "manifest_version": "1.7.3", + "manifest_adapter": "postgres", + "catalog_schema": "https://schemas.getdbt.com/dbt/catalog/v1.json", + "catalog_version": "1.7.3" + }, + "name": "customer_details", + "description": "", + "tags": [] + } + }, + { + "com.linkedin.pegasus2avro.common.Status": { + "removed": false + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "model.sample_dbt.customer_details", + "platform": "urn:li:dataPlatform:dbt", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.MySqlDDL": { + "tableSchema": "" + } + }, + "fields": [ + { + "fieldPath": "customer_id", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "full_name", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "TEXT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "initial_full_name", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NullType": {} + } + }, + "nativeDataType": "", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "email", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "TEXT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "address", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "TEXT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "city", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "TEXT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "postal_code", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "TEXT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "phone", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "TEXT", + "recursive": false, + "isPartOfKey": false + } + ] + } + }, + { + "com.linkedin.pegasus2avro.dataset.UpstreamLineage": { + "upstreams": [ + { + "auditStamp": { + "time": 1643871600000, + "actor": "urn:li:corpuser:unknown" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.address,PROD)", + "type": "TRANSFORMED" + }, + { + "auditStamp": { + "time": 1643871600000, + "actor": "urn:li:corpuser:unknown" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.city,PROD)", + "type": "TRANSFORMED" + }, + { + "auditStamp": { + "time": 1643871600000, + "actor": "urn:li:corpuser:unknown" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.customer,PROD)", + "type": "TRANSFORMED" + }, + { + "auditStamp": { + "time": 1643871600000, + "actor": "urn:li:corpuser:unknown" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.customer_snapshot,PROD)", + "type": "TRANSFORMED" + } + ], + "fineGrainedLineages": [ + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.customer,PROD),customer_id)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.customer_details,PROD),customer_id)" + ], + "confidenceScore": 0.9 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.customer,PROD),first_name)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.customer,PROD),last_name)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.customer_details,PROD),full_name)" + ], + "confidenceScore": 0.9 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.customer_snapshot,PROD),first_name)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.customer_snapshot,PROD),last_name)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.customer_details,PROD),initial_full_name)" + ], + "confidenceScore": 0.9 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.customer,PROD),email)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.customer_details,PROD),email)" + ], + "confidenceScore": 0.9 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.address,PROD),address)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.customer_details,PROD),address)" + ], + "confidenceScore": 0.9 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.city,PROD),city)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.customer_details,PROD),city)" + ], + "confidenceScore": 0.9 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.address,PROD),postal_code)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.customer_details,PROD),postal_code)" + ], + "confidenceScore": 0.9 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.address,PROD),phone)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.customer_details,PROD),phone)" + ], + "confidenceScore": 0.9 + } + ] + } + }, + { + "com.linkedin.pegasus2avro.dataset.ViewProperties": { + "materialized": false, + "viewLogic": "{{ config(\n materialized = \"ephemeral\",\n) }}\n\nSELECT\n c.customer_id,\n c.first_name || ' ' || c.last_name as \"full_name\",\n (\n select cs.first_name || ' ' || cs.last_name\n from {{ ref('customer_snapshot') }} cs where cs.customer_id = c.customer_id\n order by dbt_valid_from desc\n limit 1\n ) as \"initial_full_name\",\n c.email,\n a.address,\n m.city,\n a.postal_code,\n a.phone\nFROM\n {{ source('pagila', 'customer')}} c\n left outer join {{ source('pagila', 'address')}} a on c.address_id = a.address_id\n left outer join {{ source('pagila', 'city') }} m on a.city_id = m.city_id", + "formattedViewLogic": "SELECT\n c.customer_id,\n c.first_name || ' ' || c.last_name AS \"full_name\",\n (\n SELECT\n cs.first_name || ' ' || cs.last_name\n FROM \"pagila\".\"public\".\"customer_snapshot\" AS cs\n WHERE\n cs.customer_id = c.customer_id\n ORDER BY\n dbt_valid_from DESC\n LIMIT 1\n ) AS \"initial_full_name\",\n c.email,\n a.address,\n m.city,\n a.postal_code,\n a.phone\nFROM \"pagila\".\"public\".\"customer\" AS c\nLEFT OUTER JOIN \"pagila\".\"public\".\"address\" AS a\n ON c.address_id = a.address_id\nLEFT OUTER JOIN \"pagila\".\"public\".\"city\" AS m\n ON a.city_id = m.city_id", + "viewLanguage": "SQL" + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "dbt-prefer-sql-parser-lineage", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.an-aliased-view-for-monthly-billing,PROD)", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "Model" + ] + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "dbt-prefer-sql-parser-lineage", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.an-aliased-view-for-monthly-billing,PROD)", + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "json": { + "platform": "urn:li:dataPlatform:dbt" + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "dbt-prefer-sql-parser-lineage", + "lastRunId": "no-run-id-provided" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.an-aliased-view-for-monthly-billing,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": { + "node_type": "model", + "materialization": "table", + "dbt_file_path": "models/billing/monthly_billing_with_cust.sql", + "catalog_type": "BASE TABLE", + "language": "sql", + "dbt_unique_id": "model.sample_dbt.monthly_billing_with_cust", + "dbt_package_name": "sample_dbt", + "manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v11.json", + "manifest_version": "1.7.3", + "manifest_adapter": "postgres", + "catalog_schema": "https://schemas.getdbt.com/dbt/catalog/v1.json", + "catalog_version": "1.7.3" + }, + "name": "an-aliased-view-for-monthly-billing", + "description": "", + "tags": [] + } + }, + { + "com.linkedin.pegasus2avro.common.Status": { + "removed": false + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "model.sample_dbt.monthly_billing_with_cust", + "platform": "urn:li:dataPlatform:dbt", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.MySqlDDL": { + "tableSchema": "" + } + }, + "fields": [ + { + "fieldPath": "amount", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "numeric", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "billing_month", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "timestamp with time zone", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "customer_id", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "integer", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "email", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "text", + "recursive": false, + "isPartOfKey": false + } + ] + } + }, + { + "com.linkedin.pegasus2avro.dataset.UpstreamLineage": { + "upstreams": [ + { + "auditStamp": { + "time": 1643871600000, + "actor": "urn:li:corpuser:unknown" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.customer_details,PROD)", + "type": "TRANSFORMED" + }, + { + "auditStamp": { + "time": 1643871600000, + "actor": "urn:li:corpuser:unknown" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.payments_by_customer_by_month,PROD)", + "type": "TRANSFORMED" + } + ], + "fineGrainedLineages": [ + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.payments_by_customer_by_month,PROD),BillingMonth)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.an-aliased-view-for-monthly-billing,PROD),billing_month)" + ], + "confidenceScore": 0.9 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.payments_by_customer_by_month,PROD),customer_id)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.an-aliased-view-for-monthly-billing,PROD),customer_id)" + ], + "confidenceScore": 0.9 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.payments_by_customer_by_month,PROD),amount)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.an-aliased-view-for-monthly-billing,PROD),amount)" + ], + "confidenceScore": 0.9 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.customer_details,PROD),email)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.an-aliased-view-for-monthly-billing,PROD),email)" + ], + "confidenceScore": 0.9 + } + ] + } + }, + { + "com.linkedin.pegasus2avro.dataset.ViewProperties": { + "materialized": true, + "viewLogic": "{{ config(\n materialized = \"table\",\n alias='an-aliased-view-for-monthly-billing'\n) }}\n\nSELECT \n pbc.\"BillingMonth\" as billing_month,\n pbc.customer_id,\n pbc.amount,\n cust.email\nFROM\n {{ ref('payments_by_customer_by_month')}} pbc\n left outer join {{ ref('customer_details')}} cust on pbc.customer_id = cust.customer_id\nORDER BY\n pbc.\"BillingMonth\"", + "formattedViewLogic": "WITH __dbt__cte__customer_details AS (\n SELECT\n c.customer_id,\n c.first_name || ' ' || c.last_name AS \"full_name\",\n (\n SELECT\n cs.first_name || ' ' || cs.last_name\n FROM \"pagila\".\"public\".\"customer_snapshot\" AS cs\n WHERE\n cs.customer_id = c.customer_id\n ORDER BY\n dbt_valid_from DESC\n LIMIT 1\n ) AS \"initial_full_name\",\n c.email,\n a.address,\n m.city,\n a.postal_code,\n a.phone\n FROM \"pagila\".\"public\".\"customer\" AS c\n LEFT OUTER JOIN \"pagila\".\"public\".\"address\" AS a\n ON c.address_id = a.address_id\n LEFT OUTER JOIN \"pagila\".\"public\".\"city\" AS m\n ON a.city_id = m.city_id\n)\nSELECT\n pbc.\"BillingMonth\" AS billing_month,\n pbc.customer_id,\n pbc.amount,\n cust.email\nFROM \"pagila\".\"public\".\"payments_by_customer_by_month\" AS pbc\nLEFT OUTER JOIN __dbt__cte__customer_details AS cust\n ON pbc.customer_id = cust.customer_id\nORDER BY\n pbc.\"BillingMonth\"", + "viewLanguage": "SQL" + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "dbt-prefer-sql-parser-lineage", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataProcessInstance", + "entityUrn": "urn:li:dataProcessInstance:a42a5b1bee156e45972e12d4156fb7a2", + "changeType": "UPSERT", + "aspectName": "dataProcessInstanceProperties", + "aspect": { + "json": { + "customProperties": { + "dbt_name": "model.sample_dbt.monthly_billing_with_cust", + "dbt_urn": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.an-aliased-view-for-monthly-billing,PROD)" + }, + "name": "just-some-random-id_urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.an-aliased-view-for-monthly-billing,PROD)", + "type": "BATCH_SCHEDULED", + "created": { + "time": 1663355198240, + "actor": "urn:li:corpuser:datahub" + } + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "dbt-prefer-sql-parser-lineage", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataProcessInstance", + "entityUrn": "urn:li:dataProcessInstance:a42a5b1bee156e45972e12d4156fb7a2", + "changeType": "UPSERT", + "aspectName": "dataProcessInstanceRelationships", + "aspect": { + "json": { + "parentTemplate": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.an-aliased-view-for-monthly-billing,PROD)", + "upstreamInstances": [] + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "dbt-prefer-sql-parser-lineage", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataProcessInstance", + "entityUrn": "urn:li:dataProcessInstance:a42a5b1bee156e45972e12d4156fb7a2", + "changeType": "UPSERT", + "aspectName": "dataProcessInstanceInput", + "aspect": { + "json": { + "inputs": [ + "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.customer_details,PROD)", + "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.payments_by_customer_by_month,PROD)" + ] + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "dbt-prefer-sql-parser-lineage", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataProcessInstance", + "entityUrn": "urn:li:dataProcessInstance:a42a5b1bee156e45972e12d4156fb7a2", + "changeType": "UPSERT", + "aspectName": "dataProcessInstanceOutput", + "aspect": { + "json": { + "outputs": [ + "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.an-aliased-view-for-monthly-billing,PROD)" + ] + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "dbt-prefer-sql-parser-lineage", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataProcessInstance", + "entityUrn": "urn:li:dataProcessInstance:a42a5b1bee156e45972e12d4156fb7a2", + "changeType": "UPSERT", + "aspectName": "dataProcessInstanceRunEvent", + "aspect": { + "json": { + "timestampMillis": 1663355198240, + "partitionSpec": { + "type": "FULL_TABLE", + "partition": "FULL_TABLE_SNAPSHOT" + }, + "status": "STARTED" + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "dbt-prefer-sql-parser-lineage", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataProcessInstance", + "entityUrn": "urn:li:dataProcessInstance:a42a5b1bee156e45972e12d4156fb7a2", + "changeType": "UPSERT", + "aspectName": "dataProcessInstanceRunEvent", + "aspect": { + "json": { + "timestampMillis": 1663355198242, + "partitionSpec": { + "type": "FULL_TABLE", + "partition": "FULL_TABLE_SNAPSHOT" + }, + "status": "COMPLETE", + "result": { + "type": "SUCCESS", + "nativeResultType": "success" + }, + "durationMillis": 2 + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "dbt-prefer-sql-parser-lineage", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.an_aliased_view_for_payments,PROD)", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "Model" + ] + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "dbt-prefer-sql-parser-lineage", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.an_aliased_view_for_payments,PROD)", + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "json": { + "platform": "urn:li:dataPlatform:dbt" + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "dbt-prefer-sql-parser-lineage", + "lastRunId": "no-run-id-provided" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.an_aliased_view_for_payments,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": { + "node_type": "model", + "materialization": "view", + "dbt_file_path": "models/base/payments_base.sql", + "catalog_type": "VIEW", + "language": "sql", + "dbt_unique_id": "model.sample_dbt.payments_base", + "dbt_package_name": "sample_dbt", + "manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v11.json", + "manifest_version": "1.7.3", + "manifest_adapter": "postgres", + "catalog_schema": "https://schemas.getdbt.com/dbt/catalog/v1.json", + "catalog_version": "1.7.3" + }, + "name": "an_aliased_view_for_payments", + "description": "", + "tags": [] + } + }, + { + "com.linkedin.pegasus2avro.common.Status": { + "removed": false + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "model.sample_dbt.payments_base", + "platform": "urn:li:dataPlatform:dbt", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.MySqlDDL": { + "tableSchema": "" + } + }, + "fields": [ + { + "fieldPath": "amount", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "numeric(5,2)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "customer_id", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "integer", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "payment_date", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "timestamp with time zone", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "payment_id", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "integer", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "rental_id", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "integer", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "staff_id", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "integer", + "recursive": false, + "isPartOfKey": false + } + ] + } + }, + { + "com.linkedin.pegasus2avro.dataset.UpstreamLineage": { + "upstreams": [ + { + "auditStamp": { + "time": 1643871600000, + "actor": "urn:li:corpuser:unknown" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.payment_p2020_01,PROD)", + "type": "TRANSFORMED" + }, + { + "auditStamp": { + "time": 1643871600000, + "actor": "urn:li:corpuser:unknown" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.payment_p2020_02,PROD)", + "type": "TRANSFORMED" + }, + { + "auditStamp": { + "time": 1643871600000, + "actor": "urn:li:corpuser:unknown" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.payment_p2020_03,PROD)", + "type": "TRANSFORMED" + }, + { + "auditStamp": { + "time": 1643871600000, + "actor": "urn:li:corpuser:unknown" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.payment_p2020_04,PROD)", + "type": "TRANSFORMED" + }, + { + "auditStamp": { + "time": 1643871600000, + "actor": "urn:li:corpuser:unknown" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.payment_p2020_05,PROD)", + "type": "TRANSFORMED" + }, + { + "auditStamp": { + "time": 1643871600000, + "actor": "urn:li:corpuser:unknown" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.payment_p2020_06,PROD)", + "type": "TRANSFORMED" + } + ], + "fineGrainedLineages": [ + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.payment_p2020_01,PROD),amount)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.payment_p2020_02,PROD),amount)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.payment_p2020_03,PROD),amount)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.payment_p2020_04,PROD),amount)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.payment_p2020_05,PROD),amount)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.payment_p2020_06,PROD),amount)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.an_aliased_view_for_payments,PROD),amount)" + ], + "confidenceScore": 0.9 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.payment_p2020_01,PROD),customer_id)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.payment_p2020_02,PROD),customer_id)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.payment_p2020_03,PROD),customer_id)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.payment_p2020_04,PROD),customer_id)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.payment_p2020_05,PROD),customer_id)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.payment_p2020_06,PROD),customer_id)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.an_aliased_view_for_payments,PROD),customer_id)" + ], + "confidenceScore": 0.9 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.payment_p2020_01,PROD),payment_date)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.payment_p2020_02,PROD),payment_date)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.payment_p2020_03,PROD),payment_date)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.payment_p2020_04,PROD),payment_date)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.payment_p2020_05,PROD),payment_date)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.payment_p2020_06,PROD),payment_date)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.an_aliased_view_for_payments,PROD),payment_date)" + ], + "confidenceScore": 0.9 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.payment_p2020_01,PROD),payment_id)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.payment_p2020_02,PROD),payment_id)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.payment_p2020_03,PROD),payment_id)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.payment_p2020_04,PROD),payment_id)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.payment_p2020_05,PROD),payment_id)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.payment_p2020_06,PROD),payment_id)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.an_aliased_view_for_payments,PROD),payment_id)" + ], + "confidenceScore": 0.9 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.payment_p2020_01,PROD),rental_id)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.payment_p2020_02,PROD),rental_id)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.payment_p2020_03,PROD),rental_id)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.payment_p2020_04,PROD),rental_id)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.payment_p2020_05,PROD),rental_id)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.payment_p2020_06,PROD),rental_id)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.an_aliased_view_for_payments,PROD),rental_id)" + ], + "confidenceScore": 0.9 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.payment_p2020_01,PROD),staff_id)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.payment_p2020_02,PROD),staff_id)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.payment_p2020_03,PROD),staff_id)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.payment_p2020_04,PROD),staff_id)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.payment_p2020_05,PROD),staff_id)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.payment_p2020_06,PROD),staff_id)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.an_aliased_view_for_payments,PROD),staff_id)" + ], + "confidenceScore": 0.9 + } + ] + } + }, + { + "com.linkedin.pegasus2avro.dataset.ViewProperties": { + "materialized": false, + "viewLogic": "{{ config(\n materialized=\"view\",\n alias='an_aliased_view_for_payments'\n) }}\n\nwith payments as (\n\n select \n *\n from \n {{ source('pagila', 'payment_p2020_01')}}\n UNION ALL\n select \n *\n from \n {{ source('pagila', 'payment_p2020_02')}}\n UNION ALL\n select \n *\n from \n {{ source('pagila', 'payment_p2020_02')}}\n UNION ALL\n select \n *\n from \n {{ source('pagila', 'payment_p2020_03')}}\n UNION ALL\n select \n *\n from \n {{ source('pagila', 'payment_p2020_04')}}\n UNION ALL\n select \n *\n from \n {{ source('pagila', 'payment_p2020_05')}}\n UNION ALL\n select \n *\n from \n {{ source('pagila', 'payment_p2020_06')}}\n)\n\nselect *\nfrom payments", + "formattedViewLogic": "WITH payments AS (\n SELECT\n *\n FROM \"pagila\".\"public\".\"payment_p2020_01\"\n UNION ALL\n SELECT\n *\n FROM \"pagila\".\"public\".\"payment_p2020_02\"\n UNION ALL\n SELECT\n *\n FROM \"pagila\".\"public\".\"payment_p2020_02\"\n UNION ALL\n SELECT\n *\n FROM \"pagila\".\"public\".\"payment_p2020_03\"\n UNION ALL\n SELECT\n *\n FROM \"pagila\".\"public\".\"payment_p2020_04\"\n UNION ALL\n SELECT\n *\n FROM \"pagila\".\"public\".\"payment_p2020_05\"\n UNION ALL\n SELECT\n *\n FROM \"pagila\".\"public\".\"payment_p2020_06\"\n)\nSELECT\n *\nFROM payments", + "viewLanguage": "SQL" + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "dbt-prefer-sql-parser-lineage", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataProcessInstance", + "entityUrn": "urn:li:dataProcessInstance:fc6268f0be68fd04c310705b65efd6fe", + "changeType": "UPSERT", + "aspectName": "dataProcessInstanceProperties", + "aspect": { + "json": { + "customProperties": { + "dbt_name": "model.sample_dbt.payments_base", + "dbt_urn": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.an_aliased_view_for_payments,PROD)" + }, + "name": "just-some-random-id_urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.an_aliased_view_for_payments,PROD)", + "type": "BATCH_SCHEDULED", + "created": { + "time": 1663355198240, + "actor": "urn:li:corpuser:datahub" + } + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "dbt-prefer-sql-parser-lineage", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataProcessInstance", + "entityUrn": "urn:li:dataProcessInstance:fc6268f0be68fd04c310705b65efd6fe", + "changeType": "UPSERT", + "aspectName": "dataProcessInstanceRelationships", + "aspect": { + "json": { + "parentTemplate": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.an_aliased_view_for_payments,PROD)", + "upstreamInstances": [] + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "dbt-prefer-sql-parser-lineage", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataProcessInstance", + "entityUrn": "urn:li:dataProcessInstance:fc6268f0be68fd04c310705b65efd6fe", + "changeType": "UPSERT", + "aspectName": "dataProcessInstanceInput", + "aspect": { + "json": { + "inputs": [ + "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.payment_p2020_01,PROD)", + "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.payment_p2020_02,PROD)", + "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.payment_p2020_03,PROD)", + "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.payment_p2020_04,PROD)", + "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.payment_p2020_05,PROD)", + "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.payment_p2020_06,PROD)" + ] + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "dbt-prefer-sql-parser-lineage", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataProcessInstance", + "entityUrn": "urn:li:dataProcessInstance:fc6268f0be68fd04c310705b65efd6fe", + "changeType": "UPSERT", + "aspectName": "dataProcessInstanceOutput", + "aspect": { + "json": { + "outputs": [ + "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.an_aliased_view_for_payments,PROD)" + ] + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "dbt-prefer-sql-parser-lineage", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataProcessInstance", + "entityUrn": "urn:li:dataProcessInstance:fc6268f0be68fd04c310705b65efd6fe", + "changeType": "UPSERT", + "aspectName": "dataProcessInstanceRunEvent", + "aspect": { + "json": { + "timestampMillis": 1663355198240, + "partitionSpec": { + "type": "FULL_TABLE", + "partition": "FULL_TABLE_SNAPSHOT" + }, + "status": "STARTED" + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "dbt-prefer-sql-parser-lineage", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataProcessInstance", + "entityUrn": "urn:li:dataProcessInstance:fc6268f0be68fd04c310705b65efd6fe", + "changeType": "UPSERT", + "aspectName": "dataProcessInstanceRunEvent", + "aspect": { + "json": { + "timestampMillis": 1663355198242, + "partitionSpec": { + "type": "FULL_TABLE", + "partition": "FULL_TABLE_SNAPSHOT" + }, + "status": "COMPLETE", + "result": { + "type": "SUCCESS", + "nativeResultType": "success" + }, + "durationMillis": 2 + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "dbt-prefer-sql-parser-lineage", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payments_by_customer_by_month,PROD)", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "Model" + ] + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "dbt-prefer-sql-parser-lineage", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payments_by_customer_by_month,PROD)", + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "json": { + "platform": "urn:li:dataPlatform:dbt" + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "dbt-prefer-sql-parser-lineage", + "lastRunId": "no-run-id-provided" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payments_by_customer_by_month,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": { + "node_type": "model", + "materialization": "table", + "dbt_file_path": "models/transform/payments_by_customer_by_month.sql", + "catalog_type": "BASE TABLE", + "language": "sql", + "dbt_unique_id": "model.sample_dbt.payments_by_customer_by_month", + "dbt_package_name": "sample_dbt", + "manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v11.json", + "manifest_version": "1.7.3", + "manifest_adapter": "postgres", + "catalog_schema": "https://schemas.getdbt.com/dbt/catalog/v1.json", + "catalog_version": "1.7.3" + }, + "name": "payments_by_customer_by_month", + "description": "", + "tags": [] + } + }, + { + "com.linkedin.pegasus2avro.common.Status": { + "removed": false + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "model.sample_dbt.payments_by_customer_by_month", + "platform": "urn:li:dataPlatform:dbt", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.MySqlDDL": { + "tableSchema": "" + } + }, + "fields": [ + { + "fieldPath": "BillingMonth", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "timestamp with time zone", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "amount", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "numeric", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "customer_id", + "nullable": false, + "description": "description for customer_id from dbt", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "integer", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:dbt:tag_from_dbt" + } + ] + }, + "isPartOfKey": false + } + ] + } + }, + { + "com.linkedin.pegasus2avro.dataset.UpstreamLineage": { + "upstreams": [ + { + "auditStamp": { + "time": 1643871600000, + "actor": "urn:li:corpuser:unknown" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.an_aliased_view_for_payments,PROD)", + "type": "TRANSFORMED" + } + ], + "fineGrainedLineages": [ + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.an_aliased_view_for_payments,PROD),payment_date)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payments_by_customer_by_month,PROD),BillingMonth)" + ], + "confidenceScore": 0.9 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.an_aliased_view_for_payments,PROD),customer_id)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payments_by_customer_by_month,PROD),customer_id)" + ], + "confidenceScore": 0.9 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.an_aliased_view_for_payments,PROD),amount)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payments_by_customer_by_month,PROD),amount)" + ], + "confidenceScore": 0.9 + } + ] + } + }, + { + "com.linkedin.pegasus2avro.dataset.ViewProperties": { + "materialized": true, + "viewLogic": "{{ config(\n materialized = \"table\",\n) }}\n\nSELECT\n date_trunc('month', payment_date) as \"BillingMonth\",\n customer_id,\n sum(amount) as \"amount\"\nFROM\n {{ ref('payments_base')}}\nGROUP BY\n \"BillingMonth\",\n customer_id", + "formattedViewLogic": "SELECT\n DATE_TRUNC('MONTH', payment_date) AS \"BillingMonth\",\n customer_id,\n SUM(amount) AS \"amount\"\nFROM \"pagila\".\"public\".\"an_aliased_view_for_payments\"\nGROUP BY\n \"BillingMonth\",\n customer_id", + "viewLanguage": "SQL" + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "dbt-prefer-sql-parser-lineage", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataProcessInstance", + "entityUrn": "urn:li:dataProcessInstance:57aa623f096cf3a28af70fe94b713907", + "changeType": "UPSERT", + "aspectName": "dataProcessInstanceProperties", + "aspect": { + "json": { + "customProperties": { + "dbt_name": "model.sample_dbt.payments_by_customer_by_month", + "dbt_urn": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payments_by_customer_by_month,PROD)" + }, + "name": "just-some-random-id_urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payments_by_customer_by_month,PROD)", + "type": "BATCH_SCHEDULED", + "created": { + "time": 1663355198240, + "actor": "urn:li:corpuser:datahub" + } + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "dbt-prefer-sql-parser-lineage", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataProcessInstance", + "entityUrn": "urn:li:dataProcessInstance:57aa623f096cf3a28af70fe94b713907", + "changeType": "UPSERT", + "aspectName": "dataProcessInstanceRelationships", + "aspect": { + "json": { + "parentTemplate": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payments_by_customer_by_month,PROD)", + "upstreamInstances": [] + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "dbt-prefer-sql-parser-lineage", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataProcessInstance", + "entityUrn": "urn:li:dataProcessInstance:57aa623f096cf3a28af70fe94b713907", + "changeType": "UPSERT", + "aspectName": "dataProcessInstanceInput", + "aspect": { + "json": { + "inputs": [ + "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.an_aliased_view_for_payments,PROD)" + ] + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "dbt-prefer-sql-parser-lineage", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataProcessInstance", + "entityUrn": "urn:li:dataProcessInstance:57aa623f096cf3a28af70fe94b713907", + "changeType": "UPSERT", + "aspectName": "dataProcessInstanceOutput", + "aspect": { + "json": { + "outputs": [ + "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payments_by_customer_by_month,PROD)" + ] + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "dbt-prefer-sql-parser-lineage", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataProcessInstance", + "entityUrn": "urn:li:dataProcessInstance:57aa623f096cf3a28af70fe94b713907", + "changeType": "UPSERT", + "aspectName": "dataProcessInstanceRunEvent", + "aspect": { + "json": { + "timestampMillis": 1663355198240, + "partitionSpec": { + "type": "FULL_TABLE", + "partition": "FULL_TABLE_SNAPSHOT" + }, + "status": "STARTED" + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "dbt-prefer-sql-parser-lineage", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataProcessInstance", + "entityUrn": "urn:li:dataProcessInstance:57aa623f096cf3a28af70fe94b713907", + "changeType": "UPSERT", + "aspectName": "dataProcessInstanceRunEvent", + "aspect": { + "json": { + "timestampMillis": 1663355198242, + "partitionSpec": { + "type": "FULL_TABLE", + "partition": "FULL_TABLE_SNAPSHOT" + }, + "status": "COMPLETE", + "result": { + "type": "SUCCESS", + "nativeResultType": "success" + }, + "durationMillis": 2 + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "dbt-prefer-sql-parser-lineage", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.customer_snapshot,PROD)", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "Snapshot" + ] + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "dbt-prefer-sql-parser-lineage", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.customer_snapshot,PROD)", + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "json": { + "platform": "urn:li:dataPlatform:dbt" + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "dbt-prefer-sql-parser-lineage", + "lastRunId": "no-run-id-provided" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.customer_snapshot,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": { + "node_type": "snapshot", + "materialization": "snapshot", + "dbt_file_path": "snapshots/customer_snapshot.sql", + "catalog_type": "BASE TABLE", + "language": "sql", + "dbt_unique_id": "snapshot.sample_dbt.customer_snapshot", + "dbt_package_name": "sample_dbt", + "manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v11.json", + "manifest_version": "1.7.3", + "manifest_adapter": "postgres", + "catalog_schema": "https://schemas.getdbt.com/dbt/catalog/v1.json", + "catalog_version": "1.7.3" + }, + "name": "customer_snapshot", + "description": "", + "tags": [] + } + }, + { + "com.linkedin.pegasus2avro.common.Status": { + "removed": false + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "snapshot.sample_dbt.customer_snapshot", + "platform": "urn:li:dataPlatform:dbt", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.MySqlDDL": { + "tableSchema": "" + } + }, + "fields": [ + { + "fieldPath": "active", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "integer", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "activebool", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "boolean", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "address_id", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "integer", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "create_date", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.DateType": {} + } + }, + "nativeDataType": "date", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "customer_id", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "integer", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "dbt_scd_id", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "text", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "dbt_updated_at", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "timestamp with time zone", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "dbt_valid_from", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "timestamp with time zone", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "dbt_valid_to", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "timestamp with time zone", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "email", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "text", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "first_name", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "text", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "last_name", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "text", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "last_update", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "timestamp with time zone", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "store_id", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "integer", + "recursive": false, + "isPartOfKey": false + } + ] + } + }, + { + "com.linkedin.pegasus2avro.dataset.UpstreamLineage": { + "upstreams": [ + { + "auditStamp": { + "time": 1643871600000, + "actor": "urn:li:corpuser:unknown" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.customer,PROD)", + "type": "TRANSFORMED" + } + ], + "fineGrainedLineages": [ + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.customer,PROD),active)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.customer_snapshot,PROD),active)" + ], + "confidenceScore": 0.9 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.customer,PROD),activebool)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.customer_snapshot,PROD),activebool)" + ], + "confidenceScore": 0.9 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.customer,PROD),address_id)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.customer_snapshot,PROD),address_id)" + ], + "confidenceScore": 0.9 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.customer,PROD),create_date)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.customer_snapshot,PROD),create_date)" + ], + "confidenceScore": 0.9 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.customer,PROD),customer_id)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.customer_snapshot,PROD),customer_id)" + ], + "confidenceScore": 0.9 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.customer,PROD),email)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.customer_snapshot,PROD),email)" + ], + "confidenceScore": 0.9 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.customer,PROD),first_name)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.customer_snapshot,PROD),first_name)" + ], + "confidenceScore": 0.9 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.customer,PROD),last_name)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.customer_snapshot,PROD),last_name)" + ], + "confidenceScore": 0.9 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.customer,PROD),last_update)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.customer_snapshot,PROD),last_update)" + ], + "confidenceScore": 0.9 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.customer,PROD),store_id)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.customer_snapshot,PROD),store_id)" + ], + "confidenceScore": 0.9 + } + ] + } + }, + { + "com.linkedin.pegasus2avro.dataset.ViewProperties": { + "materialized": true, + "viewLogic": "\n\n{{\n config(\n target_database='pagila',\n target_schema='public',\n unique_key='customer_id',\n\n strategy='timestamp',\n updated_at='last_update',\n )\n}}\n\nselect * from {{ source('pagila', 'customer') }}\n\n", + "formattedViewLogic": "SELECT\n *\nFROM \"pagila\".\"public\".\"customer\"", + "viewLanguage": "SQL" + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "dbt-prefer-sql-parser-lineage", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataProcessInstance", + "entityUrn": "urn:li:dataProcessInstance:36225e795a4597b2376996774a803b0d", + "changeType": "UPSERT", + "aspectName": "dataProcessInstanceProperties", + "aspect": { + "json": { + "customProperties": { + "dbt_name": "snapshot.sample_dbt.customer_snapshot", + "dbt_urn": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.customer_snapshot,PROD)" + }, + "name": "just-some-random-id_urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.customer_snapshot,PROD)", + "type": "BATCH_SCHEDULED", + "created": { + "time": 1663355198240, + "actor": "urn:li:corpuser:datahub" + } + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "dbt-prefer-sql-parser-lineage", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataProcessInstance", + "entityUrn": "urn:li:dataProcessInstance:36225e795a4597b2376996774a803b0d", + "changeType": "UPSERT", + "aspectName": "dataProcessInstanceRelationships", + "aspect": { + "json": { + "parentTemplate": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.customer_snapshot,PROD)", + "upstreamInstances": [] + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "dbt-prefer-sql-parser-lineage", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataProcessInstance", + "entityUrn": "urn:li:dataProcessInstance:36225e795a4597b2376996774a803b0d", + "changeType": "UPSERT", + "aspectName": "dataProcessInstanceInput", + "aspect": { + "json": { + "inputs": [ + "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.customer,PROD)" + ] + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "dbt-prefer-sql-parser-lineage", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataProcessInstance", + "entityUrn": "urn:li:dataProcessInstance:36225e795a4597b2376996774a803b0d", + "changeType": "UPSERT", + "aspectName": "dataProcessInstanceOutput", + "aspect": { + "json": { + "outputs": [ + "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.customer_snapshot,PROD)" + ] + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "dbt-prefer-sql-parser-lineage", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataProcessInstance", + "entityUrn": "urn:li:dataProcessInstance:36225e795a4597b2376996774a803b0d", + "changeType": "UPSERT", + "aspectName": "dataProcessInstanceRunEvent", + "aspect": { + "json": { + "timestampMillis": 1663355198240, + "partitionSpec": { + "type": "FULL_TABLE", + "partition": "FULL_TABLE_SNAPSHOT" + }, + "status": "STARTED" + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "dbt-prefer-sql-parser-lineage", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataProcessInstance", + "entityUrn": "urn:li:dataProcessInstance:36225e795a4597b2376996774a803b0d", + "changeType": "UPSERT", + "aspectName": "dataProcessInstanceRunEvent", + "aspect": { + "json": { + "timestampMillis": 1663355198242, + "partitionSpec": { + "type": "FULL_TABLE", + "partition": "FULL_TABLE_SNAPSHOT" + }, + "status": "COMPLETE", + "result": { + "type": "SUCCESS", + "nativeResultType": "success" + }, + "durationMillis": 2 + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "dbt-prefer-sql-parser-lineage", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.an-aliased-view-for-monthly-billing,PROD)", + "changeType": "PATCH", + "aspectName": "upstreamLineage", + "aspect": { + "json": [ + { + "op": "add", + "path": "/upstreams/urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.an-aliased-view-for-monthly-billing,PROD)", + "value": { + "auditStamp": { + "time": 1643871600000, + "actor": "urn:li:corpuser:unknown" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.an-aliased-view-for-monthly-billing,PROD)", + "type": "COPY" + } + }, + { + "op": "add", + "path": "/fineGrainedLineages/NONE/urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.an-aliased-view-for-monthly-billing,PROD),amount)/NONE/urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.an-aliased-view-for-monthly-billing,PROD),amount)", + "value": { + "confidenceScore": 1.0 + } + }, + { + "op": "add", + "path": "/fineGrainedLineages/NONE/urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.an-aliased-view-for-monthly-billing,PROD),billing_month)/NONE/urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.an-aliased-view-for-monthly-billing,PROD),billing_month)", + "value": { + "confidenceScore": 1.0 + } + }, + { + "op": "add", + "path": "/fineGrainedLineages/NONE/urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.an-aliased-view-for-monthly-billing,PROD),customer_id)/NONE/urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.an-aliased-view-for-monthly-billing,PROD),customer_id)", + "value": { + "confidenceScore": 1.0 + } + }, + { + "op": "add", + "path": "/fineGrainedLineages/NONE/urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.an-aliased-view-for-monthly-billing,PROD),email)/NONE/urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.an-aliased-view-for-monthly-billing,PROD),email)", + "value": { + "confidenceScore": 1.0 + } + } + ] + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "dbt-prefer-sql-parser-lineage", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.an_aliased_view_for_payments,PROD)", + "changeType": "PATCH", + "aspectName": "upstreamLineage", + "aspect": { + "json": [ + { + "op": "add", + "path": "/upstreams/urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.an_aliased_view_for_payments,PROD)", + "value": { + "auditStamp": { + "time": 1643871600000, + "actor": "urn:li:corpuser:unknown" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.an_aliased_view_for_payments,PROD)", + "type": "COPY" + } + }, + { + "op": "add", + "path": "/fineGrainedLineages/NONE/urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.an_aliased_view_for_payments,PROD),amount)/NONE/urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.an_aliased_view_for_payments,PROD),amount)", + "value": { + "confidenceScore": 1.0 + } + }, + { + "op": "add", + "path": "/fineGrainedLineages/NONE/urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.an_aliased_view_for_payments,PROD),customer_id)/NONE/urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.an_aliased_view_for_payments,PROD),customer_id)", + "value": { + "confidenceScore": 1.0 + } + }, + { + "op": "add", + "path": "/fineGrainedLineages/NONE/urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.an_aliased_view_for_payments,PROD),payment_date)/NONE/urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.an_aliased_view_for_payments,PROD),payment_date)", + "value": { + "confidenceScore": 1.0 + } + }, + { + "op": "add", + "path": "/fineGrainedLineages/NONE/urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.an_aliased_view_for_payments,PROD),payment_id)/NONE/urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.an_aliased_view_for_payments,PROD),payment_id)", + "value": { + "confidenceScore": 1.0 + } + }, + { + "op": "add", + "path": "/fineGrainedLineages/NONE/urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.an_aliased_view_for_payments,PROD),rental_id)/NONE/urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.an_aliased_view_for_payments,PROD),rental_id)", + "value": { + "confidenceScore": 1.0 + } + }, + { + "op": "add", + "path": "/fineGrainedLineages/NONE/urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.an_aliased_view_for_payments,PROD),staff_id)/NONE/urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.an_aliased_view_for_payments,PROD),staff_id)", + "value": { + "confidenceScore": 1.0 + } + } + ] + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "dbt-prefer-sql-parser-lineage", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.payments_by_customer_by_month,PROD)", + "changeType": "PATCH", + "aspectName": "upstreamLineage", + "aspect": { + "json": [ + { + "op": "add", + "path": "/upstreams/urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payments_by_customer_by_month,PROD)", + "value": { + "auditStamp": { + "time": 1643871600000, + "actor": "urn:li:corpuser:unknown" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payments_by_customer_by_month,PROD)", + "type": "COPY" + } + }, + { + "op": "add", + "path": "/fineGrainedLineages/NONE/urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.payments_by_customer_by_month,PROD),BillingMonth)/NONE/urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payments_by_customer_by_month,PROD),BillingMonth)", + "value": { + "confidenceScore": 1.0 + } + }, + { + "op": "add", + "path": "/fineGrainedLineages/NONE/urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.payments_by_customer_by_month,PROD),amount)/NONE/urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payments_by_customer_by_month,PROD),amount)", + "value": { + "confidenceScore": 1.0 + } + }, + { + "op": "add", + "path": "/fineGrainedLineages/NONE/urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.payments_by_customer_by_month,PROD),customer_id)/NONE/urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payments_by_customer_by_month,PROD),customer_id)", + "value": { + "confidenceScore": 1.0 + } + } + ] + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "dbt-prefer-sql-parser-lineage", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.customer_snapshot,PROD)", + "changeType": "PATCH", + "aspectName": "upstreamLineage", + "aspect": { + "json": [ + { + "op": "add", + "path": "/upstreams/urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.customer_snapshot,PROD)", + "value": { + "auditStamp": { + "time": 1643871600000, + "actor": "urn:li:corpuser:unknown" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.customer_snapshot,PROD)", + "type": "COPY" + } + }, + { + "op": "add", + "path": "/fineGrainedLineages/NONE/urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.customer_snapshot,PROD),active)/NONE/urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.customer_snapshot,PROD),active)", + "value": { + "confidenceScore": 1.0 + } + }, + { + "op": "add", + "path": "/fineGrainedLineages/NONE/urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.customer_snapshot,PROD),activebool)/NONE/urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.customer_snapshot,PROD),activebool)", + "value": { + "confidenceScore": 1.0 + } + }, + { + "op": "add", + "path": "/fineGrainedLineages/NONE/urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.customer_snapshot,PROD),address_id)/NONE/urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.customer_snapshot,PROD),address_id)", + "value": { + "confidenceScore": 1.0 + } + }, + { + "op": "add", + "path": "/fineGrainedLineages/NONE/urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.customer_snapshot,PROD),create_date)/NONE/urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.customer_snapshot,PROD),create_date)", + "value": { + "confidenceScore": 1.0 + } + }, + { + "op": "add", + "path": "/fineGrainedLineages/NONE/urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.customer_snapshot,PROD),customer_id)/NONE/urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.customer_snapshot,PROD),customer_id)", + "value": { + "confidenceScore": 1.0 + } + }, + { + "op": "add", + "path": "/fineGrainedLineages/NONE/urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.customer_snapshot,PROD),dbt_scd_id)/NONE/urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.customer_snapshot,PROD),dbt_scd_id)", + "value": { + "confidenceScore": 1.0 + } + }, + { + "op": "add", + "path": "/fineGrainedLineages/NONE/urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.customer_snapshot,PROD),dbt_updated_at)/NONE/urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.customer_snapshot,PROD),dbt_updated_at)", + "value": { + "confidenceScore": 1.0 + } + }, + { + "op": "add", + "path": "/fineGrainedLineages/NONE/urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.customer_snapshot,PROD),dbt_valid_from)/NONE/urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.customer_snapshot,PROD),dbt_valid_from)", + "value": { + "confidenceScore": 1.0 + } + }, + { + "op": "add", + "path": "/fineGrainedLineages/NONE/urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.customer_snapshot,PROD),dbt_valid_to)/NONE/urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.customer_snapshot,PROD),dbt_valid_to)", + "value": { + "confidenceScore": 1.0 + } + }, + { + "op": "add", + "path": "/fineGrainedLineages/NONE/urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.customer_snapshot,PROD),email)/NONE/urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.customer_snapshot,PROD),email)", + "value": { + "confidenceScore": 1.0 + } + }, + { + "op": "add", + "path": "/fineGrainedLineages/NONE/urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.customer_snapshot,PROD),first_name)/NONE/urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.customer_snapshot,PROD),first_name)", + "value": { + "confidenceScore": 1.0 + } + }, + { + "op": "add", + "path": "/fineGrainedLineages/NONE/urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.customer_snapshot,PROD),last_name)/NONE/urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.customer_snapshot,PROD),last_name)", + "value": { + "confidenceScore": 1.0 + } + }, + { + "op": "add", + "path": "/fineGrainedLineages/NONE/urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.customer_snapshot,PROD),last_update)/NONE/urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.customer_snapshot,PROD),last_update)", + "value": { + "confidenceScore": 1.0 + } + }, + { + "op": "add", + "path": "/fineGrainedLineages/NONE/urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.customer_snapshot,PROD),store_id)/NONE/urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.customer_snapshot,PROD),store_id)", + "value": { + "confidenceScore": 1.0 + } + } + ] + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "dbt-prefer-sql-parser-lineage", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "assertion", + "entityUrn": "urn:li:assertion:ba2c6ba830d407d539452f4cf46c92a6", + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "json": { + "platform": "urn:li:dataPlatform:dbt" + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "dbt-prefer-sql-parser-lineage", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "assertion", + "entityUrn": "urn:li:assertion:ba2c6ba830d407d539452f4cf46c92a6", + "changeType": "UPSERT", + "aspectName": "assertionInfo", + "aspect": { + "json": { + "customProperties": { + "manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v11.json", + "manifest_version": "1.7.3", + "manifest_adapter": "postgres", + "catalog_schema": "https://schemas.getdbt.com/dbt/catalog/v1.json", + "catalog_version": "1.7.3" + }, + "type": "DATASET", + "datasetAssertion": { + "dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.actor,PROD)", + "scope": "DATASET_ROWS", + "aggregation": "_NATIVE_", + "operator": "_NATIVE_", + "nativeType": "assert_source_actor_last_update_is_recent", + "nativeParameters": {}, + "logic": "select\n *\nfrom \"pagila\".\"public\".\"actor\"\nwhere last_update < (now() - interval '100 years')" + } + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "dbt-prefer-sql-parser-lineage", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "assertion", + "entityUrn": "urn:li:assertion:ba2c6ba830d407d539452f4cf46c92a6", + "changeType": "UPSERT", + "aspectName": "assertionRunEvent", + "aspect": { + "json": { + "timestampMillis": 1663355198239, + "runId": "just-some-random-id", + "asserteeUrn": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.actor,PROD)", + "status": "COMPLETE", + "result": { + "type": "SUCCESS", + "nativeResults": {} + }, + "assertionUrn": "urn:li:assertion:ba2c6ba830d407d539452f4cf46c92a6", + "partitionSpec": { + "type": "FULL_TABLE", + "partition": "FULL_TABLE_SNAPSHOT" + } + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "dbt-prefer-sql-parser-lineage", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "assertion", + "entityUrn": "urn:li:assertion:10f2a119dedcaab43afc47ff13d9cb5b", + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "json": { + "platform": "urn:li:dataPlatform:dbt" + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "dbt-prefer-sql-parser-lineage", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "assertion", + "entityUrn": "urn:li:assertion:10f2a119dedcaab43afc47ff13d9cb5b", + "changeType": "UPSERT", + "aspectName": "assertionInfo", + "aspect": { + "json": { + "customProperties": { + "manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v11.json", + "manifest_version": "1.7.3", + "manifest_adapter": "postgres", + "catalog_schema": "https://schemas.getdbt.com/dbt/catalog/v1.json", + "catalog_version": "1.7.3" + }, + "type": "DATASET", + "datasetAssertion": { + "dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.an-aliased-view-for-monthly-billing,PROD)", + "scope": "DATASET_COLUMN", + "fields": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.an-aliased-view-for-monthly-billing,PROD),email)" + ], + "aggregation": "_NATIVE_", + "operator": "_NATIVE_", + "nativeType": "is_email_monthly_billing_with_cust_email", + "nativeParameters": { + "column_name": "email", + "model": "{{ get_where_subquery(ref('monthly_billing_with_cust')) }}" + }, + "logic": "\n\n select *\n from \"pagila\".\"public\".\"an-aliased-view-for-monthly-billing\"\n where email not like '%@%'\n\n" + } + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "dbt-prefer-sql-parser-lineage", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "assertion", + "entityUrn": "urn:li:assertion:10f2a119dedcaab43afc47ff13d9cb5b", + "changeType": "UPSERT", + "aspectName": "assertionRunEvent", + "aspect": { + "json": { + "timestampMillis": 1663355198239, + "runId": "just-some-random-id", + "asserteeUrn": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.an-aliased-view-for-monthly-billing,PROD)", + "status": "COMPLETE", + "result": { + "type": "SUCCESS", + "nativeResults": {} + }, + "assertionUrn": "urn:li:assertion:10f2a119dedcaab43afc47ff13d9cb5b", + "partitionSpec": { + "type": "FULL_TABLE", + "partition": "FULL_TABLE_SNAPSHOT" + } + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "dbt-prefer-sql-parser-lineage", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "assertion", + "entityUrn": "urn:li:assertion:c456eccf6440c6e3388c584689a74d91", + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "json": { + "platform": "urn:li:dataPlatform:dbt" + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "dbt-prefer-sql-parser-lineage", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "assertion", + "entityUrn": "urn:li:assertion:c456eccf6440c6e3388c584689a74d91", + "changeType": "UPSERT", + "aspectName": "assertionInfo", + "aspect": { + "json": { + "customProperties": { + "manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v11.json", + "manifest_version": "1.7.3", + "manifest_adapter": "postgres", + "catalog_schema": "https://schemas.getdbt.com/dbt/catalog/v1.json", + "catalog_version": "1.7.3" + }, + "type": "DATASET", + "datasetAssertion": { + "dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.an-aliased-view-for-monthly-billing,PROD)", + "scope": "DATASET_COLUMN", + "fields": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.an-aliased-view-for-monthly-billing,PROD),billing_month)" + ], + "aggregation": "IDENTITY", + "operator": "NOT_NULL", + "nativeType": "not_null_monthly_billing_with_cust_billing_month", + "nativeParameters": { + "column_name": "billing_month", + "model": "{{ get_where_subquery(ref('monthly_billing_with_cust')) }}" + } + } + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "dbt-prefer-sql-parser-lineage", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "assertion", + "entityUrn": "urn:li:assertion:c456eccf6440c6e3388c584689a74d91", + "changeType": "UPSERT", + "aspectName": "assertionRunEvent", + "aspect": { + "json": { + "timestampMillis": 1663355198239, + "runId": "just-some-random-id", + "asserteeUrn": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.an-aliased-view-for-monthly-billing,PROD)", + "status": "COMPLETE", + "result": { + "type": "SUCCESS", + "nativeResults": {} + }, + "assertionUrn": "urn:li:assertion:c456eccf6440c6e3388c584689a74d91", + "partitionSpec": { + "type": "FULL_TABLE", + "partition": "FULL_TABLE_SNAPSHOT" + } + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "dbt-prefer-sql-parser-lineage", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "assertion", + "entityUrn": "urn:li:assertion:f812b73477d81e6af283d918cb59e7bf", + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "json": { + "platform": "urn:li:dataPlatform:dbt" + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "dbt-prefer-sql-parser-lineage", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "assertion", + "entityUrn": "urn:li:assertion:f812b73477d81e6af283d918cb59e7bf", + "changeType": "UPSERT", + "aspectName": "assertionInfo", + "aspect": { + "json": { + "customProperties": { + "manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v11.json", + "manifest_version": "1.7.3", + "manifest_adapter": "postgres", + "catalog_schema": "https://schemas.getdbt.com/dbt/catalog/v1.json", + "catalog_version": "1.7.3" + }, + "type": "DATASET", + "datasetAssertion": { + "dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.an-aliased-view-for-monthly-billing,PROD)", + "scope": "DATASET_COLUMN", + "fields": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.an-aliased-view-for-monthly-billing,PROD),email)" + ], + "aggregation": "IDENTITY", + "operator": "NOT_NULL", + "nativeType": "not_null_monthly_billing_with_cust_email", + "nativeParameters": { + "column_name": "email", + "model": "{{ get_where_subquery(ref('monthly_billing_with_cust')) }}" + } + } + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "dbt-prefer-sql-parser-lineage", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "assertion", + "entityUrn": "urn:li:assertion:f812b73477d81e6af283d918cb59e7bf", + "changeType": "UPSERT", + "aspectName": "assertionRunEvent", + "aspect": { + "json": { + "timestampMillis": 1663355198239, + "runId": "just-some-random-id", + "asserteeUrn": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.an-aliased-view-for-monthly-billing,PROD)", + "status": "COMPLETE", + "result": { + "type": "SUCCESS", + "nativeResults": {} + }, + "assertionUrn": "urn:li:assertion:f812b73477d81e6af283d918cb59e7bf", + "partitionSpec": { + "type": "FULL_TABLE", + "partition": "FULL_TABLE_SNAPSHOT" + } + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "dbt-prefer-sql-parser-lineage", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "assertion", + "entityUrn": "urn:li:assertion:08c35a6481d3c37c93eaf9e424faa6d5", + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "json": { + "platform": "urn:li:dataPlatform:dbt" + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "dbt-prefer-sql-parser-lineage", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "assertion", + "entityUrn": "urn:li:assertion:08c35a6481d3c37c93eaf9e424faa6d5", + "changeType": "UPSERT", + "aspectName": "assertionInfo", + "aspect": { + "json": { + "customProperties": { + "manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v11.json", + "manifest_version": "1.7.3", + "manifest_adapter": "postgres", + "catalog_schema": "https://schemas.getdbt.com/dbt/catalog/v1.json", + "catalog_version": "1.7.3" + }, + "type": "DATASET", + "datasetAssertion": { + "dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.an-aliased-view-for-monthly-billing,PROD)", + "scope": "DATASET_COLUMN", + "fields": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.an-aliased-view-for-monthly-billing,PROD),customer_id)" + ], + "aggregation": "IDENTITY", + "operator": "_NATIVE_", + "parameters": { + "value": { + "value": "null", + "type": "SET" + } + }, + "nativeType": "relationships_monthly_billing_with_cust_customer_id__customer_id__ref_customer_details_", + "nativeParameters": { + "column_name": "customer_id", + "field": "customer_id", + "model": "{{ get_where_subquery(ref('monthly_billing_with_cust')) }}", + "to": "ref('customer_details')" + }, + "logic": "monthly_billing_with_cust.customer_id referential integrity to customer_details.customer_id" + } + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "dbt-prefer-sql-parser-lineage", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "assertion", + "entityUrn": "urn:li:assertion:08c35a6481d3c37c93eaf9e424faa6d5", + "changeType": "UPSERT", + "aspectName": "assertionRunEvent", + "aspect": { + "json": { + "timestampMillis": 1663355198239, + "runId": "just-some-random-id", + "asserteeUrn": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.an-aliased-view-for-monthly-billing,PROD)", + "status": "COMPLETE", + "result": { + "type": "SUCCESS", + "nativeResults": {} + }, + "assertionUrn": "urn:li:assertion:08c35a6481d3c37c93eaf9e424faa6d5", + "partitionSpec": { + "type": "FULL_TABLE", + "partition": "FULL_TABLE_SNAPSHOT" + } + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "dbt-prefer-sql-parser-lineage", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "assertion", + "entityUrn": "urn:li:assertion:08c35a6481d3c37c93eaf9e424faa6d5", + "changeType": "UPSERT", + "aspectName": "assertionInfo", + "aspect": { + "json": { + "customProperties": { + "manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v11.json", + "manifest_version": "1.7.3", + "manifest_adapter": "postgres", + "catalog_schema": "https://schemas.getdbt.com/dbt/catalog/v1.json", + "catalog_version": "1.7.3" + }, + "type": "DATASET", + "datasetAssertion": { + "dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.customer_details,PROD)", + "scope": "DATASET_COLUMN", + "fields": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.customer_details,PROD),customer_id)" + ], + "aggregation": "IDENTITY", + "operator": "_NATIVE_", + "parameters": { + "value": { + "value": "null", + "type": "SET" + } + }, + "nativeType": "relationships_monthly_billing_with_cust_customer_id__customer_id__ref_customer_details_", + "nativeParameters": { + "column_name": "customer_id", + "field": "customer_id", + "model": "{{ get_where_subquery(ref('monthly_billing_with_cust')) }}", + "to": "ref('customer_details')" + }, + "logic": "monthly_billing_with_cust.customer_id referential integrity to customer_details.customer_id" + } + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "dbt-prefer-sql-parser-lineage", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "assertion", + "entityUrn": "urn:li:assertion:08c35a6481d3c37c93eaf9e424faa6d5", + "changeType": "UPSERT", + "aspectName": "assertionRunEvent", + "aspect": { + "json": { + "timestampMillis": 1663355198239, + "runId": "just-some-random-id", + "asserteeUrn": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.customer_details,PROD)", + "status": "COMPLETE", + "result": { + "type": "SUCCESS", + "nativeResults": {} + }, + "assertionUrn": "urn:li:assertion:08c35a6481d3c37c93eaf9e424faa6d5", + "partitionSpec": { + "type": "FULL_TABLE", + "partition": "FULL_TABLE_SNAPSHOT" + } + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "dbt-prefer-sql-parser-lineage", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "assertion", + "entityUrn": "urn:li:assertion:f6a1fde3ab4919abcc04bdee93144958", + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "json": { + "platform": "urn:li:dataPlatform:dbt" + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "dbt-prefer-sql-parser-lineage", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "assertion", + "entityUrn": "urn:li:assertion:f6a1fde3ab4919abcc04bdee93144958", + "changeType": "UPSERT", + "aspectName": "assertionInfo", + "aspect": { + "json": { + "customProperties": { + "manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v11.json", + "manifest_version": "1.7.3", + "manifest_adapter": "postgres", + "catalog_schema": "https://schemas.getdbt.com/dbt/catalog/v1.json", + "catalog_version": "1.7.3" + }, + "type": "DATASET", + "datasetAssertion": { + "dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.actor,PROD)", + "scope": "DATASET_COLUMN", + "fields": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.actor,PROD),actor_id)" + ], + "aggregation": "IDENTITY", + "operator": "NOT_NULL", + "nativeType": "source_not_null_pagila_actor_actor_id", + "nativeParameters": { + "column_name": "actor_id", + "model": "{{ get_where_subquery(source('pagila', 'actor')) }}" + } + } + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "dbt-prefer-sql-parser-lineage", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "assertion", + "entityUrn": "urn:li:assertion:f6a1fde3ab4919abcc04bdee93144958", + "changeType": "UPSERT", + "aspectName": "assertionRunEvent", + "aspect": { + "json": { + "timestampMillis": 1663355198239, + "runId": "just-some-random-id", + "asserteeUrn": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.actor,PROD)", + "status": "COMPLETE", + "result": { + "type": "SUCCESS", + "nativeResults": {} + }, + "assertionUrn": "urn:li:assertion:f6a1fde3ab4919abcc04bdee93144958", + "partitionSpec": { + "type": "FULL_TABLE", + "partition": "FULL_TABLE_SNAPSHOT" + } + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "dbt-prefer-sql-parser-lineage", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "assertion", + "entityUrn": "urn:li:assertion:60ce4aad7ff6dbff7004da0f2258c9df", + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "json": { + "platform": "urn:li:dataPlatform:dbt" + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "dbt-prefer-sql-parser-lineage", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "assertion", + "entityUrn": "urn:li:assertion:60ce4aad7ff6dbff7004da0f2258c9df", + "changeType": "UPSERT", + "aspectName": "assertionInfo", + "aspect": { + "json": { + "customProperties": { + "manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v11.json", + "manifest_version": "1.7.3", + "manifest_adapter": "postgres", + "catalog_schema": "https://schemas.getdbt.com/dbt/catalog/v1.json", + "catalog_version": "1.7.3" + }, + "type": "DATASET", + "datasetAssertion": { + "dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.actor,PROD)", + "scope": "DATASET_COLUMN", + "fields": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.actor,PROD),actor_id)" + ], + "aggregation": "UNIQUE_PROPOTION", + "operator": "EQUAL_TO", + "parameters": { + "value": { + "value": "1.0", + "type": "NUMBER" + } + }, + "nativeType": "source_unique_pagila_actor_actor_id", + "nativeParameters": { + "column_name": "actor_id", + "model": "{{ get_where_subquery(source('pagila', 'actor')) }}" + } + } + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "dbt-prefer-sql-parser-lineage", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "assertion", + "entityUrn": "urn:li:assertion:60ce4aad7ff6dbff7004da0f2258c9df", + "changeType": "UPSERT", + "aspectName": "assertionRunEvent", + "aspect": { + "json": { + "timestampMillis": 1663355198239, + "runId": "just-some-random-id", + "asserteeUrn": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.actor,PROD)", + "status": "COMPLETE", + "result": { + "type": "SUCCESS", + "nativeResults": {} + }, + "assertionUrn": "urn:li:assertion:60ce4aad7ff6dbff7004da0f2258c9df", + "partitionSpec": { + "type": "FULL_TABLE", + "partition": "FULL_TABLE_SNAPSHOT" + } + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "dbt-prefer-sql-parser-lineage", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "assertion", + "entityUrn": "urn:li:assertion:08c35a6481d3c37c93eaf9e424faa6d5", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "dbt-prefer-sql-parser-lineage", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "assertion", + "entityUrn": "urn:li:assertion:10f2a119dedcaab43afc47ff13d9cb5b", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "dbt-prefer-sql-parser-lineage", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "assertion", + "entityUrn": "urn:li:assertion:60ce4aad7ff6dbff7004da0f2258c9df", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "dbt-prefer-sql-parser-lineage", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "assertion", + "entityUrn": "urn:li:assertion:ba2c6ba830d407d539452f4cf46c92a6", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "dbt-prefer-sql-parser-lineage", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "assertion", + "entityUrn": "urn:li:assertion:c456eccf6440c6e3388c584689a74d91", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "dbt-prefer-sql-parser-lineage", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "assertion", + "entityUrn": "urn:li:assertion:f6a1fde3ab4919abcc04bdee93144958", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "dbt-prefer-sql-parser-lineage", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "assertion", + "entityUrn": "urn:li:assertion:f812b73477d81e6af283d918cb59e7bf", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "dbt-prefer-sql-parser-lineage", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataProcessInstance", + "entityUrn": "urn:li:dataProcessInstance:36225e795a4597b2376996774a803b0d", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "dbt-prefer-sql-parser-lineage", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataProcessInstance", + "entityUrn": "urn:li:dataProcessInstance:57aa623f096cf3a28af70fe94b713907", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "dbt-prefer-sql-parser-lineage", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataProcessInstance", + "entityUrn": "urn:li:dataProcessInstance:a42a5b1bee156e45972e12d4156fb7a2", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "dbt-prefer-sql-parser-lineage", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataProcessInstance", + "entityUrn": "urn:li:dataProcessInstance:fc6268f0be68fd04c310705b65efd6fe", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "dbt-prefer-sql-parser-lineage", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "tag", + "entityUrn": "urn:li:tag:dbt:tag_from_dbt", + "changeType": "UPSERT", + "aspectName": "tagKey", + "aspect": { + "json": { + "name": "dbt:tag_from_dbt" + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "dbt-prefer-sql-parser-lineage", + "lastRunId": "no-run-id-provided" + } +} +] \ No newline at end of file diff --git a/metadata-ingestion/tests/integration/dbt/dbt_test_test_model_performance_golden.json b/metadata-ingestion/tests/integration/dbt/dbt_test_test_model_performance_golden.json index a9b7df7c2b2a1..bec42f460e0b5 100644 --- a/metadata-ingestion/tests/integration/dbt/dbt_test_test_model_performance_golden.json +++ b/metadata-ingestion/tests/integration/dbt/dbt_test_test_model_performance_golden.json @@ -224,7 +224,7 @@ "upstreams": [ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.customer,PROD),customer_id)" ], - "downstreamType": "FIELD_SET", + "downstreamType": "FIELD", "downstreams": [ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.customer_details,PROD),customer_id)" ], @@ -236,7 +236,7 @@ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.customer,PROD),first_name)", "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.customer,PROD),last_name)" ], - "downstreamType": "FIELD_SET", + "downstreamType": "FIELD", "downstreams": [ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.customer_details,PROD),full_name)" ], @@ -248,7 +248,7 @@ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.customer_snapshot,PROD),first_name)", "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.customer_snapshot,PROD),last_name)" ], - "downstreamType": "FIELD_SET", + "downstreamType": "FIELD", "downstreams": [ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.customer_details,PROD),initial_full_name)" ], @@ -259,7 +259,7 @@ "upstreams": [ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.customer,PROD),email)" ], - "downstreamType": "FIELD_SET", + "downstreamType": "FIELD", "downstreams": [ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.customer_details,PROD),email)" ], @@ -270,7 +270,7 @@ "upstreams": [ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.address,PROD),address)" ], - "downstreamType": "FIELD_SET", + "downstreamType": "FIELD", "downstreams": [ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.customer_details,PROD),address)" ], @@ -281,7 +281,7 @@ "upstreams": [ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.city,PROD),city)" ], - "downstreamType": "FIELD_SET", + "downstreamType": "FIELD", "downstreams": [ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.customer_details,PROD),city)" ], @@ -292,7 +292,7 @@ "upstreams": [ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.address,PROD),postal_code)" ], - "downstreamType": "FIELD_SET", + "downstreamType": "FIELD", "downstreams": [ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.customer_details,PROD),postal_code)" ], @@ -303,7 +303,7 @@ "upstreams": [ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.address,PROD),phone)" ], - "downstreamType": "FIELD_SET", + "downstreamType": "FIELD", "downstreams": [ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.customer_details,PROD),phone)" ], @@ -491,7 +491,7 @@ "upstreams": [ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.payments_by_customer_by_month,PROD),BillingMonth)" ], - "downstreamType": "FIELD_SET", + "downstreamType": "FIELD", "downstreams": [ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.an-aliased-view-for-monthly-billing,PROD),billing_month)" ], @@ -502,7 +502,7 @@ "upstreams": [ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.payments_by_customer_by_month,PROD),customer_id)" ], - "downstreamType": "FIELD_SET", + "downstreamType": "FIELD", "downstreams": [ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.an-aliased-view-for-monthly-billing,PROD),customer_id)" ], @@ -513,7 +513,7 @@ "upstreams": [ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.payments_by_customer_by_month,PROD),amount)" ], - "downstreamType": "FIELD_SET", + "downstreamType": "FIELD", "downstreams": [ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.an-aliased-view-for-monthly-billing,PROD),amount)" ], @@ -524,7 +524,7 @@ "upstreams": [ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.customer_details,PROD),email)" ], - "downstreamType": "FIELD_SET", + "downstreamType": "FIELD", "downstreams": [ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.an-aliased-view-for-monthly-billing,PROD),email)" ], @@ -899,7 +899,7 @@ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payment_p2020_05,PROD),amount)", "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payment_p2020_06,PROD),amount)" ], - "downstreamType": "FIELD_SET", + "downstreamType": "FIELD", "downstreams": [ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.an_aliased_view_for_payments,PROD),amount)" ], @@ -915,7 +915,7 @@ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payment_p2020_05,PROD),customer_id)", "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payment_p2020_06,PROD),customer_id)" ], - "downstreamType": "FIELD_SET", + "downstreamType": "FIELD", "downstreams": [ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.an_aliased_view_for_payments,PROD),customer_id)" ], @@ -931,7 +931,7 @@ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payment_p2020_05,PROD),payment_date)", "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payment_p2020_06,PROD),payment_date)" ], - "downstreamType": "FIELD_SET", + "downstreamType": "FIELD", "downstreams": [ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.an_aliased_view_for_payments,PROD),payment_date)" ], @@ -947,7 +947,7 @@ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payment_p2020_05,PROD),payment_id)", "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payment_p2020_06,PROD),payment_id)" ], - "downstreamType": "FIELD_SET", + "downstreamType": "FIELD", "downstreams": [ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.an_aliased_view_for_payments,PROD),payment_id)" ], @@ -963,7 +963,7 @@ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payment_p2020_05,PROD),rental_id)", "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payment_p2020_06,PROD),rental_id)" ], - "downstreamType": "FIELD_SET", + "downstreamType": "FIELD", "downstreams": [ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.an_aliased_view_for_payments,PROD),rental_id)" ], @@ -979,7 +979,7 @@ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payment_p2020_05,PROD),staff_id)", "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payment_p2020_06,PROD),staff_id)" ], - "downstreamType": "FIELD_SET", + "downstreamType": "FIELD", "downstreams": [ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.an_aliased_view_for_payments,PROD),staff_id)" ], @@ -1285,7 +1285,7 @@ "upstreams": [ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.an_aliased_view_for_payments,PROD),payment_date)" ], - "downstreamType": "FIELD_SET", + "downstreamType": "FIELD", "downstreams": [ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payments_by_customer_by_month,PROD),BillingMonth)" ], @@ -1296,7 +1296,7 @@ "upstreams": [ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.an_aliased_view_for_payments,PROD),customer_id)" ], - "downstreamType": "FIELD_SET", + "downstreamType": "FIELD", "downstreams": [ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payments_by_customer_by_month,PROD),customer_id)" ], @@ -1307,7 +1307,7 @@ "upstreams": [ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.an_aliased_view_for_payments,PROD),amount)" ], - "downstreamType": "FIELD_SET", + "downstreamType": "FIELD", "downstreams": [ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payments_by_customer_by_month,PROD),amount)" ], @@ -1732,7 +1732,7 @@ "upstreams": [ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.customer,PROD),active)" ], - "downstreamType": "FIELD_SET", + "downstreamType": "FIELD", "downstreams": [ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.customer_snapshot,PROD),active)" ], @@ -1743,7 +1743,7 @@ "upstreams": [ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.customer,PROD),activebool)" ], - "downstreamType": "FIELD_SET", + "downstreamType": "FIELD", "downstreams": [ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.customer_snapshot,PROD),activebool)" ], @@ -1754,7 +1754,7 @@ "upstreams": [ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.customer,PROD),address_id)" ], - "downstreamType": "FIELD_SET", + "downstreamType": "FIELD", "downstreams": [ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.customer_snapshot,PROD),address_id)" ], @@ -1765,7 +1765,7 @@ "upstreams": [ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.customer,PROD),create_date)" ], - "downstreamType": "FIELD_SET", + "downstreamType": "FIELD", "downstreams": [ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.customer_snapshot,PROD),create_date)" ], @@ -1776,7 +1776,7 @@ "upstreams": [ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.customer,PROD),customer_id)" ], - "downstreamType": "FIELD_SET", + "downstreamType": "FIELD", "downstreams": [ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.customer_snapshot,PROD),customer_id)" ], @@ -1787,7 +1787,7 @@ "upstreams": [ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.customer,PROD),email)" ], - "downstreamType": "FIELD_SET", + "downstreamType": "FIELD", "downstreams": [ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.customer_snapshot,PROD),email)" ], @@ -1798,7 +1798,7 @@ "upstreams": [ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.customer,PROD),first_name)" ], - "downstreamType": "FIELD_SET", + "downstreamType": "FIELD", "downstreams": [ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.customer_snapshot,PROD),first_name)" ], @@ -1809,7 +1809,7 @@ "upstreams": [ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.customer,PROD),last_name)" ], - "downstreamType": "FIELD_SET", + "downstreamType": "FIELD", "downstreams": [ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.customer_snapshot,PROD),last_name)" ], @@ -1820,7 +1820,7 @@ "upstreams": [ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.customer,PROD),last_update)" ], - "downstreamType": "FIELD_SET", + "downstreamType": "FIELD", "downstreams": [ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.customer_snapshot,PROD),last_update)" ], @@ -1831,7 +1831,7 @@ "upstreams": [ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.customer,PROD),store_id)" ], - "downstreamType": "FIELD_SET", + "downstreamType": "FIELD", "downstreams": [ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.customer_snapshot,PROD),store_id)" ], diff --git a/metadata-ingestion/tests/integration/dbt/dbt_test_with_complex_owner_patterns_mces_golden.json b/metadata-ingestion/tests/integration/dbt/dbt_test_with_complex_owner_patterns_mces_golden.json index 4863a1e4f7398..495fa32569f56 100644 --- a/metadata-ingestion/tests/integration/dbt/dbt_test_with_complex_owner_patterns_mces_golden.json +++ b/metadata-ingestion/tests/integration/dbt/dbt_test_with_complex_owner_patterns_mces_golden.json @@ -235,7 +235,7 @@ "upstreams": [ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.customer,PROD),customer_id)" ], - "downstreamType": "FIELD_SET", + "downstreamType": "FIELD", "downstreams": [ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.dbt_postgres.customer_details,PROD),customer_id)" ], @@ -247,7 +247,7 @@ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.customer,PROD),first_name)", "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.customer,PROD),last_name)" ], - "downstreamType": "FIELD_SET", + "downstreamType": "FIELD", "downstreams": [ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.dbt_postgres.customer_details,PROD),full_name)" ], @@ -258,7 +258,7 @@ "upstreams": [ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.customer,PROD),email)" ], - "downstreamType": "FIELD_SET", + "downstreamType": "FIELD", "downstreams": [ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.dbt_postgres.customer_details,PROD),email)" ], @@ -269,7 +269,7 @@ "upstreams": [ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.address,PROD),address)" ], - "downstreamType": "FIELD_SET", + "downstreamType": "FIELD", "downstreams": [ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.dbt_postgres.customer_details,PROD),address)" ], @@ -280,7 +280,7 @@ "upstreams": [ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.city,PROD),city)" ], - "downstreamType": "FIELD_SET", + "downstreamType": "FIELD", "downstreams": [ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.dbt_postgres.customer_details,PROD),city)" ], @@ -291,7 +291,7 @@ "upstreams": [ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.address,PROD),postal_code)" ], - "downstreamType": "FIELD_SET", + "downstreamType": "FIELD", "downstreams": [ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.dbt_postgres.customer_details,PROD),postal_code)" ], @@ -302,7 +302,7 @@ "upstreams": [ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.address,PROD),phone)" ], - "downstreamType": "FIELD_SET", + "downstreamType": "FIELD", "downstreams": [ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.dbt_postgres.customer_details,PROD),phone)" ], @@ -495,7 +495,7 @@ "upstreams": [ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.dbt_postgres.payments_by_customer_by_month,PROD),billing_month)" ], - "downstreamType": "FIELD_SET", + "downstreamType": "FIELD", "downstreams": [ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.dbt_postgres.an-aliased-view-for-monthly-billing,PROD),billing_month)" ], @@ -506,7 +506,7 @@ "upstreams": [ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.dbt_postgres.payments_by_customer_by_month,PROD),customer_id)" ], - "downstreamType": "FIELD_SET", + "downstreamType": "FIELD", "downstreams": [ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.dbt_postgres.an-aliased-view-for-monthly-billing,PROD),customer_id)" ], @@ -517,7 +517,7 @@ "upstreams": [ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.dbt_postgres.payments_by_customer_by_month,PROD),amount)" ], - "downstreamType": "FIELD_SET", + "downstreamType": "FIELD", "downstreams": [ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.dbt_postgres.an-aliased-view-for-monthly-billing,PROD),amount)" ], @@ -774,7 +774,7 @@ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payment_p2020_05,PROD),amount)", "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payment_p2020_06,PROD),amount)" ], - "downstreamType": "FIELD_SET", + "downstreamType": "FIELD", "downstreams": [ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.dbt_postgres.an-aliased-view-for-payments,PROD),amount)" ], @@ -790,7 +790,7 @@ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payment_p2020_05,PROD),customer_id)", "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payment_p2020_06,PROD),customer_id)" ], - "downstreamType": "FIELD_SET", + "downstreamType": "FIELD", "downstreams": [ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.dbt_postgres.an-aliased-view-for-payments,PROD),customer_id)" ], @@ -806,7 +806,7 @@ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payment_p2020_05,PROD),payment_date)", "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payment_p2020_06,PROD),payment_date)" ], - "downstreamType": "FIELD_SET", + "downstreamType": "FIELD", "downstreams": [ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.dbt_postgres.an-aliased-view-for-payments,PROD),payment_date)" ], @@ -822,7 +822,7 @@ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payment_p2020_05,PROD),payment_id)", "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payment_p2020_06,PROD),payment_id)" ], - "downstreamType": "FIELD_SET", + "downstreamType": "FIELD", "downstreams": [ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.dbt_postgres.an-aliased-view-for-payments,PROD),payment_id)" ], @@ -838,7 +838,7 @@ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payment_p2020_05,PROD),rental_id)", "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payment_p2020_06,PROD),rental_id)" ], - "downstreamType": "FIELD_SET", + "downstreamType": "FIELD", "downstreams": [ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.dbt_postgres.an-aliased-view-for-payments,PROD),rental_id)" ], @@ -854,7 +854,7 @@ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payment_p2020_05,PROD),staff_id)", "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payment_p2020_06,PROD),staff_id)" ], - "downstreamType": "FIELD_SET", + "downstreamType": "FIELD", "downstreams": [ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.dbt_postgres.an-aliased-view-for-payments,PROD),staff_id)" ], @@ -1022,7 +1022,7 @@ "upstreams": [ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.dbt_postgres.an-aliased-view-for-payments,PROD),payment_date)" ], - "downstreamType": "FIELD_SET", + "downstreamType": "FIELD", "downstreams": [ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.dbt_postgres.payments_by_customer_by_month,PROD),billing_month)" ], @@ -1033,7 +1033,7 @@ "upstreams": [ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.dbt_postgres.an-aliased-view-for-payments,PROD),customer_id)" ], - "downstreamType": "FIELD_SET", + "downstreamType": "FIELD", "downstreams": [ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.dbt_postgres.payments_by_customer_by_month,PROD),customer_id)" ], @@ -1044,7 +1044,7 @@ "upstreams": [ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.dbt_postgres.an-aliased-view-for-payments,PROD),amount)" ], - "downstreamType": "FIELD_SET", + "downstreamType": "FIELD", "downstreams": [ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.dbt_postgres.payments_by_customer_by_month,PROD),amount)" ], diff --git a/metadata-ingestion/tests/integration/dbt/dbt_test_with_data_platform_instance_mces_golden.json b/metadata-ingestion/tests/integration/dbt/dbt_test_with_data_platform_instance_mces_golden.json index 7c61e9fcd0e3a..20b7cf4a1c26c 100644 --- a/metadata-ingestion/tests/integration/dbt/dbt_test_with_data_platform_instance_mces_golden.json +++ b/metadata-ingestion/tests/integration/dbt/dbt_test_with_data_platform_instance_mces_golden.json @@ -237,7 +237,7 @@ "upstreams": [ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,dbt-instance-1.pagila.public.customer,PROD),customer_id)" ], - "downstreamType": "FIELD_SET", + "downstreamType": "FIELD", "downstreams": [ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,dbt-instance-1.pagila.dbt_postgres.customer_details,PROD),customer_id)" ], @@ -249,7 +249,7 @@ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,dbt-instance-1.pagila.public.customer,PROD),first_name)", "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,dbt-instance-1.pagila.public.customer,PROD),last_name)" ], - "downstreamType": "FIELD_SET", + "downstreamType": "FIELD", "downstreams": [ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,dbt-instance-1.pagila.dbt_postgres.customer_details,PROD),full_name)" ], @@ -260,7 +260,7 @@ "upstreams": [ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,dbt-instance-1.pagila.public.customer,PROD),email)" ], - "downstreamType": "FIELD_SET", + "downstreamType": "FIELD", "downstreams": [ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,dbt-instance-1.pagila.dbt_postgres.customer_details,PROD),email)" ], @@ -271,7 +271,7 @@ "upstreams": [ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,dbt-instance-1.pagila.public.address,PROD),address)" ], - "downstreamType": "FIELD_SET", + "downstreamType": "FIELD", "downstreams": [ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,dbt-instance-1.pagila.dbt_postgres.customer_details,PROD),address)" ], @@ -282,7 +282,7 @@ "upstreams": [ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,dbt-instance-1.pagila.public.city,PROD),city)" ], - "downstreamType": "FIELD_SET", + "downstreamType": "FIELD", "downstreams": [ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,dbt-instance-1.pagila.dbt_postgres.customer_details,PROD),city)" ], @@ -293,7 +293,7 @@ "upstreams": [ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,dbt-instance-1.pagila.public.address,PROD),postal_code)" ], - "downstreamType": "FIELD_SET", + "downstreamType": "FIELD", "downstreams": [ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,dbt-instance-1.pagila.dbt_postgres.customer_details,PROD),postal_code)" ], @@ -304,7 +304,7 @@ "upstreams": [ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,dbt-instance-1.pagila.public.address,PROD),phone)" ], - "downstreamType": "FIELD_SET", + "downstreamType": "FIELD", "downstreams": [ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,dbt-instance-1.pagila.dbt_postgres.customer_details,PROD),phone)" ], @@ -498,7 +498,7 @@ "upstreams": [ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.dbt_postgres.payments_by_customer_by_month,PROD),billing_month)" ], - "downstreamType": "FIELD_SET", + "downstreamType": "FIELD", "downstreams": [ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,dbt-instance-1.pagila.dbt_postgres.an-aliased-view-for-monthly-billing,PROD),billing_month)" ], @@ -509,7 +509,7 @@ "upstreams": [ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.dbt_postgres.payments_by_customer_by_month,PROD),customer_id)" ], - "downstreamType": "FIELD_SET", + "downstreamType": "FIELD", "downstreams": [ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,dbt-instance-1.pagila.dbt_postgres.an-aliased-view-for-monthly-billing,PROD),customer_id)" ], @@ -520,7 +520,7 @@ "upstreams": [ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.dbt_postgres.payments_by_customer_by_month,PROD),amount)" ], - "downstreamType": "FIELD_SET", + "downstreamType": "FIELD", "downstreams": [ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,dbt-instance-1.pagila.dbt_postgres.an-aliased-view-for-monthly-billing,PROD),amount)" ], @@ -778,7 +778,7 @@ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,dbt-instance-1.pagila.public.payment_p2020_05,PROD),amount)", "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,dbt-instance-1.pagila.public.payment_p2020_06,PROD),amount)" ], - "downstreamType": "FIELD_SET", + "downstreamType": "FIELD", "downstreams": [ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,dbt-instance-1.pagila.dbt_postgres.an-aliased-view-for-payments,PROD),amount)" ], @@ -794,7 +794,7 @@ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,dbt-instance-1.pagila.public.payment_p2020_05,PROD),customer_id)", "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,dbt-instance-1.pagila.public.payment_p2020_06,PROD),customer_id)" ], - "downstreamType": "FIELD_SET", + "downstreamType": "FIELD", "downstreams": [ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,dbt-instance-1.pagila.dbt_postgres.an-aliased-view-for-payments,PROD),customer_id)" ], @@ -810,7 +810,7 @@ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,dbt-instance-1.pagila.public.payment_p2020_05,PROD),payment_date)", "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,dbt-instance-1.pagila.public.payment_p2020_06,PROD),payment_date)" ], - "downstreamType": "FIELD_SET", + "downstreamType": "FIELD", "downstreams": [ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,dbt-instance-1.pagila.dbt_postgres.an-aliased-view-for-payments,PROD),payment_date)" ], @@ -826,7 +826,7 @@ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,dbt-instance-1.pagila.public.payment_p2020_05,PROD),payment_id)", "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,dbt-instance-1.pagila.public.payment_p2020_06,PROD),payment_id)" ], - "downstreamType": "FIELD_SET", + "downstreamType": "FIELD", "downstreams": [ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,dbt-instance-1.pagila.dbt_postgres.an-aliased-view-for-payments,PROD),payment_id)" ], @@ -842,7 +842,7 @@ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,dbt-instance-1.pagila.public.payment_p2020_05,PROD),rental_id)", "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,dbt-instance-1.pagila.public.payment_p2020_06,PROD),rental_id)" ], - "downstreamType": "FIELD_SET", + "downstreamType": "FIELD", "downstreams": [ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,dbt-instance-1.pagila.dbt_postgres.an-aliased-view-for-payments,PROD),rental_id)" ], @@ -858,7 +858,7 @@ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,dbt-instance-1.pagila.public.payment_p2020_05,PROD),staff_id)", "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,dbt-instance-1.pagila.public.payment_p2020_06,PROD),staff_id)" ], - "downstreamType": "FIELD_SET", + "downstreamType": "FIELD", "downstreams": [ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,dbt-instance-1.pagila.dbt_postgres.an-aliased-view-for-payments,PROD),staff_id)" ], @@ -1027,7 +1027,7 @@ "upstreams": [ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.dbt_postgres.an-aliased-view-for-payments,PROD),payment_date)" ], - "downstreamType": "FIELD_SET", + "downstreamType": "FIELD", "downstreams": [ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,dbt-instance-1.pagila.dbt_postgres.payments_by_customer_by_month,PROD),billing_month)" ], @@ -1038,7 +1038,7 @@ "upstreams": [ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.dbt_postgres.an-aliased-view-for-payments,PROD),customer_id)" ], - "downstreamType": "FIELD_SET", + "downstreamType": "FIELD", "downstreams": [ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,dbt-instance-1.pagila.dbt_postgres.payments_by_customer_by_month,PROD),customer_id)" ], @@ -1049,7 +1049,7 @@ "upstreams": [ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.dbt_postgres.an-aliased-view-for-payments,PROD),amount)" ], - "downstreamType": "FIELD_SET", + "downstreamType": "FIELD", "downstreams": [ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,dbt-instance-1.pagila.dbt_postgres.payments_by_customer_by_month,PROD),amount)" ], diff --git a/metadata-ingestion/tests/integration/dbt/dbt_test_with_non_incremental_lineage_mces_golden.json b/metadata-ingestion/tests/integration/dbt/dbt_test_with_non_incremental_lineage_mces_golden.json index a2a8437d551eb..80ca85a5e6c61 100644 --- a/metadata-ingestion/tests/integration/dbt/dbt_test_with_non_incremental_lineage_mces_golden.json +++ b/metadata-ingestion/tests/integration/dbt/dbt_test_with_non_incremental_lineage_mces_golden.json @@ -236,7 +236,7 @@ "upstreams": [ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.customer,PROD),customer_id)" ], - "downstreamType": "FIELD_SET", + "downstreamType": "FIELD", "downstreams": [ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.dbt_postgres.customer_details,PROD),customer_id)" ], @@ -248,7 +248,7 @@ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.customer,PROD),first_name)", "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.customer,PROD),last_name)" ], - "downstreamType": "FIELD_SET", + "downstreamType": "FIELD", "downstreams": [ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.dbt_postgres.customer_details,PROD),full_name)" ], @@ -259,7 +259,7 @@ "upstreams": [ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.customer,PROD),email)" ], - "downstreamType": "FIELD_SET", + "downstreamType": "FIELD", "downstreams": [ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.dbt_postgres.customer_details,PROD),email)" ], @@ -270,7 +270,7 @@ "upstreams": [ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.address,PROD),address)" ], - "downstreamType": "FIELD_SET", + "downstreamType": "FIELD", "downstreams": [ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.dbt_postgres.customer_details,PROD),address)" ], @@ -281,7 +281,7 @@ "upstreams": [ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.city,PROD),city)" ], - "downstreamType": "FIELD_SET", + "downstreamType": "FIELD", "downstreams": [ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.dbt_postgres.customer_details,PROD),city)" ], @@ -292,7 +292,7 @@ "upstreams": [ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.address,PROD),postal_code)" ], - "downstreamType": "FIELD_SET", + "downstreamType": "FIELD", "downstreams": [ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.dbt_postgres.customer_details,PROD),postal_code)" ], @@ -303,7 +303,7 @@ "upstreams": [ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.address,PROD),phone)" ], - "downstreamType": "FIELD_SET", + "downstreamType": "FIELD", "downstreams": [ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.dbt_postgres.customer_details,PROD),phone)" ], @@ -496,7 +496,7 @@ "upstreams": [ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.dbt_postgres.payments_by_customer_by_month,PROD),billing_month)" ], - "downstreamType": "FIELD_SET", + "downstreamType": "FIELD", "downstreams": [ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.dbt_postgres.an-aliased-view-for-monthly-billing,PROD),billing_month)" ], @@ -507,7 +507,7 @@ "upstreams": [ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.dbt_postgres.payments_by_customer_by_month,PROD),customer_id)" ], - "downstreamType": "FIELD_SET", + "downstreamType": "FIELD", "downstreams": [ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.dbt_postgres.an-aliased-view-for-monthly-billing,PROD),customer_id)" ], @@ -518,7 +518,7 @@ "upstreams": [ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.dbt_postgres.payments_by_customer_by_month,PROD),amount)" ], - "downstreamType": "FIELD_SET", + "downstreamType": "FIELD", "downstreams": [ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.dbt_postgres.an-aliased-view-for-monthly-billing,PROD),amount)" ], @@ -775,7 +775,7 @@ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payment_p2020_05,PROD),amount)", "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payment_p2020_06,PROD),amount)" ], - "downstreamType": "FIELD_SET", + "downstreamType": "FIELD", "downstreams": [ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.dbt_postgres.an-aliased-view-for-payments,PROD),amount)" ], @@ -791,7 +791,7 @@ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payment_p2020_05,PROD),customer_id)", "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payment_p2020_06,PROD),customer_id)" ], - "downstreamType": "FIELD_SET", + "downstreamType": "FIELD", "downstreams": [ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.dbt_postgres.an-aliased-view-for-payments,PROD),customer_id)" ], @@ -807,7 +807,7 @@ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payment_p2020_05,PROD),payment_date)", "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payment_p2020_06,PROD),payment_date)" ], - "downstreamType": "FIELD_SET", + "downstreamType": "FIELD", "downstreams": [ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.dbt_postgres.an-aliased-view-for-payments,PROD),payment_date)" ], @@ -823,7 +823,7 @@ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payment_p2020_05,PROD),payment_id)", "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payment_p2020_06,PROD),payment_id)" ], - "downstreamType": "FIELD_SET", + "downstreamType": "FIELD", "downstreams": [ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.dbt_postgres.an-aliased-view-for-payments,PROD),payment_id)" ], @@ -839,7 +839,7 @@ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payment_p2020_05,PROD),rental_id)", "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payment_p2020_06,PROD),rental_id)" ], - "downstreamType": "FIELD_SET", + "downstreamType": "FIELD", "downstreams": [ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.dbt_postgres.an-aliased-view-for-payments,PROD),rental_id)" ], @@ -855,7 +855,7 @@ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payment_p2020_05,PROD),staff_id)", "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payment_p2020_06,PROD),staff_id)" ], - "downstreamType": "FIELD_SET", + "downstreamType": "FIELD", "downstreams": [ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.dbt_postgres.an-aliased-view-for-payments,PROD),staff_id)" ], @@ -1023,7 +1023,7 @@ "upstreams": [ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.dbt_postgres.an-aliased-view-for-payments,PROD),payment_date)" ], - "downstreamType": "FIELD_SET", + "downstreamType": "FIELD", "downstreams": [ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.dbt_postgres.payments_by_customer_by_month,PROD),billing_month)" ], @@ -1034,7 +1034,7 @@ "upstreams": [ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.dbt_postgres.an-aliased-view-for-payments,PROD),customer_id)" ], - "downstreamType": "FIELD_SET", + "downstreamType": "FIELD", "downstreams": [ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.dbt_postgres.payments_by_customer_by_month,PROD),customer_id)" ], @@ -1045,7 +1045,7 @@ "upstreams": [ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.dbt_postgres.an-aliased-view-for-payments,PROD),amount)" ], - "downstreamType": "FIELD_SET", + "downstreamType": "FIELD", "downstreams": [ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.dbt_postgres.payments_by_customer_by_month,PROD),amount)" ], diff --git a/metadata-ingestion/tests/integration/dbt/dbt_test_with_target_platform_instance_mces_golden.json b/metadata-ingestion/tests/integration/dbt/dbt_test_with_target_platform_instance_mces_golden.json index c37f3847117f6..1e6e4d8ba94a2 100644 --- a/metadata-ingestion/tests/integration/dbt/dbt_test_with_target_platform_instance_mces_golden.json +++ b/metadata-ingestion/tests/integration/dbt/dbt_test_with_target_platform_instance_mces_golden.json @@ -236,7 +236,7 @@ "upstreams": [ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.customer,PROD),customer_id)" ], - "downstreamType": "FIELD_SET", + "downstreamType": "FIELD", "downstreams": [ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.dbt_postgres.customer_details,PROD),customer_id)" ], @@ -248,7 +248,7 @@ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.customer,PROD),first_name)", "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.customer,PROD),last_name)" ], - "downstreamType": "FIELD_SET", + "downstreamType": "FIELD", "downstreams": [ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.dbt_postgres.customer_details,PROD),full_name)" ], @@ -259,7 +259,7 @@ "upstreams": [ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.customer,PROD),email)" ], - "downstreamType": "FIELD_SET", + "downstreamType": "FIELD", "downstreams": [ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.dbt_postgres.customer_details,PROD),email)" ], @@ -270,7 +270,7 @@ "upstreams": [ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.address,PROD),address)" ], - "downstreamType": "FIELD_SET", + "downstreamType": "FIELD", "downstreams": [ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.dbt_postgres.customer_details,PROD),address)" ], @@ -281,7 +281,7 @@ "upstreams": [ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.city,PROD),city)" ], - "downstreamType": "FIELD_SET", + "downstreamType": "FIELD", "downstreams": [ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.dbt_postgres.customer_details,PROD),city)" ], @@ -292,7 +292,7 @@ "upstreams": [ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.address,PROD),postal_code)" ], - "downstreamType": "FIELD_SET", + "downstreamType": "FIELD", "downstreams": [ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.dbt_postgres.customer_details,PROD),postal_code)" ], @@ -303,7 +303,7 @@ "upstreams": [ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.address,PROD),phone)" ], - "downstreamType": "FIELD_SET", + "downstreamType": "FIELD", "downstreams": [ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.dbt_postgres.customer_details,PROD),phone)" ], @@ -496,7 +496,7 @@ "upstreams": [ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,ps-instance-1.pagila.dbt_postgres.payments_by_customer_by_month,PROD),billing_month)" ], - "downstreamType": "FIELD_SET", + "downstreamType": "FIELD", "downstreams": [ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.dbt_postgres.an-aliased-view-for-monthly-billing,PROD),billing_month)" ], @@ -507,7 +507,7 @@ "upstreams": [ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,ps-instance-1.pagila.dbt_postgres.payments_by_customer_by_month,PROD),customer_id)" ], - "downstreamType": "FIELD_SET", + "downstreamType": "FIELD", "downstreams": [ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.dbt_postgres.an-aliased-view-for-monthly-billing,PROD),customer_id)" ], @@ -518,7 +518,7 @@ "upstreams": [ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,ps-instance-1.pagila.dbt_postgres.payments_by_customer_by_month,PROD),amount)" ], - "downstreamType": "FIELD_SET", + "downstreamType": "FIELD", "downstreams": [ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.dbt_postgres.an-aliased-view-for-monthly-billing,PROD),amount)" ], @@ -775,7 +775,7 @@ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payment_p2020_05,PROD),amount)", "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payment_p2020_06,PROD),amount)" ], - "downstreamType": "FIELD_SET", + "downstreamType": "FIELD", "downstreams": [ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.dbt_postgres.an-aliased-view-for-payments,PROD),amount)" ], @@ -791,7 +791,7 @@ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payment_p2020_05,PROD),customer_id)", "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payment_p2020_06,PROD),customer_id)" ], - "downstreamType": "FIELD_SET", + "downstreamType": "FIELD", "downstreams": [ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.dbt_postgres.an-aliased-view-for-payments,PROD),customer_id)" ], @@ -807,7 +807,7 @@ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payment_p2020_05,PROD),payment_date)", "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payment_p2020_06,PROD),payment_date)" ], - "downstreamType": "FIELD_SET", + "downstreamType": "FIELD", "downstreams": [ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.dbt_postgres.an-aliased-view-for-payments,PROD),payment_date)" ], @@ -823,7 +823,7 @@ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payment_p2020_05,PROD),payment_id)", "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payment_p2020_06,PROD),payment_id)" ], - "downstreamType": "FIELD_SET", + "downstreamType": "FIELD", "downstreams": [ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.dbt_postgres.an-aliased-view-for-payments,PROD),payment_id)" ], @@ -839,7 +839,7 @@ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payment_p2020_05,PROD),rental_id)", "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payment_p2020_06,PROD),rental_id)" ], - "downstreamType": "FIELD_SET", + "downstreamType": "FIELD", "downstreams": [ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.dbt_postgres.an-aliased-view-for-payments,PROD),rental_id)" ], @@ -855,7 +855,7 @@ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payment_p2020_05,PROD),staff_id)", "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payment_p2020_06,PROD),staff_id)" ], - "downstreamType": "FIELD_SET", + "downstreamType": "FIELD", "downstreams": [ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.dbt_postgres.an-aliased-view-for-payments,PROD),staff_id)" ], @@ -1023,7 +1023,7 @@ "upstreams": [ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,ps-instance-1.pagila.dbt_postgres.an-aliased-view-for-payments,PROD),payment_date)" ], - "downstreamType": "FIELD_SET", + "downstreamType": "FIELD", "downstreams": [ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.dbt_postgres.payments_by_customer_by_month,PROD),billing_month)" ], @@ -1034,7 +1034,7 @@ "upstreams": [ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,ps-instance-1.pagila.dbt_postgres.an-aliased-view-for-payments,PROD),customer_id)" ], - "downstreamType": "FIELD_SET", + "downstreamType": "FIELD", "downstreams": [ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.dbt_postgres.payments_by_customer_by_month,PROD),customer_id)" ], @@ -1045,7 +1045,7 @@ "upstreams": [ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,ps-instance-1.pagila.dbt_postgres.an-aliased-view-for-payments,PROD),amount)" ], - "downstreamType": "FIELD_SET", + "downstreamType": "FIELD", "downstreams": [ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.dbt_postgres.payments_by_customer_by_month,PROD),amount)" ], diff --git a/metadata-ingestion/tests/integration/dbt/test_dbt.py b/metadata-ingestion/tests/integration/dbt/test_dbt.py index 941315fcfa9d5..a46da9707679c 100644 --- a/metadata-ingestion/tests/integration/dbt/test_dbt.py +++ b/metadata-ingestion/tests/integration/dbt/test_dbt.py @@ -216,6 +216,20 @@ def set_paths( run_results_files=["sample_dbt_run_results_2.json"], source_config_modifiers={}, ), + DbtTestConfig( + "dbt-prefer-sql-parser-lineage", + "dbt_test_prefer_sql_parser_lineage.json", + "dbt_test_prefer_sql_parser_lineage_golden.json", + catalog_file="sample_dbt_catalog_2.json", + manifest_file="sample_dbt_manifest_2.json", + sources_file="sample_dbt_sources_2.json", + run_results_files=["sample_dbt_run_results_2.json"], + source_config_modifiers={ + "prefer_sql_parser_lineage": True, + "skip_sources_in_lineage": True, + "entities_enabled": {"sources": "NO"}, + }, + ), ], ids=lambda dbt_test_config: dbt_test_config.run_id, ) diff --git a/metadata-ingestion/tests/unit/test_dbt_source.py b/metadata-ingestion/tests/unit/test_dbt_source.py index 99387ab4e6ae4..48a6fd0f65068 100644 --- a/metadata-ingestion/tests/unit/test_dbt_source.py +++ b/metadata-ingestion/tests/unit/test_dbt_source.py @@ -226,6 +226,32 @@ def test_dbt_config_skip_sources_in_lineage(): assert config.skip_sources_in_lineage is True +def test_dbt_config_prefer_sql_parser_lineage(): + with pytest.raises( + ValidationError, + match="prefer_sql_parser_lineage.*requires.*skip_sources_in_lineage", + ): + config_dict = { + "manifest_path": "dummy_path", + "catalog_path": "dummy_path", + "target_platform": "dummy_platform", + "prefer_sql_parser_lineage": True, + } + config = DBTCoreConfig.parse_obj(config_dict) + + config_dict = { + "manifest_path": "dummy_path", + "catalog_path": "dummy_path", + "target_platform": "dummy_platform", + "skip_sources_in_lineage": True, + "entities_enabled": {"sources": "NO"}, + "prefer_sql_parser_lineage": True, + } + config = DBTCoreConfig.parse_obj(config_dict) + assert config.skip_sources_in_lineage is True + assert config.prefer_sql_parser_lineage is True + + def test_dbt_s3_config(): # test missing aws config config_dict: dict = { diff --git a/metadata-io/src/main/java/com/linkedin/metadata/graph/elastic/ElasticSearchGraphService.java b/metadata-io/src/main/java/com/linkedin/metadata/graph/elastic/ElasticSearchGraphService.java index ada5069d0cabe..adcc64fd2bea7 100644 --- a/metadata-io/src/main/java/com/linkedin/metadata/graph/elastic/ElasticSearchGraphService.java +++ b/metadata-io/src/main/java/com/linkedin/metadata/graph/elastic/ElasticSearchGraphService.java @@ -4,7 +4,6 @@ import com.fasterxml.jackson.databind.node.JsonNodeFactory; import com.fasterxml.jackson.databind.node.ObjectNode; -import com.google.common.annotations.VisibleForTesting; import com.google.common.collect.ImmutableList; import com.linkedin.common.urn.Urn; import com.linkedin.metadata.aspect.models.graph.Edge; @@ -296,7 +295,6 @@ public List buildReindexConfigs( Collections.emptyMap())); } - @VisibleForTesting @Override public void clear() { _esBulkProcessor.deleteByQuery( diff --git a/metadata-io/src/main/java/com/linkedin/metadata/systemmetadata/ElasticSearchSystemMetadataService.java b/metadata-io/src/main/java/com/linkedin/metadata/systemmetadata/ElasticSearchSystemMetadataService.java index 13fde9e392927..57002a3bfc59d 100644 --- a/metadata-io/src/main/java/com/linkedin/metadata/systemmetadata/ElasticSearchSystemMetadataService.java +++ b/metadata-io/src/main/java/com/linkedin/metadata/systemmetadata/ElasticSearchSystemMetadataService.java @@ -2,7 +2,6 @@ import com.fasterxml.jackson.databind.node.JsonNodeFactory; import com.fasterxml.jackson.databind.node.ObjectNode; -import com.google.common.annotations.VisibleForTesting; import com.google.common.collect.ImmutableMap; import com.linkedin.common.urn.Urn; import com.linkedin.metadata.run.AspectRowSummary; @@ -250,7 +249,6 @@ public List buildReindexConfigs( Collections.emptyMap())); } - @VisibleForTesting @Override public void clear() { _esBulkProcessor.deleteByQuery( diff --git a/metadata-models/src/main/pegasus/com/linkedin/dataset/PartitionSummary.pdl b/metadata-models/src/main/pegasus/com/linkedin/dataset/PartitionSummary.pdl new file mode 100644 index 0000000000000..3984277a31417 --- /dev/null +++ b/metadata-models/src/main/pegasus/com/linkedin/dataset/PartitionSummary.pdl @@ -0,0 +1,24 @@ +namespace com.linkedin.dataset + +import com.linkedin.common.AuditStamp + +/** + * Defines how the data is partitioned + */ +record PartitionSummary { + /** + * A unique id / value for the partition for which statistics were collected, + * generated by applying the key definition to a given row. + */ + partition: string + + /** + * The created time for a given partition. + */ + created: optional AuditStamp + + /** + * The last modified / touched time for a given partition. + */ + lastModified: optional AuditStamp +} \ No newline at end of file diff --git a/metadata-models/src/main/pegasus/com/linkedin/dataset/PartitionsSummary.pdl b/metadata-models/src/main/pegasus/com/linkedin/dataset/PartitionsSummary.pdl new file mode 100644 index 0000000000000..34e696890d64f --- /dev/null +++ b/metadata-models/src/main/pegasus/com/linkedin/dataset/PartitionsSummary.pdl @@ -0,0 +1,19 @@ +namespace com.linkedin.dataset + +/** + * Defines how the data is partitioned for Data Lake tables (e.g. Hive, S3, Iceberg, Delta, Hudi, etc). + */ +@Aspect = { + "name": "partitionsSummary" +} +record PartitionsSummary { + /** + * The minimum partition as ordered + */ + minPartition: optional PartitionSummary + + /** + * The maximum partition as ordered + */ + maxPartition: optional PartitionSummary +} \ No newline at end of file diff --git a/metadata-models/src/main/pegasus/com/linkedin/metadata/key/DataHubActionKey.pdl b/metadata-models/src/main/pegasus/com/linkedin/metadata/key/DataHubActionKey.pdl new file mode 100644 index 0000000000000..8205ecbb80716 --- /dev/null +++ b/metadata-models/src/main/pegasus/com/linkedin/metadata/key/DataHubActionKey.pdl @@ -0,0 +1,14 @@ +namespace com.linkedin.metadata.key + +/** + * Key for a DataHub Action Pipeline + */ +@Aspect = { + "name": "dataHubActionKey" +} +record DataHubActionKey { + /** + * A unique id for the Action, either generated or provided + */ + id: string +} \ No newline at end of file diff --git a/metadata-models/src/main/pegasus/com/linkedin/schema/SchemaField.pdl b/metadata-models/src/main/pegasus/com/linkedin/schema/SchemaField.pdl index afb0263057b6d..f91e2004401cf 100644 --- a/metadata-models/src/main/pegasus/com/linkedin/schema/SchemaField.pdl +++ b/metadata-models/src/main/pegasus/com/linkedin/schema/SchemaField.pdl @@ -150,6 +150,8 @@ record SchemaField { /** * For Datasets which are partitioned, this determines the partitioning key. + * Note that multiple columns can be part of a partitioning key, but currently we do not support + * rendering the ordered partitioning key. */ isPartitioningKey: optional boolean diff --git a/metadata-models/src/main/pegasus/com/linkedin/timeseries/PartitionSpec.pdl b/metadata-models/src/main/pegasus/com/linkedin/timeseries/PartitionSpec.pdl index 084af1513ec80..146a285e24dba 100644 --- a/metadata-models/src/main/pegasus/com/linkedin/timeseries/PartitionSpec.pdl +++ b/metadata-models/src/main/pegasus/com/linkedin/timeseries/PartitionSpec.pdl @@ -1,24 +1,28 @@ namespace com.linkedin.timeseries /** - * Defines how the data is partitioned + * A reference to a specific partition in a dataset. */ record PartitionSpec { - - type: enum PartitionType { - FULL_TABLE, - QUERY, - PARTITION - } = "PARTITION" - /** - * String representation of the partition + * A unique id / value for the partition for which statistics were collected, + * generated by applying the key definition to a given row. */ @TimeseriesField = {} partition: string /** - * Time window of the partition if applicable + * Time window of the partition, if we are able to extract it from the partition key. */ timePartition: optional TimeWindow + + /** + * Unused! + */ + @deprecated + type: enum PartitionType { + FULL_TABLE, + QUERY, + PARTITION + } = "PARTITION" } \ No newline at end of file diff --git a/metadata-models/src/main/resources/entity-registry.yml b/metadata-models/src/main/resources/entity-registry.yml index 3af4af5e4767e..f8520990a0984 100644 --- a/metadata-models/src/main/resources/entity-registry.yml +++ b/metadata-models/src/main/resources/entity-registry.yml @@ -45,6 +45,7 @@ entities: - access - structuredProperties - forms + - partitionsSummary - name: dataHubPolicy doc: DataHub Policies represent access policies granted to users or groups on metadata operations like edit, view etc. category: internal @@ -74,6 +75,7 @@ entities: - forms - subTypes - incidentsSummary + - testResults - name: dataFlow category: core keyAspect: dataFlowKey @@ -95,12 +97,14 @@ entities: - incidentsSummary - forms - subTypes + - testResults - name: dataProcess keyAspect: dataProcessKey aspects: - dataProcessInfo - ownership - status + - testResults - name: dataProcessInstance doc: DataProcessInstance represents an instance of a datajob/jobflow run keyAspect: dataProcessInstanceKey @@ -111,6 +115,7 @@ entities: - dataProcessInstanceRelationships - dataProcessInstanceRunEvent - status + - testResults - name: chart category: core keyAspect: chartKey @@ -136,6 +141,7 @@ entities: - structuredProperties - incidentsSummary - forms + - testResults - name: dashboard keyAspect: dashboardKey aspects: @@ -159,6 +165,7 @@ entities: - structuredProperties - incidentsSummary - forms + - testResults - name: notebook doc: Notebook represents a combination of query, text, chart and etc. This is in BETA version keyAspect: notebookKey @@ -176,6 +183,7 @@ entities: - subTypes - dataPlatformInstance - browsePathsV2 + - testResults - name: corpuser doc: CorpUser represents an identity of a person (or an account) in the enterprise. keyAspect: corpUserKey @@ -193,6 +201,7 @@ entities: - roleMembership - structuredProperties - forms + - testResults - name: corpGroup doc: CorpGroup represents an identity of a group of users in the enterprise. keyAspect: corpGroupKey @@ -206,6 +215,7 @@ entities: - roleMembership - structuredProperties - forms + - testResults - name: domain doc: A data domain within an organization. category: core @@ -216,6 +226,7 @@ entities: - ownership - structuredProperties - forms + - testResults - name: container doc: A container of related data assets. category: core @@ -236,6 +247,7 @@ entities: - browsePathsV2 - structuredProperties - forms + - testResults - name: tag category: core keyAspect: tagKey @@ -244,6 +256,7 @@ entities: - ownership - deprecation - status + - testResults - name: glossaryTerm category: core keyAspect: glossaryTermKey @@ -259,6 +272,7 @@ entities: - browsePaths - structuredProperties - forms + - testResults - name: glossaryNode category: core keyAspect: glossaryNodeKey @@ -269,6 +283,7 @@ entities: - status - structuredProperties - forms + - testResults - name: dataHubIngestionSource category: internal keyAspect: dataHubIngestionSourceKey @@ -340,6 +355,7 @@ entities: - browsePathsV2 - structuredProperties - forms + - testResults - name: mlModelGroup category: core keyAspect: mlModelGroupKey @@ -357,6 +373,7 @@ entities: - browsePathsV2 - structuredProperties - forms + - testResults - name: mlModelDeployment category: core keyAspect: mlModelDeploymentKey @@ -367,6 +384,7 @@ entities: - deprecation - globalTags - dataPlatformInstance + - testResults - name: mlFeatureTable category: core keyAspect: mlFeatureTableKey @@ -385,6 +403,7 @@ entities: - browsePathsV2 - structuredProperties - forms + - testResults - name: mlFeature category: core keyAspect: mlFeatureKey @@ -403,6 +422,7 @@ entities: - browsePathsV2 - structuredProperties - forms + - testResults - name: mlPrimaryKey category: core keyAspect: mlPrimaryKeyKey @@ -419,6 +439,7 @@ entities: - dataPlatformInstance - structuredProperties - forms + - testResults - name: telemetry category: internal keyAspect: telemetryKey @@ -455,6 +476,7 @@ entities: - forms - businessAttributes - documentation + - testResults - name: globalSettings doc: Global settings for an the platform category: internal @@ -522,6 +544,7 @@ entities: - status - structuredProperties - forms + - testResults - name: ownershipType doc: Ownership Type represents a user-created ownership category for a person or group who is responsible for an asset. category: core @@ -549,6 +572,10 @@ entities: keyAspect: dataHubPersonaKey aspects: - dataHubPersonaInfo + - name: dataHubAction + category: internal + keyAspect: dataHubActionKey + aspects: [] - name: entityType doc: A type of entity in the DataHub Metadata Model. category: core diff --git a/smoke-test/tests/schema_fields/__init__.py b/smoke-test/tests/schema_fields/__init__.py new file mode 100644 index 0000000000000..e69de29bb2d1d diff --git a/smoke-test/tests/schema_fields/queries/__init__.py b/smoke-test/tests/schema_fields/queries/__init__.py new file mode 100644 index 0000000000000..e69de29bb2d1d diff --git a/smoke-test/tests/schema_fields/queries/get_chart_field.gql b/smoke-test/tests/schema_fields/queries/get_chart_field.gql new file mode 100644 index 0000000000000..424e5ad686ab9 --- /dev/null +++ b/smoke-test/tests/schema_fields/queries/get_chart_field.gql @@ -0,0 +1,20 @@ +query($urn:String!) { + chart(urn: $urn) { + inputFields { + fields { + schemaFieldUrn + schemaField { + schemaFieldEntity { + urn + fieldPath + documentation { + documentations { + documentation + } + } + } + } + } + } + } +} \ No newline at end of file diff --git a/smoke-test/tests/schema_fields/test_schemafields.py b/smoke-test/tests/schema_fields/test_schemafields.py new file mode 100644 index 0000000000000..cd282db81ff72 --- /dev/null +++ b/smoke-test/tests/schema_fields/test_schemafields.py @@ -0,0 +1,174 @@ +import logging +import os +import tempfile +import time +from random import randint + +import datahub.metadata.schema_classes as models +import pytest +from datahub.emitter.mce_builder import make_dataset_urn, make_schema_field_urn +from datahub.emitter.mcp import MetadataChangeProposalWrapper +from datahub.ingestion.api.common import PipelineContext, RecordEnvelope +from datahub.ingestion.api.sink import NoopWriteCallback +from datahub.ingestion.graph.client import DatahubClientConfig, DataHubGraph +from datahub.ingestion.sink.file import FileSink, FileSinkConfig + +from tests.utils import ( + delete_urns_from_file, + get_gms_url, + get_sleep_info, + ingest_file_via_rest, + wait_for_writes_to_sync, +) + +logger = logging.getLogger(__name__) + + +start_index = randint(10, 10000) +dataset_urns = [ + make_dataset_urn("snowflake", f"table_foo_{i}") + for i in range(start_index, start_index + 10) +] + + +class FileEmitter: + def __init__(self, filename: str) -> None: + self.sink: FileSink = FileSink( + ctx=PipelineContext(run_id="create_test_data"), + config=FileSinkConfig(filename=filename), + ) + + def emit(self, event): + self.sink.write_record_async( + record_envelope=RecordEnvelope(record=event, metadata={}), + write_callback=NoopWriteCallback(), + ) + + def close(self): + self.sink.close() + + +@pytest.fixture(scope="module") +def chart_urn(): + return "urn:li:chart:(looker,chart_foo)" + + +@pytest.fixture(scope="module") +def upstream_schema_field_urn(): + return make_schema_field_urn(make_dataset_urn("snowflake", "table_bar"), "field1") + + +def create_test_data(filename: str, chart_urn: str, upstream_schema_field_urn: str): + documentation_mcp = MetadataChangeProposalWrapper( + entityUrn=upstream_schema_field_urn, + aspect=models.DocumentationClass( + documentations=[ + models.DocumentationAssociationClass( + documentation="test documentation", + attribution=models.MetadataAttributionClass( + time=int(time.time() * 1000), + actor="urn:li:corpuser:datahub", + source="urn:li:dataHubAction:documentation_propagation", + ), + ) + ] + ), + ) + + input_fields_mcp = MetadataChangeProposalWrapper( + entityUrn=chart_urn, + aspect=models.InputFieldsClass( + fields=[ + models.InputFieldClass( + schemaFieldUrn=upstream_schema_field_urn, + schemaField=models.SchemaFieldClass( + fieldPath="field1", + type=models.SchemaFieldDataTypeClass(models.StringTypeClass()), + nativeDataType="STRING", + ), + ) + ] + ), + ) + + file_emitter = FileEmitter(filename) + for mcps in [documentation_mcp, input_fields_mcp]: + file_emitter.emit(mcps) + + file_emitter.close() + + +sleep_sec, sleep_times = get_sleep_info() + + +@pytest.fixture(scope="module", autouse=False) +def ingest_cleanup_data(request, chart_urn, upstream_schema_field_urn): + new_file, filename = tempfile.mkstemp(suffix=".json") + try: + create_test_data(filename, chart_urn, upstream_schema_field_urn) + print("ingesting schema fields test data") + ingest_file_via_rest(filename) + yield + print("removing schema fields test data") + delete_urns_from_file(filename) + wait_for_writes_to_sync() + finally: + os.remove(filename) + + +@pytest.mark.dependency() +def test_healthchecks(wait_for_healthchecks): + # Call to wait_for_healthchecks fixture will do the actual functionality. + pass + + +def get_gql_query(filename: str) -> str: + with open(filename) as fp: + return fp.read() + + +def validate_schema_field_urn_for_chart( + graph: DataHubGraph, chart_urn: str, upstream_schema_field_urn: str +) -> None: + # Validate listing + result = graph.execute_graphql( + get_gql_query("tests/schema_fields/queries/get_chart_field.gql"), + {"urn": chart_urn}, + ) + assert "chart" in result + assert "inputFields" in result["chart"] + assert len(result["chart"]["inputFields"]["fields"]) == 1 + assert ( + result["chart"]["inputFields"]["fields"][0]["schemaField"]["schemaFieldEntity"][ + "urn" + ] + == upstream_schema_field_urn + ) + assert ( + result["chart"]["inputFields"]["fields"][0]["schemaField"]["schemaFieldEntity"][ + "fieldPath" + ] + == "field1" + ) + assert ( + result["chart"]["inputFields"]["fields"][0]["schemaFieldUrn"] + == upstream_schema_field_urn + ) + assert ( + result["chart"]["inputFields"]["fields"][0]["schemaField"]["schemaFieldEntity"][ + "documentation" + ]["documentations"][0]["documentation"] + == "test documentation" + ) + + +# @tenacity.retry( +# stop=tenacity.stop_after_attempt(sleep_times), wait=tenacity.wait_fixed(sleep_sec) +# ) +@pytest.mark.dependency(depends=["test_healthchecks"]) +def test_schema_field_gql_mapper_for_charts( + ingest_cleanup_data, chart_urn, upstream_schema_field_urn +): + graph: DataHubGraph = DataHubGraph(config=DatahubClientConfig(server=get_gms_url())) + + validate_schema_field_urn_for_chart(graph, chart_urn, upstream_schema_field_urn) diff --git a/smoke-test/tests/structured_properties/test_structured_properties.py b/smoke-test/tests/structured_properties/test_structured_properties.py index bf1b5b1292750..f2327a13df6d0 100644 --- a/smoke-test/tests/structured_properties/test_structured_properties.py +++ b/smoke-test/tests/structured_properties/test_structured_properties.py @@ -119,9 +119,11 @@ def create_property_definition( qualifiedName=f"{namespace}.{property_name}", valueType=Urn.make_data_type_urn(value_type), description="The retention policy for the dataset", - entityTypes=[Urn.make_entity_type_urn(e) for e in entity_types] - if entity_types - else [Urn.make_entity_type_urn("dataset")], + entityTypes=( + [Urn.make_entity_type_urn(e) for e in entity_types] + if entity_types + else [Urn.make_entity_type_urn("dataset")] + ), cardinality=cardinality, allowedValues=allowed_values, ) @@ -137,7 +139,7 @@ def create_property_definition( def attach_property_to_entity( urn: str, property_name: str, - property_value: Union[str, float, List[str | float]], + property_value: Union[str, float, List[Union[str, float]]], graph: DataHubGraph, namespace: str = default_namespace, ):