feat(ingestion) Add frontend connection test for Snowflake

datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/ingest/IngestionResolverUtils.java

@@ -5,13 +5,15 @@
 import com.linkedin.datahub.graphql.generated.IngestionConfig;
 import com.linkedin.datahub.graphql.generated.IngestionSchedule;
 import com.linkedin.datahub.graphql.generated.IngestionSource;
+import com.linkedin.datahub.graphql.generated.StructuredReport;
 import com.linkedin.datahub.graphql.types.common.mappers.StringMapMapper;
 import com.linkedin.entity.EntityResponse;
 import com.linkedin.entity.EnvelopedAspect;
 import com.linkedin.entity.EnvelopedAspectMap;
 import com.linkedin.execution.ExecutionRequestInput;
 import com.linkedin.execution.ExecutionRequestResult;
 import com.linkedin.execution.ExecutionRequestSource;
+import com.linkedin.execution.StructuredExecutionReport;
 import com.linkedin.ingestion.DataHubIngestionSourceConfig;
 import com.linkedin.ingestion.DataHubIngestionSourceInfo;
 import com.linkedin.ingestion.DataHubIngestionSourceSchedule;
@@ -77,9 +79,20 @@ public static com.linkedin.datahub.graphql.generated.ExecutionRequestResult mapE
     result.setStartTimeMs(execRequestResult.getStartTimeMs());
     result.setDurationMs(execRequestResult.getDurationMs());
     result.setReport(execRequestResult.getReport());
+    if (execRequestResult.hasStructuredReport()) {
+      result.setStructuredReport(mapStructuredReport(execRequestResult.getStructuredReport()));
+    }
     return result;
   }
 
+  public static StructuredReport mapStructuredReport(final StructuredExecutionReport structuredReport) {
+    StructuredReport structuredReportResult = new StructuredReport();
+    structuredReportResult.setType(structuredReport.getType());
+    structuredReportResult.setSerializedValue(structuredReport.getSerializedValue());
+    structuredReportResult.setContentType(structuredReport.getContentType());
+    return structuredReportResult;
+  }
+
   public static List<IngestionSource> mapIngestionSources(final Collection<EntityResponse> entities) {
     final List<IngestionSource> results = new ArrayList<>();
     for (EntityResponse response : entities) {

datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/ingest/execution/CreateTestConnectionRequestResolver.java

@@ -74,7 +74,6 @@ public CompletableFuture<String> get(final DataFetchingEnvironment environment)
 
         Map<String, String> arguments = new HashMap<>();
         arguments.put(RECIPE_ARG_NAME, input.getRecipe());
-        arguments.put(VERSION_ARG_NAME, _ingestionConfiguration.getDefaultCliVersion());
         execInput.setArgs(new StringMap(arguments));
 
         proposal.setEntityType(Constants.EXECUTION_REQUEST_ENTITY_NAME);

datahub-graphql-core/src/main/resources/ingestion.graphql

@@ -133,6 +133,29 @@ type ExecutionRequestResult {
   """
   report: String
 
+  """
+  A structured report for this Execution Request
+  """
+  structuredReport: StructuredReport
+
+}
+
+"""
+A flexible carrier for structured results of an execution request.
+"""
+type StructuredReport {
+  """
+  The type of the structured report. (e.g. INGESTION_REPORT, TEST_CONNECTION_REPORT, etc.)
+  """
+  type: String!
+  """
+  The serialized value of the structured report
+  """
+  serializedValue: String!
+  """
+  The content-type of the serialized value (e.g. application/json, application/json;gzip etc.)
+  """
+  contentType: String!
 }
 
 """

datahub-web-react/src/app/ingest/source/builder/DefineRecipeStep.tsx

@@ -1,13 +1,14 @@
-import { Alert, Button, message, Space, Typography } from 'antd';
+import { Alert, Button, Space, Typography } from 'antd';
 import React, { useEffect, useState } from 'react';
 import styled from 'styled-components';
 import { StepProps } from './types';
-import { getSourceConfigs, jsonToYaml, yamlToJson } from '../utils';
+import { getSourceConfigs, jsonToYaml } from '../utils';
 import { YamlEditor } from './YamlEditor';
 import { ANTD_GRAY } from '../../../entity/shared/constants';
 import { IngestionSourceBuilderStep } from './steps';
 import RecipeBuilder from './RecipeBuilder';
 import { CONNECTORS_WITH_FORM } from './RecipeForm/utils';
+import { getRecipeJson } from './RecipeForm/TestConnection/TestConnectionButton';
 
 const LOOKML_DOC_LINK = 'https://datahubproject.io/docs/generated/ingestion/sources/looker#module-lookml';
 
@@ -68,14 +69,8 @@ export const DefineRecipeStep = ({ state, updateState, goTo, prev }: StepProps)
     }, [stagedRecipeYml, showLookerBanner]);
 
     const onClickNext = () => {
-        // Convert the recipe into it's json representation, and catch + report exceptions while we do it.
-        let recipeJson;
-        try {
-            recipeJson = yamlToJson(stagedRecipeYml);
-        } catch (e) {
-            message.warn('Found invalid YAML. Please check your recipe configuration.');
-            return;
-        }
+        const recipeJson = getRecipeJson(stagedRecipeYml);
+        if (!recipeJson) return;
 
         const newState = {
             ...state,

datahub-web-react/src/app/ingest/source/builder/RecipeForm/RecipeForm.tsx

@@ -7,6 +7,8 @@ import styled from 'styled-components/macro';
 import { jsonToYaml } from '../../utils';
 import { RecipeField, RECIPE_FIELDS, setFieldValueOnRecipe } from './utils';
 import FormField from './FormField';
+import TestConnectionButton from './TestConnection/TestConnectionButton';
+import { SNOWFLAKE } from '../../conf/snowflake/snowflake';
 
 export const ControlsContainer = styled.div`
     display: flex;
@@ -32,6 +34,12 @@ const MarginWrapper = styled.div`
     margin-left: 20px;
 `;
 
+const TestConnectionWrapper = styled.div`
+    display: flex;
+    justify-content: flex-end;
+    margin-top: 16px;
+`;
+
 function getInitialValues(displayRecipe: string, allFields: any[]) {
     const initialValues = {};
     let recipeObj;
@@ -108,6 +116,11 @@ function RecipeForm(props: Props) {
                     {fields.map((field, i) => (
                         <FormField field={field} removeMargin={i === fields.length - 1} />
                     ))}
+                    {type === SNOWFLAKE && (
+                        <TestConnectionWrapper>
+                            <TestConnectionButton type={type} recipe={displayRecipe} />
+                        </TestConnectionWrapper>
+                    )}
                 </Collapse.Panel>
             </StyledCollapse>
             <StyledCollapse>

datahub-web-react/src/app/ingest/source/builder/RecipeForm/TestConnection/ConnectionCapabilityView.tsx

@@ -0,0 +1,74 @@
+import { CheckOutlined, CloseOutlined, QuestionCircleOutlined } from '@ant-design/icons';
+import { Tooltip } from 'antd';
+import React from 'react';
+import { green, red } from '@ant-design/colors';
+import styled from 'styled-components/macro';
+import { ANTD_GRAY } from '../../../../../entity/shared/constants';
+
+const CapabilityWrapper = styled.div`
+    align-items: center;
+    display: flex;
+    margin: 10px 0;
+`;
+
+const CapabilityName = styled.span`
+    color: ${ANTD_GRAY[8]};
+    font-size: 18px;
+    margin-right: 12px;
+`;
+
+const CapabilityMessage = styled.span<{ success: boolean }>`
+    color: ${(props) => (props.success ? `${green[6]}` : `${red[5]}`)};
+    font-size: 12px;
+    flex: 1;
+    padding-left: 4px;
+`;
+
+const StyledQuestion = styled(QuestionCircleOutlined)`
+    color: rgba(0, 0, 0, 0.45);
+    margin-left: 4px;
+`;
+
+export const StyledCheck = styled(CheckOutlined)`
+    color: ${green[6]};
+    margin-right: 15px;
+`;
+
+export const StyledClose = styled(CloseOutlined)`
+    color: ${red[5]};
+    margin-right: 15px;
+`;
+
+const NumberWrapper = styled.span`
+    margin-right: 8px;
+`;
+
+interface Props {
+    success: boolean;
+    capability: string;
+    displayMessage: string | null;
+    tooltipMessage: string | null;
+    number?: number;
+}
+
+function ConnectionCapabilityView({ success, capability, displayMessage, tooltipMessage, number }: Props) {
+    return (
+        <CapabilityWrapper>
+            <CapabilityName>
+                {success ? <StyledCheck /> : <StyledClose />}
+                {number ? <NumberWrapper>{number}.</NumberWrapper> : ''}
+                {capability}
+            </CapabilityName>
+            <CapabilityMessage success={success}>
+                {displayMessage}
+                {tooltipMessage && (
+                    <Tooltip overlay={tooltipMessage}>
+                        <StyledQuestion />
+                    </Tooltip>
+                )}
+            </CapabilityMessage>
+        </CapabilityWrapper>
+    );
+}
+
+export default ConnectionCapabilityView;

datahub-web-react/src/app/ingest/source/builder/RecipeForm/TestConnection/TestConnectionButton.tsx

@@ -0,0 +1,128 @@
+import { CheckCircleOutlined } from '@ant-design/icons';
+import { Button, message } from 'antd';
+import React, { useEffect, useState } from 'react';
+import { green } from '@ant-design/colors';
+import {
+    useCreateTestConnectionRequestMutation,
+    useGetIngestionExecutionRequestLazyQuery,
+} from '../../../../../../graphql/ingestion.generated';
+import { FAILURE, getSourceConfigs, RUNNING, yamlToJson } from '../../../utils';
+import { TestConnectionResult } from './types';
+import TestConnectionModal from './TestConnectionModal';
+
+export function getRecipeJson(recipeYaml: string) {
+    // Convert the recipe into it's json representation, and catch + report exceptions while we do it.
+    let recipeJson;
+    try {
+        recipeJson = yamlToJson(recipeYaml);
+    } catch (e) {
+        const messageText = (e as any).parsedLine
+            ? `Please fix line ${(e as any).parsedLine} in your recipe.`
+            : 'Please check your recipe configuration.';
+        message.warn(`Found invalid YAML. ${messageText}`);
+        return null;
+    }
+    return recipeJson;
+}
+
+interface Props {
+    type: string;
+    recipe: string;
+}
+
+function TestConnectionButton(props: Props) {
+    const { type, recipe } = props;
+    const [isLoading, setIsLoading] = useState(false);
+    const [isModalVisible, setIsModalVisible] = useState(false);
+    const [pollingInterval, setPollingInterval] = useState<null | NodeJS.Timeout>(null);
+    const [testConnectionResult, setTestConnectionResult] = useState<null | TestConnectionResult>(null);
+    const [createTestConnectionRequest, { data: requestData }] = useCreateTestConnectionRequestMutation();
+    const [getIngestionExecutionRequest, { data: resultData, loading }] = useGetIngestionExecutionRequestLazyQuery();
+
+    const sourceConfigs = getSourceConfigs(type);
+
+    useEffect(() => {
+        if (requestData && requestData.createTestConnectionRequest) {
+            const interval = setInterval(
+                () =>
+                    getIngestionExecutionRequest({
+                        variables: { urn: requestData.createTestConnectionRequest as string },
+                    }),
+                2000,
+            );
+            setIsLoading(true);
+            setIsModalVisible(true);
+            setPollingInterval(interval);
+        }
+    }, [requestData, getIngestionExecutionRequest]);
+
+    useEffect(() => {
+        if (!loading && resultData) {
+            const result = resultData.executionRequest?.result;
+            if (result && result.status !== RUNNING) {
+                if (result.status === FAILURE) {
+                    message.error(
+                        'Something went wrong with your connection test. Please check your recipe and try again.',
+                    );
+                    setIsModalVisible(false);
+                }
+                if (result.structuredReport) {
+                    const testConnectionReport = JSON.parse(result.structuredReport.serializedValue);
+                    setTestConnectionResult(testConnectionReport);
+                }
+                if (pollingInterval) clearInterval(pollingInterval);
+                setIsLoading(false);
+            }
+        }
+    }, [resultData, pollingInterval, loading]);
+
+    useEffect(() => {
+        if (!isModalVisible && pollingInterval) {
+            clearInterval(pollingInterval);
+        }
+    }, [isModalVisible, pollingInterval]);
+
+    function testConnection() {
+        const recipeJson = getRecipeJson(recipe);
+        if (recipeJson) {
+            createTestConnectionRequest({ variables: { input: { recipe: recipeJson } } })
+                .then((res) =>
+                    getIngestionExecutionRequest({
+                        variables: { urn: res.data?.createTestConnectionRequest as string },
+                    }),
+                )
+                .catch(() => {
+                    message.error(
+                        'There was an unexpected error when trying to test your connection. Please try again.',
+                    );
+                });
+
+            setIsLoading(true);
+            setIsModalVisible(true);
+        }
+    }
+
+    const internalFailure = !!testConnectionResult?.internal_failure;
+    const basicConnectivityFailure = testConnectionResult?.basic_connectivity?.capable === false;
+    const testConnectionFailed = internalFailure || basicConnectivityFailure;
+
+    return (
+        <>
+            <Button onClick={testConnection}>
+                <CheckCircleOutlined style={{ color: green[5] }} />
+                Test Connection
+            </Button>
+            {isModalVisible && (
+                <TestConnectionModal
+                    isLoading={isLoading}
+                    testConnectionFailed={testConnectionFailed}
+                    sourceConfig={sourceConfigs}
+                    testConnectionResult={testConnectionResult}
+                    hideModal={() => setIsModalVisible(false)}
+                />
+            )}
+        </>
+    );
+}
+
+export default TestConnectionButton;