-
Notifications
You must be signed in to change notification settings - Fork 2.9k
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Merge branch 'datahub-project:master' into master
- Loading branch information
Showing
97 changed files
with
5,398 additions
and
274 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,18 +1,33 @@ | ||
| import setuptools | ||
| import os | ||
|
|
||
| folders = ["./smoke-test/tests"] | ||
|
|
||
| for folder in folders: | ||
| print(f"Checking folder {folder}") | ||
| a = [i for i in setuptools.find_packages(folder) if "cypress" not in i] | ||
| b = [i for i in setuptools.find_namespace_packages(folder) if "cypress" not in i] | ||
| packages = [i for i in setuptools.find_packages(folder) if "cypress" not in i] | ||
| namespace_packages = [ | ||
| i for i in setuptools.find_namespace_packages(folder) if "cypress" not in i | ||
| ] | ||
|
|
||
| in_a_not_b = set(a) - set(b) | ||
| in_b_not_a = set(b) - set(a) | ||
| print("Packages found:", packages) | ||
| print("Namespace packages found:", namespace_packages) | ||
|
|
||
| in_packages_not_namespace = set(packages) - set(namespace_packages) | ||
| in_namespace_not_packages = set(namespace_packages) - set(packages) | ||
|
|
||
| if in_packages_not_namespace: | ||
| print(f"Packages not in namespace packages: {in_packages_not_namespace}") | ||
| if in_namespace_not_packages: | ||
| print(f"Namespace packages not in packages: {in_namespace_not_packages}") | ||
| for pkg in in_namespace_not_packages: | ||
| pkg_path = os.path.join(folder, pkg.replace(".", os.path.sep)) | ||
| print(f"Contents of {pkg_path}:") | ||
| print(os.listdir(pkg_path)) | ||
|
|
||
| assert ( | ||
| len(in_a_not_b) == 0 | ||
| ), f"Found packages in {folder} that are not in namespace packages: {in_a_not_b}" | ||
| len(in_packages_not_namespace) == 0 | ||
| ), f"Found packages in {folder} that are not in namespace packages: {in_packages_not_namespace}" | ||
| assert ( | ||
| len(in_b_not_a) == 0 | ||
| ), f"Found namespace packages in {folder} that are not in packages: {in_b_not_a}" | ||
| len(in_namespace_not_packages) == 0 | ||
| ), f"Found namespace packages in {folder} that are not in packages: {in_namespace_not_packages}" |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
56 changes: 56 additions & 0 deletions
56
...c/main/java/com/linkedin/datahub/upgrade/common/steps/ClearSystemMetadataServiceStep.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,56 @@ | ||
| package com.linkedin.datahub.upgrade.common.steps; | ||
|
|
||
| import com.linkedin.datahub.upgrade.UpgradeContext; | ||
| import com.linkedin.datahub.upgrade.UpgradeStep; | ||
| import com.linkedin.datahub.upgrade.UpgradeStepResult; | ||
| import com.linkedin.datahub.upgrade.impl.DefaultUpgradeStepResult; | ||
| import com.linkedin.datahub.upgrade.nocode.NoCodeUpgrade; | ||
| import com.linkedin.metadata.systemmetadata.SystemMetadataService; | ||
| import java.util.function.Function; | ||
|
|
||
| public class ClearSystemMetadataServiceStep implements UpgradeStep { | ||
|
|
||
| private final SystemMetadataService _systemMetadataService; | ||
| private final boolean _alwaysRun; | ||
|
|
||
| public ClearSystemMetadataServiceStep( | ||
| final SystemMetadataService systemMetadataService, final boolean alwaysRun) { | ||
| _systemMetadataService = systemMetadataService; | ||
| _alwaysRun = alwaysRun; | ||
| } | ||
|
|
||
| @Override | ||
| public String id() { | ||
| return "ClearSystemMetadataServiceStep"; | ||
| } | ||
|
|
||
| @Override | ||
| public boolean skip(UpgradeContext context) { | ||
| if (_alwaysRun) { | ||
| return false; | ||
| } | ||
| if (context.parsedArgs().containsKey(NoCodeUpgrade.CLEAN_ARG_NAME)) { | ||
| return false; | ||
| } | ||
| context.report().addLine("Cleanup has not been requested."); | ||
| return true; | ||
| } | ||
|
|
||
| @Override | ||
| public int retryCount() { | ||
| return 1; | ||
| } | ||
|
|
||
| @Override | ||
| public Function<UpgradeContext, UpgradeStepResult> executable() { | ||
| return (context) -> { | ||
| try { | ||
| _systemMetadataService.clear(); | ||
| } catch (Exception e) { | ||
| context.report().addLine("Failed to clear system metadata service", e); | ||
| return new DefaultUpgradeStepResult(id(), UpgradeStepResult.Result.FAILED); | ||
| } | ||
| return new DefaultUpgradeStepResult(id(), UpgradeStepResult.Result.SUCCEEDED); | ||
| }; | ||
| } | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.