From 8055e46830801f21a4ee9f2e29616d96b0c53094 Mon Sep 17 00:00:00 2001 From: david-leifker <114954101+david-leifker@users.noreply.github.com> Date: Fri, 21 Apr 2023 15:45:20 -0500 Subject: [PATCH] Logging and test models fixes (#7884) --- build.gradle | 8 ++++---- datahub-upgrade/build.gradle | 1 + .../models/EntitySpecBuilderTest.java | 10 +++++----- .../java/datahub-client/build.gradle | 1 + metadata-io/build.gradle | 2 ++ .../metadata/ESTestConfiguration.java | 5 +---- .../com/linkedin/metadata/TestEntityUtil.java | 4 ++-- .../src/test/resources/logback-test.xml | 19 +++++++++++++++++++ metadata-jobs/mae-consumer/build.gradle | 1 + metadata-jobs/pe-consumer/build.gradle | 1 + metadata-service/war/build.gradle | 1 + .../{BrowsePaths.pdl => TestBrowsePaths.pdl} | 4 ++-- .../com/datahub/test/TestEntityAspect.pdl | 2 +- .../test/testing/PizzaSearchDocument.pdl | 2 +- 14 files changed, 42 insertions(+), 19 deletions(-) create mode 100644 metadata-io/src/test/resources/logback-test.xml rename test-models/src/main/pegasus/com/datahub/test/{BrowsePaths.pdl => TestBrowsePaths.pdl} (88%) diff --git a/build.gradle b/build.gradle index 5fe09cfe3fd96..6d5d2f3b04c1c 100644 --- a/build.gradle +++ b/build.gradle @@ -3,8 +3,8 @@ buildscript { // Releases: https://github.com/linkedin/rest.li/blob/master/CHANGELOG.md ext.pegasusVersion = '29.22.16' ext.mavenVersion = '3.6.3' - ext.springVersion = '5.3.20' - ext.springBootVersion = '2.7.10' + ext.springVersion = '5.3.27' + ext.springBootVersion = '2.7.11' ext.openTelemetryVersion = '1.18.0' ext.neo4jVersion = '4.4.9' ext.graphQLJavaVersion = '19.0' @@ -15,8 +15,8 @@ buildscript { ext.jettyVersion = '9.4.46.v20220331' ext.playVersion = '2.8.18' ext.log4jVersion = '2.19.0' - ext.slf4jVersion = '1.7.32' - ext.logbackClassic = '1.2.11' + ext.slf4jVersion = '1.7.36' + ext.logbackClassic = '1.2.12' ext.hadoop3Version = '3.3.5' ext.docker_registry = 'linkedin' diff --git a/datahub-upgrade/build.gradle b/datahub-upgrade/build.gradle index 5e2a38cee70fb..d6252e2f60b83 100644 --- a/datahub-upgrade/build.gradle +++ b/datahub-upgrade/build.gradle @@ -63,6 +63,7 @@ dependencies { testImplementation externalDependency.springBootTest testCompile externalDependency.mockito testCompile externalDependency.testng + testRuntime externalDependency.logbackClassic } bootJar { diff --git a/entity-registry/src/test/java/com/linkedin/metadata/models/EntitySpecBuilderTest.java b/entity-registry/src/test/java/com/linkedin/metadata/models/EntitySpecBuilderTest.java index 9858145133e82..135043a23d158 100644 --- a/entity-registry/src/test/java/com/linkedin/metadata/models/EntitySpecBuilderTest.java +++ b/entity-registry/src/test/java/com/linkedin/metadata/models/EntitySpecBuilderTest.java @@ -1,6 +1,6 @@ package com.linkedin.metadata.models; -import com.datahub.test.BrowsePaths; +import com.datahub.test.TestBrowsePaths; import com.datahub.test.SearchFeatures; import com.datahub.test.Snapshot; import com.datahub.test.TestEntityInfo; @@ -91,7 +91,7 @@ public void testBuildEntitySpecs() { final Map aspectSpecMap = testEntitySpec.getAspectSpecMap(); assertEquals(4, aspectSpecMap.size()); assertTrue(aspectSpecMap.containsKey("testEntityKey")); - assertTrue(aspectSpecMap.containsKey("browsePaths")); + assertTrue(aspectSpecMap.containsKey("testBrowsePaths")); assertTrue(aspectSpecMap.containsKey("testEntityInfo")); assertTrue(aspectSpecMap.containsKey("searchFeatures")); @@ -99,7 +99,7 @@ public void testBuildEntitySpecs() { validateTestEntityKey(aspectSpecMap.get("testEntityKey")); // Assert on BrowsePaths Aspect - validateBrowsePaths(aspectSpecMap.get("browsePaths")); + validateBrowsePaths(aspectSpecMap.get("testBrowsePaths")); // Assert on TestEntityInfo Aspect validateTestEntityInfo(aspectSpecMap.get("testEntityInfo")); @@ -130,8 +130,8 @@ private void validateTestEntityKey(final AspectSpec keyAspectSpec) { private void validateBrowsePaths(final AspectSpec browsePathAspectSpec) { - assertEquals("browsePaths", browsePathAspectSpec.getName()); - assertEquals(new BrowsePaths().schema().getFullName(), browsePathAspectSpec.getPegasusSchema().getFullName()); + assertEquals("testBrowsePaths", browsePathAspectSpec.getName()); + assertEquals(new TestBrowsePaths().schema().getFullName(), browsePathAspectSpec.getPegasusSchema().getFullName()); assertEquals(1, browsePathAspectSpec.getSearchableFieldSpecs().size()); assertEquals(SearchableAnnotation.FieldType.BROWSE_PATH, browsePathAspectSpec.getSearchableFieldSpecs().get(0) .getSearchableAnnotation().getFieldType()); diff --git a/metadata-integration/java/datahub-client/build.gradle b/metadata-integration/java/datahub-client/build.gradle index 6f44685ca897b..4286946e6b904 100644 --- a/metadata-integration/java/datahub-client/build.gradle +++ b/metadata-integration/java/datahub-client/build.gradle @@ -51,6 +51,7 @@ dependencies { testCompile externalDependency.mockServerClient testCompile externalDependency.testContainers testCompile externalDependency.httpAsyncClient + testRuntime externalDependency.logbackClassic swaggerCodegen 'io.swagger.codegen.v3:swagger-codegen-cli:3.0.33' } diff --git a/metadata-io/build.gradle b/metadata-io/build.gradle index edd9944035fe6..7720ee9f815d3 100644 --- a/metadata-io/build.gradle +++ b/metadata-io/build.gradle @@ -59,6 +59,8 @@ dependencies { testCompile project(':test-models') testImplementation externalDependency.springBootTest testImplementation project(':datahub-graphql-core') + // logback >=1.3 required due to `testcontainers` only + testImplementation 'ch.qos.logback:logback-classic:1.4.7' testAnnotationProcessor externalDependency.lombok diff --git a/metadata-io/src/test/java/com/linkedin/metadata/ESTestConfiguration.java b/metadata-io/src/test/java/com/linkedin/metadata/ESTestConfiguration.java index 219eca967f7c0..7cae130d93ece 100644 --- a/metadata-io/src/test/java/com/linkedin/metadata/ESTestConfiguration.java +++ b/metadata-io/src/test/java/com/linkedin/metadata/ESTestConfiguration.java @@ -7,8 +7,6 @@ import com.linkedin.metadata.models.registry.ConfigEntityRegistry; import com.linkedin.metadata.models.registry.EntityRegistry; import com.linkedin.metadata.models.registry.EntityRegistryException; -import com.linkedin.metadata.models.registry.MergedEntityRegistry; -import com.linkedin.metadata.models.registry.SnapshotEntityRegistry; import com.linkedin.metadata.search.elasticsearch.indexbuilder.ESIndexBuilder; import com.linkedin.metadata.search.elasticsearch.update.ESBulkProcessor; import com.linkedin.metadata.version.GitVersion; @@ -125,8 +123,7 @@ protected ESIndexBuilder getIndexBuilder(@Qualifier("elasticSearchRestHighLevelC @Bean(name = "entityRegistry") public EntityRegistry entityRegistry() throws EntityRegistryException { - ConfigEntityRegistry configEntityRegistry = new ConfigEntityRegistry( + return new ConfigEntityRegistry( ESTestConfiguration.class.getClassLoader().getResourceAsStream("entity-registry.yml")); - return new MergedEntityRegistry(SnapshotEntityRegistry.getInstance()).apply(configEntityRegistry); } } diff --git a/metadata-io/src/test/java/com/linkedin/metadata/TestEntityUtil.java b/metadata-io/src/test/java/com/linkedin/metadata/TestEntityUtil.java index 11dd5f4786d36..52375f197bbf8 100644 --- a/metadata-io/src/test/java/com/linkedin/metadata/TestEntityUtil.java +++ b/metadata-io/src/test/java/com/linkedin/metadata/TestEntityUtil.java @@ -1,6 +1,6 @@ package com.linkedin.metadata; -import com.datahub.test.BrowsePaths; +import com.datahub.test.TestBrowsePaths; import com.datahub.test.KeyPartEnum; import com.datahub.test.SearchFeatures; import com.datahub.test.SimpleNestedRecord1; @@ -52,7 +52,7 @@ public static TestEntitySnapshot getSnapshot() { Urn urn = getTestEntityUrn(); snapshot.setUrn(urn); - BrowsePaths browsePaths = new BrowsePaths().setPaths(new StringArray(ImmutableList.of("/a/b/c", "d/e/f"))); + TestBrowsePaths browsePaths = new TestBrowsePaths().setPaths(new StringArray(ImmutableList.of("/a/b/c", "d/e/f"))); SearchFeatures searchFeatures = new SearchFeatures().setFeature1(2).setFeature2(1); TestEntityAspectArray aspects = new TestEntityAspectArray( diff --git a/metadata-io/src/test/resources/logback-test.xml b/metadata-io/src/test/resources/logback-test.xml new file mode 100644 index 0000000000000..b474d56dbfcf2 --- /dev/null +++ b/metadata-io/src/test/resources/logback-test.xml @@ -0,0 +1,19 @@ + + + + + %date{ISO8601} [%thread] %-5level %logger{36} - %msg%n + + + WARN + + + + + + + + + + + \ No newline at end of file diff --git a/metadata-jobs/mae-consumer/build.gradle b/metadata-jobs/mae-consumer/build.gradle index a02e3d01789f8..405c40914a89f 100644 --- a/metadata-jobs/mae-consumer/build.gradle +++ b/metadata-jobs/mae-consumer/build.gradle @@ -45,6 +45,7 @@ dependencies { implementation externalDependency.awsMskIamAuth testImplementation externalDependency.springBootTest + testRuntime externalDependency.logbackClassic } task avroSchemaSources(type: Copy) { diff --git a/metadata-jobs/pe-consumer/build.gradle b/metadata-jobs/pe-consumer/build.gradle index 38b451fafc2f9..517b021353f9d 100644 --- a/metadata-jobs/pe-consumer/build.gradle +++ b/metadata-jobs/pe-consumer/build.gradle @@ -23,6 +23,7 @@ dependencies { annotationProcessor externalDependency.lombok runtime externalDependency.logbackClassic testCompile externalDependency.mockito + testRuntime externalDependency.logbackClassic } task avroSchemaSources(type: Copy) { diff --git a/metadata-service/war/build.gradle b/metadata-service/war/build.gradle index 68c1357ced11f..84c9d04e59c56 100644 --- a/metadata-service/war/build.gradle +++ b/metadata-service/war/build.gradle @@ -37,6 +37,7 @@ dependencies { runtime externalDependency.log4j2Api runtime externalDependency.logbackClassic implementation externalDependency.awsMskIamAuth + testRuntime externalDependency.logbackClassic } configurations { diff --git a/test-models/src/main/pegasus/com/datahub/test/BrowsePaths.pdl b/test-models/src/main/pegasus/com/datahub/test/TestBrowsePaths.pdl similarity index 88% rename from test-models/src/main/pegasus/com/datahub/test/BrowsePaths.pdl rename to test-models/src/main/pegasus/com/datahub/test/TestBrowsePaths.pdl index 1b115b25613ec..ac8ac8453e0ca 100644 --- a/test-models/src/main/pegasus/com/datahub/test/BrowsePaths.pdl +++ b/test-models/src/main/pegasus/com/datahub/test/TestBrowsePaths.pdl @@ -4,9 +4,9 @@ namespace com.datahub.test * Shared aspect containing Browse Paths to be indexed for an entity. */ @Aspect = { - "name": "browsePaths" + "name": "testBrowsePaths" } -record BrowsePaths { +record TestBrowsePaths { /** * A list of valid browse paths for the entity. * diff --git a/test-models/src/main/pegasus/com/datahub/test/TestEntityAspect.pdl b/test-models/src/main/pegasus/com/datahub/test/TestEntityAspect.pdl index 4807c247de1bd..bea978463618a 100644 --- a/test-models/src/main/pegasus/com/datahub/test/TestEntityAspect.pdl +++ b/test-models/src/main/pegasus/com/datahub/test/TestEntityAspect.pdl @@ -6,6 +6,6 @@ namespace com.datahub.test typeref TestEntityAspect = union[ TestEntityKey, TestEntityInfo, - BrowsePaths, + TestBrowsePaths, SearchFeatures ] \ No newline at end of file diff --git a/test-models/src/main/pegasus/com/datahub/test/testing/PizzaSearchDocument.pdl b/test-models/src/main/pegasus/com/datahub/test/testing/PizzaSearchDocument.pdl index e63614020b184..da137f14dd86d 100644 --- a/test-models/src/main/pegasus/com/datahub/test/testing/PizzaSearchDocument.pdl +++ b/test-models/src/main/pegasus/com/datahub/test/testing/PizzaSearchDocument.pdl @@ -27,7 +27,7 @@ record PizzaSearchDocument { /** * The path parts to browse for this pizza. */ - browsePaths: optional array[string] + testBrowsePaths: optional array[string] /** * Whether this pizza still exists.