Logging and test models fixes (#7884)

datahub-project · Apr 21, 2023 · 2ffa39f · 2ffa39f
1 parent e9c2f9a
commit 2ffa39f
Show file tree

Hide file tree

Showing 14 changed files with 42 additions and 19 deletions.
diff --git a/build.gradle b/build.gradle
@@ -3,8 +3,8 @@ buildscript {
   // Releases: https://github.com/linkedin/rest.li/blob/master/CHANGELOG.md
   ext.pegasusVersion = '29.22.16'
   ext.mavenVersion = '3.6.3'
-  ext.springVersion = '5.3.20'
-  ext.springBootVersion = '2.7.10'
+  ext.springVersion = '5.3.27'
+  ext.springBootVersion = '2.7.11'
   ext.openTelemetryVersion = '1.18.0'
   ext.neo4jVersion = '4.4.9'
   ext.graphQLJavaVersion = '19.0'
@@ -15,8 +15,8 @@ buildscript {
   ext.jettyVersion = '9.4.46.v20220331'
   ext.playVersion = '2.8.18'
   ext.log4jVersion = '2.19.0'
-  ext.slf4jVersion = '1.7.32'
-  ext.logbackClassic = '1.2.11'
+  ext.slf4jVersion = '1.7.36'
+  ext.logbackClassic = '1.2.12'
   ext.hadoop3Version = '3.3.5'
 
   ext.docker_registry = 'linkedin'

diff --git a/datahub-upgrade/build.gradle b/datahub-upgrade/build.gradle
@@ -63,6 +63,7 @@ dependencies {
   testImplementation externalDependency.springBootTest
   testCompile externalDependency.mockito
   testCompile externalDependency.testng
+  testRuntime externalDependency.logbackClassic
 }
 
 bootJar {

diff --git a/entity-registry/src/test/java/com/linkedin/metadata/models/EntitySpecBuilderTest.java b/entity-registry/src/test/java/com/linkedin/metadata/models/EntitySpecBuilderTest.java
@@ -1,6 +1,6 @@
 package com.linkedin.metadata.models;
 
-import com.datahub.test.BrowsePaths;
+import com.datahub.test.TestBrowsePaths;
 import com.datahub.test.SearchFeatures;
 import com.datahub.test.Snapshot;
 import com.datahub.test.TestEntityInfo;
@@ -91,15 +91,15 @@ public void testBuildEntitySpecs() {
     final Map<String, AspectSpec> aspectSpecMap = testEntitySpec.getAspectSpecMap();
     assertEquals(4, aspectSpecMap.size());
     assertTrue(aspectSpecMap.containsKey("testEntityKey"));
-    assertTrue(aspectSpecMap.containsKey("browsePaths"));
+    assertTrue(aspectSpecMap.containsKey("testBrowsePaths"));
     assertTrue(aspectSpecMap.containsKey("testEntityInfo"));
     assertTrue(aspectSpecMap.containsKey("searchFeatures"));
 
     // Assert on TestEntityKey
     validateTestEntityKey(aspectSpecMap.get("testEntityKey"));
 
     // Assert on BrowsePaths Aspect
-    validateBrowsePaths(aspectSpecMap.get("browsePaths"));
+    validateBrowsePaths(aspectSpecMap.get("testBrowsePaths"));
 
     // Assert on TestEntityInfo Aspect
     validateTestEntityInfo(aspectSpecMap.get("testEntityInfo"));
@@ -130,8 +130,8 @@ private void validateTestEntityKey(final AspectSpec keyAspectSpec) {
 
 
   private void validateBrowsePaths(final AspectSpec browsePathAspectSpec) {
-    assertEquals("browsePaths", browsePathAspectSpec.getName());
-    assertEquals(new BrowsePaths().schema().getFullName(), browsePathAspectSpec.getPegasusSchema().getFullName());
+    assertEquals("testBrowsePaths", browsePathAspectSpec.getName());
+    assertEquals(new TestBrowsePaths().schema().getFullName(), browsePathAspectSpec.getPegasusSchema().getFullName());
     assertEquals(1, browsePathAspectSpec.getSearchableFieldSpecs().size());
     assertEquals(SearchableAnnotation.FieldType.BROWSE_PATH, browsePathAspectSpec.getSearchableFieldSpecs().get(0)
         .getSearchableAnnotation().getFieldType());

diff --git a/metadata-integration/java/datahub-client/build.gradle b/metadata-integration/java/datahub-client/build.gradle
@@ -51,6 +51,7 @@ dependencies {
     testCompile externalDependency.mockServerClient
     testCompile externalDependency.testContainers
     testCompile externalDependency.httpAsyncClient
+    testRuntime externalDependency.logbackClassic
 
     swaggerCodegen 'io.swagger.codegen.v3:swagger-codegen-cli:3.0.33'
 }

diff --git a/metadata-io/build.gradle b/metadata-io/build.gradle
@@ -59,6 +59,8 @@ dependencies {
   testCompile project(':test-models')
   testImplementation externalDependency.springBootTest
   testImplementation project(':datahub-graphql-core')
+  // logback >=1.3 required due to `testcontainers` only
+  testImplementation 'ch.qos.logback:logback-classic:1.4.7'
 
   testAnnotationProcessor externalDependency.lombok
 

diff --git a/metadata-io/src/test/java/com/linkedin/metadata/ESTestConfiguration.java b/metadata-io/src/test/java/com/linkedin/metadata/ESTestConfiguration.java
@@ -7,8 +7,6 @@
 import com.linkedin.metadata.models.registry.ConfigEntityRegistry;
 import com.linkedin.metadata.models.registry.EntityRegistry;
 import com.linkedin.metadata.models.registry.EntityRegistryException;
-import com.linkedin.metadata.models.registry.MergedEntityRegistry;
-import com.linkedin.metadata.models.registry.SnapshotEntityRegistry;
 import com.linkedin.metadata.search.elasticsearch.indexbuilder.ESIndexBuilder;
 import com.linkedin.metadata.search.elasticsearch.update.ESBulkProcessor;
 import com.linkedin.metadata.version.GitVersion;
@@ -125,8 +123,7 @@ protected ESIndexBuilder getIndexBuilder(@Qualifier("elasticSearchRestHighLevelC
 
     @Bean(name = "entityRegistry")
     public EntityRegistry entityRegistry() throws EntityRegistryException {
-        ConfigEntityRegistry configEntityRegistry = new ConfigEntityRegistry(
+        return new ConfigEntityRegistry(
                 ESTestConfiguration.class.getClassLoader().getResourceAsStream("entity-registry.yml"));
-        return new MergedEntityRegistry(SnapshotEntityRegistry.getInstance()).apply(configEntityRegistry);
     }
 }
diff --git a/metadata-io/src/test/java/com/linkedin/metadata/TestEntityUtil.java b/metadata-io/src/test/java/com/linkedin/metadata/TestEntityUtil.java
@@ -1,6 +1,6 @@
 package com.linkedin.metadata;
 
-import com.datahub.test.BrowsePaths;
+import com.datahub.test.TestBrowsePaths;
 import com.datahub.test.KeyPartEnum;
 import com.datahub.test.SearchFeatures;
 import com.datahub.test.SimpleNestedRecord1;
@@ -52,7 +52,7 @@ public static TestEntitySnapshot getSnapshot() {
     Urn urn = getTestEntityUrn();
     snapshot.setUrn(urn);
 
-    BrowsePaths browsePaths = new BrowsePaths().setPaths(new StringArray(ImmutableList.of("/a/b/c", "d/e/f")));
+    TestBrowsePaths browsePaths = new TestBrowsePaths().setPaths(new StringArray(ImmutableList.of("/a/b/c", "d/e/f")));
     SearchFeatures searchFeatures = new SearchFeatures().setFeature1(2).setFeature2(1);
 
     TestEntityAspectArray aspects = new TestEntityAspectArray(

diff --git a/metadata-io/src/test/resources/logback-test.xml b/metadata-io/src/test/resources/logback-test.xml
@@ -0,0 +1,19 @@
+<configuration debug="false" scan="true">
+
+    <appender name="STDOUT" target="System.out" class="ch.qos.logback.core.ConsoleAppender">
+        <encoder>
+            <pattern>%date{ISO8601} [%thread] %-5level %logger{36} - %msg%n</pattern>
+        </encoder>
+        <filter class="ch.qos.logback.classic.filter.ThresholdFilter">
+            <level>WARN</level>
+        </filter>
+    </appender>
+
+    <logger name="com.datastax.oss" level="ERROR" />
+    <logger name="com.datastax.driver" level="FATAL" />
+
+    <root level="WARN">
+        <appender-ref ref="STDOUT" />
+    </root>
+
+</configuration>
diff --git a/metadata-jobs/mae-consumer/build.gradle b/metadata-jobs/mae-consumer/build.gradle
@@ -45,6 +45,7 @@ dependencies {
     implementation externalDependency.awsMskIamAuth
 
     testImplementation externalDependency.springBootTest
+    testRuntime externalDependency.logbackClassic
 }
 
 task avroSchemaSources(type: Copy) {

diff --git a/metadata-jobs/pe-consumer/build.gradle b/metadata-jobs/pe-consumer/build.gradle
@@ -23,6 +23,7 @@ dependencies {
   annotationProcessor externalDependency.lombok
   runtime externalDependency.logbackClassic
   testCompile externalDependency.mockito
+  testRuntime externalDependency.logbackClassic
 }
 
 task avroSchemaSources(type: Copy) {

diff --git a/metadata-service/war/build.gradle b/metadata-service/war/build.gradle
@@ -37,6 +37,7 @@ dependencies {
   runtime externalDependency.log4j2Api
   runtime externalDependency.logbackClassic
   implementation externalDependency.awsMskIamAuth
+  testRuntime externalDependency.logbackClassic
 }
 
 configurations {

diff --git a/.../pegasus/com/datahub/test/BrowsePaths.pdl → ...asus/com/datahub/test/TestBrowsePaths.pdl b/.../pegasus/com/datahub/test/BrowsePaths.pdl → ...asus/com/datahub/test/TestBrowsePaths.pdl
@@ -4,9 +4,9 @@ namespace com.datahub.test
  * Shared aspect containing Browse Paths to be indexed for an entity.
  */
 @Aspect = {
-    "name": "browsePaths"
+    "name": "testBrowsePaths"
 }
-record BrowsePaths {
+record TestBrowsePaths {
   /**
    * A list of valid browse paths for the entity.
    *

diff --git a/test-models/src/main/pegasus/com/datahub/test/TestEntityAspect.pdl b/test-models/src/main/pegasus/com/datahub/test/TestEntityAspect.pdl
@@ -6,6 +6,6 @@ namespace com.datahub.test
 typeref TestEntityAspect = union[
   TestEntityKey,
   TestEntityInfo,
-  BrowsePaths,
+  TestBrowsePaths,
   SearchFeatures
 ]
diff --git a/test-models/src/main/pegasus/com/datahub/test/testing/PizzaSearchDocument.pdl b/test-models/src/main/pegasus/com/datahub/test/testing/PizzaSearchDocument.pdl
@@ -27,7 +27,7 @@ record PizzaSearchDocument {
   /**
    * The path parts to browse for this pizza.
    */
-  browsePaths: optional array[string]
+  testBrowsePaths: optional array[string]
 
   /**
    * Whether this pizza still exists.