feat(bootstrap): add bootstrap step to clear out unknown aspect rows from the database #5148
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,15 @@ | ||
| package com.linkedin.datahub.upgrade.config; | ||
|
|
||
| import com.linkedin.datahub.upgrade.removeunknownaspects.RemoveUnknownAspects; | ||
| import com.linkedin.metadata.entity.EntityService; | ||
| import org.springframework.context.annotation.Bean; | ||
| import org.springframework.context.annotation.Configuration; | ||
|
|
||
|
|
||
| @Configuration | ||
| public class RemoveUnknownAspectsConfig { | ||
| @Bean(name = "removeUnknownAspects") | ||
| public RemoveUnknownAspects removeUnknownAspects(EntityService entityService) { | ||
| return new RemoveUnknownAspects(entityService); | ||
| } | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,41 @@ | ||
| package com.linkedin.datahub.upgrade.removeunknownaspects; | ||
|
|
||
| import com.linkedin.datahub.upgrade.UpgradeContext; | ||
| import com.linkedin.datahub.upgrade.UpgradeStep; | ||
| import com.linkedin.datahub.upgrade.UpgradeStepResult; | ||
| import com.linkedin.datahub.upgrade.impl.DefaultUpgradeStepResult; | ||
| import com.linkedin.gms.factory.telemetry.TelemetryUtils; | ||
| import com.linkedin.metadata.entity.EntityService; | ||
| import java.util.HashMap; | ||
| import java.util.function.Function; | ||
| import lombok.RequiredArgsConstructor; | ||
| import lombok.extern.slf4j.Slf4j; | ||
|
|
||
|
|
||
| @Slf4j | ||
| @RequiredArgsConstructor | ||
| public class RemoveClientIdAspectStep implements UpgradeStep { | ||
|
|
||
| private static final String INVALID_CLIENT_ID_ASPECT = "clientId"; | ||
|
|
||
| private final EntityService _entityService; | ||
|
|
||
| @Override | ||
| public String id() { | ||
| return this.getClass().getSimpleName(); | ||
| } | ||
|
|
||
| @Override | ||
| public boolean skip(UpgradeContext context) { | ||
| return false; | ||
| } | ||
|
|
||
| @Override | ||
| public Function<UpgradeContext, UpgradeStepResult> executable() { | ||
| return upgradeContext -> { | ||
| _entityService.deleteAspect(TelemetryUtils.CLIENT_ID_URN, INVALID_CLIENT_ID_ASPECT, | ||
| new HashMap<>(), true); | ||
| return (UpgradeStepResult) new DefaultUpgradeStepResult(id(), UpgradeStepResult.Result.SUCCEEDED); | ||
| }; | ||
| } | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,40 @@ | ||
| package com.linkedin.datahub.upgrade.removeunknownaspects; | ||
|
|
||
| import com.google.common.collect.ImmutableList; | ||
| import com.linkedin.datahub.upgrade.Upgrade; | ||
| import com.linkedin.datahub.upgrade.UpgradeCleanupStep; | ||
| import com.linkedin.datahub.upgrade.UpgradeStep; | ||
| import com.linkedin.metadata.entity.EntityService; | ||
| import java.util.ArrayList; | ||
| import java.util.List; | ||
|
|
||
|
|
||
| public class RemoveUnknownAspects implements Upgrade { | ||
|
|
||
| private final List<UpgradeStep> _steps; | ||
|
|
||
| public RemoveUnknownAspects(final EntityService entityService) { | ||
| _steps = buildSteps(entityService); | ||
| } | ||
|
|
||
| @Override | ||
| public String id() { | ||
| return this.getClass().getSimpleName(); | ||
| } | ||
|
|
||
| @Override | ||
| public List<UpgradeStep> steps() { | ||
| return _steps; | ||
| } | ||
|
|
||
| private List<UpgradeStep> buildSteps(final EntityService entityService) { | ||
| final List<UpgradeStep> steps = new ArrayList<>(); | ||
| steps.add(new RemoveClientIdAspectStep(entityService)); | ||
| return steps; | ||
| } | ||
|
|
||
| @Override | ||
| public List<UpgradeCleanupStep> cleanupSteps() { | ||
| return ImmutableList.of(); | ||
| } | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,16 +1,28 @@ | ||
| package com.linkedin.metadata.entity; | ||
|
|
||
| import com.linkedin.metadata.entity.cassandra.CassandraAspect; | ||
| import com.linkedin.metadata.entity.ebean.EbeanAspectV2; | ||
| import javax.annotation.Nonnull; | ||
| import lombok.Value; | ||
| import lombok.extern.slf4j.Slf4j; | ||
|
|
||
|
|
||
| /** | ||
| * This class holds values required to construct a unique key to identify an entity aspect record in a database. | ||
| * Its existence started mainly for compatibility with {@link com.linkedin.metadata.entity.ebean.EbeanAspectV2.PrimaryKey} | ||
| */ | ||
| @Value | ||
| @Slf4j | ||
| public class EntityAspectIdentifier { | ||
| @Nonnull String urn; | ||
| @Nonnull String aspect; | ||
| long version; | ||
|
|
||
| public static EntityAspectIdentifier fromEbean(EbeanAspectV2 ebeanAspectV2) { | ||
| return new EntityAspectIdentifier(ebeanAspectV2.getUrn(), ebeanAspectV2.getAspect(), ebeanAspectV2.getVersion()); | ||
| } | ||
|
|
||
| public static EntityAspectIdentifier fromCassandra(CassandraAspect cassandraAspect) { | ||
| return new EntityAspectIdentifier(cassandraAspect.getUrn(), cassandraAspect.getAspect(), cassandraAspect.getVersion()); | ||
| } | ||
| } | ||
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
I really dislike the naming inconsistency of
EbeanAspectV2andCassandraAspect:( Seems very strange that this class needs to know about specific DB implementations, also.There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
It's a utility conversion from the lower level aspect, made more sense here to me tied to the actual higher level object it will always be used with than in a separate utility class that will never be found by others trying to do the same thing.