-
Notifications
You must be signed in to change notification settings - Fork 2.9k
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
feat(plugins): spring custom plugins #10389
Changes from 28 commits
a2c4abb
b70d28b
45f563b
fce41e8
f6d5f82
226794c
b7417c2
519b8d2
24d8d57
74cd310
5e81000
69a195b
8df7d3f
9da5da3
38f3fe6
dda060f
2b6a64e
33fc3ce
26b4597
4b2a195
c1fce8f
7780bc1
61f8345
6364c10
caaf722
b22b0cf
00e5a7f
6961836
d1e9dfb
b66d2c9
c6a4bf0
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -16,6 +16,8 @@ | |
import java.util.List; | ||
import java.util.Map; | ||
import java.util.Objects; | ||
import java.util.concurrent.ConcurrentHashMap; | ||
import java.util.function.BiFunction; | ||
import java.util.function.Function; | ||
import java.util.stream.Collectors; | ||
import java.util.stream.IntStream; | ||
|
@@ -24,21 +26,13 @@ | |
import javax.annotation.Nullable; | ||
import lombok.Getter; | ||
import lombok.extern.slf4j.Slf4j; | ||
import org.apache.commons.lang3.ArrayUtils; | ||
|
||
@Slf4j | ||
public class PluginFactory { | ||
|
||
private static final String[] VALIDATOR_PACKAGES = { | ||
"com.linkedin.metadata.aspect.plugins.validation", "com.linkedin.metadata.aspect.validation" | ||
}; | ||
private static final String[] HOOK_PACKAGES = { | ||
"com.linkedin.metadata.aspect.plugins.hooks", "com.linkedin.metadata.aspect.hooks" | ||
}; | ||
|
||
public static PluginFactory withCustomClasspath( | ||
@Nullable PluginConfiguration pluginConfiguration, @Nonnull List<ClassLoader> classLoaders) { | ||
return new PluginFactory(pluginConfiguration, classLoaders); | ||
return new PluginFactory(pluginConfiguration, classLoaders).loadPlugins(); | ||
} | ||
|
||
public static PluginFactory withConfig(@Nullable PluginConfiguration pluginConfiguration) { | ||
|
@@ -49,44 +43,135 @@ public static PluginFactory empty() { | |
return PluginFactory.withConfig(PluginConfiguration.EMPTY); | ||
} | ||
|
||
public static PluginFactory merge(PluginFactory a, PluginFactory b) { | ||
return PluginFactory.withCustomClasspath( | ||
PluginConfiguration.merge(a.getPluginConfiguration(), b.getPluginConfiguration()), | ||
public static PluginFactory merge( | ||
PluginFactory a, | ||
PluginFactory b, | ||
@Nullable | ||
BiFunction<PluginConfiguration, List<ClassLoader>, PluginFactory> pluginFactoryProvider) { | ||
PluginConfiguration mergedPluginConfig = | ||
PluginConfiguration.merge(a.pluginConfiguration, b.pluginConfiguration); | ||
List<ClassLoader> mergedClassLoaders = | ||
Stream.concat(a.getClassLoaders().stream(), b.getClassLoaders().stream()) | ||
.collect(Collectors.toList())); | ||
.collect(Collectors.toList()); | ||
|
||
if (pluginFactoryProvider != null) { | ||
return pluginFactoryProvider.apply(mergedPluginConfig, mergedClassLoaders); | ||
} else { | ||
return PluginFactory.withCustomClasspath(mergedPluginConfig, mergedClassLoaders); | ||
} | ||
} | ||
|
||
@Getter private final PluginConfiguration pluginConfiguration; | ||
@Nonnull @Getter private final List<ClassLoader> classLoaders; | ||
@Getter private final List<AspectPayloadValidator> aspectPayloadValidators; | ||
@Getter private final List<MutationHook> mutationHooks; | ||
@Getter private final List<MCLSideEffect> mclSideEffects; | ||
@Getter private final List<MCPSideEffect> mcpSideEffects; | ||
@Getter private List<AspectPayloadValidator> aspectPayloadValidators; | ||
@Getter private List<MutationHook> mutationHooks; | ||
@Getter private List<MCLSideEffect> mclSideEffects; | ||
@Getter private List<MCPSideEffect> mcpSideEffects; | ||
|
||
private final ClassGraph classGraph; | ||
private static final Map<Long, List<PluginSpec>> pluginCache = new ConcurrentHashMap<>(); | ||
|
||
public PluginFactory( | ||
@Nullable PluginConfiguration pluginConfiguration, @Nonnull List<ClassLoader> classLoaders) { | ||
this.classGraph = | ||
new ClassGraph() | ||
.acceptPackages(ArrayUtils.addAll(HOOK_PACKAGES, VALIDATOR_PACKAGES)) | ||
.enableRemoteJarScanning() | ||
.enableExternalClasses() | ||
.enableClassInfo() | ||
.enableMethodInfo(); | ||
|
||
this.classLoaders = classLoaders; | ||
|
||
if (!this.classLoaders.isEmpty()) { | ||
classLoaders.forEach(this.classGraph::addClassLoader); | ||
} | ||
|
||
this.pluginConfiguration = | ||
pluginConfiguration == null ? PluginConfiguration.EMPTY : pluginConfiguration; | ||
} | ||
|
||
public PluginFactory loadPlugins() { | ||
this.aspectPayloadValidators = buildAspectPayloadValidators(this.pluginConfiguration); | ||
this.mutationHooks = buildMutationHooks(this.pluginConfiguration); | ||
this.mclSideEffects = buildMCLSideEffects(this.pluginConfiguration); | ||
this.mcpSideEffects = buildMCPSideEffects(this.pluginConfiguration); | ||
return this; | ||
} | ||
|
||
/** | ||
* Memory intensive operation because of the size of the jars. Limit packages, classes scanned, | ||
* cache results | ||
* | ||
* @param configs plugin configurations | ||
* @return auto-closeable scan result | ||
*/ | ||
protected static <T extends PluginSpec> List<T> initPlugins( | ||
@Nonnull List<ClassLoader> classLoaders, | ||
@Nonnull Class<?> baseClazz, | ||
@Nonnull List<String> packageNames, | ||
@Nonnull List<AspectPluginConfig> configs) { | ||
|
||
List<String> classNames = | ||
configs.stream().map(AspectPluginConfig::getClassName).collect(Collectors.toList()); | ||
|
||
if (classNames.isEmpty()) { | ||
return Collections.emptyList(); | ||
} else { | ||
long key = | ||
IntStream.concat( | ||
classLoaders.stream().mapToInt(Object::hashCode), | ||
IntStream.concat( | ||
IntStream.of(baseClazz.getName().hashCode()), | ||
configs.stream().mapToInt(AspectPluginConfig::hashCode))) | ||
.sum(); | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Interesting, so this is creating a key like Is this guaranteed to be unique? There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. This is unique within a given jvm execution. It might leak a bit when/if jars are being constantly replaced since each time that happens we'll be creating orphans. This was primarily added to help with tests where we run multiple concurrent test threads and memory usage was triggering OOM. |
||
|
||
return (List<T>) | ||
pluginCache.computeIfAbsent( | ||
key, | ||
k -> { | ||
try { | ||
ClassGraph classGraph = | ||
new ClassGraph() | ||
.acceptPackages(packageNames.stream().distinct().toArray(String[]::new)) | ||
.acceptClasses(classNames.stream().distinct().toArray(String[]::new)) | ||
.enableRemoteJarScanning() | ||
.enableExternalClasses() | ||
.enableClassInfo() | ||
.enableMethodInfo(); | ||
if (!classLoaders.isEmpty()) { | ||
classLoaders.forEach(classGraph::addClassLoader); | ||
} | ||
|
||
try (ScanResult scanResult = classGraph.scan()) { | ||
Map<String, ClassInfo> classMap = | ||
scanResult.getSubclasses(baseClazz).stream() | ||
.collect(Collectors.toMap(ClassInfo::getName, Function.identity())); | ||
|
||
return configs.stream() | ||
.map( | ||
config -> { | ||
try { | ||
ClassInfo classInfo = classMap.get(config.getClassName()); | ||
if (classInfo == null) { | ||
throw new IllegalStateException( | ||
String.format( | ||
"The following class cannot be loaded: %s", | ||
config.getClassName())); | ||
} | ||
MethodInfo constructorMethod = | ||
classInfo.getConstructorInfo().get(0); | ||
return ((T) | ||
constructorMethod | ||
.loadClassAndGetConstructor() | ||
.newInstance()) | ||
.setConfig(config); | ||
} catch (Exception e) { | ||
log.error( | ||
"Error constructing entity registry plugin class: {}", | ||
config.getClassName(), | ||
e); | ||
return Stream.<T>empty(); | ||
} | ||
}) | ||
.map(plugin -> (T) plugin) | ||
.filter(PluginSpec::enabled) | ||
.collect(Collectors.toList()); | ||
} | ||
} catch (Exception e) { | ||
throw new IllegalArgumentException( | ||
String.format( | ||
"Failed to load entity registry plugins: %s.", baseClazz.getName()), | ||
e); | ||
} | ||
}); | ||
} | ||
} | ||
|
||
/** | ||
|
@@ -187,68 +272,67 @@ private List<AspectPayloadValidator> buildAspectPayloadValidators( | |
: applyDisable( | ||
build( | ||
AspectPayloadValidator.class, | ||
pluginConfiguration.getAspectPayloadValidators(), | ||
VALIDATOR_PACKAGES)); | ||
pluginConfiguration.validatorPackages(), | ||
pluginConfiguration.getAspectPayloadValidators())); | ||
} | ||
|
||
private List<MutationHook> buildMutationHooks(@Nullable PluginConfiguration pluginConfiguration) { | ||
return pluginConfiguration == null | ||
? Collections.emptyList() | ||
: applyDisable( | ||
build(MutationHook.class, pluginConfiguration.getMutationHooks(), HOOK_PACKAGES)); | ||
build( | ||
MutationHook.class, | ||
pluginConfiguration.mutationPackages(), | ||
pluginConfiguration.getMutationHooks())); | ||
} | ||
|
||
private List<MCLSideEffect> buildMCLSideEffects( | ||
@Nullable PluginConfiguration pluginConfiguration) { | ||
return pluginConfiguration == null | ||
? Collections.emptyList() | ||
: applyDisable( | ||
build(MCLSideEffect.class, pluginConfiguration.getMclSideEffects(), HOOK_PACKAGES)); | ||
build( | ||
MCLSideEffect.class, | ||
pluginConfiguration.mclSideEffectPackages(), | ||
pluginConfiguration.getMclSideEffects())); | ||
} | ||
|
||
private List<MCPSideEffect> buildMCPSideEffects( | ||
@Nullable PluginConfiguration pluginConfiguration) { | ||
return pluginConfiguration == null | ||
? Collections.emptyList() | ||
: applyDisable( | ||
build(MCPSideEffect.class, pluginConfiguration.getMcpSideEffects(), HOOK_PACKAGES)); | ||
build( | ||
MCPSideEffect.class, | ||
pluginConfiguration.mcpSideEffectPackages(), | ||
pluginConfiguration.getMcpSideEffects())); | ||
} | ||
|
||
private <T> List<T> build( | ||
Class<?> baseClazz, List<AspectPluginConfig> configs, String... packageNames) { | ||
try (ScanResult scanResult = classGraph.acceptPackages(packageNames).scan()) { | ||
|
||
Map<String, ClassInfo> classMap = | ||
scanResult.getSubclasses(baseClazz).stream() | ||
.collect(Collectors.toMap(ClassInfo::getName, Function.identity())); | ||
|
||
return configs.stream() | ||
.flatMap( | ||
config -> { | ||
try { | ||
ClassInfo classInfo = classMap.get(config.getClassName()); | ||
if (classInfo == null) { | ||
throw new IllegalStateException( | ||
String.format( | ||
"The following class cannot be loaded: %s", config.getClassName())); | ||
} | ||
MethodInfo constructorMethod = classInfo.getConstructorInfo().get(0); | ||
return Stream.of( | ||
(T) constructorMethod.loadClassAndGetConstructor().newInstance(config)); | ||
} catch (Exception e) { | ||
log.error( | ||
"Error constructing entity registry plugin class: {}", | ||
config.getClassName(), | ||
e); | ||
return Stream.empty(); | ||
} | ||
}) | ||
.collect(Collectors.toList()); | ||
/** | ||
* Load plugins given the base class (i.e. a validator) and the name of the implementing class | ||
* found in the configuration objects. | ||
* | ||
* <p>For performance reasons, scan the packages found in packageNames | ||
* | ||
* <p>Designed to avoid any Spring dependency, see alternative implementation for Spring | ||
* | ||
* @param baseClazz base class for the plugin | ||
* @param configs configuration with implementing class information | ||
* @param packageNames package names to scan | ||
* @return list of plugin instances | ||
* @param <T> the plugin class | ||
*/ | ||
protected <T extends PluginSpec> List<T> build( | ||
Class<?> baseClazz, List<String> packageNames, List<AspectPluginConfig> configs) { | ||
List<AspectPluginConfig> nonSpringConfigs = | ||
configs.stream() | ||
.filter( | ||
config -> | ||
config.getSpring() == null | ||
|| Boolean.FALSE.equals(config.getSpring().isEnabled())) | ||
.collect(Collectors.toList()); | ||
|
||
} catch (Exception e) { | ||
throw new IllegalArgumentException( | ||
String.format("Failed to load entity registry plugins: %s.", baseClazz.getName()), e); | ||
} | ||
return initPlugins(classLoaders, baseClazz, packageNames, nonSpringConfigs); | ||
} | ||
|
||
@Nonnull | ||
|
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Where is the limit being applied?
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
The packages and class names are used in the acceptPackages and acceptClass