diff options
author | Øyvind Grønnesby <oyving@verizonmedia.com> | 2021-04-28 12:21:57 +0200 |
---|---|---|
committer | Øyvind Grønnesby <oyving@verizonmedia.com> | 2021-04-28 12:21:57 +0200 |
commit | dd2ef6cfc4d3d6e3735d1cb553f7ae2560a7f1ff (patch) | |
tree | e1ba3a56439bb3c16022b60d2a7ab3534037827e /container-core/src/main | |
parent | a0db2b1020ea53aa356a7547a23d4e1dfaa851c0 (diff) | |
parent | e79af49a3159e5505cd3e5f2605c299d38fe40cd (diff) |
Merge remote-tracking branch 'origin/master' into ogronnesby/billing-api-v2
Diffstat (limited to 'container-core/src/main')
106 files changed, 6189 insertions, 198 deletions
diff --git a/container-core/src/main/java/com/yahoo/container/bundle/BundleInstantiationSpecification.java b/container-core/src/main/java/com/yahoo/container/bundle/BundleInstantiationSpecification.java new file mode 100644 index 00000000000..0fb8a99a957 --- /dev/null +++ b/container-core/src/main/java/com/yahoo/container/bundle/BundleInstantiationSpecification.java @@ -0,0 +1,86 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.container.bundle; + +import com.yahoo.component.ComponentId; +import com.yahoo.component.ComponentSpecification; + + +/** + * Specifies how a component should be instantiated from a bundle. + * + * Immutable + * + * @author Tony Vaagenes + */ +public final class BundleInstantiationSpecification { + + public final ComponentId id; + public final ComponentSpecification classId; + public final ComponentSpecification bundle; + + public BundleInstantiationSpecification(ComponentSpecification id, ComponentSpecification classId, ComponentSpecification bundle) { + this.id = id.toId(); + this.classId = (classId != null) ? classId : id.withoutNamespace(); + this.bundle = (bundle != null) ? bundle : this.classId; + } + + // Must only be used when classId != null, otherwise the id must be handled as a ComponentSpecification + // (since converting a spec string to a ComponentId and then to a ComponentSpecification causes loss of information). + public BundleInstantiationSpecification(ComponentId id, ComponentSpecification classId, ComponentSpecification bundle) { + this(id.toSpecification(), classId, bundle); + assert (classId!= null); + } + + private static final String defaultInternalBundle = "container-search-and-docproc"; + + private static BundleInstantiationSpecification getInternalSpecificationFromString(String idSpec, String classSpec) { + return new BundleInstantiationSpecification( + new ComponentSpecification(idSpec), + (classSpec == null || classSpec.isEmpty())? null : new ComponentSpecification(classSpec), + new ComponentSpecification(defaultInternalBundle)); + } + + public static BundleInstantiationSpecification getInternalSearcherSpecification(ComponentSpecification idSpec, + ComponentSpecification classSpec) { + return new BundleInstantiationSpecification(idSpec, classSpec, new ComponentSpecification(defaultInternalBundle)); + } + + // TODO: These are the same for now because they are in the same bundle. + public static BundleInstantiationSpecification getInternalHandlerSpecificationFromStrings(String idSpec, String classSpec) { + return getInternalSpecificationFromString(idSpec, classSpec); + } + + public static BundleInstantiationSpecification getInternalProcessingSpecificationFromStrings(String idSpec, String classSpec) { + return getInternalSpecificationFromString(idSpec, classSpec); + } + + public static BundleInstantiationSpecification getInternalSearcherSpecificationFromStrings(String idSpec, String classSpec) { + return getInternalSpecificationFromString(idSpec, classSpec); + } + + public static BundleInstantiationSpecification getFromStrings(String idSpec, String classSpec, String bundleSpec) { + return new BundleInstantiationSpecification( + new ComponentSpecification(idSpec), + (classSpec == null || classSpec.isEmpty())? null : new ComponentSpecification(classSpec), + (bundleSpec == null || bundleSpec.isEmpty())? null : new ComponentSpecification(bundleSpec)); + } + + /** + * Return a new instance of the specification with bundle name altered + * + * @param bundleName the new name of the bundle + * @return the new instance of the specification + */ + public BundleInstantiationSpecification inBundle(String bundleName) { + return new BundleInstantiationSpecification(this.id, this.classId, new ComponentSpecification(bundleName)); + } + + public String getClassName() { + return classId.getName(); + } + + public BundleInstantiationSpecification nestInNamespace(ComponentId namespace) { + return new BundleInstantiationSpecification(id.nestInNamespace(namespace), classId, bundle); + } + +} diff --git a/container-core/src/main/java/com/yahoo/container/bundle/MockBundle.java b/container-core/src/main/java/com/yahoo/container/bundle/MockBundle.java new file mode 100644 index 00000000000..a6524b41886 --- /dev/null +++ b/container-core/src/main/java/com/yahoo/container/bundle/MockBundle.java @@ -0,0 +1,264 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.container.bundle; + +import org.osgi.framework.Bundle; +import org.osgi.framework.BundleContext; +import org.osgi.framework.ServiceReference; +import org.osgi.framework.Version; +import org.osgi.framework.wiring.BundleCapability; +import org.osgi.framework.wiring.BundleRequirement; +import org.osgi.framework.wiring.BundleRevision; +import org.osgi.framework.wiring.BundleWire; +import org.osgi.framework.wiring.BundleWiring; +import org.osgi.resource.Capability; +import org.osgi.resource.Requirement; +import org.osgi.resource.Wire; + +import java.io.File; +import java.io.InputStream; +import java.net.URL; +import java.security.cert.X509Certificate; +import java.util.Collection; +import java.util.Collections; +import java.util.Dictionary; +import java.util.Enumeration; +import java.util.Hashtable; +import java.util.List; +import java.util.Map; + +/** + * @author gjoranv + * @author ollivir + */ +public class MockBundle implements Bundle, BundleWiring { + public static final String SymbolicName = "mock-bundle"; + public static final Version BundleVersion = new Version(1, 0, 0); + + private static final Class<BundleWiring> bundleWiringClass = BundleWiring.class; + + @Override + public int getState() { + return Bundle.ACTIVE; + } + + @Override + public void start(int options) { + } + + @Override + public void start() { + } + + @Override + public void stop(int options) { + } + + @Override + public void stop() { + } + + @Override + public void update(InputStream input) { + } + + @Override + public void update() { + } + + @Override + public void uninstall() { + } + + @Override + public Dictionary<String, String> getHeaders(String locale) { + return getHeaders(); + } + + @Override + public String getSymbolicName() { + return SymbolicName; + } + + @Override + public Version getVersion() { + return BundleVersion; + } + + @Override + public String getLocation() { + return getSymbolicName(); + } + + @Override + public long getBundleId() { + return 0L; + } + + @Override + public Dictionary<String, String> getHeaders() { + return new Hashtable<>(); + } + + @Override + public ServiceReference<?>[] getRegisteredServices() { + return new ServiceReference<?>[0]; + } + + @Override + public ServiceReference<?>[] getServicesInUse() { + return getRegisteredServices(); + } + + @Override + public boolean hasPermission(Object permission) { + return true; + } + + @Override + public URL getResource(String name) { + throw new UnsupportedOperationException(); + } + + @Override + public Class<?> loadClass(String name) { + throw new UnsupportedOperationException(); + } + + @Override + public Enumeration<URL> getResources(String name) { + throw new UnsupportedOperationException(); + } + + @Override + public Enumeration<String> getEntryPaths(String path) { + throw new UnsupportedOperationException(); + } + + @Override + public URL getEntry(String path) { + throw new UnsupportedOperationException(); + } + + @Override + public Enumeration<URL> findEntries(String path, String filePattern, boolean recurse) { + return Collections.emptyEnumeration(); + } + + + @Override + public long getLastModified() { + return 1L; + } + + @Override + public BundleContext getBundleContext() { + throw new UnsupportedOperationException(); + } + + @Override + public Map<X509Certificate, List<X509Certificate>> getSignerCertificates(int signersType) { + return Collections.emptyMap(); + } + + @SuppressWarnings("unchecked") + @Override + public <T> T adapt(Class<T> type) { + if (type.equals(bundleWiringClass)) { + return (T) this; + } else { + throw new UnsupportedOperationException(); + } + } + + @Override + public File getDataFile(String filename) { + return null; + } + + @Override + public int compareTo(Bundle o) { + return Long.compare(getBundleId(), o.getBundleId()); + } + + + //TODO: replace with mockito + @Override + public List<URL> findEntries(String p1, String p2, int p3) { + throw new UnsupportedOperationException(); + } + + @Override + public List<Wire> getRequiredResourceWires(String p1) { + throw new UnsupportedOperationException(); + } + + @Override + public List<Capability> getResourceCapabilities(String p1) { + throw new UnsupportedOperationException(); + } + + @Override + public boolean isCurrent() { + throw new UnsupportedOperationException(); + } + + @Override + public List<BundleWire> getRequiredWires(String p1) { + throw new UnsupportedOperationException(); + } + + @Override + public List<BundleCapability> getCapabilities(String p1) { + throw new UnsupportedOperationException(); + } + + @Override + public List<Wire> getProvidedResourceWires(String p1) { + throw new UnsupportedOperationException(); + } + + @Override + public List<BundleWire> getProvidedWires(String p1) { + throw new UnsupportedOperationException(); + } + + @Override + public BundleRevision getRevision() { + throw new UnsupportedOperationException(); + } + + @Override + public List<Requirement> getResourceRequirements(String p1) { + throw new UnsupportedOperationException(); + } + + @Override + public boolean isInUse() { + throw new UnsupportedOperationException(); + } + + @Override + public Collection<String> listResources(String p1, String p2, int p3) { + return Collections.emptyList(); + } + + @Override + public ClassLoader getClassLoader() { + return MockBundle.class.getClassLoader(); + } + + @Override + public List<BundleRequirement> getRequirements(String p1) { + throw new UnsupportedOperationException(); + } + + @Override + public BundleRevision getResource() { + throw new UnsupportedOperationException(); + } + + @Override + public Bundle getBundle() { + throw new UnsupportedOperationException(); + } +} diff --git a/container-core/src/main/java/com/yahoo/container/bundle/package-info.java b/container-core/src/main/java/com/yahoo/container/bundle/package-info.java new file mode 100644 index 00000000000..c9707371626 --- /dev/null +++ b/container-core/src/main/java/com/yahoo/container/bundle/package-info.java @@ -0,0 +1,5 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +@ExportPackage +package com.yahoo.container.bundle; + +import com.yahoo.osgi.annotation.ExportPackage; diff --git a/container-core/src/main/java/com/yahoo/container/di/CloudSubscriberFactory.java b/container-core/src/main/java/com/yahoo/container/di/CloudSubscriberFactory.java new file mode 100644 index 00000000000..065733a719a --- /dev/null +++ b/container-core/src/main/java/com/yahoo/container/di/CloudSubscriberFactory.java @@ -0,0 +1,142 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.container.di; + +import com.yahoo.config.ConfigInstance; +import com.yahoo.config.subscription.ConfigHandle; +import com.yahoo.config.subscription.ConfigSource; +import com.yahoo.config.subscription.ConfigSourceSet; +import com.yahoo.config.subscription.ConfigSubscriber; +import com.yahoo.container.di.config.Subscriber; +import com.yahoo.container.di.config.SubscriberFactory; +import com.yahoo.vespa.config.ConfigKey; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.Set; +import java.util.WeakHashMap; +import java.util.logging.Level; +import java.util.logging.Logger; + +/** + * @author Tony Vaagenes + * @author ollivir + */ +public class CloudSubscriberFactory implements SubscriberFactory { + + private static final Logger log = Logger.getLogger(CloudSubscriberFactory.class.getName()); + + private final ConfigSource configSource; + private final Map<CloudSubscriber, Integer> activeSubscribers = new WeakHashMap<>(); + + private Optional<Long> testGeneration = Optional.empty(); + + public CloudSubscriberFactory(ConfigSource configSource) { + this.configSource = configSource; + } + + @Override + public Subscriber getSubscriber(Set<? extends ConfigKey<?>> configKeys) { + Set<ConfigKey<ConfigInstance>> subscriptionKeys = new HashSet<>(); + for(ConfigKey<?> key: configKeys) { + @SuppressWarnings("unchecked") // ConfigKey is defined as <CONFIGCLASS extends ConfigInstance> + ConfigKey<ConfigInstance> invariant = (ConfigKey<ConfigInstance>) key; + subscriptionKeys.add(invariant); + } + CloudSubscriber subscriber = new CloudSubscriber(subscriptionKeys, configSource); + + testGeneration.ifPresent(subscriber.subscriber::reload); // TODO: test specific code, remove + activeSubscribers.put(subscriber, 0); + + return subscriber; + } + + //TODO: test specific code, remove + @Override + public void reloadActiveSubscribers(long generation) { + testGeneration = Optional.of(generation); + + List<CloudSubscriber> subscribers = new ArrayList<>(activeSubscribers.keySet()); + subscribers.forEach(s -> s.subscriber.reload(generation)); + } + + private static class CloudSubscriber implements Subscriber { + + private final ConfigSubscriber subscriber; + private final Map<ConfigKey<ConfigInstance>, ConfigHandle<ConfigInstance>> handles = new HashMap<>(); + + // if waitNextGeneration has not yet been called, -1 should be returned + private long generation = -1L; + + private CloudSubscriber(Set<ConfigKey<ConfigInstance>> keys, ConfigSource configSource) { + this.subscriber = new ConfigSubscriber(configSource); + keys.forEach(k -> handles.put(k, subscriber.subscribe(k.getConfigClass(), k.getConfigId()))); + } + + @Override + public boolean configChanged() { + return handles.values().stream().anyMatch(ConfigHandle::isChanged); + } + + @Override + public long generation() { + return generation; + } + + //mapValues returns a view,, so we need to force evaluation of it here to prevent deferred evaluation. + @Override + public Map<ConfigKey<ConfigInstance>, ConfigInstance> config() { + Map<ConfigKey<ConfigInstance>, ConfigInstance> ret = new HashMap<>(); + handles.forEach((k, v) -> ret.put(k, v.getConfig())); + return ret; + } + + @Override + public long waitNextGeneration(boolean isInitializing) { + if (handles.isEmpty()) + throw new IllegalStateException("No config keys registered"); + + // Catch and just log config exceptions due to missing config values for parameters that do + // not have a default value. These exceptions occur when the user has removed a component + // from services.xml, and the component takes a config that has parameters without a + // default value in the def-file. There is a new 'components' config underway, where the + // component is removed, so this old config generation will soon be replaced by a new one. + boolean gotNextGen = false; + int numExceptions = 0; + while ( ! gotNextGen) { + try { + if (subscriber.nextGeneration(isInitializing)) + gotNextGen = true; + } + catch (IllegalArgumentException e) { + numExceptions++; + log.log(Level.WARNING, "Got exception from the config system (ignore if you just removed a " + + "component from your application that used the mentioned config) Subscriber info: " + + subscriber.toString(), e); + if (numExceptions >= 5) + throw new IllegalArgumentException("Failed retrieving the next config generation", e); + } + } + + generation = subscriber.getGeneration(); + return generation; + } + + @Override + public void close() { + subscriber.close(); + } + + } + + public static class Provider implements com.google.inject.Provider<SubscriberFactory> { + @Override + public SubscriberFactory get() { + return new CloudSubscriberFactory(ConfigSourceSet.createDefault()); + } + } + +} diff --git a/container-core/src/main/java/com/yahoo/container/di/ComponentDeconstructor.java b/container-core/src/main/java/com/yahoo/container/di/ComponentDeconstructor.java new file mode 100644 index 00000000000..4e3881a6fe6 --- /dev/null +++ b/container-core/src/main/java/com/yahoo/container/di/ComponentDeconstructor.java @@ -0,0 +1,18 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.container.di; + +import org.osgi.framework.Bundle; + +import java.util.Collection; +import java.util.List; + +/** + * @author gjoranv + * @author Tony Vaagenes + */ +public interface ComponentDeconstructor { + + /** Deconstructs the given components in order, then the given bundles. */ + void deconstruct(List<Object> components, Collection<Bundle> bundles); + +} diff --git a/container-core/src/main/java/com/yahoo/container/di/ConfigRetriever.java b/container-core/src/main/java/com/yahoo/container/di/ConfigRetriever.java new file mode 100644 index 00000000000..a7ff6c46a8b --- /dev/null +++ b/container-core/src/main/java/com/yahoo/container/di/ConfigRetriever.java @@ -0,0 +1,185 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.container.di; + +import com.google.common.collect.Sets; +import com.yahoo.config.ConfigInstance; +import com.yahoo.container.di.componentgraph.core.Keys; +import com.yahoo.container.di.config.Subscriber; +import com.yahoo.vespa.config.ConfigKey; + +import java.util.Collections; +import java.util.HashSet; +import java.util.Map; +import java.util.Optional; +import java.util.Set; +import java.util.function.Function; +import java.util.logging.Level; +import java.util.logging.Logger; + +import static java.util.logging.Level.FINE; + +/** + * @author Tony Vaagenes + * @author gjoranv + * @author ollivir + */ +public final class ConfigRetriever { + + private static final Logger log = Logger.getLogger(ConfigRetriever.class.getName()); + + private final Set<ConfigKey<? extends ConfigInstance>> bootstrapKeys; + private Set<ConfigKey<? extends ConfigInstance>> componentSubscriberKeys; + private final Subscriber bootstrapSubscriber; + private Subscriber componentSubscriber; + private final Function<Set<ConfigKey<? extends ConfigInstance>>, Subscriber> subscribe; + + public ConfigRetriever(Set<ConfigKey<? extends ConfigInstance>> bootstrapKeys, + Function<Set<ConfigKey<? extends ConfigInstance>>, Subscriber> subscribe) { + this.bootstrapKeys = bootstrapKeys; + this.componentSubscriberKeys = new HashSet<>(); + this.subscribe = subscribe; + if (bootstrapKeys.isEmpty()) { + throw new IllegalArgumentException("Bootstrap key set is empty"); + } + this.bootstrapSubscriber = subscribe.apply(bootstrapKeys); + this.componentSubscriber = subscribe.apply(componentSubscriberKeys); + } + + public ConfigSnapshot getConfigs(Set<ConfigKey<? extends ConfigInstance>> componentConfigKeys, + long leastGeneration, boolean isInitializing) { + // Loop until we get config. + while (true) { + Optional<ConfigSnapshot> maybeSnapshot = getConfigsOnce(componentConfigKeys, leastGeneration, isInitializing); + if (maybeSnapshot.isPresent()) { + var configSnapshot = maybeSnapshot.get(); + resetComponentSubscriberIfBootstrap(configSnapshot); + return configSnapshot; + } + } + } + + Optional<ConfigSnapshot> getConfigsOnce(Set<ConfigKey<? extends ConfigInstance>> componentConfigKeys, + long leastGeneration, boolean isInitializing) { + if (!Sets.intersection(componentConfigKeys, bootstrapKeys).isEmpty()) { + throw new IllegalArgumentException( + "Component config keys [" + componentConfigKeys + "] overlaps with bootstrap config keys [" + bootstrapKeys + "]"); + } + log.log(FINE, "getConfigsOnce: " + componentConfigKeys); + + Set<ConfigKey<? extends ConfigInstance>> allKeys = new HashSet<>(componentConfigKeys); + allKeys.addAll(bootstrapKeys); + setupComponentSubscriber(allKeys); + + return getConfigsOptional(leastGeneration, isInitializing); + } + + private Optional<ConfigSnapshot> getConfigsOptional(long leastGeneration, boolean isInitializing) { + long newestComponentGeneration = componentSubscriber.waitNextGeneration(isInitializing); + log.log(FINE, "getConfigsOptional: new component generation: " + newestComponentGeneration); + + // leastGeneration is only used to ensure newer generation when the previous generation was invalidated due to an exception + if (newestComponentGeneration < leastGeneration) { + return Optional.empty(); + } else if (bootstrapSubscriber.generation() < newestComponentGeneration) { + long newestBootstrapGeneration = bootstrapSubscriber.waitNextGeneration(isInitializing); + log.log(FINE, "getConfigsOptional: new bootstrap generation: " + bootstrapSubscriber.generation()); + Optional<ConfigSnapshot> bootstrapConfig = bootstrapConfigIfChanged(); + if (bootstrapConfig.isPresent()) { + return bootstrapConfig; + } else { + if (newestBootstrapGeneration == newestComponentGeneration) { + log.log(FINE, "Got new components configs with unchanged bootstrap configs."); + return componentsConfigIfChanged(); + } else { + // This should not be a normal case, and hence a warning to allow investigation. + log.warning("Did not get same generation for bootstrap (" + newestBootstrapGeneration + + ") and components configs (" + newestComponentGeneration + ")."); + return Optional.empty(); + } + } + } else { + // bootstrapGen==componentGen (happens only when a new component subscriber returns first config after bootstrap) + return componentsConfigIfChanged(); + } + } + + private Optional<ConfigSnapshot> bootstrapConfigIfChanged() { + return configIfChanged(bootstrapSubscriber, BootstrapConfigs::new); + } + + private Optional<ConfigSnapshot> componentsConfigIfChanged() { + return configIfChanged(componentSubscriber, ComponentsConfigs::new); + } + + private Optional<ConfigSnapshot> configIfChanged(Subscriber subscriber, + Function<Map<ConfigKey<? extends ConfigInstance>, ConfigInstance>, ConfigSnapshot> constructor) { + if (subscriber.configChanged()) { + return Optional.of(constructor.apply(Keys.covariantCopy(subscriber.config()))); + } else { + return Optional.empty(); + } + } + + private void resetComponentSubscriberIfBootstrap(ConfigSnapshot snapshot) { + if (snapshot instanceof BootstrapConfigs) { + setupComponentSubscriber(Collections.emptySet()); + } + } + + private void setupComponentSubscriber(Set<ConfigKey<? extends ConfigInstance>> keys) { + if (! componentSubscriberKeys.equals(keys)) { + componentSubscriber.close(); + componentSubscriberKeys = keys; + try { + log.log(FINE, "Setting up new component subscriber for keys: " + keys); + componentSubscriber = subscribe.apply(keys); + } catch (Throwable e) { + log.log(Level.WARNING, "Failed setting up subscriptions for component configs: " + e.getMessage()); + log.log(Level.WARNING, "Config keys: " + keys); + throw e; + } + } + } + + public void shutdown() { + bootstrapSubscriber.close(); + componentSubscriber.close(); + } + + //TODO: check if these are really needed + public long getBootstrapGeneration() { + return bootstrapSubscriber.generation(); + } + + public long getComponentsGeneration() { + return componentSubscriber.generation(); + } + + public static class ConfigSnapshot { + private final Map<ConfigKey<? extends ConfigInstance>, ConfigInstance> configs; + + ConfigSnapshot(Map<ConfigKey<? extends ConfigInstance>, ConfigInstance> configs) { + this.configs = configs; + } + + public Map<ConfigKey<? extends ConfigInstance>, ConfigInstance> configs() { + return configs; + } + + public int size() { + return configs.size(); + } + } + + public static class BootstrapConfigs extends ConfigSnapshot { + BootstrapConfigs(Map<ConfigKey<? extends ConfigInstance>, ConfigInstance> configs) { + super(configs); + } + } + + public static class ComponentsConfigs extends ConfigSnapshot { + ComponentsConfigs(Map<ConfigKey<? extends ConfigInstance>, ConfigInstance> configs) { + super(configs); + } + } +} diff --git a/container-core/src/main/java/com/yahoo/container/di/Container.java b/container-core/src/main/java/com/yahoo/container/di/Container.java new file mode 100644 index 00000000000..82c7f65bc2a --- /dev/null +++ b/container-core/src/main/java/com/yahoo/container/di/Container.java @@ -0,0 +1,289 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.container.di; + +import com.google.inject.Injector; +import com.yahoo.config.ConfigInstance; +import com.yahoo.config.ConfigurationRuntimeException; +import com.yahoo.config.subscription.ConfigInterruptedException; +import com.yahoo.container.ComponentsConfig; +import com.yahoo.container.bundle.BundleInstantiationSpecification; +import com.yahoo.container.di.ConfigRetriever.BootstrapConfigs; +import com.yahoo.container.di.ConfigRetriever.ComponentsConfigs; +import com.yahoo.container.di.ConfigRetriever.ConfigSnapshot; +import com.yahoo.container.di.componentgraph.core.ComponentGraph; +import com.yahoo.container.di.componentgraph.core.ComponentNode; +import com.yahoo.container.di.componentgraph.core.JerseyNode; +import com.yahoo.container.di.componentgraph.core.Node; +import com.yahoo.container.di.config.ApplicationBundlesConfig; +import com.yahoo.container.di.config.PlatformBundlesConfig; +import com.yahoo.container.di.config.RestApiContext; +import com.yahoo.container.di.config.SubscriberFactory; +import com.yahoo.vespa.config.ConfigKey; +import org.osgi.framework.Bundle; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.HashSet; +import java.util.IdentityHashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.logging.Level; +import java.util.logging.Logger; + +import static java.util.logging.Level.FINE; + +/** + * @author gjoranv + * @author Tony Vaagenes + * @author ollivir + */ +public class Container { + + private static final Logger log = Logger.getLogger(Container.class.getName()); + + private final SubscriberFactory subscriberFactory; + private final ConfigKey<ApplicationBundlesConfig> applicationBundlesConfigKey; + private final ConfigKey<PlatformBundlesConfig> platformBundlesConfigKey; + private final ConfigKey<ComponentsConfig> componentsConfigKey; + private final ComponentDeconstructor componentDeconstructor; + private final Osgi osgi; + + private final ConfigRetriever configurer; + private List<String> platformBundles; // Used to verify that platform bundles don't change. + private long previousConfigGeneration = -1L; + private long leastGeneration = -1L; + + public Container(SubscriberFactory subscriberFactory, String configId, ComponentDeconstructor componentDeconstructor, Osgi osgi) { + this.subscriberFactory = subscriberFactory; + this.componentDeconstructor = componentDeconstructor; + this.osgi = osgi; + + applicationBundlesConfigKey = new ConfigKey<>(ApplicationBundlesConfig.class, configId); + platformBundlesConfigKey = new ConfigKey<>(PlatformBundlesConfig.class, configId); + componentsConfigKey = new ConfigKey<>(ComponentsConfig.class, configId); + var bootstrapKeys = Set.of(applicationBundlesConfigKey, platformBundlesConfigKey, componentsConfigKey); + this.configurer = new ConfigRetriever(bootstrapKeys, subscriberFactory::getSubscriber); + } + + public Container(SubscriberFactory subscriberFactory, String configId, ComponentDeconstructor componentDeconstructor) { + this(subscriberFactory, configId, componentDeconstructor, new Osgi() { + }); + } + + public ComponentGraph getNewComponentGraph(ComponentGraph oldGraph, Injector fallbackInjector, boolean isInitializing) { + try { + Collection<Bundle> obsoleteBundles = new HashSet<>(); + ComponentGraph newGraph = getConfigAndCreateGraph(oldGraph, fallbackInjector, isInitializing, obsoleteBundles); + newGraph.reuseNodes(oldGraph); + constructComponents(newGraph); + deconstructObsoleteComponents(oldGraph, newGraph, obsoleteBundles); + return newGraph; + } catch (Throwable t) { + invalidateGeneration(oldGraph.generation(), t); + throw t; + } + } + + private ComponentGraph getConfigAndCreateGraph(ComponentGraph graph, + Injector fallbackInjector, + boolean isInitializing, + Collection<Bundle> obsoleteBundles) // NOTE: Return value + { + ConfigSnapshot snapshot; + while (true) { + snapshot = configurer.getConfigs(graph.configKeys(), leastGeneration, isInitializing); + + log.log(FINE, String.format("createNewGraph:\n" + "graph.configKeys = %s\n" + "graph.generation = %s\n" + "snapshot = %s\n", + graph.configKeys(), graph.generation(), snapshot)); + + if (snapshot instanceof BootstrapConfigs) { + if (getBootstrapGeneration() <= previousConfigGeneration) { + throw new IllegalStateException(String.format( + "Got bootstrap configs out of sequence for old config generation %d.\n" + "Previous config generation is %d", + getBootstrapGeneration(), previousConfigGeneration)); + } + log.log(FINE, "Got new bootstrap generation\n" + configGenerationsString()); + + if (graph.generation() == 0) { + platformBundles = getConfig(platformBundlesConfigKey, snapshot.configs()).bundlePaths(); + osgi.installPlatformBundles(platformBundles); + } else { + throwIfPlatformBundlesChanged(snapshot); + } + Collection<Bundle> bundlesToRemove = installApplicationBundles(snapshot.configs()); + obsoleteBundles.addAll(bundlesToRemove); + + graph = createComponentsGraph(snapshot.configs(), getBootstrapGeneration(), fallbackInjector); + + // Continues loop + + } else if (snapshot instanceof ComponentsConfigs) { + break; + } + } + log.log(FINE, "Got components configs,\n" + configGenerationsString()); + return createAndConfigureComponentsGraph(snapshot.configs(), fallbackInjector); + } + + private long getBootstrapGeneration() { + return configurer.getBootstrapGeneration(); + } + + private long getComponentsGeneration() { + return configurer.getComponentsGeneration(); + } + + private String configGenerationsString() { + return String.format("bootstrap generation = %d\n" + "components generation: %d\n" + "previous generation: %d", + getBootstrapGeneration(), getComponentsGeneration(), previousConfigGeneration); + } + + private void throwIfPlatformBundlesChanged(ConfigSnapshot snapshot) { + var checkPlatformBundles = getConfig(platformBundlesConfigKey, snapshot.configs()).bundlePaths(); + if (! checkPlatformBundles.equals(platformBundles)) + throw new RuntimeException("Platform bundles are not allowed to change!\nOld: " + platformBundles + "\nNew: " + checkPlatformBundles); + } + + private ComponentGraph createAndConfigureComponentsGraph(Map<ConfigKey<? extends ConfigInstance>, ConfigInstance> componentsConfigs, + Injector fallbackInjector) { + ComponentGraph componentGraph = createComponentsGraph(componentsConfigs, getComponentsGeneration(), fallbackInjector); + componentGraph.setAvailableConfigs(componentsConfigs); + return componentGraph; + } + + private void constructComponents(ComponentGraph graph) { + graph.nodes().forEach(Node::constructInstance); + } + + private void deconstructObsoleteComponents(ComponentGraph oldGraph, + ComponentGraph newGraph, + Collection<Bundle> obsoleteBundles) { + Map<Object, ?> newComponents = new IdentityHashMap<>(newGraph.size()); + for (Object component : newGraph.allConstructedComponentsAndProviders()) + newComponents.put(component, null); + + List<Object> obsoleteComponents = new ArrayList<>(); + for (Object component : oldGraph.allConstructedComponentsAndProviders()) + if ( ! newComponents.containsKey(component)) + obsoleteComponents.add(component); + + componentDeconstructor.deconstruct(obsoleteComponents, obsoleteBundles); + } + + private Set<Bundle> installApplicationBundles(Map<ConfigKey<? extends ConfigInstance>, ConfigInstance> configsIncludingBootstrapConfigs) { + ApplicationBundlesConfig applicationBundlesConfig = getConfig(applicationBundlesConfigKey, configsIncludingBootstrapConfigs); + return osgi.useApplicationBundles(applicationBundlesConfig.bundles()); + } + + private ComponentGraph createComponentsGraph(Map<ConfigKey<? extends ConfigInstance>, ConfigInstance> configsIncludingBootstrapConfigs, + long generation, Injector fallbackInjector) { + previousConfigGeneration = generation; + + ComponentGraph graph = new ComponentGraph(generation); + ComponentsConfig componentsConfig = getConfig(componentsConfigKey, configsIncludingBootstrapConfigs); + if (componentsConfig == null) { + throw new ConfigurationRuntimeException("The set of all configs does not include a valid 'components' config. Config set: " + + configsIncludingBootstrapConfigs.keySet()); + } + addNodes(componentsConfig, graph); + injectNodes(componentsConfig, graph); + + graph.complete(fallbackInjector); + return graph; + } + + private void addNodes(ComponentsConfig componentsConfig, ComponentGraph graph) { + + for (ComponentsConfig.Components config : componentsConfig.components()) { + BundleInstantiationSpecification specification = bundleInstantiationSpecification(config); + Class<?> componentClass = osgi.resolveClass(specification); + Node componentNode; + + if (RestApiContext.class.isAssignableFrom(componentClass)) { + Class<? extends RestApiContext> nodeClass = componentClass.asSubclass(RestApiContext.class); + componentNode = new JerseyNode(specification.id, config.configId(), nodeClass, osgi); + } else { + componentNode = new ComponentNode(specification.id, config.configId(), componentClass, null); + } + graph.add(componentNode); + } + } + + private void injectNodes(ComponentsConfig config, ComponentGraph graph) { + for (ComponentsConfig.Components component : config.components()) { + Node componentNode = ComponentGraph.getNode(graph, component.id()); + + for (ComponentsConfig.Components.Inject inject : component.inject()) { + //TODO: Support inject.name() + componentNode.inject(ComponentGraph.getNode(graph, inject.id())); + } + } + } + + private void invalidateGeneration(long generation, Throwable cause) { + leastGeneration = Math.max(configurer.getComponentsGeneration(), configurer.getBootstrapGeneration()) + 1; + if (!(cause instanceof InterruptedException) && !(cause instanceof ConfigInterruptedException)) { + log.log(Level.WARNING, newGraphErrorMessage(generation, cause), cause); + } + } + + private static String newGraphErrorMessage(long generation, Throwable cause) { + String failedFirstMessage = "Failed to set up first component graph"; + String failedNewMessage = "Failed to set up new component graph"; + String constructMessage = " due to error when constructing one of the components"; + String retainMessage = ". Retaining previous component generation."; + + if (generation == 0) { + if (cause instanceof ComponentNode.ComponentConstructorException) { + return failedFirstMessage + constructMessage; + } else { + return failedFirstMessage; + } + } else { + if (cause instanceof ComponentNode.ComponentConstructorException) { + return failedNewMessage + constructMessage + retainMessage; + } else { + return failedNewMessage + retainMessage; + } + } + } + + public void shutdown(ComponentGraph graph, ComponentDeconstructor deconstructor) { + shutdownConfigurer(); + if (graph != null) { + deconstructAllComponents(graph, deconstructor); + } + } + + void shutdownConfigurer() { + configurer.shutdown(); + } + + // Reload config manually, when subscribing to non-configserver sources + public void reloadConfig(long generation) { + subscriberFactory.reloadActiveSubscribers(generation); + } + + private void deconstructAllComponents(ComponentGraph graph, ComponentDeconstructor deconstructor) { + // This is only used for shutdown, so no need to uninstall any bundles. + deconstructor.deconstruct(graph.allConstructedComponentsAndProviders(), Collections.emptyList()); + } + + public static <T extends ConfigInstance> T getConfig(ConfigKey<T> key, + Map<ConfigKey<? extends ConfigInstance>, ConfigInstance> configs) { + ConfigInstance inst = configs.get(key); + + if (inst == null || key.getConfigClass() == null) { + throw new RuntimeException("Missing config " + key); + } + + return key.getConfigClass().cast(inst); + } + + private static BundleInstantiationSpecification bundleInstantiationSpecification(ComponentsConfig.Components config) { + return BundleInstantiationSpecification.getFromStrings(config.id(), config.classId(), config.bundle()); + } + +} diff --git a/container-core/src/main/java/com/yahoo/container/di/Osgi.java b/container-core/src/main/java/com/yahoo/container/di/Osgi.java new file mode 100644 index 00000000000..940986e2f38 --- /dev/null +++ b/container-core/src/main/java/com/yahoo/container/di/Osgi.java @@ -0,0 +1,54 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.container.di; + +import com.yahoo.component.ComponentSpecification; +import com.yahoo.config.FileReference; +import com.yahoo.container.bundle.BundleInstantiationSpecification; +import com.yahoo.container.bundle.MockBundle; +import com.yahoo.container.di.osgi.BundleClasses; +import org.osgi.framework.Bundle; + +import java.util.Collection; +import java.util.Collections; +import java.util.Set; +import java.util.stream.Collectors; + +import static java.util.Collections.emptySet; + +/** + * @author gjoranv + * @author Tony Vaagenes + * @author ollivir + */ +public interface Osgi { + default BundleClasses getBundleClasses(ComponentSpecification bundle, Set<String> packagesToScan) { + return new BundleClasses(new MockBundle(), Collections.emptySet()); + } + + default void installPlatformBundles(Collection<String> bundlePaths) { + System.out.println("installPlatformBundles " + bundlePaths); + } + + /** + * Returns the set of bundles that is not used by the current application generation, + * and therefore should be scheduled for uninstalling. + */ + default Set<Bundle> useApplicationBundles(Collection<FileReference> bundles) { + System.out.println("useBundles " + bundles.stream().map(Object::toString).collect(Collectors.joining(", "))); + return emptySet(); + } + + default Class<?> resolveClass(BundleInstantiationSpecification spec) { + System.out.println("resolving class " + spec.classId); + try { + return Class.forName(spec.classId.getName()); + } catch (ClassNotFoundException e) { + throw new RuntimeException(e); + } + } + + default Bundle getBundle(ComponentSpecification spec) { + System.out.println("resolving bundle " + spec); + return new MockBundle(); + } +} diff --git a/container-core/src/main/java/com/yahoo/container/di/componentgraph/core/ComponentGraph.java b/container-core/src/main/java/com/yahoo/container/di/componentgraph/core/ComponentGraph.java new file mode 100644 index 00000000000..71d0e539b5a --- /dev/null +++ b/container-core/src/main/java/com/yahoo/container/di/componentgraph/core/ComponentGraph.java @@ -0,0 +1,430 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.container.di.componentgraph.core; + +import com.google.common.collect.Iterables; +import com.google.common.collect.Sets; +import com.google.inject.BindingAnnotation; +import com.google.inject.ConfigurationException; +import com.google.inject.Guice; +import com.google.inject.Injector; +import com.google.inject.Key; +import com.yahoo.collections.Pair; +import com.yahoo.component.ComponentId; +import com.yahoo.component.provider.ComponentRegistry; +import com.yahoo.config.ConfigInstance; +import com.yahoo.container.di.componentgraph.Provider; +import com.yahoo.container.di.componentgraph.cycle.CycleFinder; +import com.yahoo.container.di.componentgraph.cycle.Graph; +import com.yahoo.vespa.config.ConfigKey; + +import java.lang.annotation.Annotation; +import java.lang.reflect.ParameterizedType; +import java.lang.reflect.Type; +import java.lang.reflect.TypeVariable; +import java.lang.reflect.WildcardType; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.Collections; +import java.util.HashMap; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.Set; +import java.util.logging.Level; +import java.util.logging.Logger; +import java.util.stream.Collectors; + +import static com.yahoo.container.di.componentgraph.core.Exceptions.removeStackTrace; + +/** + * @author Tony Vaagenes + * @author gjoranv + * @author ollivir + * + * Not thread safe. + */ +public class ComponentGraph { + + private static final Logger log = Logger.getLogger(ComponentGraph.class.getName()); + + private final long generation; + private final Map<ComponentId, Node> nodesById = new HashMap<>(); + + public ComponentGraph(long generation) { + this.generation = generation; + } + + public ComponentGraph() { + this(0L); + } + + public long generation() { + return generation; + } + + public int size() { + return nodesById.size(); + } + + public Collection<Node> nodes() { + return nodesById.values(); + } + + public void add(Node component) { + if (nodesById.containsKey(component.componentId())) { + throw new IllegalStateException("Multiple components with the same id " + component.componentId()); + } + nodesById.put(component.componentId(), component); + } + + private Optional<Node> lookupGlobalComponent(Key<?> key) { + if (!(key.getTypeLiteral().getType() instanceof Class)) { + throw new RuntimeException("Type not supported " + key.getTypeLiteral()); + } + Class<?> clazz = key.getTypeLiteral().getRawType(); + + Collection<ComponentNode> components = matchingComponentNodes(nodes(), key); + if (components.isEmpty()) { + return Optional.empty(); + } else if (components.size() == 1) { + return Optional.ofNullable(Iterables.get(components, 0)); + } else { + + List<Node> nonProviderComponents = components.stream().filter(c -> !Provider.class.isAssignableFrom(c.instanceType())) + .collect(Collectors.toList()); + if (nonProviderComponents.isEmpty()) { + throw new IllegalStateException("Multiple global component providers for class '" + clazz.getName() + "' found"); + } else if (nonProviderComponents.size() == 1) { + return Optional.of(nonProviderComponents.get(0)); + } else { + throw new IllegalStateException("Multiple global components with class '" + clazz.getName() + "' found"); + } + } + } + + public <T> T getInstance(Class<T> clazz) { + return getInstance(Key.get(clazz)); + } + + @SuppressWarnings("unchecked") + public <T> T getInstance(Key<T> key) { + // TODO: Combine exception handling with lookupGlobalComponent. + Object ob = lookupGlobalComponent(key).map(Node::component) + .orElseThrow(() -> new IllegalStateException(String.format("No global component with key '%s' ", key))); + return (T) ob; + } + + private Collection<ComponentNode> componentNodes() { + return nodesOfType(nodes(), ComponentNode.class); + } + + private Collection<ComponentRegistryNode> componentRegistryNodes() { + return nodesOfType(nodes(), ComponentRegistryNode.class); + } + + private Collection<ComponentNode> osgiComponentsOfClass(Class<?> clazz) { + return componentNodes().stream().filter(node -> clazz.isAssignableFrom(node.componentType())).collect(Collectors.toList()); + } + + public List<Node> complete(Injector fallbackInjector) { + componentNodes().forEach(node -> completeNode(node, fallbackInjector)); + componentRegistryNodes().forEach(this::completeComponentRegistryNode); + return topologicalSort(nodes()); + } + + public List<Node> complete() { + return complete(Guice.createInjector()); + } + + public Set<ConfigKey<? extends ConfigInstance>> configKeys() { + return nodes().stream().flatMap(node -> node.configKeys().stream()).collect(Collectors.toSet()); + } + + public void setAvailableConfigs(Map<ConfigKey<? extends ConfigInstance>, ConfigInstance> configs) { + componentNodes().forEach(node -> node.setAvailableConfigs(Keys.invariantCopy(configs))); + } + + public void reuseNodes(ComponentGraph old) { + // copy instances if node equal + Set<ComponentId> commonComponentIds = Sets.intersection(nodesById.keySet(), old.nodesById.keySet()); + for (ComponentId id : commonComponentIds) { + if (nodesById.get(id).equals(old.nodesById.get(id))) { + nodesById.get(id).instance = old.nodesById.get(id).instance; + } + } + + // reset instances with modified dependencies + for (Node node : topologicalSort(nodes())) { + for (Node usedComponent : node.usedComponents()) { + if (usedComponent.instance.isEmpty()) { + node.instance = Optional.empty(); + } + } + } + } + + /** All constructed components and providers of this, in reverse creation order, i.e., suited for ordered deconstruction. */ + public List<Object> allConstructedComponentsAndProviders() { + List<Node> orderedNodes = topologicalSort(nodes()); + Collections.reverse(orderedNodes); + return orderedNodes.stream().map(node -> node.constructedInstance().get()).collect(Collectors.toList()); + } + + private void completeComponentRegistryNode(ComponentRegistryNode registry) { + registry.injectAll(osgiComponentsOfClass(registry.componentClass())); + } + + private void completeNode(ComponentNode node, Injector fallbackInjector) { + try { + Object[] arguments = node.getAnnotatedConstructorParams().stream().map(param -> handleParameter(node, fallbackInjector, param)) + .toArray(); + + node.setArguments(arguments); + } catch (Exception e) { + throw removeStackTrace(new RuntimeException("When resolving dependencies of " + node.idAndType(), e)); + } + } + + private Object handleParameter(Node node, Injector fallbackInjector, Pair<Type, List<Annotation>> annotatedParameterType) { + Type parameterType = annotatedParameterType.getFirst(); + List<Annotation> annotations = annotatedParameterType.getSecond(); + + if (parameterType instanceof Class && parameterType.equals(ComponentId.class)) { + return node.componentId(); + } else if (parameterType instanceof Class && ConfigInstance.class.isAssignableFrom((Class<?>) parameterType)) { + return handleConfigParameter((ComponentNode) node, (Class<?>) parameterType); + } else if (parameterType instanceof ParameterizedType + && ((ParameterizedType) parameterType).getRawType().equals(ComponentRegistry.class)) { + ParameterizedType registry = (ParameterizedType) parameterType; + return getComponentRegistry(registry.getActualTypeArguments()[0]); + } else if (parameterType instanceof Class) { + return handleComponentParameter(node, fallbackInjector, (Class<?>) parameterType, annotations); + } else if (parameterType instanceof ParameterizedType) { + throw new RuntimeException("Injection of parameterized type " + parameterType + " is not supported."); + } else { + throw new RuntimeException("Injection of type " + parameterType + " is not supported"); + } + } + + private ComponentRegistryNode newComponentRegistryNode(Class<?> componentClass) { + ComponentRegistryNode registry = new ComponentRegistryNode(componentClass); + add(registry); //TODO: don't mutate nodes here. + return registry; + } + + private ComponentRegistryNode getComponentRegistry(Type componentType) { + Class<?> componentClass; + if (componentType instanceof WildcardType) { + WildcardType wildcardType = (WildcardType) componentType; + if (wildcardType.getLowerBounds().length > 0 || wildcardType.getUpperBounds().length > 1) { + throw new RuntimeException("Can't create ComponentRegistry of unknown wildcard type" + wildcardType); + } + componentClass = (Class<?>) wildcardType.getUpperBounds()[0]; + } else if (componentType instanceof Class) { + componentClass = (Class<?>) componentType; + } else if (componentType instanceof TypeVariable) { + throw new RuntimeException("Can't create ComponentRegistry of unknown type variable " + componentType); + } else { + throw new RuntimeException("Can't create ComponentRegistry of unknown type " + componentType); + } + + for (ComponentRegistryNode node : componentRegistryNodes()) { + if (node.componentClass().equals(componentType)) { + return node; + } + } + return newComponentRegistryNode(componentClass); + } + + @SuppressWarnings("unchecked") + private ConfigKey<ConfigInstance> handleConfigParameter(ComponentNode node, Class<?> clazz) { + Class<ConfigInstance> castClass = (Class<ConfigInstance>) clazz; + return new ConfigKey<>(castClass, node.configId()); + } + + private <T> Key<T> getKey(Class<T> clazz, Optional<Annotation> bindingAnnotation) { + return bindingAnnotation.map(annotation -> Key.get(clazz, annotation)).orElseGet(() -> Key.get(clazz)); + } + + private Optional<GuiceNode> matchingGuiceNode(Key<?> key, Object instance) { + return matchingNodes(nodes(), GuiceNode.class, key).stream().filter(node -> node.component() == instance). // TODO: assert that there is only one (after filter) + findFirst(); + } + + private Node lookupOrCreateGlobalComponent(Node node, Injector fallbackInjector, Class<?> clazz, Key<?> key) { + Optional<Node> component = lookupGlobalComponent(key); + if (component.isEmpty()) { + Object instance; + try { + log.log(Level.INFO, "Trying the fallback injector to create" + messageForNoGlobalComponent(clazz, node)); + instance = fallbackInjector.getInstance(key); + } catch (ConfigurationException e) { + throw removeStackTrace(new IllegalStateException( + (messageForMultipleClassLoaders(clazz).isEmpty()) ? "No global" + messageForNoGlobalComponent(clazz, node) + : messageForMultipleClassLoaders(clazz))); + } + component = Optional.of(matchingGuiceNode(key, instance).orElseGet(() -> { + GuiceNode guiceNode = new GuiceNode(instance, key.getAnnotation()); + add(guiceNode); + return guiceNode; + })); + } + return component.get(); + } + + private Node handleComponentParameter(Node node, Injector fallbackInjector, Class<?> clazz, Collection<Annotation> annotations) { + + List<Annotation> bindingAnnotations = annotations.stream().filter(ComponentGraph::isBindingAnnotation).collect(Collectors.toList()); + Key<?> key = getKey(clazz, bindingAnnotations.stream().findFirst()); + + if (bindingAnnotations.size() > 1) { + throw new RuntimeException(String.format("More than one binding annotation used in class '%s'", node.instanceType())); + } + + Collection<ComponentNode> injectedNodesOfCorrectType = matchingComponentNodes(node.componentsToInject, key); + if (injectedNodesOfCorrectType.size() == 0) { + return lookupOrCreateGlobalComponent(node, fallbackInjector, clazz, key); + } else if (injectedNodesOfCorrectType.size() == 1) { + return Iterables.get(injectedNodesOfCorrectType, 0); + } else { + //TODO: !className for last parameter + throw new RuntimeException( + String.format("Multiple components of type '%s' injected into component '%s'", clazz.getName(), node.instanceType())); + } + } + + private static String messageForNoGlobalComponent(Class<?> clazz, Node node) { + return String.format(" component of class %s to inject into component %s.", clazz.getName(), node.idAndType()); + } + + private String messageForMultipleClassLoaders(Class<?> clazz) { + String errMsg = "Class " + clazz.getName() + " is provided by the framework, and cannot be embedded in a user bundle. " + + "To resolve this problem, please refer to osgi-classloading.html#multiple-implementations in the documentation"; + + try { + Class<?> resolvedClass = Class.forName(clazz.getName(), false, this.getClass().getClassLoader()); + if (!resolvedClass.equals(clazz)) { + return errMsg; + } + } catch (ClassNotFoundException ignored) { + + } + return ""; + } + + public static Node getNode(ComponentGraph graph, String componentId) { + return graph.nodesById.get(new ComponentId(componentId)); + } + + private static <T> Collection<T> nodesOfType(Collection<Node> nodes, Class<T> clazz) { + List<T> ret = new ArrayList<>(); + for (Node node : nodes) { + if (clazz.isInstance(node)) { + ret.add(clazz.cast(node)); + } + } + return ret; + } + + private static Collection<ComponentNode> matchingComponentNodes(Collection<Node> nodes, Key<?> key) { + return matchingNodes(nodes, ComponentNode.class, key); + } + + // Finds all nodes with a given nodeType and instance with given key + private static <T extends Node> Collection<T> matchingNodes(Collection<Node> nodes, Class<T> nodeType, Key<?> key) { + Class<?> clazz = key.getTypeLiteral().getRawType(); + Annotation annotation = key.getAnnotation(); + + List<T> filteredByClass = nodesOfType(nodes, nodeType).stream().filter(node -> clazz.isAssignableFrom(node.componentType())) + .collect(Collectors.toList()); + + if (filteredByClass.size() == 1) { + return filteredByClass; + } else { + List<T> filteredByClassAndAnnotation = filteredByClass.stream() + .filter(node -> (annotation == null && node.instanceKey().getAnnotation() == null) + || annotation.equals(node.instanceKey().getAnnotation())) + .collect(Collectors.toList()); + if (filteredByClassAndAnnotation.size() > 0) { + return filteredByClassAndAnnotation; + } else { + return filteredByClass; + } + } + } + + // Returns true if annotation is a BindingAnnotation, e.g. com.google.inject.name.Named + public static boolean isBindingAnnotation(Annotation annotation) { + LinkedList<Class<?>> queue = new LinkedList<>(); + queue.add(annotation.getClass()); + queue.addAll(Arrays.asList(annotation.getClass().getInterfaces())); + + while (!queue.isEmpty()) { + Class<?> clazz = queue.removeFirst(); + if (clazz.getAnnotation(BindingAnnotation.class) != null) { + return true; + } else { + if (clazz.getSuperclass() != null) { + queue.addFirst(clazz.getSuperclass()); + } + } + } + return false; + } + + /** + * The returned list is the nodes from the graph bottom-up. + * + * For each iteration, the algorithm finds the components that are not "wanted by" any other component, + * and prepends those components into the resulting 'sorted' list. Hence, the first element in the returned + * list is the component that is directly or indirectly wanted by "most" other components. + * + * @return A list where a earlier than b in the list implies that there is no path from a to b + */ + private static List<Node> topologicalSort(Collection<Node> nodes) { + Map<ComponentId, Integer> numIncoming = new HashMap<>(); + + nodes.forEach( + node -> node.usedComponents().forEach( + injectedNode -> numIncoming.merge(injectedNode.componentId(), 1, (a, b) -> a + b))); + + LinkedList<Node> sorted = new LinkedList<>(); + List<Node> unsorted = new ArrayList<>(nodes); + + while (!unsorted.isEmpty()) { + List<Node> ready = new ArrayList<>(); + List<Node> notReady = new ArrayList<>(); + unsorted.forEach(node -> { + if (numIncoming.getOrDefault(node.componentId(), 0) == 0) { + ready.add(node); + } else { + notReady.add(node); + } + }); + + if (ready.isEmpty()) { + throw new IllegalStateException("There is a cycle in the component injection graph: " + findCycle(notReady)); + } + + ready.forEach(node -> node.usedComponents() + .forEach(injectedNode -> numIncoming.merge(injectedNode.componentId(), -1, (a, b) -> a + b))); + sorted.addAll(0, ready); + unsorted = notReady; + } + return sorted; + } + + private static List<String> findCycle(List<Node> nodes) { + var cyclicGraph = new Graph<String>(); + for (var node : nodes) { + for (var adjacent : node.usedComponents()) { + cyclicGraph.edge(node.componentId().stringValue(), + adjacent.componentId().stringValue()); + } + } + return new CycleFinder<>(cyclicGraph).findCycle(); + } + +} diff --git a/container-core/src/main/java/com/yahoo/container/di/componentgraph/core/ComponentNode.java b/container-core/src/main/java/com/yahoo/container/di/componentgraph/core/ComponentNode.java new file mode 100644 index 00000000000..b6fa4241e26 --- /dev/null +++ b/container-core/src/main/java/com/yahoo/container/di/componentgraph/core/ComponentNode.java @@ -0,0 +1,313 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.container.di.componentgraph.core; + +import com.google.inject.Inject; +import com.google.inject.Key; +import com.yahoo.collections.Pair; +import com.yahoo.component.AbstractComponent; +import com.yahoo.component.ComponentId; +import com.yahoo.config.ConfigInstance; +import com.yahoo.container.di.componentgraph.Provider; +import com.yahoo.vespa.config.ConfigKey; + +import java.lang.annotation.Annotation; +import java.lang.reflect.Constructor; +import java.lang.reflect.InvocationTargetException; +import java.lang.reflect.Modifier; +import java.lang.reflect.ParameterizedType; +import java.lang.reflect.Type; +import java.time.Duration; +import java.time.Instant; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Comparator; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.logging.Logger; +import java.util.stream.Collectors; + +import static com.yahoo.container.di.componentgraph.core.Exceptions.cutStackTraceAtConstructor; +import static com.yahoo.container.di.componentgraph.core.Exceptions.removeStackTrace; +import static com.yahoo.container.di.componentgraph.core.Keys.createKey; +import static java.util.logging.Level.FINE; +import static java.util.logging.Level.INFO; + +/** + * @author Tony Vaagenes + * @author gjoranv + * @author ollivir + */ +public class ComponentNode extends Node { + + private static final Logger log = Logger.getLogger(ComponentNode.class.getName()); + + private final Class<?> clazz; + private final Annotation key; + private Object[] arguments = null; + private final String configId; + + private final Constructor<?> constructor; + + private Map<ConfigKey<ConfigInstance>, ConfigInstance> availableConfigs = null; + + + public ComponentNode(ComponentId componentId, + String configId, + Class<?> clazz, Annotation XXX_key) // TODO expose key, not javaAnnotation + { + super(componentId); + if (isAbstract(clazz)) { + throw new IllegalArgumentException("Can't instantiate abstract class " + clazz.getName()); + } + this.configId = configId; + this.clazz = clazz; + this.key = XXX_key; + this.constructor = bestConstructor(clazz); + } + + public ComponentNode(ComponentId componentId, String configId, Class<?> clazz) { + this(componentId, configId, clazz, null); + } + + public String configId() { + return configId; + } + + @Override + public Key<?> instanceKey() { + return createKey(clazz, key); + } + + @Override + public Class<?> instanceType() { + return clazz; + } + + @Override + public List<Node> usedComponents() { + if (arguments == null) { + throw new IllegalStateException("Arguments must be set first."); + } + List<Node> ret = new ArrayList<>(); + for (Object arg : arguments) { + if (arg instanceof Node) { + ret.add((Node) arg); + } + } + return ret; + } + + private static List<Class<?>> allSuperClasses(Class<?> clazz) { + List<Class<?>> ret = new ArrayList<>(); + while (clazz != null) { + ret.add(clazz); + clazz = clazz.getSuperclass(); + } + return ret; + } + + @Override + public Class<?> componentType() { + if (Provider.class.isAssignableFrom(clazz)) { + //TODO: Test what happens if you ask for something that isn't a class, e.g. a parameterized type. + + List<Type> allGenericInterfaces = allSuperClasses(clazz).stream().flatMap(c -> Arrays.stream(c.getGenericInterfaces())).collect(Collectors.toList()); + for (Type t : allGenericInterfaces) { + if (t instanceof ParameterizedType && ((ParameterizedType) t).getRawType().equals(Provider.class)) { + Type[] typeArgs = ((ParameterizedType) t).getActualTypeArguments(); + if (typeArgs != null && typeArgs.length > 0) { + return (Class<?>) typeArgs[0]; + } + } + } + throw new IllegalStateException("Component type cannot be resolved"); + } else { + return clazz; + } + } + + public void setArguments(Object[] arguments) { + this.arguments = arguments; + } + + @Override + protected Object newInstance() { + if (arguments == null) { + throw new IllegalStateException("graph.complete must be called before retrieving instances."); + } + + List<Object> actualArguments = new ArrayList<>(); + for (Object ob : arguments) { + if (ob instanceof Node) { + actualArguments.add(((Node) ob).component()); + } else if (ob instanceof ConfigKey) { + actualArguments.add(availableConfigs.get(ob)); + } else { + actualArguments.add(ob); + } + } + + Object instance; + try { + log.log(FINE, () -> "Constructing " + idAndType()); + Instant start = Instant.now(); + instance = constructor.newInstance(actualArguments.toArray()); + Duration duration = Duration.between(start, Instant.now()); + log.log(duration.compareTo(Duration.ofMinutes(1)) > 0 ? INFO : FINE, + () -> "Finished constructing " + idAndType() + " in " + duration); + } catch (InvocationTargetException | InstantiationException | IllegalAccessException e) { + StackTraceElement dependencyInjectorMarker = new StackTraceElement("============= Dependency Injection =============", "newInstance", null, -1); + throw removeStackTrace(new ComponentConstructorException("Error constructing " + idAndType() + ": " + e.getMessage(), cutStackTraceAtConstructor(e.getCause(), dependencyInjectorMarker))); + } + + return initId(instance); + } + + private Object initId(Object component) { + if (component instanceof AbstractComponent) { + AbstractComponent abstractComponent = (AbstractComponent) component; + if (abstractComponent.hasInitializedId() && !abstractComponent.getId().equals(componentId())) { + throw new IllegalStateException( + "Component with id '" + componentId() + "' is trying to set its component id explicitly: '" + abstractComponent.getId() + "'. " + + "This is not allowed, so please remove any call to super() in your component's constructor."); + } + abstractComponent.initId(componentId()); + } + return component; + } + + + @Override + public int hashCode() { + final int prime = 31; + int result = super.hashCode(); + result = prime * result + Arrays.hashCode(arguments); + result = prime * result + ((availableConfigs == null) ? 0 : availableConfigs.hashCode()); + result = prime * result + ((configId == null) ? 0 : configId.hashCode()); + return result; + } + + @Override + public boolean equals(Object other) { + if (other instanceof ComponentNode) { + ComponentNode that = (ComponentNode) other; + return super.equals(that) && equalEdges(Arrays.asList(this.arguments), Arrays.asList(that.arguments)) && this.usedConfigs().equals(that.usedConfigs()); + } else { + return false; + } + } + + private List<ConfigInstance> usedConfigs() { + if (availableConfigs == null) { + throw new IllegalStateException("setAvailableConfigs must be called!"); + } + List<ConfigInstance> ret = new ArrayList<>(); + for (Object arg : arguments) { + if (arg instanceof ConfigKey) { + ret.add(availableConfigs.get(arg)); + } + } + return ret; + } + + protected List<Pair<Type, List<Annotation>>> getAnnotatedConstructorParams() { + Type[] types = constructor.getGenericParameterTypes(); + Annotation[][] annotations = constructor.getParameterAnnotations(); + + List<Pair<Type, List<Annotation>>> ret = new ArrayList<>(); + + for (int i = 0; i < types.length; i++) { + ret.add(new Pair<>(types[i], Arrays.asList(annotations[i]))); + } + return ret; + } + + public void setAvailableConfigs(Map<ConfigKey<ConfigInstance>, ConfigInstance> configs) { + if (arguments == null) { + throw new IllegalStateException("graph.complete must be called before graph.setAvailableConfigs."); + } + this.availableConfigs = configs; + } + + @Override + public Set<ConfigKey<ConfigInstance>> configKeys() { + return configParameterClasses().stream().map(par -> new ConfigKey<>(par, configId)).collect(Collectors.toSet()); + } + + @SuppressWarnings("unchecked") + private List<Class<ConfigInstance>> configParameterClasses() { + List<Class<ConfigInstance>> ret = new ArrayList<>(); + for (Type type : constructor.getGenericParameterTypes()) { + if (type instanceof Class && ConfigInstance.class.isAssignableFrom((Class<?>) type)) { + ret.add((Class<ConfigInstance>) type); + } + } + return ret; + } + + @Override + public String label() { + LinkedList<String> configNames = configKeys().stream().map(k -> k.getName() + ".def").collect(Collectors.toCollection(LinkedList::new)); + + configNames.addFirst(instanceType().getSimpleName()); + configNames.addFirst(Node.packageName(instanceType())); + + return "{" + String.join("|", configNames) + "}"; + } + + private static Constructor<?> bestConstructor(Class<?> clazz) { + Constructor<?>[] publicConstructors = clazz.getConstructors(); + + Constructor<?> annotated = null; + for (Constructor<?> ctor : publicConstructors) { + Annotation annotation = ctor.getAnnotation(Inject.class); + if (annotation != null) { + if (annotated == null) { + annotated = ctor; + } else { + throw componentConstructorException("Multiple constructor annotated with @Inject in class " + clazz.getName()); + } + } + } + if (annotated != null) { + return annotated; + } + + if (publicConstructors.length == 0) { + throw componentConstructorException("No public constructors in class " + clazz.getName()); + } else if (publicConstructors.length == 1) { + return publicConstructors[0]; + } else { + log.warning(String.format("Multiple public constructors found in class %s, there should only be one. " + + "If more than one public constructor is needed, the primary one must be annotated with @Inject.", clazz.getName())); + List<Pair<Constructor<?>, Integer>> withParameterCount = new ArrayList<>(); + for (Constructor<?> ctor : publicConstructors) { + long count = Arrays.stream(ctor.getParameterTypes()).filter(ConfigInstance.class::isAssignableFrom).count(); + withParameterCount.add(new Pair<>(ctor, (int) count)); + } + withParameterCount.sort(Comparator.comparingInt(Pair::getSecond)); + return withParameterCount.get(withParameterCount.size() - 1).getFirst(); + } + } + + private static ComponentConstructorException componentConstructorException(String message) { + return removeStackTrace(new ComponentConstructorException(message)); + } + + public static class ComponentConstructorException extends RuntimeException { + ComponentConstructorException(String message) { + super(message); + } + + ComponentConstructorException(String message, Throwable cause) { + super(message, cause); + } + } + + + private static boolean isAbstract(Class<?> clazz) { + return Modifier.isAbstract(clazz.getModifiers()); + } +}
\ No newline at end of file diff --git a/container-core/src/main/java/com/yahoo/container/di/componentgraph/core/ComponentRegistryNode.java b/container-core/src/main/java/com/yahoo/container/di/componentgraph/core/ComponentRegistryNode.java new file mode 100644 index 00000000000..429052c0039 --- /dev/null +++ b/container-core/src/main/java/com/yahoo/container/di/componentgraph/core/ComponentRegistryNode.java @@ -0,0 +1,107 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.container.di.componentgraph.core; + +import com.google.inject.Key; +import com.google.inject.util.Types; +import com.yahoo.component.ComponentId; +import com.yahoo.component.provider.ComponentRegistry; +import com.yahoo.config.ConfigInstance; +import com.yahoo.vespa.config.ConfigKey; + +import java.util.Collections; +import java.util.List; +import java.util.Set; +import java.util.stream.Collectors; + +/** + * @author Tony Vaagenes + * @author gjoranv + * @author ollivir + */ +public class ComponentRegistryNode extends Node { + + private static ComponentId componentRegistryNamespace = ComponentId.fromString("ComponentRegistry"); + + private final Class<?> componentClass; + + public ComponentRegistryNode(Class<?> componentClass) { + super(componentId(componentClass)); + this.componentClass = componentClass; + } + + @Override + public List<Node> usedComponents() { + return componentsToInject; + } + + @Override + protected Object newInstance() { + ComponentRegistry<Object> registry = new ComponentRegistry<>(); + componentsToInject.forEach(component -> registry.register(component.componentId(), component.component())); + + return registry; + } + + @Override + public Key<?> instanceKey() { + return Key.get(Types.newParameterizedType(ComponentRegistry.class, componentClass)); + } + + @Override + public Class<?> instanceType() { + return instanceKey().getTypeLiteral().getRawType(); + } + + @Override + public Class<?> componentType() { + return instanceType(); + } + + public Class<?> componentClass() { + return componentClass; + } + + @Override + public Set<ConfigKey<ConfigInstance>> configKeys() { + return Collections.emptySet(); + } + + @Override + public int hashCode() { + final int prime = 31; + int result = super.hashCode(); + result = prime * result + ((componentClass == null) ? 0 : componentClass.hashCode()); + return result; + } + + @Override + public boolean equals(Object other) { + if (other instanceof ComponentRegistryNode) { + ComponentRegistryNode that = (ComponentRegistryNode) other; + return this.componentId().equals(that.componentId()) && this.instanceType().equals(that.instanceType()) + && equalNodeEdges(this.usedComponents(), that.usedComponents()); + } else { + return false; + } + } + + @Override + public String label() { + return String.format("{ComponentRegistry\\<%s\\>|%s}", componentClass.getSimpleName(), Node.packageName(componentClass)); + } + + private static ComponentId componentId(Class<?> componentClass) { + return syntheticComponentId(componentClass.getName(), componentClass, componentRegistryNamespace); + } + + public static boolean equalNodeEdges(List<Node> edges, List<Node> otherEdges) { + if (edges.size() == otherEdges.size()) { + List<ComponentId> left = edges.stream().map(Node::componentId).sorted().collect(Collectors.toList()); + List<ComponentId> right = otherEdges.stream().map(Node::componentId).sorted().collect(Collectors.toList()); + return left.equals(right); + } else { + return false; + } + } + +} diff --git a/container-core/src/main/java/com/yahoo/container/di/componentgraph/core/Exceptions.java b/container-core/src/main/java/com/yahoo/container/di/componentgraph/core/Exceptions.java new file mode 100644 index 00000000000..b0d9d1f3921 --- /dev/null +++ b/container-core/src/main/java/com/yahoo/container/di/componentgraph/core/Exceptions.java @@ -0,0 +1,47 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.container.di.componentgraph.core; + +import java.util.Arrays; + +class Exceptions { + + static <E extends Throwable> E removeStackTrace(E exception) { + if (preserveStackTrace()) { + return exception; + } else { + exception.setStackTrace(new StackTraceElement[0]); + return exception; + } + } + + static boolean preserveStackTrace() { + String preserve = System.getProperty("jdisc.container.preserveStackTrace"); + return (preserve != null && !preserve.isEmpty()); + } + + static Throwable cutStackTraceAtConstructor(Throwable throwable, StackTraceElement marker) { + if (throwable != null && !preserveStackTrace()) { + StackTraceElement[] stackTrace = throwable.getStackTrace(); + int upTo = stackTrace.length - 1; + + // take until ComponentNode is reached + while (upTo >= 0 && !stackTrace[upTo].getClassName().equals(ComponentNode.class.getName())) { + upTo--; + } + + // then drop until <init> is reached + while (upTo >= 0 && !stackTrace[upTo].getMethodName().equals("<init>")) { + upTo--; + } + if (upTo < 0) { + throwable.setStackTrace(new StackTraceElement[0]); + } else { + throwable.setStackTrace(Arrays.copyOfRange(stackTrace, 0, upTo)); + } + + cutStackTraceAtConstructor(throwable.getCause(), marker); + } + return throwable; + } + +} diff --git a/container-core/src/main/java/com/yahoo/container/di/componentgraph/core/GuiceNode.java b/container-core/src/main/java/com/yahoo/container/di/componentgraph/core/GuiceNode.java new file mode 100644 index 00000000000..61d0d9bba8d --- /dev/null +++ b/container-core/src/main/java/com/yahoo/container/di/componentgraph/core/GuiceNode.java @@ -0,0 +1,78 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.container.di.componentgraph.core; + +import com.google.inject.Key; +import com.yahoo.component.ComponentId; +import com.yahoo.config.ConfigInstance; +import com.yahoo.vespa.config.ConfigKey; + +import java.lang.annotation.Annotation; +import java.util.Collections; +import java.util.List; +import java.util.Set; + +import static com.yahoo.container.di.componentgraph.core.Keys.createKey; + +/** + * @author Tony Vaagenes + * @author gjoranv + * @author ollivir + */ +public final class GuiceNode extends Node { + private static final ComponentId guiceNamespace = ComponentId.fromString("Guice"); + + private final Object myInstance; + private final Annotation annotation; + + public GuiceNode(Object myInstance, + Annotation annotation) { + super(componentId(myInstance)); + this.myInstance = myInstance; + this.annotation = annotation; + } + + @Override + public Set<ConfigKey<ConfigInstance>> configKeys() { + return Collections.emptySet(); + } + + @Override + public Key<?> instanceKey() { + return createKey(myInstance.getClass(), annotation); + } + + @Override + public Class<?> instanceType() { + return myInstance.getClass(); + } + + @Override + public Class<?> componentType() { + return instanceType(); + } + + + @Override + public List<Node> usedComponents() { + return Collections.emptyList(); + } + + @Override + protected Object newInstance() { + return myInstance; + } + + @Override + public void inject(Node component) { + throw new UnsupportedOperationException("Illegal to inject components to a GuiceNode!"); + } + + @Override + public String label() { + return String.format("{{%s|Guice}|%s}", instanceType().getSimpleName(), Node.packageName(instanceType())); + } + + private static ComponentId componentId(Object instance) { + return Node.syntheticComponentId(instance.getClass().getName(), instance, guiceNamespace); + } +} diff --git a/container-core/src/main/java/com/yahoo/container/di/componentgraph/core/JerseyNode.java b/container-core/src/main/java/com/yahoo/container/di/componentgraph/core/JerseyNode.java new file mode 100644 index 00000000000..0f8aa678934 --- /dev/null +++ b/container-core/src/main/java/com/yahoo/container/di/componentgraph/core/JerseyNode.java @@ -0,0 +1,92 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.container.di.componentgraph.core; + +import com.yahoo.component.ComponentId; +import com.yahoo.component.ComponentSpecification; +import com.yahoo.container.di.Osgi; +import com.yahoo.container.di.config.JerseyBundlesConfig; +import com.yahoo.container.di.config.RestApiContext; +import com.yahoo.container.di.config.RestApiContext.BundleInfo; +import com.yahoo.container.di.osgi.BundleClasses; +import org.osgi.framework.Bundle; +import org.osgi.framework.wiring.BundleWiring; + +import java.net.URL; +import java.util.Collection; +import java.util.HashSet; +import java.util.List; + +/** + * Represents an instance of RestApiContext + * + * @author gjoranv + * @author Tony Vaagenes + * @author ollivir + */ +public class JerseyNode extends ComponentNode { + private static final String WEB_INF_URL = "WebInfUrl"; + + private final Osgi osgi; + + public JerseyNode(ComponentId componentId, String configId, Class<?> clazz, Osgi osgi) { + super(componentId, configId, clazz, null); + this.osgi = osgi; + } + + @Override + protected RestApiContext newInstance() { + Object instance = super.newInstance(); + RestApiContext restApiContext = (RestApiContext) instance; + + List<JerseyBundlesConfig.Bundles> bundles = restApiContext.bundlesConfig.bundles(); + for (JerseyBundlesConfig.Bundles bundleConfig : bundles) { + BundleClasses bundleClasses = osgi.getBundleClasses(ComponentSpecification.fromString(bundleConfig.spec()), + new HashSet<>(bundleConfig.packages())); + + restApiContext.addBundle(createBundleInfo(bundleClasses.bundle(), bundleClasses.classEntries())); + } + + componentsToInject.forEach(component -> restApiContext.addInjectableComponent(component.instanceKey(), component.componentId(), + component.component())); + + return restApiContext; + } + + @Override + public int hashCode() { + return super.hashCode(); + } + + @Override + public boolean equals(Object other) { + return super.equals(other) + && (other instanceof JerseyNode && this.componentsToInject.equals(((JerseyNode) other).componentsToInject)); + } + + public static BundleInfo createBundleInfo(Bundle bundle, Collection<String> classEntries) { + BundleInfo bundleInfo = new BundleInfo(bundle.getSymbolicName(), bundle.getVersion(), bundle.getLocation(), webInfUrl(bundle), + bundle.adapt(BundleWiring.class).getClassLoader()); + + bundleInfo.setClassEntries(classEntries); + return bundleInfo; + } + + public static Bundle getBundle(Osgi osgi, String bundleSpec) { + Bundle bundle = osgi.getBundle(ComponentSpecification.fromString(bundleSpec)); + if (bundle == null) { + throw new IllegalArgumentException("Bundle not found: " + bundleSpec); + } + return bundle; + } + + private static URL webInfUrl(Bundle bundle) { + String webInfUrlHeader = bundle.getHeaders().get(WEB_INF_URL); + + if (webInfUrlHeader == null) { + return null; + } else { + return bundle.getEntry(webInfUrlHeader); + } + } + +} diff --git a/container-core/src/main/java/com/yahoo/container/di/componentgraph/core/Keys.java b/container-core/src/main/java/com/yahoo/container/di/componentgraph/core/Keys.java new file mode 100644 index 00000000000..be80fc1616d --- /dev/null +++ b/container-core/src/main/java/com/yahoo/container/di/componentgraph/core/Keys.java @@ -0,0 +1,39 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.container.di.componentgraph.core; + +import com.google.inject.Key; +import com.yahoo.config.ConfigInstance; +import com.yahoo.vespa.config.ConfigKey; + +import java.lang.annotation.Annotation; +import java.lang.reflect.Type; +import java.util.HashMap; +import java.util.Map; + +/** + * @author ollivir + */ +public class Keys { + + static Key<?> createKey(Type instanceType, Annotation annotation) { + if (annotation == null) { + return Key.get(instanceType); + } else { + return Key.get(instanceType, annotation); + } + } + + @SuppressWarnings("unchecked") + public static Map<ConfigKey<ConfigInstance>, ConfigInstance> invariantCopy(Map<ConfigKey<? extends ConfigInstance>, ConfigInstance> configs) { + Map<ConfigKey<ConfigInstance>, ConfigInstance> ret = new HashMap<>(); + configs.forEach((k, v) -> ret.put((ConfigKey<ConfigInstance>) k, v)); + return ret; + } + + public static Map<ConfigKey<? extends ConfigInstance>, ConfigInstance> covariantCopy(Map<ConfigKey<ConfigInstance>, ConfigInstance> configs) { + Map<ConfigKey<? extends ConfigInstance>, ConfigInstance> ret = new HashMap<>(); + configs.forEach((k, v) -> ret.put(k, v)); + return ret; + } + +} diff --git a/container-core/src/main/java/com/yahoo/container/di/componentgraph/core/Node.java b/container-core/src/main/java/com/yahoo/container/di/componentgraph/core/Node.java new file mode 100644 index 00000000000..3afc8bb817c --- /dev/null +++ b/container-core/src/main/java/com/yahoo/container/di/componentgraph/core/Node.java @@ -0,0 +1,162 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.container.di.componentgraph.core; + +import com.google.inject.Key; +import com.yahoo.component.ComponentId; +import com.yahoo.config.ConfigInstance; +import com.yahoo.container.di.componentgraph.Provider; +import com.yahoo.vespa.config.ConfigKey; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.Iterator; +import java.util.List; +import java.util.Optional; +import java.util.Set; +import java.util.logging.Logger; + +import static java.util.logging.Level.FINE; + +/** + * @author Tony Vaagenes + * @author gjoranv + * @author ollivir + */ +public abstract class Node { + + private final static Logger log = Logger.getLogger(Node.class.getName()); + + private final ComponentId componentId; + protected Optional<Object> instance = Optional.empty(); + List<Node> componentsToInject = new ArrayList<>(); + + public Node(ComponentId componentId) { + this.componentId = componentId; + } + + public abstract Key<?> instanceKey(); + + /** + * The components actually used by this node. Consist of a subset of the injected nodes + subset of the global nodes. + */ + public abstract List<Node> usedComponents(); + + protected abstract Object newInstance(); + + /** Constructs the instance represented by this node, if not already done. */ + public void constructInstance() { + if ( ! instance.isPresent()) + instance = Optional.of(newInstance()); + } + + /** + * Returns the component represented by this - which is either the instance, or if the instance is a provider, + * the component returned by it. + */ + public Object component() { + constructInstance(); + if (instance.get() instanceof Provider) { + Provider<?> provider = (Provider<?>) instance.get(); + return provider.get(); + } else { + return instance.get(); + } + } + + public abstract Set<ConfigKey<ConfigInstance>> configKeys(); + + public void inject(Node component) { + componentsToInject.add(component); + } + + public void injectAll(Collection<ComponentNode> componentNodes) { + componentNodes.forEach(this::inject); + } + + public abstract Class<?> instanceType(); + + public abstract Class<?> componentType(); + + public abstract String label(); + + public String idAndType() { + String className = instanceType().getName(); + + if (className.equals(componentId.getName())) { + return "'" + componentId + "'"; + } else { + return "'" + componentId + "' of type '" + className + "'"; + } + } + + private static boolean equalNodes(Object a, Object b) { + if (a instanceof Node && b instanceof Node) { + Node l = (Node) a; + Node r = (Node) b; + return l.componentId.equals(r.componentId); + } else { + return a.equals(b); + } + } + + public static boolean equalEdges(List<?> edges1, List<?> edges2) { + Iterator<?> right = edges2.iterator(); + for (Object l : edges1) { + if (!right.hasNext()) { + return false; + } + Object r = right.next(); + if (!equalNodes(l, r)) { + return false; + } + } + return !right.hasNext(); + } + + @Override + public int hashCode() { + final int prime = 31; + int result = 1; + result = prime * result + ((componentId == null) ? 0 : componentId.hashCode()); + result = prime * result + ((componentsToInject == null) ? 0 : componentsToInject.hashCode()); + return result; + } + + @Override + public boolean equals(Object other) { + if (other instanceof Node) { + Node that = (Node) other; + return getClass().equals(that.getClass()) && this.componentId.equals(that.componentId) + && this.instanceType().equals(that.instanceType()) && equalEdges(this.usedComponents(), that.usedComponents()); + } else { + return false; + } + } + + public ComponentId componentId() { + return componentId; + } + + /** Returns the already constructed instance in this, if any */ + public Optional<?> constructedInstance() { + return instance; + } + + /** + * @param identityObject he identifying object that makes the Node unique + */ + protected static ComponentId syntheticComponentId(String className, Object identityObject, ComponentId namespace) { + String name = className + "_" + System.identityHashCode(identityObject); + return ComponentId.fromString(name).nestInNamespace(namespace); + } + + public static String packageName(Class<?> componentClass) { + String fullClassName = componentClass.getName(); + int index = fullClassName.lastIndexOf('.'); + if (index < 0) { + return ""; + } else { + return fullClassName.substring(0, index); + } + } +} diff --git a/container-core/src/main/java/com/yahoo/container/di/componentgraph/core/package-info.java b/container-core/src/main/java/com/yahoo/container/di/componentgraph/core/package-info.java new file mode 100644 index 00000000000..e9b5b14d5d8 --- /dev/null +++ b/container-core/src/main/java/com/yahoo/container/di/componentgraph/core/package-info.java @@ -0,0 +1,5 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +@ExportPackage +package com.yahoo.container.di.componentgraph.core; + +import com.yahoo.osgi.annotation.ExportPackage; diff --git a/container-core/src/main/java/com/yahoo/container/di/componentgraph/cycle/CycleFinder.java b/container-core/src/main/java/com/yahoo/container/di/componentgraph/cycle/CycleFinder.java new file mode 100644 index 00000000000..327949bb8d0 --- /dev/null +++ b/container-core/src/main/java/com/yahoo/container/di/componentgraph/cycle/CycleFinder.java @@ -0,0 +1,95 @@ +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.container.di.componentgraph.cycle; + +import java.util.ArrayList; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.logging.Logger; +import java.util.stream.Collectors; + +import static com.yahoo.container.di.componentgraph.cycle.CycleFinder.State.BLACK; +import static com.yahoo.container.di.componentgraph.cycle.CycleFinder.State.GRAY; +import static com.yahoo.container.di.componentgraph.cycle.CycleFinder.State.WHITE; +import static java.util.logging.Level.FINE; +import static java.util.Collections.singletonList; + + +/** + * <p>Applies the + * <a href="https://www.geeksforgeeks.org/detect-cycle-direct-graph-using-colors/"> three-color algorithm</a> + * to detect a cycle in a directed graph. If there are multiple cycles, this implementation only detects one + * of them and does not guarantee that the shortest cycle is found. + * </p> + * + * @author gjoranv + */ +public class CycleFinder<T> { + private static final Logger log = Logger.getLogger(CycleFinder.class.getName()); + + enum State { + WHITE, GRAY, BLACK; + } + + private final Graph<T> graph; + + private Map<T, State> colors; + + private List<T> cycle; + + public CycleFinder(Graph<T> graph) { + this.graph = graph; + } + + private void resetState() { + cycle = null; + colors = new LinkedHashMap<>(); + graph.getVertices().forEach(v -> colors.put(v, WHITE)); + } + + /** + * Returns a list of vertices constituting a cycle in the graph, or an empty + * list if no cycle was found. Only the first encountered cycle is returned. + */ + public List<T> findCycle() { + resetState(); + for (T vertex : graph.getVertices()) { + if (colors.get(vertex) == WHITE) { + if (visitDepthFirst(vertex, new ArrayList<>(singletonList(vertex)))) { + if (cycle == null) throw new IllegalStateException("Null cycle - this should never happen"); + if (cycle.isEmpty()) throw new IllegalStateException("Empty cycle - this should never happen"); + log.log(FINE, "Cycle detected: " + cycle); + return cycle; + } + } + } + return new ArrayList<>(); + } + + private boolean visitDepthFirst(T vertex, List<T> path) { + colors.put(vertex, GRAY); + log.log(FINE, "Vertex start " + vertex + " - colors: " + colors + " - path: " + path); + for (T adjacent : graph.getAdjacent(vertex)) { + path.add(adjacent); + if (colors.get(adjacent) == GRAY) { + cycle = removePathIntoCycle(path); + return true; + } + if (colors.get(adjacent) == WHITE && visitDepthFirst(adjacent, path)) { + return true; + } + path.remove(adjacent); + } + colors.put(vertex, BLACK); + log.log(FINE, "Vertex end " + vertex + " - colors: " + colors + " - path: " + path); + return false; + } + + private List<T> removePathIntoCycle(List<T> pathWithCycle) { + T cycleStart = pathWithCycle.get(pathWithCycle.size() - 1); + return pathWithCycle.stream() + .dropWhile(vertex -> ! vertex.equals(cycleStart)) + .collect(Collectors.toList()); + } + +} diff --git a/container-core/src/main/java/com/yahoo/container/di/componentgraph/cycle/Graph.java b/container-core/src/main/java/com/yahoo/container/di/componentgraph/cycle/Graph.java new file mode 100644 index 00000000000..946330668bd --- /dev/null +++ b/container-core/src/main/java/com/yahoo/container/di/componentgraph/cycle/Graph.java @@ -0,0 +1,41 @@ +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +package com.yahoo.container.di.componentgraph.cycle; + +import java.util.LinkedHashMap; +import java.util.LinkedHashSet; +import java.util.Map; +import java.util.Set; + +/** + * Class representing a directed graph. + * + * @author gjoranv + */ +public class Graph<T> { + + private final Map<T, LinkedHashSet<T>> adjMap = new LinkedHashMap<>(); + + public void edge(T from, T to) { + if (from == null || to == null) + throw new IllegalArgumentException("Null vertices are not allowed, edge: " + from + "->" + to); + + adjMap.computeIfAbsent(from, k -> new LinkedHashSet<>()).add(to); + adjMap.computeIfAbsent(to, k -> new LinkedHashSet<>()); + } + + Set<T> getVertices() { + return adjMap.keySet(); + } + + /** + * Returns the outgoing edges of the given vertex. + */ + Set<T> getAdjacent(T vertex) { + return adjMap.get(vertex); + } + + private void throwIfMissingVertex(T vertex) { + if (! adjMap.containsKey(vertex)) throw new IllegalArgumentException("No such vertex in the graph: " + vertex); + } +} diff --git a/container-core/src/main/java/com/yahoo/container/di/config/ResolveDependencyException.java b/container-core/src/main/java/com/yahoo/container/di/config/ResolveDependencyException.java new file mode 100644 index 00000000000..c88f851909c --- /dev/null +++ b/container-core/src/main/java/com/yahoo/container/di/config/ResolveDependencyException.java @@ -0,0 +1,13 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.container.di.config; + +/** + * @author gjoranv + */ +public class ResolveDependencyException extends RuntimeException { + + public ResolveDependencyException(String message) { + super(message); + } + +} diff --git a/container-core/src/main/java/com/yahoo/container/di/config/RestApiContext.java b/container-core/src/main/java/com/yahoo/container/di/config/RestApiContext.java new file mode 100644 index 00000000000..bfb9a8f9160 --- /dev/null +++ b/container-core/src/main/java/com/yahoo/container/di/config/RestApiContext.java @@ -0,0 +1,98 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.container.di.config; + +import com.google.common.collect.ImmutableSet; +import com.google.inject.Inject; +import com.google.inject.Key; +import com.yahoo.component.ComponentId; +import org.osgi.framework.Version; + +import java.net.URL; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.List; +import java.util.Set; + +/** + * Only for internal JDisc use. + * + * @author gjoranv + */ +public class RestApiContext { + + private final List<BundleInfo> bundles = new ArrayList<>(); + private final List<Injectable> injectableComponents = new ArrayList<>(); + + public final JerseyBundlesConfig bundlesConfig; + public final JerseyInjectionConfig injectionConfig; + + @Inject + public RestApiContext(JerseyBundlesConfig bundlesConfig, JerseyInjectionConfig injectionConfig) { + this.bundlesConfig = bundlesConfig; + this.injectionConfig = injectionConfig; + } + + public List<BundleInfo> getBundles() { + return Collections.unmodifiableList(bundles); + } + + public void addBundle(BundleInfo bundle) { + bundles.add(bundle); + } + + public List<Injectable> getInjectableComponents() { + return Collections.unmodifiableList(injectableComponents); + } + + public void addInjectableComponent(Key<?> key, ComponentId id, Object component) { + injectableComponents.add(new Injectable(key, id, component)); + } + + public static class Injectable { + public final Key<?> key; + public final ComponentId id; + public final Object instance; + + public Injectable(Key<?> key, ComponentId id, Object instance) { + this.key = key; + this.id = id; + this.instance = instance; + } + @Override + public String toString() { + return id.toString(); + } + } + + public static class BundleInfo { + public final String symbolicName; + public final Version version; + public final String fileLocation; + public final URL webInfUrl; + public final ClassLoader classLoader; + + private Set<String> classEntries; + + public BundleInfo(String symbolicName, Version version, String fileLocation, URL webInfUrl, ClassLoader classLoader) { + this.symbolicName = symbolicName; + this.version = version; + this.fileLocation = fileLocation; + this.webInfUrl = webInfUrl; + this.classLoader = classLoader; + } + + @Override + public String toString() { + return symbolicName + ":" + version; + } + + public void setClassEntries(Collection<String> entries) { + this.classEntries = ImmutableSet.copyOf(entries); + } + + public Set<String> getClassEntries() { + return classEntries; + } + } +} diff --git a/container-core/src/main/java/com/yahoo/container/di/config/Subscriber.java b/container-core/src/main/java/com/yahoo/container/di/config/Subscriber.java new file mode 100644 index 00000000000..60207447bfd --- /dev/null +++ b/container-core/src/main/java/com/yahoo/container/di/config/Subscriber.java @@ -0,0 +1,23 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.container.di.config; + +import com.yahoo.config.ConfigInstance; +import com.yahoo.vespa.config.ConfigKey; + +import java.util.Map; + +/** + * @author Tony Vaagenes + * @author gjoranv + */ +public interface Subscriber { + + long waitNextGeneration(boolean isInitializing); + long generation(); + + boolean configChanged(); + Map<ConfigKey<ConfigInstance>, ConfigInstance> config(); + + void close(); + +} diff --git a/container-core/src/main/java/com/yahoo/container/di/config/SubscriberFactory.java b/container-core/src/main/java/com/yahoo/container/di/config/SubscriberFactory.java new file mode 100644 index 00000000000..c1c36a1b3de --- /dev/null +++ b/container-core/src/main/java/com/yahoo/container/di/config/SubscriberFactory.java @@ -0,0 +1,20 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.container.di.config; + +import com.google.inject.ProvidedBy; +import com.yahoo.container.di.CloudSubscriberFactory; +import com.yahoo.vespa.config.ConfigKey; + +import java.util.Set; + +/** + * @author Tony Vaagenes + * @author gjoranv + */ +@ProvidedBy(CloudSubscriberFactory.Provider.class) +public interface SubscriberFactory { + + Subscriber getSubscriber(Set<? extends ConfigKey<?>> configKeys); + void reloadActiveSubscribers(long generation); + +} diff --git a/container-core/src/main/java/com/yahoo/container/di/config/package-info.java b/container-core/src/main/java/com/yahoo/container/di/config/package-info.java new file mode 100644 index 00000000000..b8f65b1c3c8 --- /dev/null +++ b/container-core/src/main/java/com/yahoo/container/di/config/package-info.java @@ -0,0 +1,5 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +@ExportPackage +package com.yahoo.container.di.config; + +import com.yahoo.osgi.annotation.ExportPackage; diff --git a/container-core/src/main/java/com/yahoo/container/di/osgi/BundleClasses.java b/container-core/src/main/java/com/yahoo/container/di/osgi/BundleClasses.java new file mode 100644 index 00000000000..bca3ed73d0b --- /dev/null +++ b/container-core/src/main/java/com/yahoo/container/di/osgi/BundleClasses.java @@ -0,0 +1,27 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.container.di.osgi; + +import org.osgi.framework.Bundle; + +import java.util.Collection; + +/** + * @author ollivir + */ +public class BundleClasses { + private final Bundle bundle; + private final Collection<String> classEntries; + + public BundleClasses(Bundle bundle, Collection<String> classEntries) { + this.bundle = bundle; + this.classEntries = classEntries; + } + + public Bundle bundle() { + return bundle; + } + + public Collection<String> classEntries() { + return classEntries; + } +} diff --git a/container-core/src/main/java/com/yahoo/container/di/osgi/OsgiUtil.java b/container-core/src/main/java/com/yahoo/container/di/osgi/OsgiUtil.java new file mode 100644 index 00000000000..e1854155e5b --- /dev/null +++ b/container-core/src/main/java/com/yahoo/container/di/osgi/OsgiUtil.java @@ -0,0 +1,168 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.container.di.osgi; + +import com.yahoo.component.ComponentSpecification; +import com.yahoo.osgi.maven.ProjectBundleClassPaths; +import com.yahoo.osgi.maven.ProjectBundleClassPaths.BundleClasspathMapping; +import org.osgi.framework.Bundle; +import org.osgi.framework.wiring.BundleWiring; + +import java.io.File; +import java.io.IOException; +import java.net.URISyntaxException; +import java.net.URL; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; +import java.util.Set; +import java.util.function.Predicate; +import java.util.jar.JarEntry; +import java.util.jar.JarFile; +import java.util.logging.Level; +import java.util.logging.Logger; +import java.util.stream.Collectors; + +import static com.google.common.io.Files.fileTreeTraverser; + +/** + * Tested by com.yahoo.application.container.jersey.JerseyTest + * + * @author Tony Vaagenes + * @author ollivir + */ +public class OsgiUtil { + private static final Logger log = Logger.getLogger(OsgiUtil.class.getName()); + private static final String CLASS_FILE_TYPE_SUFFIX = ".class"; + + public static Collection<String> getClassEntriesInBundleClassPath(Bundle bundle, Set<String> packagesToScan) { + BundleWiring bundleWiring = bundle.adapt(BundleWiring.class); + + if (packagesToScan.isEmpty()) { + return bundleWiring.listResources("/", "*" + CLASS_FILE_TYPE_SUFFIX, + BundleWiring.LISTRESOURCES_LOCAL | BundleWiring.LISTRESOURCES_RECURSE); + } else { + List<String> ret = new ArrayList<>(); + for (String pkg : packagesToScan) { + ret.addAll(bundleWiring.listResources(packageToPath(pkg), "*" + CLASS_FILE_TYPE_SUFFIX, BundleWiring.LISTRESOURCES_LOCAL)); + } + return ret; + } + } + + public static Collection<String> getClassEntriesForBundleUsingProjectClassPathMappings(ClassLoader classLoader, + ComponentSpecification bundleSpec, Set<String> packagesToScan) { + return classEntriesFrom(bundleClassPathMapping(bundleSpec, classLoader).classPathElements, packagesToScan); + } + + private static BundleClasspathMapping bundleClassPathMapping(ComponentSpecification bundleSpec, ClassLoader classLoader) { + ProjectBundleClassPaths projectBundleClassPaths = loadProjectBundleClassPaths(classLoader); + + if (projectBundleClassPaths.mainBundle.bundleSymbolicName.equals(bundleSpec.getName())) { + return projectBundleClassPaths.mainBundle; + } else { + log.log(Level.WARNING, + "Dependencies of the bundle " + bundleSpec + " will not be scanned. Please file a feature request if you need this"); + return matchingBundleClassPathMapping(bundleSpec, projectBundleClassPaths.providedDependencies); + } + } + + public static BundleClasspathMapping matchingBundleClassPathMapping(ComponentSpecification bundleSpec, + Collection<BundleClasspathMapping> providedBundlesClassPathMappings) { + for (BundleClasspathMapping mapping : providedBundlesClassPathMappings) { + if (mapping.bundleSymbolicName.equals(bundleSpec.getName())) { + return mapping; + } + } + throw new RuntimeException("No such bundle: " + bundleSpec); + } + + private static ProjectBundleClassPaths loadProjectBundleClassPaths(ClassLoader classLoader) { + URL classPathMappingsFileLocation = classLoader.getResource(ProjectBundleClassPaths.CLASSPATH_MAPPINGS_FILENAME); + if (classPathMappingsFileLocation == null) { + throw new RuntimeException("Couldn't find " + ProjectBundleClassPaths.CLASSPATH_MAPPINGS_FILENAME + " in the class path."); + } + + try { + return ProjectBundleClassPaths.load(Paths.get(classPathMappingsFileLocation.toURI())); + } catch (IOException | URISyntaxException e) { + throw new RuntimeException(e); + } + } + + private static Collection<String> classEntriesFrom(List<String> classPathEntries, Set<String> packagesToScan) { + Set<String> packagePathsToScan = packagesToScan.stream().map(OsgiUtil::packageToPath).collect(Collectors.toSet()); + List<String> ret = new ArrayList<>(); + + for (String entry : classPathEntries) { + Path path = Paths.get(entry); + if (Files.isDirectory(path)) { + ret.addAll(classEntriesInPath(path, packagePathsToScan)); + } else if (Files.isRegularFile(path) && path.getFileName().toString().endsWith(".jar")) { + ret.addAll(classEntriesInJar(path, packagePathsToScan)); + } else { + throw new RuntimeException("Unsupported path " + path + " in the class path"); + } + } + return ret; + } + + private static String relativePathToClass(Path rootPath, Path pathToClass) { + Path relativePath = rootPath.relativize(pathToClass); + return relativePath.toString(); + } + + private static Collection<String> classEntriesInPath(Path rootPath, Collection<String> packagePathsToScan) { + Iterable<File> fileIterator; + if (packagePathsToScan.isEmpty()) { + fileIterator = fileTreeTraverser().preOrderTraversal(rootPath.toFile()); + } else { + List<File> files = new ArrayList<>(); + for (String packagePath : packagePathsToScan) { + for (File file : fileTreeTraverser().children(rootPath.resolve(packagePath).toFile())) { + files.add(file); + } + } + fileIterator = files; + } + + List<String> ret = new ArrayList<>(); + for (File file : fileIterator) { + if (file.isFile() && file.getName().endsWith(CLASS_FILE_TYPE_SUFFIX)) { + ret.add(relativePathToClass(rootPath, file.toPath())); + } + } + return ret; + } + + private static String packagePath(String name) { + int index = name.lastIndexOf('/'); + if (index < 0) { + return name; + } else { + return name.substring(0, index); + } + } + + private static Collection<String> classEntriesInJar(Path jarPath, Set<String> packagePathsToScan) { + Predicate<String> acceptedPackage; + if (packagePathsToScan.isEmpty()) { + acceptedPackage = ign -> true; + } else { + acceptedPackage = name -> packagePathsToScan.contains(packagePath(name)); + } + + try (JarFile jarFile = new JarFile(jarPath.toFile())) { + return jarFile.stream().map(JarEntry::getName).filter(name -> name.endsWith(CLASS_FILE_TYPE_SUFFIX)).filter(acceptedPackage) + .collect(Collectors.toList()); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + + private static String packageToPath(String packageName) { + return packageName.replace('.', '/'); + } +} diff --git a/container-core/src/main/java/com/yahoo/container/di/osgi/package-info.java b/container-core/src/main/java/com/yahoo/container/di/osgi/package-info.java new file mode 100644 index 00000000000..9685cf571bd --- /dev/null +++ b/container-core/src/main/java/com/yahoo/container/di/osgi/package-info.java @@ -0,0 +1,8 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +/** + * @author Tony Vaagenes + */ +@ExportPackage +package com.yahoo.container.di.osgi; + +import com.yahoo.osgi.annotation.ExportPackage; diff --git a/container-core/src/main/java/com/yahoo/container/handler/LogHandler.java b/container-core/src/main/java/com/yahoo/container/handler/LogHandler.java index 991cd83ffa8..f1ba68ff3c8 100644 --- a/container-core/src/main/java/com/yahoo/container/handler/LogHandler.java +++ b/container-core/src/main/java/com/yahoo/container/handler/LogHandler.java @@ -11,6 +11,7 @@ import com.yahoo.jdisc.handler.CompletionHandler; import com.yahoo.jdisc.handler.ContentChannel; import java.io.IOException; +import java.io.InterruptedIOException; import java.io.OutputStream; import java.nio.ByteBuffer; import java.time.Instant; @@ -23,7 +24,7 @@ import java.util.logging.Level; public class LogHandler extends ThreadedHttpRequestHandler { private final LogReader logReader; - private static final long MB = 1024*1024; + private static final long MB = 1024 * 1024; @Inject public LogHandler(Executor executor, LogHandlerConfig config) { @@ -45,11 +46,11 @@ public class LogHandler extends ThreadedHttpRequestHandler { return new AsyncHttpResponse(200) { @Override + public long maxPendingBytes() { return MB; } + @Override public void render(OutputStream output, ContentChannel networkChannel, CompletionHandler handler) { - try { - OutputStream blockingOutput = new MaxPendingContentChannelOutputStream(networkChannel, 1*MB); - logReader.writeLogs(blockingOutput, from, to, hostname); - blockingOutput.close(); + try (output) { + logReader.writeLogs(output, from, to, hostname); } catch (Throwable t) { log.log(Level.WARNING, "Failed reading logs from " + from + " to " + to, t); @@ -62,74 +63,5 @@ public class LogHandler extends ThreadedHttpRequestHandler { } - private static class MaxPendingContentChannelOutputStream extends ContentChannelOutputStream { - private final long maxPending; - private final AtomicLong sent = new AtomicLong(0); - private final AtomicLong acked = new AtomicLong(0); - - public MaxPendingContentChannelOutputStream(ContentChannel endpoint, long maxPending) { - super(endpoint); - this.maxPending = maxPending; - } - - private long pendingBytes() { - return sent.get() - acked.get(); - } - - private class TrackCompletition implements CompletionHandler { - private final long written; - private final AtomicBoolean replied = new AtomicBoolean(false); - TrackCompletition(long written) { - this.written = written; - sent.addAndGet(written); - } - @Override - public void completed() { - if (!replied.getAndSet(true)) { - acked.addAndGet(written); - } - } - - @Override - public void failed(Throwable t) { - if (!replied.getAndSet(true)) { - acked.addAndGet(written); - } - } - } - @Override - public void send(ByteBuffer src) throws IOException { - try { - stallWhilePendingAbove(maxPending); - } catch (InterruptedException ignored) { - throw new IOException("Interrupted waiting for IO"); - } - CompletionHandler pendingTracker = new TrackCompletition(src.remaining()); - try { - send(src, pendingTracker); - } catch (Throwable throwable) { - pendingTracker.failed(throwable); - throw throwable; - } - } - - private void stallWhilePendingAbove(long pending) throws InterruptedException { - while (pendingBytes() > pending) { - Thread.sleep(1); - } - } - - @Override - public void flush() throws IOException { - super.flush(); - try { - stallWhilePendingAbove(0); - } - catch (InterruptedException e) { - throw new IOException("Interrupted waiting for IO"); - } - } - - } } diff --git a/container-core/src/main/java/com/yahoo/container/jdisc/AclMapping.java b/container-core/src/main/java/com/yahoo/container/jdisc/AclMapping.java new file mode 100644 index 00000000000..e7c3d71ba44 --- /dev/null +++ b/container-core/src/main/java/com/yahoo/container/jdisc/AclMapping.java @@ -0,0 +1,50 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +package com.yahoo.container.jdisc; + +import java.util.Objects; + +/** + * Mapping from request to action + * + * @author mortent + */ +public interface AclMapping { + class Action { + public static final Action READ = new Action("read"); + public static final Action WRITE = new Action("write"); + private final String name; + public static Action custom(String name) { + return new Action(name); + } + private Action(String name) { + if(Objects.requireNonNull(name).isBlank()) { + throw new IllegalArgumentException("Name cannot be blank"); + } + this.name = Objects.requireNonNull(name); + } + public String name() { return name; } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + Action action = (Action) o; + return Objects.equals(name, action.name); + } + + @Override + public int hashCode() { + return Objects.hash(name); + } + + @Override + public String toString() { + return "Action{" + + "name='" + name + '\'' + + '}'; + } + } + + Action get(RequestView requestView); +} diff --git a/container-core/src/main/java/com/yahoo/container/jdisc/HttpMethodAclMapping.java b/container-core/src/main/java/com/yahoo/container/jdisc/HttpMethodAclMapping.java new file mode 100644 index 00000000000..2ca19f689ee --- /dev/null +++ b/container-core/src/main/java/com/yahoo/container/jdisc/HttpMethodAclMapping.java @@ -0,0 +1,71 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +package com.yahoo.container.jdisc; + +import com.yahoo.jdisc.http.HttpRequest; + +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; +import java.util.Optional; + +import static com.yahoo.jdisc.http.HttpRequest.Method.CONNECT; +import static com.yahoo.jdisc.http.HttpRequest.Method.DELETE; +import static com.yahoo.jdisc.http.HttpRequest.Method.GET; +import static com.yahoo.jdisc.http.HttpRequest.Method.HEAD; +import static com.yahoo.jdisc.http.HttpRequest.Method.OPTIONS; +import static com.yahoo.jdisc.http.HttpRequest.Method.PATCH; +import static com.yahoo.jdisc.http.HttpRequest.Method.POST; +import static com.yahoo.jdisc.http.HttpRequest.Method.PUT; +import static com.yahoo.jdisc.http.HttpRequest.Method.TRACE; + +/** + * Acl Mapping based on http method. + * Defaults to:<br> + * {GET, HEAD, OPTIONS} -> READ<br> + * {POST, DELETE, PUT, PATCH, CONNECT, TRACE} -> WRITE + * @author mortent + */ +public class HttpMethodAclMapping implements AclMapping { + + private final Map<HttpRequest.Method, Action> mappings; + + private HttpMethodAclMapping(Map<HttpRequest.Method, Action> overrides) { + HashMap<HttpRequest.Method, Action> tmp = new HashMap<>(defaultMappings()); + tmp.putAll(overrides); + mappings = Map.copyOf(tmp); + } + + private static Map<HttpRequest.Method, Action> defaultMappings() { + return Map.of(GET, Action.READ, + HEAD, Action.READ, + OPTIONS, Action.READ, + POST, Action.WRITE, + DELETE, Action.WRITE, + PUT, Action.WRITE, + PATCH, Action.WRITE, + CONNECT, Action.WRITE, + TRACE, Action.WRITE); + } + + @Override + public Action get(RequestView requestView) { + return Optional.ofNullable(mappings.get(requestView.method())) + .orElseThrow(() -> new IllegalArgumentException("Illegal request method: " + requestView.method())); + } + + public static HttpMethodAclMapping.Builder standard() { + return new HttpMethodAclMapping.Builder(); + } + + public static class Builder { + private final Map<com.yahoo.jdisc.http.HttpRequest.Method, Action> overrides = new HashMap<>(); + public HttpMethodAclMapping.Builder override(HttpRequest.Method method, Action action) { + overrides.put(method, action); + return this; + } + public HttpMethodAclMapping build() { + return new HttpMethodAclMapping(overrides); + } + } +} diff --git a/container-core/src/main/java/com/yahoo/container/jdisc/HttpRequestBuilder.java b/container-core/src/main/java/com/yahoo/container/jdisc/HttpRequestBuilder.java new file mode 100644 index 00000000000..3f70f4b75bb --- /dev/null +++ b/container-core/src/main/java/com/yahoo/container/jdisc/HttpRequestBuilder.java @@ -0,0 +1,71 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.container.jdisc; + +import com.yahoo.jdisc.http.HttpRequest.Method; + +import java.io.InputStream; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.TreeMap; + +/** + * Builder for creating a {@link HttpRequest} to be used in test context + * + * @author bjorncs + */ +public class HttpRequestBuilder { + private final Method method; + private final String path; + private final Map<String, List<String>> queryParameters = new TreeMap<>(); + private final Map<String, String> headers = new TreeMap<>(); + private String scheme; + private String hostname; + private InputStream content; + + private HttpRequestBuilder(Method method, String path) { + this.method = method; + this.path = path; + } + + public static HttpRequestBuilder create(Method method, String path) { return new HttpRequestBuilder(method, path); } + + public HttpRequestBuilder withQueryParameter(String name, String value) { + this.queryParameters.computeIfAbsent(name, ignored -> new ArrayList<>()).add(value); + return this; + } + + public HttpRequestBuilder withHeader(String name, String value) { this.headers.put(name, value); return this; } + + public HttpRequestBuilder withRequestContent(InputStream content) { this.content = content; return this; } + + public HttpRequestBuilder withScheme(String scheme) { this.scheme = scheme; return this; } + + public HttpRequestBuilder withHostname(String hostname) { this.hostname = hostname; return this; } + + public HttpRequest build() { + String scheme = this.scheme != null ? this.scheme : "http"; + String hostname = this.hostname != null ? this.hostname : "localhost"; + StringBuilder uriBuilder = new StringBuilder(scheme).append("://").append(hostname).append(path); + if (queryParameters.size() > 0) { + uriBuilder.append('?'); + queryParameters.forEach((name, values) -> { + for (String value : values) { + uriBuilder.append(name).append('=').append(value).append('&'); + } + }); + int lastIndex = uriBuilder.length() - 1; + if (uriBuilder.charAt(lastIndex) == '&') { + uriBuilder.setLength(lastIndex); + } + } + HttpRequest request; + if (content != null) { + request = HttpRequest.createTestRequest(uriBuilder.toString(), method, content); + } else { + request = HttpRequest.createTestRequest(uriBuilder.toString(), method); + } + headers.forEach((name, value) -> request.getJDiscRequest().headers().put(name, value)); + return request; + } +} diff --git a/container-core/src/main/java/com/yahoo/container/jdisc/HttpRequestHandler.java b/container-core/src/main/java/com/yahoo/container/jdisc/HttpRequestHandler.java new file mode 100644 index 00000000000..f322c9c5b6f --- /dev/null +++ b/container-core/src/main/java/com/yahoo/container/jdisc/HttpRequestHandler.java @@ -0,0 +1,20 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +package com.yahoo.container.jdisc; + +import com.yahoo.jdisc.handler.RequestHandler; + +/** + * Extends a request handler with a http specific + * + * @author mortent + */ +public interface HttpRequestHandler extends RequestHandler { + + /** + * @return handler specification + */ + default RequestHandlerSpec requestHandlerSpec() { + return RequestHandlerSpec.DEFAULT_INSTANCE; + } +} diff --git a/container-core/src/main/java/com/yahoo/container/jdisc/HttpResponse.java b/container-core/src/main/java/com/yahoo/container/jdisc/HttpResponse.java index a6042c541c0..5df40a90fe6 100644 --- a/container-core/src/main/java/com/yahoo/container/jdisc/HttpResponse.java +++ b/container-core/src/main/java/com/yahoo/container/jdisc/HttpResponse.java @@ -40,6 +40,9 @@ public abstract class HttpResponse { /** Marshals this response to the network layer. The caller is responsible for flushing and closing outputStream. */ public abstract void render(OutputStream outputStream) throws IOException; + /** The amount of content bytes this response may have in-flight (if positive) before response rendering blocks. */ + public long maxPendingBytes() { return -1; } + /** * Returns the numeric HTTP status code, e.g. 200, 404 and so on. * diff --git a/container-core/src/main/java/com/yahoo/container/jdisc/RequestHandlerSpec.java b/container-core/src/main/java/com/yahoo/container/jdisc/RequestHandlerSpec.java new file mode 100644 index 00000000000..0ebb0bb99d9 --- /dev/null +++ b/container-core/src/main/java/com/yahoo/container/jdisc/RequestHandlerSpec.java @@ -0,0 +1,46 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +package com.yahoo.container.jdisc; + +import java.util.Objects; + +/** + * A specification provided by a request handler. + * Available through request context attribute + * + * @author mortent + */ +public class RequestHandlerSpec { + + public static final String ATTRIBUTE_NAME = RequestHandlerSpec.class.getName(); + public static final RequestHandlerSpec DEFAULT_INSTANCE = RequestHandlerSpec.builder().build(); + + private final AclMapping aclMapping; + + private RequestHandlerSpec(AclMapping aclMapping) { + this.aclMapping = aclMapping; + } + + public AclMapping aclMapping() { + return aclMapping; + } + + public static Builder builder(){ + return new Builder(); + } + + public static class Builder { + + private AclMapping aclMapping = HttpMethodAclMapping.standard().build(); + + public Builder withAclMapping(AclMapping aclMapping) { + this.aclMapping = Objects.requireNonNull(aclMapping); + return this; + } + + public RequestHandlerSpec build() { + return new RequestHandlerSpec(aclMapping); + } + } +} + diff --git a/container-core/src/main/java/com/yahoo/container/jdisc/RequestView.java b/container-core/src/main/java/com/yahoo/container/jdisc/RequestView.java new file mode 100644 index 00000000000..51a5fdc8959 --- /dev/null +++ b/container-core/src/main/java/com/yahoo/container/jdisc/RequestView.java @@ -0,0 +1,18 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +package com.yahoo.container.jdisc; + +import com.yahoo.jdisc.http.HttpRequest; + +import java.net.URI; + +/** + * Read-only view of the request + * + * @author mortent + */ +public interface RequestView { + HttpRequest.Method method(); + + URI uri(); +} diff --git a/container-core/src/main/java/com/yahoo/container/jdisc/ThreadedHttpRequestHandler.java b/container-core/src/main/java/com/yahoo/container/jdisc/ThreadedHttpRequestHandler.java index 9687697d6f6..be708f2fc94 100644 --- a/container-core/src/main/java/com/yahoo/container/jdisc/ThreadedHttpRequestHandler.java +++ b/container-core/src/main/java/com/yahoo/container/jdisc/ThreadedHttpRequestHandler.java @@ -9,6 +9,10 @@ import com.yahoo.jdisc.handler.CompletionHandler; import com.yahoo.jdisc.handler.ContentChannel; import com.yahoo.jdisc.handler.UnsafeContentInputStream; import com.yahoo.jdisc.handler.ResponseHandler; + +import java.io.InterruptedIOException; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicLong; import java.util.logging.Level; import java.io.IOException; @@ -28,7 +32,7 @@ import java.util.logging.Logger; * @author Steinar Knutsen * @author bratseth */ -public abstract class ThreadedHttpRequestHandler extends ThreadedRequestHandler { +public abstract class ThreadedHttpRequestHandler extends ThreadedRequestHandler implements HttpRequestHandler { public static final String CONTENT_TYPE = "Content-Type"; private static final String RENDERING_ERRORS = "rendering_errors"; @@ -97,7 +101,8 @@ public abstract class ThreadedHttpRequestHandler extends ThreadedRequestHandler LoggingCompletionHandler logOnCompletion = null; ContentChannelOutputStream output = null; try { - output = new ContentChannelOutputStream(channel); + output = httpResponse.maxPendingBytes() > 0 ? new MaxPendingContentChannelOutputStream(channel, httpResponse.maxPendingBytes()) + : new ContentChannelOutputStream(channel); logOnCompletion = createLoggingCompletionHandler(startTime, System.currentTimeMillis(), httpResponse, request, output); @@ -247,4 +252,82 @@ public abstract class ThreadedHttpRequestHandler extends ThreadedRequestHandler return (com.yahoo.jdisc.http.HttpRequest) request; } + + /** + * @author baldersheim + */ + static class MaxPendingContentChannelOutputStream extends ContentChannelOutputStream { + private final long maxPending; + private final AtomicLong sent = new AtomicLong(0); + private final AtomicLong acked = new AtomicLong(0); + + public MaxPendingContentChannelOutputStream(ContentChannel endpoint, long maxPending) { + super(endpoint); + this.maxPending = maxPending; + } + + private long pendingBytes() { + return sent.get() - acked.get(); + } + + private class TrackCompletion implements CompletionHandler { + + private final long written; + private final AtomicBoolean replied = new AtomicBoolean(false); + + TrackCompletion(long written) { + this.written = written; + sent.addAndGet(written); + } + + @Override + public void completed() { + if ( ! replied.getAndSet(true)) { + acked.addAndGet(written); + } + } + + @Override + public void failed(Throwable t) { + if ( ! replied.getAndSet(true)) { + acked.addAndGet(written); + } + } + } + + @Override + public void send(ByteBuffer src) throws IOException { + try { + stallWhilePendingAbove(maxPending); + } catch (InterruptedException ignored) { + throw new InterruptedIOException("Interrupted waiting for IO"); + } + CompletionHandler pendingTracker = new TrackCompletion(src.remaining()); + try { + send(src, pendingTracker); + } catch (Throwable throwable) { + pendingTracker.failed(throwable); + throw throwable; + } + } + + private void stallWhilePendingAbove(long pending) throws InterruptedException { + while (pendingBytes() > pending) { + Thread.sleep(1); + } + } + + @Override + public void flush() throws IOException { + super.flush(); + try { + stallWhilePendingAbove(0); + } + catch (InterruptedException e) { + throw new InterruptedIOException("Interrupted waiting for IO"); + } + } + + } + } diff --git a/container-core/src/main/java/com/yahoo/container/logging/AccessLogHandler.java b/container-core/src/main/java/com/yahoo/container/logging/AccessLogHandler.java index 89aab1513ee..f14479899f5 100644 --- a/container-core/src/main/java/com/yahoo/container/logging/AccessLogHandler.java +++ b/container-core/src/main/java/com/yahoo/container/logging/AccessLogHandler.java @@ -12,7 +12,7 @@ class AccessLogHandler { AccessLogHandler(AccessLogConfig.FileHandler config, LogWriter<RequestLogEntry> logWriter) { logFileHandler = new LogFileHandler<>( - toCompression(config), config.pattern(), config.rotation(), + toCompression(config), config.bufferSize(), config.pattern(), config.rotation(), config.symlink(), config.queueSize(), "request-logger", logWriter); } diff --git a/container-core/src/main/java/com/yahoo/container/logging/ConnectionLogEntry.java b/container-core/src/main/java/com/yahoo/container/logging/ConnectionLogEntry.java index 6afe3b74329..5b30ce5963d 100644 --- a/container-core/src/main/java/com/yahoo/container/logging/ConnectionLogEntry.java +++ b/container-core/src/main/java/com/yahoo/container/logging/ConnectionLogEntry.java @@ -33,6 +33,8 @@ public class ConnectionLogEntry { private final Instant sslPeerNotAfter; private final String sslSniServerName; private final SslHandshakeFailure sslHandshakeFailure; + private final String httpProtocol; + private final String proxyProtocolVersion; private ConnectionLogEntry(Builder builder) { @@ -57,6 +59,8 @@ public class ConnectionLogEntry { this.sslPeerNotAfter = builder.sslPeerNotAfter; this.sslSniServerName = builder.sslSniServerName; this.sslHandshakeFailure = builder.sslHandshakeFailure; + this.httpProtocol = builder.httpProtocol; + this.proxyProtocolVersion = builder.proxyProtocolVersion; } public static Builder builder(UUID id, Instant timestamp) { @@ -84,6 +88,8 @@ public class ConnectionLogEntry { public Optional<Instant> sslPeerNotAfter() { return Optional.ofNullable(sslPeerNotAfter); } public Optional<String> sslSniServerName() { return Optional.ofNullable(sslSniServerName); } public Optional<SslHandshakeFailure> sslHandshakeFailure() { return Optional.ofNullable(sslHandshakeFailure); } + public Optional<String> httpProtocol() { return Optional.ofNullable(httpProtocol); } + public Optional<String> proxyProtocolVersion() { return Optional.ofNullable(proxyProtocolVersion); } public static class SslHandshakeFailure { private final String type; @@ -133,6 +139,8 @@ public class ConnectionLogEntry { private Instant sslPeerNotAfter; private String sslSniServerName; private SslHandshakeFailure sslHandshakeFailure; + private String httpProtocol; + private String proxyProtocolVersion; Builder(UUID id, Instant timestamp) { @@ -217,9 +225,18 @@ public class ConnectionLogEntry { this.sslHandshakeFailure = sslHandshakeFailure; return this; } + public Builder withHttpProtocol(String protocol) { + this.httpProtocol = protocol; + return this; + } + public Builder withProxyProtocolVersion(String version) { + this.proxyProtocolVersion = version; + return this; + } public ConnectionLogEntry build(){ return new ConnectionLogEntry(this); } + } } diff --git a/container-core/src/main/java/com/yahoo/container/logging/ConnectionLogHandler.java b/container-core/src/main/java/com/yahoo/container/logging/ConnectionLogHandler.java index 7a0e8aca95e..7b130884667 100644 --- a/container-core/src/main/java/com/yahoo/container/logging/ConnectionLogHandler.java +++ b/container-core/src/main/java/com/yahoo/container/logging/ConnectionLogHandler.java @@ -8,9 +8,11 @@ package com.yahoo.container.logging; class ConnectionLogHandler { private final LogFileHandler<ConnectionLogEntry> logFileHandler; - public ConnectionLogHandler(String logDirectoryName, String clusterName, int queueSize, LogWriter<ConnectionLogEntry> logWriter) { + public ConnectionLogHandler(String logDirectoryName, int bufferSize, String clusterName, + int queueSize, LogWriter<ConnectionLogEntry> logWriter) { logFileHandler = new LogFileHandler<>( LogFileHandler.Compression.ZSTD, + bufferSize, String.format("logs/vespa/%s/ConnectionLog.%s.%s", logDirectoryName, clusterName, "%Y%m%d%H%M%S"), "0 60 ...", String.format("ConnectionLog.%s", clusterName), diff --git a/container-core/src/main/java/com/yahoo/container/logging/FileConnectionLog.java b/container-core/src/main/java/com/yahoo/container/logging/FileConnectionLog.java index 7432c313286..749426d3da9 100644 --- a/container-core/src/main/java/com/yahoo/container/logging/FileConnectionLog.java +++ b/container-core/src/main/java/com/yahoo/container/logging/FileConnectionLog.java @@ -14,7 +14,7 @@ public class FileConnectionLog extends AbstractComponent implements ConnectionLo @Inject public FileConnectionLog(ConnectionLogConfig config) { - logHandler = new ConnectionLogHandler(config.logDirectoryName(), config.cluster(), config.queueSize(), new JsonConnectionLogWriter()); + logHandler = new ConnectionLogHandler(config.logDirectoryName(), config.bufferSize(), config.cluster(), config.queueSize(), new JsonConnectionLogWriter()); } @Override diff --git a/container-core/src/main/java/com/yahoo/container/logging/JsonConnectionLogWriter.java b/container-core/src/main/java/com/yahoo/container/logging/JsonConnectionLogWriter.java index 158d2ec4ea6..dfdc5f1b55a 100644 --- a/container-core/src/main/java/com/yahoo/container/logging/JsonConnectionLogWriter.java +++ b/container-core/src/main/java/com/yahoo/container/logging/JsonConnectionLogWriter.java @@ -33,12 +33,32 @@ class JsonConnectionLogWriter implements LogWriter<ConnectionLogEntry> { writeOptionalInteger(generator, "peerPort", unwrap(record.peerPort())); writeOptionalString(generator, "localAddress", unwrap(record.localAddress())); writeOptionalInteger(generator, "localPort", unwrap(record.localPort())); - writeOptionalString(generator, "remoteAddress", unwrap(record.remoteAddress())); - writeOptionalInteger(generator, "remotePort", unwrap(record.remotePort())); - writeOptionalLong(generator, "httpBytesReceived", unwrap(record.httpBytesReceived())); - writeOptionalLong(generator, "httpBytesSent", unwrap(record.httpBytesSent())); - writeOptionalLong(generator, "requests", unwrap(record.requests())); - writeOptionalLong(generator, "responses", unwrap(record.responses())); + + String proxyProtocolVersion = unwrap(record.proxyProtocolVersion()); + String proxyProtocolRemoteAddress = unwrap(record.remoteAddress()); + Integer proxyProtocolRemotePort = unwrap(record.remotePort()); + if (isAnyValuePresent(proxyProtocolVersion, proxyProtocolRemoteAddress, proxyProtocolRemotePort)) { + generator.writeObjectFieldStart("proxyProtocol"); + writeOptionalString(generator, "version", proxyProtocolVersion); + writeOptionalString(generator, "remoteAddress", proxyProtocolRemoteAddress); + writeOptionalInteger(generator, "remotePort", proxyProtocolRemotePort); + generator.writeEndObject(); + } + + String httpVersion = unwrap(record.httpProtocol()); + Long httpBytesReceived = unwrap(record.httpBytesReceived()); + Long httpBytesSent = unwrap(record.httpBytesSent()); + Long httpRequests = unwrap(record.requests()); + Long httpResponses = unwrap(record.responses()); + if (isAnyValuePresent(httpVersion, httpBytesReceived, httpBytesSent, httpRequests, httpResponses)) { + generator.writeObjectFieldStart("http"); + writeOptionalString(generator, "version", httpVersion); + writeOptionalLong(generator, "bytesReceived", httpBytesReceived); + writeOptionalLong(generator, "responses", httpResponses); + writeOptionalLong(generator, "bytesSent", httpBytesSent); + writeOptionalLong(generator, "requests", httpRequests); + generator.writeEndObject(); + } String sslProtocol = unwrap(record.sslProtocol()); String sslSessionId = unwrap(record.sslSessionId()); diff --git a/container-core/src/main/java/com/yahoo/container/logging/LogFileHandler.java b/container-core/src/main/java/com/yahoo/container/logging/LogFileHandler.java index 0f2a9e42eb8..85c211c0e3a 100644 --- a/container-core/src/main/java/com/yahoo/container/logging/LogFileHandler.java +++ b/container-core/src/main/java/com/yahoo/container/logging/LogFileHandler.java @@ -47,21 +47,15 @@ class LogFileHandler <LOGTYPE> { @FunctionalInterface private interface Pollable<T> { Operation<T> poll() throws InterruptedException; } - LogFileHandler(Compression compression, String filePattern, String rotationTimes, String symlinkName, int queueSize, - String threadName, LogWriter<LOGTYPE> logWriter) { - this(compression, filePattern, calcTimesMinutes(rotationTimes), symlinkName, queueSize, threadName, logWriter); + LogFileHandler(Compression compression, int bufferSize, String filePattern, String rotationTimes, String symlinkName, + int queueSize, String threadName, LogWriter<LOGTYPE> logWriter) { + this(compression, bufferSize, filePattern, calcTimesMinutes(rotationTimes), symlinkName, queueSize, threadName, logWriter); } - LogFileHandler( - Compression compression, - String filePattern, - long[] rotationTimes, - String symlinkName, - int queueSize, - String threadName, - LogWriter<LOGTYPE> logWriter) { + LogFileHandler(Compression compression, int bufferSize, String filePattern, long[] rotationTimes, String symlinkName, + int queueSize, String threadName, LogWriter<LOGTYPE> logWriter) { this.logQueue = new LinkedBlockingQueue<>(queueSize); - this.logThread = new LogThread<>(logWriter, filePattern, compression, rotationTimes, symlinkName, threadName, this::poll); + this.logThread = new LogThread<>(logWriter, filePattern, compression, bufferSize, rotationTimes, symlinkName, threadName, this::poll); this.logThread.start(); } @@ -197,6 +191,7 @@ class LogFileHandler <LOGTYPE> { private volatile String fileName; private final LogWriter<LOGTYPE> logWriter; private final Compression compression; + private final int bufferSize; private final long[] rotationTimes; private final String symlinkName; private final ExecutorService executor = createCompressionTaskExecutor(); @@ -206,6 +201,7 @@ class LogFileHandler <LOGTYPE> { LogThread(LogWriter<LOGTYPE> logWriter, String filePattern, Compression compression, + int bufferSize, long[] rotationTimes, String symlinkName, String threadName, @@ -215,6 +211,7 @@ class LogFileHandler <LOGTYPE> { this.logWriter = logWriter; this.filePattern = filePattern; this.compression = compression; + this.bufferSize = bufferSize; this.rotationTimes = rotationTimes; this.symlinkName = (symlinkName != null && !symlinkName.isBlank()) ? symlinkName : null; this.operationProvider = operationProvider; @@ -360,7 +357,7 @@ class LogFileHandler <LOGTYPE> { internalClose(); try { checkAndCreateDir(fileName); - fileOutput = new PageCacheFriendlyFileOutputStream(nativeIO, Paths.get(fileName), 4 * 1024 * 1024); + fileOutput = new PageCacheFriendlyFileOutputStream(nativeIO, Paths.get(fileName), bufferSize); LogFileDb.nowLoggingTo(fileName); } catch (IOException e) { throw new RuntimeException("Couldn't open log file '" + fileName + "'", e); diff --git a/container-core/src/main/java/com/yahoo/jdisc/http/HttpRequest.java b/container-core/src/main/java/com/yahoo/jdisc/http/HttpRequest.java index 118c34245c0..0b5e9ddde58 100644 --- a/container-core/src/main/java/com/yahoo/jdisc/http/HttpRequest.java +++ b/container-core/src/main/java/com/yahoo/jdisc/http/HttpRequest.java @@ -47,7 +47,8 @@ public class HttpRequest extends Request implements ServletOrJdiscHttpRequest { public enum Version { HTTP_1_0("HTTP/1.0"), - HTTP_1_1("HTTP/1.1"); + HTTP_1_1("HTTP/1.1"), + HTTP_2_0("HTTP/2.0"); private final String str; @@ -70,6 +71,7 @@ public class HttpRequest extends Request implements ServletOrJdiscHttpRequest { } } + private final long jvmRelativeCreatedAt = System.nanoTime(); private final HeaderFields trailers = new HeaderFields(); private final Map<String, List<String>> parameters = new HashMap<>(); private Principal principal; @@ -296,6 +298,11 @@ public class HttpRequest extends Request implements ServletOrJdiscHttpRequest { return version == Version.HTTP_1_1; } + /** + * @return the relative created timestamp (using {@link System#nanoTime()} + */ + public long relativeCreatedAtNanoTime() { return jvmRelativeCreatedAt; } + public Principal getUserPrincipal() { return principal; } diff --git a/container-core/src/main/java/com/yahoo/jdisc/http/filter/DiscFilterRequest.java b/container-core/src/main/java/com/yahoo/jdisc/http/filter/DiscFilterRequest.java index f7ab399574c..72068bd2dd5 100644 --- a/container-core/src/main/java/com/yahoo/jdisc/http/filter/DiscFilterRequest.java +++ b/container-core/src/main/java/com/yahoo/jdisc/http/filter/DiscFilterRequest.java @@ -1,6 +1,7 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.jdisc.http.filter; +import com.yahoo.container.jdisc.RequestView; import com.yahoo.jdisc.HeaderFields; import com.yahoo.jdisc.http.Cookie; import com.yahoo.jdisc.http.HttpHeaders; @@ -254,6 +255,19 @@ public abstract class DiscFilterRequest { } } + public RequestView asRequestView() { + return new RequestView() { + @Override + public HttpRequest.Method method() { + return HttpRequest.Method.valueOf(getMethod()); + } + + @Override + public URI uri() { + return getUri(); + } + }; + } public List<Cookie> getCookies() { return parent.decodeCookieHeader(); diff --git a/container-core/src/main/java/com/yahoo/jdisc/http/server/jetty/AccessLogRequestLog.java b/container-core/src/main/java/com/yahoo/jdisc/http/server/jetty/AccessLogRequestLog.java index 2f9fc0d07b2..11898381f0a 100644 --- a/container-core/src/main/java/com/yahoo/jdisc/http/server/jetty/AccessLogRequestLog.java +++ b/container-core/src/main/java/com/yahoo/jdisc/http/server/jetty/AccessLogRequestLog.java @@ -24,7 +24,7 @@ import java.util.function.BiConsumer; import java.util.logging.Level; import java.util.logging.Logger; -import static com.yahoo.jdisc.http.server.jetty.HttpServletRequestUtils.getConnectorLocalPort; +import static com.yahoo.jdisc.http.server.jetty.RequestUtils.getConnectorLocalPort; /** * This class is a bridge between Jetty's {@link org.eclipse.jetty.server.handler.RequestLogHandler} diff --git a/container-core/src/main/java/com/yahoo/jdisc/http/server/jetty/AccessLoggingRequestHandler.java b/container-core/src/main/java/com/yahoo/jdisc/http/server/jetty/AccessLoggingRequestHandler.java index 842ab75a312..5b628d73ab8 100644 --- a/container-core/src/main/java/com/yahoo/jdisc/http/server/jetty/AccessLoggingRequestHandler.java +++ b/container-core/src/main/java/com/yahoo/jdisc/http/server/jetty/AccessLoggingRequestHandler.java @@ -6,6 +6,7 @@ import com.yahoo.container.logging.AccessLogEntry; import com.yahoo.jdisc.Request; import com.yahoo.jdisc.handler.AbstractRequestHandler; import com.yahoo.jdisc.handler.ContentChannel; +import com.yahoo.jdisc.handler.DelegatedRequestHandler; import com.yahoo.jdisc.handler.RequestHandler; import com.yahoo.jdisc.handler.ResponseHandler; import com.yahoo.jdisc.http.HttpRequest; @@ -23,7 +24,7 @@ import java.util.Optional; * * @author bakksjo */ -public class AccessLoggingRequestHandler extends AbstractRequestHandler { +public class AccessLoggingRequestHandler extends AbstractRequestHandler implements DelegatedRequestHandler { public static final String CONTEXT_KEY_ACCESS_LOG_ENTRY = AccessLoggingRequestHandler.class.getName() + "_access-log-entry"; @@ -56,4 +57,8 @@ public class AccessLoggingRequestHandler extends AbstractRequestHandler { } + @Override + public RequestHandler getDelegate() { + return delegate; + } } diff --git a/container-core/src/main/java/com/yahoo/jdisc/http/server/jetty/ConnectorFactory.java b/container-core/src/main/java/com/yahoo/jdisc/http/server/jetty/ConnectorFactory.java index d7ad12a5c64..71c0b3a0225 100644 --- a/container-core/src/main/java/com/yahoo/jdisc/http/server/jetty/ConnectorFactory.java +++ b/container-core/src/main/java/com/yahoo/jdisc/http/server/jetty/ConnectorFactory.java @@ -7,6 +7,8 @@ import com.yahoo.jdisc.http.ConnectorConfig; import com.yahoo.jdisc.http.ssl.SslContextFactoryProvider; import com.yahoo.security.tls.MixedMode; import com.yahoo.security.tls.TransportSecurityUtils; +import org.eclipse.jetty.alpn.server.ALPNServerConnectionFactory; +import org.eclipse.jetty.http2.server.HTTP2ServerConnectionFactory; import org.eclipse.jetty.server.ConnectionFactory; import org.eclipse.jetty.server.DetectorConnectionFactory; import org.eclipse.jetty.server.HttpConfiguration; @@ -18,6 +20,7 @@ import org.eclipse.jetty.server.ServerConnector; import org.eclipse.jetty.server.SslConnectionFactory; import org.eclipse.jetty.util.ssl.SslContextFactory; +import java.util.Collection; import java.util.List; /** @@ -76,41 +79,68 @@ public class ConnectorFactory { } private List<ConnectionFactory> createConnectionFactories(Metric metric) { - HttpConnectionFactory httpFactory = newHttpConnectionFactory(); if (!isSslEffectivelyEnabled(connectorConfig)) { - return List.of(httpFactory); + return List.of(newHttp1ConnectionFactory()); } else if (connectorConfig.ssl().enabled()) { - return connectionFactoriesForHttps(metric, httpFactory); + return connectionFactoriesForHttps(metric); } else if (TransportSecurityUtils.isTransportSecurityEnabled()) { switch (TransportSecurityUtils.getInsecureMixedMode()) { case TLS_CLIENT_MIXED_SERVER: case PLAINTEXT_CLIENT_MIXED_SERVER: - return List.of(new DetectorConnectionFactory(newSslConnectionFactory(metric, httpFactory)), httpFactory); + return connectionFactoriesForHttpsMixedMode(metric); case DISABLED: - return connectionFactoriesForHttps(metric, httpFactory); + return connectionFactoriesForHttps(metric); default: throw new IllegalStateException(); } } else { - return List.of(httpFactory); + return List.of(newHttp1ConnectionFactory()); } } - private List<ConnectionFactory> connectionFactoriesForHttps(Metric metric, HttpConnectionFactory httpFactory) { + private List<ConnectionFactory> connectionFactoriesForHttps(Metric metric) { ConnectorConfig.ProxyProtocol proxyProtocolConfig = connectorConfig.proxyProtocol(); - SslConnectionFactory sslFactory = newSslConnectionFactory(metric, httpFactory); - if (proxyProtocolConfig.enabled()) { - if (proxyProtocolConfig.mixedMode()) { - return List.of(new DetectorConnectionFactory(sslFactory, new ProxyConnectionFactory(sslFactory.getProtocol())), sslFactory, httpFactory); + HttpConnectionFactory http1Factory = newHttp1ConnectionFactory(); + if (connectorConfig.http2Enabled()) { + HTTP2ServerConnectionFactory http2Factory = newHttp2ConnectionFactory(); + ALPNServerConnectionFactory alpnFactory = newAlpnConnectionFactory(List.of(http1Factory, http2Factory), http1Factory); + SslConnectionFactory sslFactory = newSslConnectionFactory(metric, alpnFactory); + if (proxyProtocolConfig.enabled()) { + ProxyConnectionFactory proxyProtocolFactory = newProxyProtocolConnectionFactory(sslFactory); + if (proxyProtocolConfig.mixedMode()) { + DetectorConnectionFactory detectorFactory = newDetectorConnectionFactory(sslFactory); + return List.of(detectorFactory, proxyProtocolFactory, sslFactory, alpnFactory, http1Factory, http2Factory); + } else { + return List.of(proxyProtocolFactory, sslFactory, alpnFactory, http1Factory, http2Factory); + } } else { - return List.of(new ProxyConnectionFactory(sslFactory.getProtocol()), sslFactory, httpFactory); + return List.of(sslFactory, alpnFactory, http1Factory, http2Factory); } } else { - return List.of(sslFactory, httpFactory); + SslConnectionFactory sslFactory = newSslConnectionFactory(metric, http1Factory); + if (proxyProtocolConfig.enabled()) { + ProxyConnectionFactory proxyProtocolFactory = newProxyProtocolConnectionFactory(sslFactory); + if (proxyProtocolConfig.mixedMode()) { + DetectorConnectionFactory detectorFactory = newDetectorConnectionFactory(sslFactory); + return List.of(detectorFactory, proxyProtocolFactory, sslFactory, http1Factory); + } else { + return List.of(proxyProtocolFactory, sslFactory, http1Factory); + } + } else { + return List.of(sslFactory, http1Factory); + } } } - private HttpConnectionFactory newHttpConnectionFactory() { + private List<ConnectionFactory> connectionFactoriesForHttpsMixedMode(Metric metric) { + // No support for proxy-protocol/http2 when using HTTP with TLS mixed mode + HttpConnectionFactory httpFactory = newHttp1ConnectionFactory(); + SslConnectionFactory sslFactory = newSslConnectionFactory(metric, httpFactory); + DetectorConnectionFactory detectorFactory = newDetectorConnectionFactory(sslFactory); + return List.of(detectorFactory, httpFactory, sslFactory); + } + + private HttpConfiguration newHttpConfiguration() { HttpConfiguration httpConfig = new HttpConfiguration(); httpConfig.setSendDateHeader(true); httpConfig.setSendServerVersion(false); @@ -122,16 +152,41 @@ public class ConnectorFactory { if (isSslEffectivelyEnabled(connectorConfig)) { httpConfig.addCustomizer(new SecureRequestCustomizer()); } - return new HttpConnectionFactory(httpConfig); + return httpConfig; + } + + private HttpConnectionFactory newHttp1ConnectionFactory() { + return new HttpConnectionFactory(newHttpConfiguration()); } - private SslConnectionFactory newSslConnectionFactory(Metric metric, HttpConnectionFactory httpFactory) { + private HTTP2ServerConnectionFactory newHttp2ConnectionFactory() { + return new HTTP2ServerConnectionFactory(newHttpConfiguration()); + } + + private SslConnectionFactory newSslConnectionFactory(Metric metric, ConnectionFactory wrappedFactory) { SslContextFactory ctxFactory = sslContextFactoryProvider.getInstance(connectorConfig.name(), connectorConfig.listenPort()); - SslConnectionFactory connectionFactory = new SslConnectionFactory(ctxFactory, httpFactory.getProtocol()); + SslConnectionFactory connectionFactory = new SslConnectionFactory(ctxFactory, wrappedFactory.getProtocol()); connectionFactory.addBean(new SslHandshakeFailedListener(metric, connectorConfig.name(), connectorConfig.listenPort())); return connectionFactory; } + private ALPNServerConnectionFactory newAlpnConnectionFactory(Collection<ConnectionFactory> alternatives, + ConnectionFactory defaultFactory) { + String[] protocols = alternatives.stream().map(ConnectionFactory::getProtocol).toArray(String[]::new); + ALPNServerConnectionFactory factory = new ALPNServerConnectionFactory(protocols); + factory.setDefaultProtocol(defaultFactory.getProtocol()); + return factory; + } + + private DetectorConnectionFactory newDetectorConnectionFactory(ConnectionFactory.Detecting... alternatives) { + // Note: Detector connection factory with single alternative will fallback to next protocol in connection factory list + return new DetectorConnectionFactory(alternatives); + } + + private ProxyConnectionFactory newProxyProtocolConnectionFactory(ConnectionFactory wrappedFactory) { + return new ProxyConnectionFactory(wrappedFactory.getProtocol()); + } + private static boolean isSslEffectivelyEnabled(ConnectorConfig config) { return config.ssl().enabled() || (config.implicitTlsEnabled() && TransportSecurityUtils.isTransportSecurityEnabled()); diff --git a/container-core/src/main/java/com/yahoo/jdisc/http/server/jetty/FilterResolver.java b/container-core/src/main/java/com/yahoo/jdisc/http/server/jetty/FilterResolver.java index 1e2686aa184..a9639ba4da7 100644 --- a/container-core/src/main/java/com/yahoo/jdisc/http/server/jetty/FilterResolver.java +++ b/container-core/src/main/java/com/yahoo/jdisc/http/server/jetty/FilterResolver.java @@ -11,13 +11,13 @@ import com.yahoo.jdisc.http.HttpRequest; import com.yahoo.jdisc.http.filter.RequestFilter; import com.yahoo.jdisc.http.filter.ResponseFilter; import com.yahoo.jdisc.http.servlet.ServletRequest; +import org.eclipse.jetty.server.Request; -import javax.servlet.http.HttpServletRequest; import java.net.URI; import java.util.Map; import java.util.Optional; -import static com.yahoo.jdisc.http.server.jetty.JDiscHttpServlet.getConnector; +import static com.yahoo.jdisc.http.server.jetty.RequestUtils.getConnector; /** * Resolve request/response filter (chain) based on {@link FilterBindings}. @@ -36,38 +36,38 @@ class FilterResolver { this.strictFiltering = strictFiltering; } - Optional<RequestFilter> resolveRequestFilter(HttpServletRequest servletRequest, URI jdiscUri) { - Optional<String> maybeFilterId = bindings.resolveRequestFilter(jdiscUri, getConnector(servletRequest).listenPort()); + Optional<RequestFilter> resolveRequestFilter(Request request, URI jdiscUri) { + Optional<String> maybeFilterId = bindings.resolveRequestFilter(jdiscUri, getConnector(request).listenPort()); if (maybeFilterId.isPresent()) { - metric.add(MetricDefinitions.FILTERING_REQUEST_HANDLED, 1L, createMetricContext(servletRequest, maybeFilterId.get())); - servletRequest.setAttribute(ServletRequest.JDISC_REQUEST_CHAIN, maybeFilterId.get()); + metric.add(MetricDefinitions.FILTERING_REQUEST_HANDLED, 1L, createMetricContext(request, maybeFilterId.get())); + request.setAttribute(ServletRequest.JDISC_REQUEST_CHAIN, maybeFilterId.get()); } else if (!strictFiltering) { - metric.add(MetricDefinitions.FILTERING_REQUEST_UNHANDLED, 1L, createMetricContext(servletRequest, null)); + metric.add(MetricDefinitions.FILTERING_REQUEST_UNHANDLED, 1L, createMetricContext(request, null)); } else { String syntheticFilterId = RejectingRequestFilter.SYNTHETIC_FILTER_CHAIN_ID; - metric.add(MetricDefinitions.FILTERING_REQUEST_HANDLED, 1L, createMetricContext(servletRequest, syntheticFilterId)); - servletRequest.setAttribute(ServletRequest.JDISC_REQUEST_CHAIN, syntheticFilterId); + metric.add(MetricDefinitions.FILTERING_REQUEST_HANDLED, 1L, createMetricContext(request, syntheticFilterId)); + request.setAttribute(ServletRequest.JDISC_REQUEST_CHAIN, syntheticFilterId); return Optional.of(RejectingRequestFilter.INSTANCE); } return maybeFilterId.map(bindings::getRequestFilter); } - Optional<ResponseFilter> resolveResponseFilter(HttpServletRequest servletRequest, URI jdiscUri) { - Optional<String> maybeFilterId = bindings.resolveResponseFilter(jdiscUri, getConnector(servletRequest).listenPort()); + Optional<ResponseFilter> resolveResponseFilter(Request request, URI jdiscUri) { + Optional<String> maybeFilterId = bindings.resolveResponseFilter(jdiscUri, getConnector(request).listenPort()); if (maybeFilterId.isPresent()) { - metric.add(MetricDefinitions.FILTERING_RESPONSE_HANDLED, 1L, createMetricContext(servletRequest, maybeFilterId.get())); - servletRequest.setAttribute(ServletRequest.JDISC_RESPONSE_CHAIN, maybeFilterId.get()); + metric.add(MetricDefinitions.FILTERING_RESPONSE_HANDLED, 1L, createMetricContext(request, maybeFilterId.get())); + request.setAttribute(ServletRequest.JDISC_RESPONSE_CHAIN, maybeFilterId.get()); } else { - metric.add(MetricDefinitions.FILTERING_RESPONSE_UNHANDLED, 1L, createMetricContext(servletRequest, null)); + metric.add(MetricDefinitions.FILTERING_RESPONSE_UNHANDLED, 1L, createMetricContext(request, null)); } return maybeFilterId.map(bindings::getResponseFilter); } - private Metric.Context createMetricContext(HttpServletRequest request, String filterId) { + private Metric.Context createMetricContext(Request request, String filterId) { Map<String, String> extraDimensions = filterId != null ? Map.of(MetricDefinitions.FILTER_CHAIN_ID_DIMENSION, filterId) : Map.of(); - return JDiscHttpServlet.getConnector(request).createRequestMetricContext(request, extraDimensions); + return getConnector(request).createRequestMetricContext(request, extraDimensions); } private static class RejectingRequestFilter extends NoopSharedResource implements RequestFilter { diff --git a/container-core/src/main/java/com/yahoo/jdisc/http/server/jetty/FilteringRequestHandler.java b/container-core/src/main/java/com/yahoo/jdisc/http/server/jetty/FilteringRequestHandler.java index de768f979a1..43acbb9b096 100644 --- a/container-core/src/main/java/com/yahoo/jdisc/http/server/jetty/FilteringRequestHandler.java +++ b/container-core/src/main/java/com/yahoo/jdisc/http/server/jetty/FilteringRequestHandler.java @@ -2,12 +2,15 @@ package com.yahoo.jdisc.http.server.jetty; import com.google.common.base.Preconditions; +import com.yahoo.container.jdisc.RequestHandlerSpec; +import com.yahoo.container.jdisc.HttpRequestHandler; import com.yahoo.jdisc.Request; import com.yahoo.jdisc.Response; import com.yahoo.jdisc.handler.AbstractRequestHandler; import com.yahoo.jdisc.handler.BindingNotFoundException; import com.yahoo.jdisc.handler.CompletionHandler; import com.yahoo.jdisc.handler.ContentChannel; +import com.yahoo.jdisc.handler.DelegatedRequestHandler; import com.yahoo.jdisc.handler.RequestDeniedException; import com.yahoo.jdisc.handler.RequestHandler; import com.yahoo.jdisc.handler.ResponseHandler; @@ -15,9 +18,9 @@ import com.yahoo.jdisc.http.HttpRequest; import com.yahoo.jdisc.http.filter.RequestFilter; import com.yahoo.jdisc.http.filter.ResponseFilter; -import javax.servlet.http.HttpServletRequest; import java.nio.ByteBuffer; import java.util.Objects; +import java.util.Optional; import java.util.concurrent.atomic.AtomicBoolean; /** @@ -42,11 +45,11 @@ class FilteringRequestHandler extends AbstractRequestHandler { }; private final FilterResolver filterResolver; - private final HttpServletRequest servletRequest; + private final org.eclipse.jetty.server.Request jettyRequest; - public FilteringRequestHandler(FilterResolver filterResolver, HttpServletRequest servletRequest) { + public FilteringRequestHandler(FilterResolver filterResolver, org.eclipse.jetty.server.Request jettyRequest) { this.filterResolver = filterResolver; - this.servletRequest = servletRequest; + this.jettyRequest = jettyRequest; } @Override @@ -54,9 +57,9 @@ class FilteringRequestHandler extends AbstractRequestHandler { Preconditions.checkArgument(request instanceof HttpRequest, "Expected HttpRequest, got " + request); Objects.requireNonNull(originalResponseHandler, "responseHandler"); - RequestFilter requestFilter = filterResolver.resolveRequestFilter(servletRequest, request.getUri()) + RequestFilter requestFilter = filterResolver.resolveRequestFilter(jettyRequest, request.getUri()) .orElse(null); - ResponseFilter responseFilter = filterResolver.resolveResponseFilter(servletRequest, request.getUri()) + ResponseFilter responseFilter = filterResolver.resolveResponseFilter(jettyRequest, request.getUri()) .orElse(null); // Not using request.connect() here - it adds logic for error handling that we'd rather leave to the framework. @@ -66,6 +69,9 @@ class FilteringRequestHandler extends AbstractRequestHandler { throw new BindingNotFoundException(request.getUri()); } + getRequestHandlerSpec(resolvedRequestHandler) + .ifPresent(requestHandlerSpec -> request.context().put(RequestHandlerSpec.ATTRIBUTE_NAME, requestHandlerSpec)); + RequestHandler requestHandler = new ReferenceCountingRequestHandler(resolvedRequestHandler); ResponseHandler responseHandler; @@ -90,6 +96,18 @@ class FilteringRequestHandler extends AbstractRequestHandler { return contentChannel; } + private Optional<RequestHandlerSpec> getRequestHandlerSpec(RequestHandler resolvedRequestHandler) { + RequestHandler delegate = resolvedRequestHandler; + if (delegate instanceof DelegatedRequestHandler) { + delegate = ((DelegatedRequestHandler) delegate).getDelegateRecursive(); + } + if(delegate instanceof HttpRequestHandler) { + return Optional.ofNullable(((HttpRequestHandler) delegate).requestHandlerSpec()); + } else { + return Optional.empty(); + } + } + private static class FilteringResponseHandler implements ResponseHandler { private final ResponseHandler delegate; diff --git a/container-core/src/main/java/com/yahoo/jdisc/http/server/jetty/FormPostRequestHandler.java b/container-core/src/main/java/com/yahoo/jdisc/http/server/jetty/FormPostRequestHandler.java index 38f84438526..57fb32f89f0 100644 --- a/container-core/src/main/java/com/yahoo/jdisc/http/server/jetty/FormPostRequestHandler.java +++ b/container-core/src/main/java/com/yahoo/jdisc/http/server/jetty/FormPostRequestHandler.java @@ -7,6 +7,7 @@ import com.yahoo.jdisc.ResourceReference; import com.yahoo.jdisc.handler.AbstractRequestHandler; import com.yahoo.jdisc.handler.CompletionHandler; import com.yahoo.jdisc.handler.ContentChannel; +import com.yahoo.jdisc.handler.DelegatedRequestHandler; import com.yahoo.jdisc.handler.RequestHandler; import com.yahoo.jdisc.handler.ResponseHandler; import com.yahoo.jdisc.http.HttpRequest; @@ -38,7 +39,7 @@ import static com.yahoo.jdisc.http.server.jetty.CompletionHandlerUtils.NOOP_COMP * @author bakksjo * $Id$ */ -class FormPostRequestHandler extends AbstractRequestHandler implements ContentChannel { +class FormPostRequestHandler extends AbstractRequestHandler implements ContentChannel, DelegatedRequestHandler { private final ByteArrayOutputStream accumulatedRequestContent = new ByteArrayOutputStream(); private final RequestHandler delegateHandler; @@ -185,4 +186,9 @@ class FormPostRequestHandler extends AbstractRequestHandler implements ContentCh } } } + + @Override + public RequestHandler getDelegate() { + return delegateHandler; + } } diff --git a/container-core/src/main/java/com/yahoo/jdisc/http/server/jetty/HealthCheckProxyHandler.java b/container-core/src/main/java/com/yahoo/jdisc/http/server/jetty/HealthCheckProxyHandler.java index 0f7ce77e4cd..8b6192bb455 100644 --- a/container-core/src/main/java/com/yahoo/jdisc/http/server/jetty/HealthCheckProxyHandler.java +++ b/container-core/src/main/java/com/yahoo/jdisc/http/server/jetty/HealthCheckProxyHandler.java @@ -40,7 +40,7 @@ import java.util.concurrent.Executors; import java.util.logging.Level; import java.util.logging.Logger; -import static com.yahoo.jdisc.http.server.jetty.HttpServletRequestUtils.getConnectorLocalPort; +import static com.yahoo.jdisc.http.server.jetty.RequestUtils.getConnectorLocalPort; /** * A handler that proxies status.html health checks @@ -91,7 +91,7 @@ class HealthCheckProxyHandler extends HandlerWrapper { @Override public void handle(String target, Request request, HttpServletRequest servletRequest, HttpServletResponse servletResponse) throws IOException, ServletException { - int localPort = getConnectorLocalPort(servletRequest); + int localPort = getConnectorLocalPort(request); ProxyTarget proxyTarget = portToProxyTargetMapping.get(localPort); if (proxyTarget != null) { AsyncContext asyncContext = servletRequest.startAsync(); diff --git a/container-core/src/main/java/com/yahoo/jdisc/http/server/jetty/HttpRequestDispatch.java b/container-core/src/main/java/com/yahoo/jdisc/http/server/jetty/HttpRequestDispatch.java index 05715b13d10..7828751df5a 100644 --- a/container-core/src/main/java/com/yahoo/jdisc/http/server/jetty/HttpRequestDispatch.java +++ b/container-core/src/main/java/com/yahoo/jdisc/http/server/jetty/HttpRequestDispatch.java @@ -14,7 +14,6 @@ import com.yahoo.jdisc.http.ConnectorConfig; import com.yahoo.jdisc.http.HttpHeaders; import com.yahoo.jdisc.http.HttpRequest; import org.eclipse.jetty.io.EofException; -import org.eclipse.jetty.server.HttpConnection; import org.eclipse.jetty.server.Request; import javax.servlet.AsyncContext; @@ -34,8 +33,8 @@ import java.util.logging.Level; import java.util.logging.Logger; import static com.yahoo.jdisc.http.HttpHeaders.Values.APPLICATION_X_WWW_FORM_URLENCODED; -import static com.yahoo.jdisc.http.server.jetty.HttpServletRequestUtils.getConnection; -import static com.yahoo.jdisc.http.server.jetty.JDiscHttpServlet.getConnector; +import static com.yahoo.jdisc.http.server.jetty.RequestUtils.getConnector; +import static com.yahoo.jdisc.http.server.jetty.RequestUtils.getHttp1Connection; import static com.yahoo.yolean.Exceptions.throwUnchecked; /** @@ -72,7 +71,7 @@ class HttpRequestDispatch { jDiscContext.janitor, metricReporter, jDiscContext.developerMode()); - markConnectionAsNonPersistentIfThresholdReached(servletRequest); + markHttp1ConnectionAsNonPersistentIfThresholdReached(jettyRequest); this.async = servletRequest.startAsync(); async.setTimeout(0); metricReporter.uriLength(jettyRequest.getOriginalURI().length()); @@ -139,22 +138,24 @@ class HttpRequestDispatch { }; } - private static void markConnectionAsNonPersistentIfThresholdReached(HttpServletRequest request) { + private static void markHttp1ConnectionAsNonPersistentIfThresholdReached(Request request) { ConnectorConfig connectorConfig = getConnector(request).connectorConfig(); int maxRequestsPerConnection = connectorConfig.maxRequestsPerConnection(); if (maxRequestsPerConnection > 0) { - HttpConnection connection = getConnection(request); - if (connection.getMessagesIn() >= maxRequestsPerConnection) { - connection.getGenerator().setPersistent(false); - } + getHttp1Connection(request).ifPresent(connection -> { + if (connection.getMessagesIn() >= maxRequestsPerConnection) { + connection.getGenerator().setPersistent(false); + } + }); } double maxConnectionLifeInSeconds = connectorConfig.maxConnectionLife(); if (maxConnectionLifeInSeconds > 0) { - HttpConnection connection = getConnection(request); - Instant expireAt = Instant.ofEpochMilli((long)(connection.getCreatedTimeStamp() + maxConnectionLifeInSeconds * 1000)); - if (Instant.now().isAfter(expireAt)) { - connection.getGenerator().setPersistent(false); - } + getHttp1Connection(request).ifPresent(connection -> { + Instant expireAt = Instant.ofEpochMilli((long) (connection.getCreatedTimeStamp() + maxConnectionLifeInSeconds * 1000)); + if (Instant.now().isAfter(expireAt)) { + connection.getGenerator().setPersistent(false); + } + }); } } @@ -212,7 +213,7 @@ class HttpRequestDispatch { AccessLogEntry accessLogEntry, HttpServletRequest servletRequest) { RequestHandler requestHandler = wrapHandlerIfFormPost( - new FilteringRequestHandler(context.filterResolver, servletRequest), + new FilteringRequestHandler(context.filterResolver, (Request)servletRequest), servletRequest, context.serverConfig.removeRawPostBodyForWwwUrlEncodedPost()); return new AccessLoggingRequestHandler(requestHandler, accessLogEntry); diff --git a/container-core/src/main/java/com/yahoo/jdisc/http/server/jetty/HttpRequestFactory.java b/container-core/src/main/java/com/yahoo/jdisc/http/server/jetty/HttpRequestFactory.java index e8d37cfadb5..8b223c45827 100644 --- a/container-core/src/main/java/com/yahoo/jdisc/http/server/jetty/HttpRequestFactory.java +++ b/container-core/src/main/java/com/yahoo/jdisc/http/server/jetty/HttpRequestFactory.java @@ -4,6 +4,7 @@ package com.yahoo.jdisc.http.server.jetty; import com.yahoo.jdisc.http.HttpRequest; import com.yahoo.jdisc.http.servlet.ServletRequest; import com.yahoo.jdisc.service.CurrentContainer; +import org.eclipse.jetty.server.Request; import org.eclipse.jetty.util.Utf8Appendable; import javax.servlet.http.HttpServletRequest; @@ -13,8 +14,8 @@ import java.security.cert.X509Certificate; import java.util.Enumeration; import static com.yahoo.jdisc.Response.Status.BAD_REQUEST; -import static com.yahoo.jdisc.http.server.jetty.HttpServletRequestUtils.getConnection; -import static com.yahoo.jdisc.http.server.jetty.HttpServletRequestUtils.getConnectorLocalPort; +import static com.yahoo.jdisc.http.server.jetty.RequestUtils.getConnection; +import static com.yahoo.jdisc.http.server.jetty.RequestUtils.getConnectorLocalPort; /** * @author Simon Thoresen Hult @@ -30,7 +31,7 @@ class HttpRequestFactory { HttpRequest.Method.valueOf(servletRequest.getMethod()), HttpRequest.Version.fromString(servletRequest.getProtocol()), new InetSocketAddress(servletRequest.getRemoteAddr(), servletRequest.getRemotePort()), - getConnection(servletRequest).getCreatedTimeStamp()); + getConnection((Request) servletRequest).getCreatedTimeStamp()); httpRequest.context().put(ServletRequest.JDISC_REQUEST_X509CERT, getCertChain(servletRequest)); return httpRequest; } catch (Utf8Appendable.NotUtf8Exception e) { @@ -43,7 +44,7 @@ class HttpRequestFactory { try { String scheme = servletRequest.getScheme(); String host = servletRequest.getServerName(); - int port = getConnectorLocalPort(servletRequest); + int port = getConnectorLocalPort((Request) servletRequest); String path = servletRequest.getRequestURI(); String query = servletRequest.getQueryString(); diff --git a/container-core/src/main/java/com/yahoo/jdisc/http/server/jetty/JDiscFilterInvokerFilter.java b/container-core/src/main/java/com/yahoo/jdisc/http/server/jetty/JDiscFilterInvokerFilter.java index a89c115a1c2..2904d79ad41 100644 --- a/container-core/src/main/java/com/yahoo/jdisc/http/server/jetty/JDiscFilterInvokerFilter.java +++ b/container-core/src/main/java/com/yahoo/jdisc/http/server/jetty/JDiscFilterInvokerFilter.java @@ -4,6 +4,7 @@ package com.yahoo.jdisc.http.server.jetty; import com.yahoo.container.logging.AccessLogEntry; import com.yahoo.jdisc.handler.ResponseHandler; import com.yahoo.jdisc.http.filter.RequestFilter; +import org.eclipse.jetty.server.Request; import javax.servlet.AsyncContext; import javax.servlet.AsyncListener; @@ -26,7 +27,7 @@ import java.util.Map; import java.util.Optional; import java.util.concurrent.atomic.AtomicReference; -import static com.yahoo.jdisc.http.server.jetty.JDiscHttpServlet.getConnector; +import static com.yahoo.jdisc.http.server.jetty.RequestUtils.getConnector; import static com.yahoo.yolean.Exceptions.throwUnchecked; /** @@ -77,7 +78,7 @@ class JDiscFilterInvokerFilter implements Filter { private void runChainAndResponseFilters(URI uri, HttpServletRequest request, HttpServletResponse response, FilterChain chain) throws IOException, ServletException { Optional<OneTimeRunnable> responseFilterInvoker = - jDiscContext.filterResolver.resolveResponseFilter(request, uri) + jDiscContext.filterResolver.resolveResponseFilter(toJettyRequest(request), uri) .map(responseFilter -> new OneTimeRunnable(() -> filterInvoker.invokeResponseFilterChain(responseFilter, uri, request, response))); @@ -107,7 +108,7 @@ class JDiscFilterInvokerFilter implements Filter { private HttpServletRequest runRequestFilterWithMatchingBinding(AtomicReference<Boolean> responseReturned, URI uri, HttpServletRequest request, HttpServletResponse response) throws IOException { try { - RequestFilter requestFilter = jDiscContext.filterResolver.resolveRequestFilter(request, uri).orElse(null); + RequestFilter requestFilter = jDiscContext.filterResolver.resolveRequestFilter(toJettyRequest(request), uri).orElse(null); if (requestFilter == null) return request; @@ -134,13 +135,20 @@ class JDiscFilterInvokerFilter implements Filter { final AccessLogEntry accessLogEntry = null; // Not used in this context. return new HttpRequestDispatch(jDiscContext, accessLogEntry, - getConnector(request).createRequestMetricContext(request, Map.of()), + getConnector(toJettyRequest(request)).createRequestMetricContext(request, Map.of()), request, response); } catch (IOException e) { throw throwUnchecked(e); } } + private static Request toJettyRequest(HttpServletRequest request) { + if (request instanceof com.yahoo.jdisc.http.servlet.ServletRequest) { + return (Request) ((com.yahoo.jdisc.http.servlet.ServletRequest)request).getRequest(); + } + return (Request) request; + } + @Override public void destroy() {} diff --git a/container-core/src/main/java/com/yahoo/jdisc/http/server/jetty/JDiscHttpServlet.java b/container-core/src/main/java/com/yahoo/jdisc/http/server/jetty/JDiscHttpServlet.java index 41a1ffc2709..7e1445ffa4f 100644 --- a/container-core/src/main/java/com/yahoo/jdisc/http/server/jetty/JDiscHttpServlet.java +++ b/container-core/src/main/java/com/yahoo/jdisc/http/server/jetty/JDiscHttpServlet.java @@ -5,6 +5,7 @@ import com.yahoo.container.logging.AccessLogEntry; import com.yahoo.jdisc.Metric; import com.yahoo.jdisc.handler.OverloadException; import com.yahoo.jdisc.http.HttpRequest.Method; +import org.eclipse.jetty.server.Request; import javax.servlet.ServletException; import javax.servlet.annotation.WebServlet; @@ -20,7 +21,7 @@ import java.util.logging.Logger; import java.util.stream.Collectors; import java.util.stream.Stream; -import static com.yahoo.jdisc.http.server.jetty.HttpServletRequestUtils.getConnection; +import static com.yahoo.jdisc.http.server.jetty.RequestUtils.getConnector; /** * @author Simon Thoresen Hult @@ -85,7 +86,7 @@ class JDiscHttpServlet extends HttpServlet { @Override protected void service(HttpServletRequest request, HttpServletResponse response) throws ServletException, IOException { - request.setAttribute(JDiscServerConnector.REQUEST_ATTRIBUTE, getConnector(request)); + request.setAttribute(JDiscServerConnector.REQUEST_ATTRIBUTE, getConnector((Request) request)); Metric.Context metricContext = getMetricContext(request); context.metric.add(MetricDefinitions.NUM_REQUESTS, 1, metricContext); @@ -103,10 +104,6 @@ class JDiscHttpServlet extends HttpServlet { } } - static JDiscServerConnector getConnector(HttpServletRequest request) { - return (JDiscServerConnector)getConnection(request).getConnector(); - } - private void dispatchHttpRequest(HttpServletRequest request, HttpServletResponse response) throws IOException { AccessLogEntry accessLogEntry = new AccessLogEntry(); request.setAttribute(ATTRIBUTE_NAME_ACCESS_LOG_ENTRY, accessLogEntry); diff --git a/container-core/src/main/java/com/yahoo/jdisc/http/server/jetty/JettyConnectionLogger.java b/container-core/src/main/java/com/yahoo/jdisc/http/server/jetty/JettyConnectionLogger.java index cd1ca490f61..e7cdb13425f 100644 --- a/container-core/src/main/java/com/yahoo/jdisc/http/server/jetty/JettyConnectionLogger.java +++ b/container-core/src/main/java/com/yahoo/jdisc/http/server/jetty/JettyConnectionLogger.java @@ -6,6 +6,7 @@ import com.yahoo.container.logging.ConnectionLogEntry; import com.yahoo.container.logging.ConnectionLogEntry.SslHandshakeFailure.ExceptionEntry; import com.yahoo.io.HexDump; import com.yahoo.jdisc.http.ServerConfig; +import org.eclipse.jetty.http2.server.HTTP2ServerConnection; import org.eclipse.jetty.io.Connection; import org.eclipse.jetty.io.EndPoint; import org.eclipse.jetty.io.SocketChannelEndPoint; @@ -94,9 +95,18 @@ class JettyConnectionLogger extends AbstractLifeCycle implements Connection.List info = ConnectionInfo.from(endpoint); connectionInfo.put(IdentityKey.of(endpoint), info); } + String connectionClassName = connection.getClass().getSimpleName(); // For hidden implementations of Connection if (connection instanceof SslConnection) { SSLEngine sslEngine = ((SslConnection) connection).getSSLEngine(); sslToConnectionInfo.put(IdentityKey.of(sslEngine), info); + } else if (connection instanceof HttpConnection) { + info.setHttpProtocol("HTTP/1.1"); + } else if (connection instanceof HTTP2ServerConnection) { + info.setHttpProtocol("HTTP/2.0"); + } else if (connectionClassName.endsWith("ProxyProtocolV1Connection")) { + info.setProxyProtocolVersion("v1"); + } else if (connectionClassName.endsWith("ProxyProtocolV2Connection")) { + info.setProxyProtocolVersion("v2"); } if (connection.getEndPoint() instanceof ProxyConnectionFactory.ProxyEndPoint) { InetSocketAddress remoteAddress = connection.getEndPoint().getRemoteAddress(); @@ -227,6 +237,8 @@ class JettyConnectionLogger extends AbstractLifeCycle implements Connection.List private Date sslPeerNotAfter; private List<SNIServerName> sslSniServerNames; private SSLHandshakeException sslHandshakeException; + private String proxyProtocolVersion; + private String httpProtocol; private ConnectionInfo(UUID uuid, long createdAt, InetSocketAddress localAddress, InetSocketAddress peerAddress) { this.uuid = uuid; @@ -290,6 +302,10 @@ class JettyConnectionLogger extends AbstractLifeCycle implements Connection.List return this; } + synchronized ConnectionInfo setHttpProtocol(String protocol) { this.httpProtocol = protocol; return this; } + + synchronized ConnectionInfo setProxyProtocolVersion(String version) { this.proxyProtocolVersion = version; return this; } + synchronized ConnectionLogEntry toLogEntry() { ConnectionLogEntry.Builder builder = ConnectionLogEntry.builder(uuid, Instant.ofEpochMilli(createdAt)); if (closedAt > 0) { @@ -348,6 +364,12 @@ class JettyConnectionLogger extends AbstractLifeCycle implements Connection.List .orElse("UNKNOWN"); builder.withSslHandshakeFailure(new ConnectionLogEntry.SslHandshakeFailure(type, exceptionChain)); } + if (httpProtocol != null) { + builder.withHttpProtocol(httpProtocol); + } + if (proxyProtocolVersion != null) { + builder.withProxyProtocolVersion(proxyProtocolVersion); + } return builder.build(); } diff --git a/container-core/src/main/java/com/yahoo/jdisc/http/server/jetty/ReferenceCountingRequestHandler.java b/container-core/src/main/java/com/yahoo/jdisc/http/server/jetty/ReferenceCountingRequestHandler.java index f2bf5b56d5c..71cca62ce9c 100644 --- a/container-core/src/main/java/com/yahoo/jdisc/http/server/jetty/ReferenceCountingRequestHandler.java +++ b/container-core/src/main/java/com/yahoo/jdisc/http/server/jetty/ReferenceCountingRequestHandler.java @@ -7,6 +7,7 @@ import com.yahoo.jdisc.Response; import com.yahoo.jdisc.SharedResource; import com.yahoo.jdisc.handler.CompletionHandler; import com.yahoo.jdisc.handler.ContentChannel; +import com.yahoo.jdisc.handler.DelegatedRequestHandler; import com.yahoo.jdisc.handler.NullContent; import com.yahoo.jdisc.handler.RequestHandler; import com.yahoo.jdisc.handler.ResponseHandler; @@ -26,7 +27,7 @@ import java.util.logging.Logger; * @author bakksjo */ @SuppressWarnings("try") -class ReferenceCountingRequestHandler implements RequestHandler { +class ReferenceCountingRequestHandler implements DelegatedRequestHandler { private static final Logger log = Logger.getLogger(ReferenceCountingRequestHandler.class.getName()); @@ -79,6 +80,11 @@ class ReferenceCountingRequestHandler implements RequestHandler { return delegate.toString(); } + @Override + public RequestHandler getDelegate() { + return delegate; + } + private static class ReferenceCountingResponseHandler implements ResponseHandler { final SharedResource request; diff --git a/container-core/src/main/java/com/yahoo/jdisc/http/server/jetty/HttpServletRequestUtils.java b/container-core/src/main/java/com/yahoo/jdisc/http/server/jetty/RequestUtils.java index e7b9f459d2e..5fca7a8d778 100644 --- a/container-core/src/main/java/com/yahoo/jdisc/http/server/jetty/HttpServletRequestUtils.java +++ b/container-core/src/main/java/com/yahoo/jdisc/http/server/jetty/RequestUtils.java @@ -1,26 +1,39 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.jdisc.http.server.jetty; +import org.eclipse.jetty.io.Connection; import org.eclipse.jetty.server.HttpConnection; +import org.eclipse.jetty.server.Request; import javax.servlet.http.HttpServletRequest; +import java.util.Optional; /** * @author bjorncs */ -public class HttpServletRequestUtils { - private HttpServletRequestUtils() {} +public class RequestUtils { + private RequestUtils() {} - public static HttpConnection getConnection(HttpServletRequest request) { - return (HttpConnection)request.getAttribute("org.eclipse.jetty.server.HttpConnection"); + public static Connection getConnection(Request request) { + return request.getHttpChannel().getConnection(); + } + + public static Optional<HttpConnection> getHttp1Connection(Request request) { + Connection connection = getConnection(request); + if (connection instanceof HttpConnection) return Optional.of((HttpConnection) connection); + return Optional.empty(); + } + + public static JDiscServerConnector getConnector(Request request) { + return (JDiscServerConnector) request.getHttpChannel().getConnector(); } /** * Note: {@link HttpServletRequest#getLocalPort()} may return the local port of the load balancer / reverse proxy if proxy-protocol is enabled. * @return the actual local port of the underlying Jetty connector */ - public static int getConnectorLocalPort(HttpServletRequest request) { - JDiscServerConnector connector = (JDiscServerConnector) getConnection(request).getConnector(); + public static int getConnectorLocalPort(Request request) { + JDiscServerConnector connector = getConnector(request); int actualLocalPort = connector.getLocalPort(); int localPortIfConnectorUnopened = -1; int localPortIfConnectorClosed = -2; diff --git a/container-core/src/main/java/com/yahoo/jdisc/http/server/jetty/SecuredRedirectHandler.java b/container-core/src/main/java/com/yahoo/jdisc/http/server/jetty/SecuredRedirectHandler.java index e32c9d46deb..dad274ae520 100644 --- a/container-core/src/main/java/com/yahoo/jdisc/http/server/jetty/SecuredRedirectHandler.java +++ b/container-core/src/main/java/com/yahoo/jdisc/http/server/jetty/SecuredRedirectHandler.java @@ -14,7 +14,7 @@ import java.util.HashMap; import java.util.List; import java.util.Map; -import static com.yahoo.jdisc.http.server.jetty.HttpServletRequestUtils.getConnectorLocalPort; +import static com.yahoo.jdisc.http.server.jetty.RequestUtils.getConnectorLocalPort; /** * A secure redirect handler inspired by {@link org.eclipse.jetty.server.handler.SecuredRedirectHandler}. @@ -33,7 +33,7 @@ class SecuredRedirectHandler extends HandlerWrapper { @Override public void handle(String target, Request request, HttpServletRequest servletRequest, HttpServletResponse servletResponse) throws IOException, ServletException { - int localPort = getConnectorLocalPort(servletRequest); + int localPort = getConnectorLocalPort(request); if (!redirectMap.containsKey(localPort)) { _handler.handle(target, request, servletRequest, servletResponse); return; diff --git a/container-core/src/main/java/com/yahoo/jdisc/http/server/jetty/TlsClientAuthenticationEnforcer.java b/container-core/src/main/java/com/yahoo/jdisc/http/server/jetty/TlsClientAuthenticationEnforcer.java index 10a6c4702b5..7299ab4b500 100644 --- a/container-core/src/main/java/com/yahoo/jdisc/http/server/jetty/TlsClientAuthenticationEnforcer.java +++ b/container-core/src/main/java/com/yahoo/jdisc/http/server/jetty/TlsClientAuthenticationEnforcer.java @@ -16,7 +16,7 @@ import java.util.HashMap; import java.util.List; import java.util.Map; -import static com.yahoo.jdisc.http.server.jetty.HttpServletRequestUtils.getConnectorLocalPort; +import static com.yahoo.jdisc.http.server.jetty.RequestUtils.getConnectorLocalPort; /** * A Jetty handler that enforces TLS client authentication with configurable white list. @@ -34,7 +34,7 @@ class TlsClientAuthenticationEnforcer extends HandlerWrapper { @Override public void handle(String target, Request request, HttpServletRequest servletRequest, HttpServletResponse servletResponse) throws IOException, ServletException { if (isHttpsRequest(request) - && !isRequestToWhitelistedBinding(servletRequest) + && !isRequestToWhitelistedBinding(request) && !isClientAuthenticated(servletRequest)) { servletResponse.sendError( Response.Status.UNAUTHORIZED, @@ -60,14 +60,14 @@ class TlsClientAuthenticationEnforcer extends HandlerWrapper { return request.getDispatcherType() == DispatcherType.REQUEST && request.getScheme().equalsIgnoreCase("https"); } - private boolean isRequestToWhitelistedBinding(HttpServletRequest servletRequest) { - int localPort = getConnectorLocalPort(servletRequest); + private boolean isRequestToWhitelistedBinding(Request jettyRequest) { + int localPort = getConnectorLocalPort(jettyRequest); List<String> whiteListedPaths = getWhitelistedPathsForPort(localPort); if (whiteListedPaths == null) { return true; // enforcer not enabled } // Note: Same path definition as HttpRequestFactory.getUri() - return whiteListedPaths.contains(servletRequest.getRequestURI()); + return whiteListedPaths.contains(jettyRequest.getRequestURI()); } private List<String> getWhitelistedPathsForPort(int localPort) { diff --git a/container-core/src/main/java/com/yahoo/jdisc/http/server/jetty/testutils/ConnectorFactoryRegistryModule.java b/container-core/src/main/java/com/yahoo/jdisc/http/server/jetty/testutils/ConnectorFactoryRegistryModule.java new file mode 100644 index 00000000000..9d475309955 --- /dev/null +++ b/container-core/src/main/java/com/yahoo/jdisc/http/server/jetty/testutils/ConnectorFactoryRegistryModule.java @@ -0,0 +1,52 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.jdisc.http.server.jetty.testutils; + +import com.google.inject.Binder; +import com.google.inject.Module; +import com.google.inject.Provides; +import com.yahoo.component.ComponentId; +import com.yahoo.component.provider.ComponentRegistry; +import com.yahoo.jdisc.http.ConnectorConfig; + +import com.yahoo.jdisc.http.server.jetty.ConnectorFactory; +import com.yahoo.jdisc.http.ssl.impl.ConfiguredSslContextFactoryProvider; + +/** + * Guice module for test ConnectorFactories + * + * @author Tony Vaagenes + */ +public class ConnectorFactoryRegistryModule implements Module { + + private final ConnectorConfig config; + + public ConnectorFactoryRegistryModule(ConnectorConfig config) { + this.config = config; + } + + public ConnectorFactoryRegistryModule() { + this(new ConnectorConfig(new ConnectorConfig.Builder())); + } + + @SuppressWarnings("unused") + @Provides + public ComponentRegistry<ConnectorFactory> connectorFactoryComponentRegistry() { + ComponentRegistry<ConnectorFactory> registry = new ComponentRegistry<>(); + registry.register(ComponentId.createAnonymousComponentId("connector-factory"), + new StaticKeyDbConnectorFactory(config)); + + registry.freeze(); + return registry; + } + + @Override public void configure(Binder binder) {} + + private static class StaticKeyDbConnectorFactory extends ConnectorFactory { + + public StaticKeyDbConnectorFactory(ConnectorConfig connectorConfig) { + super(connectorConfig, new ConfiguredSslContextFactoryProvider(connectorConfig)); + } + + } + +} diff --git a/container-core/src/main/java/com/yahoo/jdisc/http/server/jetty/testutils/ServletModule.java b/container-core/src/main/java/com/yahoo/jdisc/http/server/jetty/testutils/ServletModule.java new file mode 100644 index 00000000000..a507255c9b7 --- /dev/null +++ b/container-core/src/main/java/com/yahoo/jdisc/http/server/jetty/testutils/ServletModule.java @@ -0,0 +1,23 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.jdisc.http.server.jetty.testutils; + +import com.google.inject.Binder; +import com.google.inject.Module; +import com.google.inject.Provides; +import com.yahoo.component.provider.ComponentRegistry; + +import org.eclipse.jetty.servlet.ServletHolder; + +/** + * @author Tony Vaagenes + */ +public class ServletModule implements Module { + + @SuppressWarnings("unused") + @Provides + public ComponentRegistry<ServletHolder> servletHolderComponentRegistry() { + return new ComponentRegistry<>(); + } + + @Override public void configure(Binder binder) { } +} diff --git a/container-core/src/main/java/com/yahoo/jdisc/http/server/jetty/testutils/TestDriver.java b/container-core/src/main/java/com/yahoo/jdisc/http/server/jetty/testutils/TestDriver.java new file mode 100644 index 00000000000..7f3d54f1d34 --- /dev/null +++ b/container-core/src/main/java/com/yahoo/jdisc/http/server/jetty/testutils/TestDriver.java @@ -0,0 +1,122 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.jdisc.http.server.jetty.testutils; + +import com.google.inject.AbstractModule; +import com.google.inject.Module; +import com.google.inject.util.Modules; +import com.yahoo.container.logging.ConnectionLog; +import com.yahoo.container.logging.RequestLog; +import com.yahoo.jdisc.application.ContainerBuilder; +import com.yahoo.jdisc.handler.RequestHandler; +import com.yahoo.jdisc.http.ConnectorConfig; +import com.yahoo.jdisc.http.ServerConfig; +import com.yahoo.jdisc.http.ServletPathsConfig; +import com.yahoo.jdisc.http.server.jetty.FilterBindings; +import com.yahoo.jdisc.http.server.jetty.JettyHttpServer; +import com.yahoo.jdisc.http.server.jetty.VoidConnectionLog; +import com.yahoo.jdisc.http.server.jetty.VoidRequestLog; +import com.yahoo.security.SslContextBuilder; + +import javax.net.ssl.SSLContext; +import java.nio.file.Paths; +import java.util.ArrayList; +import java.util.List; +import java.util.SortedMap; +import java.util.TreeMap; + +/** + * A {@link com.yahoo.jdisc.test.TestDriver} that is configured with {@link JettyHttpServer}. + * + * @author bjorncs + */ +public class TestDriver implements AutoCloseable { + + private final com.yahoo.jdisc.test.TestDriver jdiscCoreTestDriver; + private final JettyHttpServer server; + private final SSLContext sslContext; + + private TestDriver(Builder builder) { + ServerConfig serverConfig = + builder.serverConfig != null ? builder.serverConfig : new ServerConfig(new ServerConfig.Builder()); + ConnectorConfig connectorConfig = + builder.connectorConfig != null ? builder.connectorConfig : new ConnectorConfig(new ConnectorConfig.Builder()); + Module baseModule = createBaseModule(serverConfig, connectorConfig); + Module combinedModule = + builder.extraGuiceModules.isEmpty() ? baseModule : Modules.override(baseModule).with(builder.extraGuiceModules); + com.yahoo.jdisc.test.TestDriver jdiscCoreTestDriver = + com.yahoo.jdisc.test.TestDriver.newSimpleApplicationInstance(combinedModule); + ContainerBuilder containerBuilder = jdiscCoreTestDriver.newContainerBuilder(); + JettyHttpServer server = containerBuilder.getInstance(JettyHttpServer.class); + containerBuilder.serverProviders().install(server); + builder.handlers.forEach((binding, handler) -> containerBuilder.serverBindings().bind(binding, handler)); + jdiscCoreTestDriver.activateContainer(containerBuilder); + server.start(); + this.jdiscCoreTestDriver = jdiscCoreTestDriver; + this.server = server; + this.sslContext = newSslContext(containerBuilder); + } + + public static Builder newBuilder() { return new Builder(); } + + public SSLContext sslContext() { return sslContext; } + public JettyHttpServer server() { return server; } + + @Override public void close() { shutdown(); } + + public boolean shutdown() { + server.close(); + server.release(); + return jdiscCoreTestDriver.close(); + } + + private static SSLContext newSslContext(ContainerBuilder builder) { + ConnectorConfig.Ssl sslConfig = builder.getInstance(ConnectorConfig.class).ssl(); + if (!sslConfig.enabled()) return null; + + return new SslContextBuilder() + .withKeyStore(Paths.get(sslConfig.privateKeyFile()), Paths.get(sslConfig.certificateFile())) + .withTrustStore(Paths.get(sslConfig.caCertificateFile())) + .build(); + } + + private static Module createBaseModule(ServerConfig serverConfig, ConnectorConfig connectorConfig) { + return Modules.combine( + new AbstractModule() { + @Override + protected void configure() { + bind(ServletPathsConfig.class).toInstance(new ServletPathsConfig(new ServletPathsConfig.Builder())); + bind(ServerConfig.class).toInstance(serverConfig); + bind(ConnectorConfig.class).toInstance(connectorConfig); + bind(FilterBindings.class).toInstance(new FilterBindings.Builder().build()); + bind(ConnectionLog.class).toInstance(new VoidConnectionLog()); + bind(RequestLog.class).toInstance(new VoidRequestLog()); + } + }, + new ConnectorFactoryRegistryModule(connectorConfig), + new ServletModule()); + } + + public static class Builder { + private final SortedMap<String, RequestHandler> handlers = new TreeMap<>(); + private final List<Module> extraGuiceModules = new ArrayList<>(); + private ServerConfig serverConfig; + private ConnectorConfig connectorConfig; + + private Builder() {} + + public Builder withRequestHandler(String binding, RequestHandler handler) { + this.handlers.put(binding, handler); return this; + } + + public Builder withRequestHandler(RequestHandler handler) { return withRequestHandler("http://*/*", handler); } + + public Builder withServerConfig(ServerConfig config) { this.serverConfig = config; return this; } + + public Builder withConnectorConfig(ConnectorConfig config) { this.connectorConfig = config; return this; } + + public Builder withGuiceModule(Module module) { this.extraGuiceModules.add(module); return this; } + + public TestDriver build() { return new TestDriver(this); } + + } +} diff --git a/container-core/src/main/java/com/yahoo/jdisc/http/servlet/ServletRequest.java b/container-core/src/main/java/com/yahoo/jdisc/http/servlet/ServletRequest.java index c945dc6d8b6..bb78511a17f 100644 --- a/container-core/src/main/java/com/yahoo/jdisc/http/servlet/ServletRequest.java +++ b/container-core/src/main/java/com/yahoo/jdisc/http/servlet/ServletRequest.java @@ -6,6 +6,7 @@ import com.yahoo.jdisc.HeaderFields; import com.yahoo.jdisc.http.Cookie; import com.yahoo.jdisc.http.HttpHeaders; import com.yahoo.jdisc.http.HttpRequest; +import org.eclipse.jetty.server.Request; import javax.servlet.http.HttpServletRequest; import javax.servlet.http.HttpServletRequestWrapper; @@ -24,7 +25,7 @@ import java.util.Map; import java.util.Set; import java.util.concurrent.TimeUnit; -import static com.yahoo.jdisc.http.server.jetty.HttpServletRequestUtils.getConnection; +import static com.yahoo.jdisc.http.server.jetty.RequestUtils.getConnection; /** * Mutable wrapper to use a {@link javax.servlet.http.HttpServletRequest} @@ -68,7 +69,7 @@ public class ServletRequest extends HttpServletRequestWrapper implements Servlet remoteHostAddress = request.getRemoteAddr(); remoteHostName = request.getRemoteHost(); remotePort = request.getRemotePort(); - connectedAt = getConnection(request).getCreatedTimeStamp(); + connectedAt = getConnection((Request) request).getCreatedTimeStamp(); headerFields = new HeaderFields(); Enumeration<String> parentHeaders = request.getHeaderNames(); diff --git a/container-core/src/main/java/com/yahoo/metrics/simple/Bucket.java b/container-core/src/main/java/com/yahoo/metrics/simple/Bucket.java new file mode 100644 index 00000000000..b75a0529a03 --- /dev/null +++ b/container-core/src/main/java/com/yahoo/metrics/simple/Bucket.java @@ -0,0 +1,209 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.metrics.simple; + +import java.util.AbstractMap.SimpleImmutableEntry; +import java.util.ArrayList; +import java.util.Collection; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.logging.Logger; + +import com.yahoo.collections.LazyMap; +import com.yahoo.collections.LazySet; +import java.util.logging.Level; + +/** + * An aggregation of data which is only written to from a single thread. + * + * @author Steinar Knutsen + */ +public class Bucket { + + private static final Logger log = Logger.getLogger(Bucket.class.getName()); + private final Map<Identifier, UntypedMetric> values = LazyMap.newHashMap(); + + boolean gotTimeStamps; + long fromMillis; + long toMillis; + + public Bucket() { + this.gotTimeStamps = false; + this.fromMillis = 0; + this.toMillis = 0; + } + + public Bucket(long fromMillis, long toMillis) { + this.gotTimeStamps = true; + this.fromMillis = fromMillis; + this.toMillis = toMillis; + } + + public Set<Map.Entry<Identifier, UntypedMetric>> entrySet() { + return values.entrySet(); + } + + void put(Sample x) { + UntypedMetric value = get(x); + Measurement m = x.getMeasurement(); + switch (x.getMetricType()) { + case GAUGE: + value.put(m.getMagnitude()); + break; + case COUNTER: + value.add(m.getMagnitude()); + break; + default: + throw new IllegalArgumentException("Unsupported metric type: " + x.getMetricType()); + } + } + + void put(Identifier id, UntypedMetric value) { + values.put(id, value); + } + + boolean hasIdentifier(Identifier id) { + return values.containsKey(id); + } + + void merge(Bucket other, boolean otherIsNewer) { + LazySet<String> malformedMetrics = LazySet.newHashSet(); + for (Map.Entry<Identifier, UntypedMetric> entry : other.values.entrySet()) { + String metricName = entry.getKey().getName(); + try { + if (!malformedMetrics.contains(metricName)) { + get(entry.getKey(), entry.getValue()).merge(entry.getValue(), otherIsNewer); + } + } catch (IllegalArgumentException e) { + log.log(Level.WARNING, "Problems merging metric " + metricName + ", possibly ignoring data."); + // avoid spamming the log if there are a lot of mismatching + // threads + malformedMetrics.add(metricName); + } + } + } + + void merge(Bucket other) { + boolean otherIsNewer = resolveTimeStamps(other); + merge(other, otherIsNewer); + } + + private boolean resolveTimeStamps(Bucket other) { + boolean otherIsNewer = other.fromMillis > this.fromMillis; + if (! gotTimeStamps) { + fromMillis = other.fromMillis; + toMillis = other.toMillis; + gotTimeStamps = other.gotTimeStamps; + } else if (other.gotTimeStamps) { + fromMillis = Math.min(fromMillis, other.fromMillis); + toMillis = Math.max(toMillis, other.toMillis); + } + return otherIsNewer; + } + + private UntypedMetric get(Sample sample) { + Identifier dim = sample.getIdentifier(); + UntypedMetric v = values.get(dim); + + if (v == null) { + // please keep inside guard, as sample.getHistogramDefinition(String) touches a volatile + v = new UntypedMetric(sample.getHistogramDefinition(dim.getName())); + values.put(dim, v); + } + return v; + } + + private UntypedMetric get(Identifier dim, UntypedMetric other) { + UntypedMetric v = values.get(dim); + + if (v == null) { + v = new UntypedMetric(other.getMetricDefinition()); + values.put(dim, v); + } + return v; + } + + public Collection<String> getAllMetricNames() { + Set<String> names = new HashSet<>(); + for (Identifier id : values.keySet()) { + names.add(id.getName()); + } + return names; + } + + public Collection<Map.Entry<Point, UntypedMetric>> getValuesForMetric(String metricName) { + List<Map.Entry<Point, UntypedMetric>> singleMetric = new ArrayList<>(); + for (Map.Entry<Identifier, UntypedMetric> entry : values.entrySet()) { + if (metricName.equals(entry.getKey().getName())) { + singleMetric.add(locationValuePair(entry)); + } + } + return singleMetric; + } + + public Map<Point, UntypedMetric> getMapForMetric(String metricName) { + Map<Point, UntypedMetric> result = new HashMap<>(); + for (Map.Entry<Identifier, UntypedMetric> entry : values.entrySet()) { + if (metricName.equals(entry.getKey().getName())) { + result.put(entry.getKey().getLocation(), entry.getValue()); + } + } + return result; + } + + public Map<String, List<Map.Entry<Point, UntypedMetric>>> getValuesByMetricName() { + Map<String, List<Map.Entry<Point, UntypedMetric>>> result = new HashMap<>(); + for (Map.Entry<Identifier, UntypedMetric> entry : values.entrySet()) { + List<Map.Entry<Point, UntypedMetric>> singleMetric; + if (result.containsKey(entry.getKey().getName())) { + singleMetric = result.get(entry.getKey().getName()); + } else { + singleMetric = new ArrayList<>(); + result.put(entry.getKey().getName(), singleMetric); + } + singleMetric.add(locationValuePair(entry)); + } + return result; + } + + private SimpleImmutableEntry<Point, UntypedMetric> locationValuePair(Map.Entry<Identifier, UntypedMetric> entry) { + return new SimpleImmutableEntry<>(entry.getKey().getLocation(), entry.getValue()); + } + + @Override + public String toString() { + return "Bucket [values=" + (values != null ? toString(values.entrySet(), 3) : null) + "]"; + } + + private String toString(Collection<?> collection, int maxLen) { + StringBuilder builder = new StringBuilder(); + builder.append("["); + int i = 0; + for (Iterator<?> iterator = collection.iterator(); iterator.hasNext() && i < maxLen; i++) { + if (i > 0) { + builder.append(", "); + } + builder.append(iterator.next()); + } + builder.append("]"); + return builder.toString(); + } + + /** + * This bucket contains data newer than approximately this point in time. + */ + public long getFromMillis() { + return fromMillis; + } + + /** + * This bucket contains data older than approximately this point in time. + */ + public long getToMillis() { + return toMillis; + } + +} diff --git a/container-core/src/main/java/com/yahoo/metrics/simple/Counter.java b/container-core/src/main/java/com/yahoo/metrics/simple/Counter.java new file mode 100644 index 00000000000..21cdbd3c219 --- /dev/null +++ b/container-core/src/main/java/com/yahoo/metrics/simple/Counter.java @@ -0,0 +1,73 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.metrics.simple; + +import com.google.common.annotations.Beta; +import com.yahoo.metrics.simple.UntypedMetric.AssumedType; + +/** + * A counter metric. Create a counter by declaring it with + * {@link MetricReceiver#declareCounter(String)} or + * {@link MetricReceiver#declareCounter(String, Point)}. + * + * @author steinar + */ +@Beta +public class Counter { + private final Point defaultPosition; + private final String name; + private final MetricReceiver metricReceiver; + + Counter(String name, Point defaultPosition, MetricReceiver receiver) { + this.name = name; + this.defaultPosition = defaultPosition; + this.metricReceiver = receiver; + } + + /** + * Increase the dimension-less/zero-point value of this counter by 1. + */ + public void add() { + add(1L, defaultPosition); + } + + /** + * Add to the dimension-less/zero-point value of this counter. + * + * @param n the amount by which to increase this counter + */ + public void add(long n) { + add(n, defaultPosition); + } + + /** + * Increase this metric at the given point by 1. + * + * @param p the point in the metric space at which to increase this metric by 1 + */ + public void add(Point p) { + add(1L, p); + } + + /** + * Add to this metric at the given point. + * + * @param n + * the amount by which to increase this counter + * @param p + * the point in the metric space at which to add to the metric + */ + public void add(long n, Point p) { + metricReceiver.update(new Sample(new Measurement(Long.valueOf(n)), new Identifier(name, p), AssumedType.COUNTER)); + } + + /** + * Create a PointBuilder with default dimension values as given when this + * counter was declared. + * + * @return a PointBuilder reflecting the default dimension values of this + * counter + */ + public PointBuilder builder() { + return new PointBuilder(defaultPosition); + } +} diff --git a/container-core/src/main/java/com/yahoo/metrics/simple/DimensionCache.java b/container-core/src/main/java/com/yahoo/metrics/simple/DimensionCache.java new file mode 100644 index 00000000000..8893a88d94c --- /dev/null +++ b/container-core/src/main/java/com/yahoo/metrics/simple/DimensionCache.java @@ -0,0 +1,110 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.metrics.simple; + +import java.util.HashMap; +import java.util.HashSet; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Map.Entry; +import java.util.Set; + +/** + * The persistence layer for metrics. Both CPU and memory hungry, but + * it runs in its own little world. + * + * @author Steinar Knutsen + */ +class DimensionCache { + + private static class TimeStampedMetric { + public final long millis; + public final UntypedMetric metric; + public TimeStampedMetric(long millis, UntypedMetric metric) { + this.millis = millis; + this.metric = metric; + } + } + + private final Map<String, LinkedHashMap<Point, TimeStampedMetric>> persistentData = new HashMap<>(); + private final int pointsToKeep; + + public DimensionCache(int pointsToKeep) { + this.pointsToKeep = pointsToKeep; + } + + void updateDimensionPersistence(Bucket toDelete, Bucket toPresent) { + updatePersistentData(toDelete); + padPresentation(toPresent); + } + + private void padPresentation(Bucket toPresent) { + Map<String, List<Entry<Point, UntypedMetric>>> currentMetricNames = toPresent.getValuesByMetricName(); + + for (Map.Entry<String, List<Entry<Point, UntypedMetric>>> metric : currentMetricNames.entrySet()) { + final int currentDataPoints = metric.getValue().size(); + if (currentDataPoints < pointsToKeep) { + padMetric(metric.getKey(), toPresent, currentDataPoints); + } + } + Set<String> keysMissingFromPresentation = new HashSet<>(persistentData.keySet()); + keysMissingFromPresentation.removeAll(currentMetricNames.keySet()); + for (String cachedMetric : keysMissingFromPresentation) { + padMetric(cachedMetric, toPresent, 0); + } + } + + private void updatePersistentData(Bucket toDelete) { + if (toDelete == null) { + return; + } + long millis = toDelete.gotTimeStamps ? toDelete.toMillis : System.currentTimeMillis(); + for (Map.Entry<String, List<Entry<Point, UntypedMetric>>> metric : toDelete.getValuesByMetricName().entrySet()) { + LinkedHashMap<Point, TimeStampedMetric> cachedPoints = getCachedMetric(metric.getKey()); + + for (Entry<Point, UntypedMetric> newestInterval : metric.getValue()) { + // overwriting an existing entry does not update the order + // in the map + cachedPoints.remove(newestInterval.getKey()); + TimeStampedMetric toInsert = new TimeStampedMetric(millis, newestInterval.getValue()); + cachedPoints.put(newestInterval.getKey(), toInsert); + } + } + } + + private static final long MAX_AGE_MILLIS = 4 * 3600 * 1000; + + private void padMetric(String metric, Bucket toPresent, int currentDataPoints) { + LinkedHashMap<Point, TimeStampedMetric> cachedPoints = getCachedMetric(metric); + int toAdd = pointsToKeep - currentDataPoints; + @SuppressWarnings({"unchecked","rawtypes"}) + Entry<Point, TimeStampedMetric>[] cachedEntries = cachedPoints.entrySet().toArray(new Entry[0]); + long nowMillis = System.currentTimeMillis(); + for (int i = cachedEntries.length - 1; i >= 0 && toAdd > 0; --i) { + Entry<Point, TimeStampedMetric> leastOld = cachedEntries[i]; + if (leastOld.getValue().millis + MAX_AGE_MILLIS < nowMillis) { + continue; + } + Identifier id = new Identifier(metric, leastOld.getKey()); + if ( ! toPresent.hasIdentifier(id)) { + toPresent.put(id, leastOld.getValue().metric.pruneData()); + --toAdd; + } + } + } + + @SuppressWarnings("serial") + private LinkedHashMap<Point, TimeStampedMetric> getCachedMetric(String metricName) { + LinkedHashMap<Point, TimeStampedMetric> points = persistentData.get(metricName); + if (points == null) { + points = new LinkedHashMap<>(16, 0.75f, false) { + protected @Override boolean removeEldestEntry(Map.Entry<Point, TimeStampedMetric> eldest) { + return size() > pointsToKeep; + } + }; + persistentData.put(metricName, points); + } + return points; + } + +} diff --git a/container-core/src/main/java/com/yahoo/metrics/simple/Gauge.java b/container-core/src/main/java/com/yahoo/metrics/simple/Gauge.java new file mode 100644 index 00000000000..1edefd0ae5a --- /dev/null +++ b/container-core/src/main/java/com/yahoo/metrics/simple/Gauge.java @@ -0,0 +1,58 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.metrics.simple; + +import com.google.common.annotations.Beta; +import com.yahoo.metrics.simple.UntypedMetric.AssumedType; + +/** + * A gauge metric, i.e. a bucket of arbitrary sample values. Create a gauge + * metric by declaring it with {@link MetricReceiver#declareGauge(String)} or + * {@link MetricReceiver#declareGauge(String, Point)}. + * + * @author steinar + */ +@Beta +public class Gauge { + + private final Point defaultPosition; + private final String name; + private final MetricReceiver receiver; + + Gauge(String name, Point defaultPosition, MetricReceiver receiver) { + this.name = name; + this.defaultPosition = defaultPosition; + this.receiver = receiver; + } + + /** + * Record a sample with default or no position. + * + * @param x + * sample value + */ + public void sample(double x) { + sample(x, defaultPosition); + } + + /** + * Record a sample at the given position. + * + * @param x + * sample value + * @param p + * position/dimension values for the sample + */ + public void sample(double x, Point p) { + receiver.update(new Sample(new Measurement(Double.valueOf(x)), new Identifier(name, p), AssumedType.GAUGE)); + } + + /** + * Create a PointBuilder with the default dimension values reflecting those + * given when this gauge was declared. + * + * @return a builder initialized with defaults from this metric instance + */ + public PointBuilder builder() { + return new PointBuilder(defaultPosition); + } +} diff --git a/container-core/src/main/java/com/yahoo/metrics/simple/Identifier.java b/container-core/src/main/java/com/yahoo/metrics/simple/Identifier.java new file mode 100644 index 00000000000..4d0f470534a --- /dev/null +++ b/container-core/src/main/java/com/yahoo/metrics/simple/Identifier.java @@ -0,0 +1,61 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.metrics.simple; + +/** + * The name of the metric and its n-dimensional position. Basically a pair of a + * Point and a metric name. Written to be robust against null input as the API + * gives very little guidance, converting null to empty string/point. Immutable. + * + * @author Steinar Knutsen + */ +public class Identifier { + + private final String name; + private final Point location; + + public Identifier(String name, Point location) { + this.name = (name == null ? "" : name); + this.location = (location == null ? Point.emptyPoint() : location); + } + + @Override + public int hashCode() { + final int prime = 31; + int result = 1; + result = prime * result + location.hashCode(); + result = prime * result + name.hashCode(); + return result; + } + + @Override + public boolean equals(Object obj) { + if (this == obj) return true; + if (obj == null) return false; + if (getClass() != obj.getClass()) return false; + + Identifier other = (Identifier) obj; + if (!location.equals(other.location)) { + return false; + } + if (!name.equals(other.name)) { + return false; + } + return true; + } + + @Override + public String toString() { + StringBuilder builder = new StringBuilder(); + builder.append("Identifier [name=").append(name).append(", location=").append(location).append("]"); + return builder.toString(); + } + + public String getName() { + return name; + } + + public Point getLocation() { + return location; + } + +} diff --git a/container-core/src/main/java/com/yahoo/metrics/simple/Measurement.java b/container-core/src/main/java/com/yahoo/metrics/simple/Measurement.java new file mode 100644 index 00000000000..4098ac1bdea --- /dev/null +++ b/container-core/src/main/java/com/yahoo/metrics/simple/Measurement.java @@ -0,0 +1,21 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.metrics.simple; + +/** + * Wrapper class for the actually measured value. + * + * @author Steinar Knutsen + */ +public class Measurement { + + private final Number magnitude; + + public Measurement(Number magnitude) { + this.magnitude = magnitude; + } + + Number getMagnitude() { + return magnitude; + } + +} diff --git a/container-core/src/main/java/com/yahoo/metrics/simple/MetricAggregator.java b/container-core/src/main/java/com/yahoo/metrics/simple/MetricAggregator.java new file mode 100644 index 00000000000..7168eb49676 --- /dev/null +++ b/container-core/src/main/java/com/yahoo/metrics/simple/MetricAggregator.java @@ -0,0 +1,71 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.metrics.simple; + +import java.util.List; +import java.util.concurrent.atomic.AtomicReference; + +import com.yahoo.concurrent.ThreadLocalDirectory; +import com.yahoo.metrics.ManagerConfig; + +/** + * Worker thread to collect the data stored in worker threads and build + * snapshots for external consumption. Using the correct executor gives the + * necessary guarantees for this being invoked from only a single thread. + * + * @author Steinar Knutsen + */ +class MetricAggregator implements Runnable { + + private final ThreadLocalDirectory<Bucket, Sample> metricsCollection; + private final AtomicReference<Bucket> currentSnapshot; + private int generation = 0; + private final Bucket[] buffer; + private long fromMillis; + private final DimensionCache dimensions; + + MetricAggregator(ThreadLocalDirectory<Bucket, Sample> metricsCollection, AtomicReference<Bucket> currentSnapshot, + ManagerConfig settings) { + if (settings.reportPeriodSeconds() < 10) { + throw new IllegalArgumentException("Do not use this metrics implementation" + + " if report periods of less than 10 seconds is desired."); + } + buffer = new Bucket[settings.reportPeriodSeconds()]; + dimensions = new DimensionCache(settings.pointsToKeepPerMetric()); + fromMillis = System.currentTimeMillis(); + this.metricsCollection = metricsCollection; + this.currentSnapshot = currentSnapshot; + } + + @Override + public void run() { + Bucket toDelete = updateBuffer(); + createSnapshot(toDelete); + } + + private void createSnapshot(Bucket toDelete) { + Bucket toPresent = new Bucket(); + for (Bucket b : buffer) { + if (b == null) { + continue; + } + toPresent.merge(b); + } + dimensions.updateDimensionPersistence(toDelete, toPresent); + currentSnapshot.set(toPresent); + } + + private Bucket updateBuffer() { + List<Bucket> buckets = metricsCollection.fetch(); + long toMillis = System.currentTimeMillis(); + int bucketIndex = generation++ % buffer.length; + Bucket bucketToDelete = buffer[bucketIndex]; + Bucket latest = new Bucket(fromMillis, toMillis); + for (Bucket b : buckets) { + latest.merge(b, true); + } + buffer[bucketIndex] = latest; + this.fromMillis = toMillis; + return bucketToDelete; + } + +} diff --git a/container-core/src/main/java/com/yahoo/metrics/simple/MetricManager.java b/container-core/src/main/java/com/yahoo/metrics/simple/MetricManager.java new file mode 100644 index 00000000000..1956783b4c0 --- /dev/null +++ b/container-core/src/main/java/com/yahoo/metrics/simple/MetricManager.java @@ -0,0 +1,64 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.metrics.simple; + +import java.util.concurrent.ScheduledThreadPoolExecutor; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicReference; +import java.util.logging.Logger; + +import com.yahoo.component.AbstractComponent; +import com.yahoo.concurrent.ThreadLocalDirectory; +import com.yahoo.concurrent.ThreadLocalDirectory.Updater; +import com.yahoo.container.di.componentgraph.Provider; +import com.yahoo.metrics.ManagerConfig; +import java.util.logging.Level; + +/** + * This is the coordinating class owning the executor and the top level objects + * for measured metrics. + * + * @author Steinar Knutsen + */ +public class MetricManager extends AbstractComponent implements Provider<MetricReceiver> { + + private static Logger log = Logger.getLogger(MetricManager.class.getName()); + + private final ScheduledThreadPoolExecutor executor; + private final MetricReceiver receiver; + private ThreadLocalDirectory<Bucket, Sample> metricsCollection; + + public MetricManager(ManagerConfig settings) { + this(settings, new MetricUpdater()); + } + + private MetricManager(ManagerConfig settings, Updater<Bucket, Sample> updater) { + log.log(Level.CONFIG, "setting up simple metrics gathering." + + " reportPeriodSeconds=" + settings.reportPeriodSeconds() + + ", pointsToKeepPerMetric=" + settings.pointsToKeepPerMetric()); + metricsCollection = new ThreadLocalDirectory<>(updater); + final AtomicReference<Bucket> currentSnapshot = new AtomicReference<>(null); + executor = new ScheduledThreadPoolExecutor(1); + // Fixed rate, not fixed delay, is it is not too important that each + // bucket has data for exactly one second, but one should strive for + // this.buffer to contain data for as close a period to the report + // interval as possible + executor.scheduleAtFixedRate(new MetricAggregator(metricsCollection, currentSnapshot, settings), 1, 1, TimeUnit.SECONDS); + receiver = new MetricReceiver(metricsCollection, currentSnapshot); + } + + static MetricManager constructWithCustomUpdater(ManagerConfig settings, Updater<Bucket, Sample> updater) { + return new MetricManager(settings, updater); + } + + + @Override + public void deconstruct() { + executor.shutdown(); + } + + @Override + public MetricReceiver get() { + return receiver; + } + +} diff --git a/container-core/src/main/java/com/yahoo/metrics/simple/MetricReceiver.java b/container-core/src/main/java/com/yahoo/metrics/simple/MetricReceiver.java new file mode 100644 index 00000000000..e0e3469e257 --- /dev/null +++ b/container-core/src/main/java/com/yahoo/metrics/simple/MetricReceiver.java @@ -0,0 +1,298 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.metrics.simple; + +import java.util.HashMap; +import java.util.Map; +import java.util.Optional; +import java.util.concurrent.atomic.AtomicReference; + +import com.google.common.annotations.Beta; +import com.google.common.collect.ImmutableMap; +import com.yahoo.concurrent.ThreadLocalDirectory; + +/** + * The reception point for measurements. This is the class users should inject + * in constructors for declaring instances of {@link Counter} and {@link Gauge} + * for the actual measurement of metrics. + * + * @author Steinar Knutsen + */ +@Beta +public class MetricReceiver { + + public static final MetricReceiver nullImplementation = new NullReceiver(); + private final ThreadLocalDirectory<Bucket, Sample> metricsCollection; + private final AtomicReference<Bucket> currentSnapshot; + + // metricSettings is volatile for reading, the lock is for updates + private final Object histogramDefinitionsLock = new Object(); + private volatile Map<String, MetricSettings> metricSettings; + + private static final class NullCounter extends Counter { + + NullCounter() { + super(null, null, null); + } + + @Override + public void add() { + } + + @Override + public void add(long n) { + } + + @Override + public void add(Point p) { + } + + @Override + public void add(long n, Point p) { + } + + @Override + public PointBuilder builder() { + return super.builder(); + } + } + + private static final class NullGauge extends Gauge { + NullGauge() { + super(null, null, null); + } + + @Override + public void sample(double x) { + } + + @Override + public void sample(double x, Point p) { + } + + @Override + public PointBuilder builder() { + return super.builder(); + } + + } + + public static final class MockReceiver extends MetricReceiver { + + private final ThreadLocalDirectory<Bucket, Sample> collection; + + private MockReceiver(ThreadLocalDirectory<Bucket, Sample> collection) { + super(collection, null); + this.collection = collection; + } + + public MockReceiver() { + this(new ThreadLocalDirectory<>(new MetricUpdater())); + } + + /** Gathers all data since last snapshot */ + public Bucket getSnapshot() { + final Bucket merged = new Bucket(); + for (Bucket b : collection.fetch()) { + merged.merge(b, true); + } + return merged; + } + + /** Utility method for testing */ + public Point point(String dim, String val) { + return pointBuilder().set(dim, val).build(); + } + + } + + private static final class NullReceiver extends MetricReceiver { + + NullReceiver() { + super(null, null); + } + + @Override + public void update(Sample s) { + } + + @Override + public Counter declareCounter(String name) { + return new NullCounter(); + } + + @Override + public Counter declareCounter(String name, Point boundDimensions) { + return new NullCounter(); + } + + @Override + public Gauge declareGauge(String name) { + return new NullGauge(); + } + + @Override + public Gauge declareGauge(String name, Point boundDimensions) { + return new NullGauge(); + } + + @Override + public Gauge declareGauge(String name, Optional<Point> boundDimensions, MetricSettings customSettings) { + return null; + } + + @Override + public PointBuilder pointBuilder() { + return null; + } + + @Override + public Bucket getSnapshot() { + return null; + } + + @Override + void addMetricDefinition(String metricName, MetricSettings definition) { + } + + @Override + MetricSettings getMetricDefinition(String metricName) { + return null; + } + } + + public MetricReceiver(ThreadLocalDirectory<Bucket, Sample> metricsCollection, AtomicReference<Bucket> currentSnapshot) { + this.metricsCollection = metricsCollection; + this.currentSnapshot = currentSnapshot; + metricSettings = new ImmutableMap.Builder<String, MetricSettings>().build(); + } + + /** + * Update a metric. This API is not intended for clients for the + * simplemetrics API, declare a Counter or a Gauge using + * {@link #declareCounter(String)}, {@link #declareCounter(String, Point)}, + * {@link #declareGauge(String)}, or {@link #declareGauge(String, Point)} + * instead. + * + * @param sample a single simple containing all meta data necessary to update a metric + */ + public void update(Sample sample) { + // pass around the receiver instead of histogram settings to avoid reading any volatile if unnecessary + sample.setReceiver(this); + metricsCollection.update(sample); + } + + /** + * Declare a counter metric without setting any default position. + * + * @param name the name of the metric + * @return a thread-safe counter + */ + public Counter declareCounter(String name) { + return declareCounter(name, null); + } + + /** + * Declare a counter metric, with default dimension values as given. Create + * the point argument by using a builder from {@link #pointBuilder()}. + * + * @param name the name of the metric + * @param boundDimensions dimensions which have a fixed value in the life cycle of the metric object or null + * @return a thread-safe counter with given default values + */ + public Counter declareCounter(String name, Point boundDimensions) { + return new Counter(name, boundDimensions, this); + } + + /** + * Declare a gauge metric with any default position. + * + * @param name the name of the metric + * @return a thread-safe gauge instance + */ + public Gauge declareGauge(String name) { + return declareGauge(name, null); + } + + /** + * Declare a gauge metric, with default dimension values as given. Create + * the point argument by using a builder from {@link #pointBuilder()}. + * + * @param name the name of the metric + * @param boundDimensions dimensions which have a fixed value in the life cycle of the metric object or null + * @return a thread-safe gauge metric + */ + public Gauge declareGauge(String name, Point boundDimensions) { + return declareGauge(name, Optional.ofNullable(boundDimensions), null); + } + + /** + * Declare a gauge metric, with default dimension values as given. Create + * the point argument by using a builder from {@link #pointBuilder()}. + * MetricSettings instances are built using + * {@link MetricSettings.Builder}. + * + * @param name the name of the metric + * @param boundDimensions an optional of dimensions which have a fixed value in the life cycle of the metric object + * @param customSettings any optional settings + * @return a thread-safe gauge metric + */ + public Gauge declareGauge(String name, Optional<Point> boundDimensions, MetricSettings customSettings) { + if (customSettings != null) { + addMetricDefinition(name, customSettings); + } + Point defaultDimensions = null; + if (boundDimensions.isPresent()) { + defaultDimensions = boundDimensions.get(); + } + return new Gauge(name, defaultDimensions, this); + } + + /** + * Create a PointBuilder instance with no default settings. PointBuilder + * instances are not thread-safe. + * + * @return an "empty" point builder instance + */ + public PointBuilder pointBuilder() { + return new PointBuilder(); + } + + /** + * Fetch the latest metric values, aggregated over all threads for the + * configured sample history (by default five minutes). The values will be + * less than 1 second old, and this method has only a memory barrier as side + * effect. + * + * @return the latest five minutes of metrics + */ + public Bucket getSnapshot() { + return currentSnapshot.get(); + } + + /** + * Add how to build a histogram for a given metric. + * + * @param metricName the metric where samples should be put in a histogram + * @param definition settings for a histogram + */ + void addMetricDefinition(String metricName, MetricSettings definition) { + synchronized (histogramDefinitionsLock) { + // read the volatile _after_ acquiring the lock + Map<String, MetricSettings> oldMetricDefinitions = metricSettings; + Map<String, MetricSettings> builderMap = new HashMap<>(oldMetricDefinitions.size() + 1); + builderMap.putAll(oldMetricDefinitions); + builderMap.put(metricName, definition); + metricSettings = ImmutableMap.copyOf(builderMap); + } + } + + /** + * Get how to build a histogram for a given metric, or null if no histogram should be created. + * + * @param metricName the name of an arbitrary metric + * @return the corresponding histogram definition or null + */ + MetricSettings getMetricDefinition(String metricName) { + return metricSettings.get(metricName); + } +} diff --git a/container-core/src/main/java/com/yahoo/metrics/simple/MetricSettings.java b/container-core/src/main/java/com/yahoo/metrics/simple/MetricSettings.java new file mode 100644 index 00000000000..924e311015b --- /dev/null +++ b/container-core/src/main/java/com/yahoo/metrics/simple/MetricSettings.java @@ -0,0 +1,70 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.metrics.simple; + +import com.google.common.annotations.Beta; + +/** + * All information needed for creating any extra data structures associated with + * a single metric, outside of its basic type. + * + * @author steinar + */ +@Beta +public final class MetricSettings { + + /** + * A builder for the immutable MetricSettings instances. + */ + @Beta + public static final class Builder { + private boolean histogram = false; + + /** + * Create a new builder for a MetricSettings instance with default + * settings. + */ + public Builder() { + } + + /** + * Set whether a resulting metric should have a histogram. Default is + * false. + * + * @param histogram + * whether to generate a histogram + * @return this, to facilitate chaining + */ + public Builder histogram(boolean histogram) { + this.histogram = histogram; + return this; + } + + /** + * Build a fresh MetricSettings instance. + * + * @return a MetricSettings instance containing the values set in this + * builder + */ + public MetricSettings build() { + return new MetricSettings(histogram); + } + } + + private final int significantDigits; // could have been static, but would + // just introduce bugs when we must + // expose this setting + private final boolean histogram; + + private MetricSettings(boolean histogram) { + this.histogram = histogram; + this.significantDigits = 2; + } + + int getSignificantdigits() { + return significantDigits; + } + + boolean isHistogram() { + return histogram; + } +} diff --git a/container-core/src/main/java/com/yahoo/metrics/simple/MetricUpdater.java b/container-core/src/main/java/com/yahoo/metrics/simple/MetricUpdater.java new file mode 100644 index 00000000000..848132c9bea --- /dev/null +++ b/container-core/src/main/java/com/yahoo/metrics/simple/MetricUpdater.java @@ -0,0 +1,24 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.metrics.simple; + +import com.yahoo.concurrent.ThreadLocalDirectory.Updater; + +/** + * The link between each single thread and the central data store. + * + * @author Steinar Knutsen + */ +class MetricUpdater implements Updater<Bucket, Sample> { + + @Override + public Bucket createGenerationInstance(Bucket previous) { + return new Bucket(); + } + + @Override + public Bucket update(Bucket current, Sample x) { + current.put(x); + return current; + } + +} diff --git a/container-core/src/main/java/com/yahoo/metrics/simple/Point.java b/container-core/src/main/java/com/yahoo/metrics/simple/Point.java new file mode 100644 index 00000000000..672d05c1874 --- /dev/null +++ b/container-core/src/main/java/com/yahoo/metrics/simple/Point.java @@ -0,0 +1,133 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.metrics.simple; + +import java.util.Arrays; +import java.util.List; +import java.util.Map; + +import com.google.common.annotations.Beta; +import com.google.common.collect.ImmutableList; +import com.yahoo.collections.Tuple2; +import com.yahoo.jdisc.Metric.Context; + +/** + * An efficiently comparable point in a sparse vector space. + * + * @author steinar + */ +@Beta +public final class Point implements Context { + + private final Value[] location; + private final String[] dimensions; + + public Point(Map<String, ?> properties) { + this(buildParameters(properties)); + } + + private Point(Tuple2<String[], Value[]> dimensionsAndLocation) { + this(dimensionsAndLocation.first, dimensionsAndLocation.second); + } + + /** + * Only to be used by simplemetrics itself. + * + * @param dimensions dimension name, Point takes ownership of the array + * @param location dimension values, Point takes ownership of the array + */ + Point(String[] dimensions, Value[] location) { + this.dimensions = dimensions; + this.location = location; + } + + private static final Point theEmptyPoint = new Point(new String[0], new Value[0]); + + /** the canonical 0-dimensional Point. */ + public static Point emptyPoint() { return theEmptyPoint; } + + private static Tuple2<String[], Value[]> buildParameters(Map<String, ?> properties) { + String[] dimensions = properties.keySet().toArray(new String[0]); + Arrays.sort(dimensions); + Value[] location = new Value[dimensions.length]; + for (int i = 0; i < dimensions.length; ++i) { + location[i] = Value.of(String.valueOf(properties.get(dimensions[i]))); + } + return new Tuple2<>(dimensions, location); + + } + + @Override + public boolean equals(Object obj) { + if (this == obj) { + return true; + } + if (obj == null) { + return false; + } + if (getClass() != obj.getClass()) { + return false; + } + Point other = (Point) obj; + if (!Arrays.equals(dimensions, other.dimensions)) { + return false; + } + if (!Arrays.equals(location, other.location)) { + return false; + } + return true; + } + + @Override + public int hashCode() { + final int prime = 31; + int result = 1; + result = prime * result + Arrays.hashCode(dimensions); + result = prime * result + Arrays.hashCode(location); + return result; + } + + @Override + public String toString() { + final int maxLen = 3; + StringBuilder builder = new StringBuilder(); + builder.append("Point [location=") + .append(Arrays.asList(location).subList(0, Math.min(location.length, maxLen))) + .append(", dimensions=") + .append(Arrays.asList(dimensions).subList(0, Math.min(dimensions.length, maxLen))) + .append("]"); + return builder.toString(); + } + + /** + * Get an immutable list view of the values for each dimension. + */ + public List<Value> location() { + return ImmutableList.copyOf(location); + } + + /** + * Get an immutable list view of the names of each dimension. + */ + public List<String> dimensions() { + return ImmutableList.copyOf(dimensions); + } + + /** + * Get the number of dimensions defined for this Point, i.e. the size of the + * collection returned by {@link #dimensions()}. + */ + public int dimensionality() { + return dimensions.length; + } + + /** package private accessor only for simplemetrics itself */ + String[] getDimensions() { + return dimensions; + } + + /** package private accessor only for simplemetrics itself */ + Value[] getLocation() { + return location; + } + +} diff --git a/container-core/src/main/java/com/yahoo/metrics/simple/PointBuilder.java b/container-core/src/main/java/com/yahoo/metrics/simple/PointBuilder.java new file mode 100644 index 00000000000..f613aab26a2 --- /dev/null +++ b/container-core/src/main/java/com/yahoo/metrics/simple/PointBuilder.java @@ -0,0 +1,123 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.metrics.simple; + +import java.util.ArrayList; +import java.util.Collections; + +import com.google.common.annotations.Beta; + +/** + * Single-use builder for the immutable Point instances used to set dimensions + * for a metric. Get a fresh instance either from a corresponding Gauge or Counter, + * or through the MetricReceiver API. + * + * @author steinar + */ +@Beta +public final class PointBuilder { + private ArrayList<String> dimensions; + private ArrayList<Value> location; + + public enum Discriminator { + LONG, DOUBLE, STRING; + } + + PointBuilder() { + this(null); + } + + PointBuilder(Point p) { + dimensions = new ArrayList<>(); + location = new ArrayList<>(); + if (p != null) { + int size = p.dimensionality(); + dimensions = new ArrayList<>(size+2); + location = new ArrayList<>(size+2); + for (String dimensionName : p.getDimensions()) { + dimensions.add(dimensionName); + } + for (Value dimensionValue : p.getLocation()) { + location.add(dimensionValue); + } + } else { + dimensions = new ArrayList<>(4); + location = new ArrayList<>(4); + } + } + + /** + * Set a named dimension to an integer value. + * + * @param dimensionName the name of the dimension to set + * @param dimensionValue to value for the given dimension + * @return this, to facilitate chaining + */ + public PointBuilder set(String dimensionName, long dimensionValue) { + return set(dimensionName, Value.of(dimensionValue)); + } + + /** + * Set a named dimension to a floating point value. + * + * @param dimensionName the name of the dimension to set + * @param dimensionValue to value for the given dimension + * @return this, to facilitate chaining + */ + public PointBuilder set(String dimensionName, double dimensionValue) { + return set(dimensionName, Value.of(dimensionValue)); + } + + /** + * Set a named dimension to a string value. + * + * @param dimensionName the name of the dimension to set + * @param dimensionValue to value for the given dimension + * @return this, to facilitate chaining + */ + public PointBuilder set(String dimensionName, String dimensionValue) { + return set(dimensionName, Value.of(dimensionValue)); + } + + private PointBuilder set(String axisName, Value w) { + // handle setting same axis multiple times nicely + int i = Collections.binarySearch(dimensions, axisName); + if (i < 0) { + dimensions.add(~i, axisName); + location.add(~i, w); + } else { + // only set location, dim obviously exists + location.set(i, w); + } + return this; + } + + /** + * Create a new Point instance using the settings stored in this + * PointBuilder. PointBuilder instances cannot be re-used after build() has + * been invoked. + * + * @return a Point instance reflecting this builder + */ + public Point build() { + Point p = Point.emptyPoint(); + int size = dimensions.size(); + if (size != 0) { + p = new Point(dimensions.toArray(new String[size]), location.toArray(new Value[size])); + } + // deny builder re-use + dimensions = null; + location = null; + return p; + } + + @Override + public String toString() { + final int maxLen = 3; + StringBuilder builder = new StringBuilder(); + builder.append("PointBuilder [dimensions=") + .append(dimensions != null ? dimensions.subList(0, Math.min(dimensions.size(), maxLen)) : null) + .append(", location=").append(location != null ? location.subList(0, Math.min(location.size(), maxLen)) : null) + .append("]"); + return builder.toString(); + } +} diff --git a/container-core/src/main/java/com/yahoo/metrics/simple/Sample.java b/container-core/src/main/java/com/yahoo/metrics/simple/Sample.java new file mode 100644 index 00000000000..0d2144deeb4 --- /dev/null +++ b/container-core/src/main/java/com/yahoo/metrics/simple/Sample.java @@ -0,0 +1,56 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.metrics.simple; + +import com.yahoo.metrics.simple.UntypedMetric.AssumedType; + +/** + * A single metric measurement and all the meta data needed to route it + * correctly. + * + * @author Steinar Knutsen + */ +public class Sample { + + private final Identifier identifier; + private final Measurement measurement; + private final AssumedType metricType; + private MetricReceiver metricReceiver = null; + + public Sample(Measurement measurement, Identifier id, AssumedType t) { + this.identifier = id; + this.measurement = measurement; + this.metricType = t; + } + + Identifier getIdentifier() { + return identifier; + } + + Measurement getMeasurement() { + return measurement; + } + + AssumedType getMetricType() { + return metricType; + } + + void setReceiver(MetricReceiver metricReceiver) { + this.metricReceiver = metricReceiver; + } + + /** + * Get histogram definition for an arbitrary metric. Caveat emptor: This + * involves reading a volatile. + * + * @param metricName name of the metric to get histogram definition for + * @return how to define a new histogram or null + */ + MetricSettings getHistogramDefinition(String metricName) { + if (metricReceiver == null) { + return null; + } else { + return metricReceiver.getMetricDefinition(metricName); + } + } + +} diff --git a/container-core/src/main/java/com/yahoo/metrics/simple/UnitTestSetup.java b/container-core/src/main/java/com/yahoo/metrics/simple/UnitTestSetup.java new file mode 100644 index 00000000000..e6856ee2970 --- /dev/null +++ b/container-core/src/main/java/com/yahoo/metrics/simple/UnitTestSetup.java @@ -0,0 +1,71 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.metrics.simple; + +import com.yahoo.metrics.ManagerConfig; + +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; + +/** + * Common code for running unit tests of simplemetrics + * + * @author ean + */ +public class UnitTestSetup { + + MetricManager metricManager; + MetricReceiver receiver; + ObservableUpdater updater; + + static class ObservableUpdater extends MetricUpdater { + CountDownLatch gotData = new CountDownLatch(1); + private volatile boolean hasBeenAccessed = false; + + @Override + public Bucket createGenerationInstance(Bucket previous) { + if (hasBeenAccessed) { + gotData.countDown(); + } + return super.createGenerationInstance(previous); + } + + @Override + public Bucket update(Bucket current, Sample x) { + hasBeenAccessed = true; + return super.update(current, x); + } + } + + void init() { + updater = new ObservableUpdater(); + metricManager = MetricManager.constructWithCustomUpdater(new ManagerConfig(new ManagerConfig.Builder()), updater); + receiver = metricManager.get(); + } + + void fini() { + receiver = null; + metricManager.deconstruct(); + metricManager = null; + updater = null; + } + + public Bucket getUpdatedSnapshot() throws InterruptedException { + updater.gotData.await(10, TimeUnit.SECONDS); + Bucket s = receiver.getSnapshot(); + long startedWaitingForSnapshot = System.currentTimeMillis(); + // just waiting for the correct snapshot being constructed (yes, this is + // necessary) + while (s == null || s.entrySet().size() == 0) { + if (System.currentTimeMillis() - startedWaitingForSnapshot > (10L * 1000L)) { + throw new RuntimeException("Test timed out."); + } + Thread.sleep(10); + s = receiver.getSnapshot(); + } + return s; + } + + public MetricReceiver getReceiver() { + return receiver; + } +} diff --git a/container-core/src/main/java/com/yahoo/metrics/simple/UntypedMetric.java b/container-core/src/main/java/com/yahoo/metrics/simple/UntypedMetric.java new file mode 100644 index 00000000000..f757ab15022 --- /dev/null +++ b/container-core/src/main/java/com/yahoo/metrics/simple/UntypedMetric.java @@ -0,0 +1,142 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.metrics.simple; + +import java.util.logging.Logger; + +import org.HdrHistogram.DoubleHistogram; + +import java.util.logging.Level; + +/** + * A gauge or a counter or... who knows? The class for storing a metric when the + * metric has not been declared. + * + * @author Steinar Knutsen + */ +public class UntypedMetric { + + private static final Logger log = Logger.getLogger(UntypedMetric.class.getName()); + + private long count = 0L; + private double current = 0.0d; + private double max; + private double min; + private double sum; + private AssumedType outputFormat = AssumedType.NONE; + private final DoubleHistogram histogram; + private final MetricSettings metricSettings; + + public enum AssumedType { NONE, GAUGE, COUNTER }; + + UntypedMetric(MetricSettings metricSettings) { + this.metricSettings = metricSettings; + if (metricSettings == null || !metricSettings.isHistogram()) { + histogram = null; + } else { + histogram = new DoubleHistogram(metricSettings.getSignificantdigits()); + } + } + + void add(Number x) { + outputFormat = AssumedType.COUNTER; + count += x.longValue(); + } + + void put(Number x) { + outputFormat = AssumedType.GAUGE; + current = x.doubleValue(); + if (histogram != null) { + histogram.recordValue(current); + } + if (count > 0) { + max = Math.max(current, max); + min = Math.min(current, min); + sum += current; + } else { + max = current; + min = current; + sum = current; + } + ++count; + } + + UntypedMetric pruneData() { + UntypedMetric pruned = new UntypedMetric(null); + pruned.outputFormat = this.outputFormat; + pruned.current = this.current; + return pruned; + } + + void merge(UntypedMetric other, boolean otherIsNewer) throws IllegalArgumentException { + if (outputFormat == AssumedType.NONE) { + outputFormat = other.outputFormat; + } + if (outputFormat != other.outputFormat) { + throw new IllegalArgumentException("Mismatching output formats: " + outputFormat + " and " + other.outputFormat + "."); + } + if (count > 0) { + if (other.count > 0) { + max = Math.max(other.max, max); + min = Math.min(other.min, min); + if (otherIsNewer) { + current = other.current; + } + } + } else { + max = other.max; + min = other.min; + current = other.current; + } + count += other.count; + sum += other.sum; + if (histogram != null) { + // some config scenarios may lead to differing histogram settings, + // so doing this defensively + if (other.histogram != null) { + try { + histogram.add(other.histogram); + } catch (ArrayIndexOutOfBoundsException e) { + log.log(Level.WARNING, "Had trouble merging histograms: " + e.getMessage()); + } + } + } + } + + public boolean isCounter() { return outputFormat == AssumedType.COUNTER; } + + public long getCount() { return count; } + public double getLast() { return current; } + public double getMax() { return max; } + public double getMin() { return min; } + public double getSum() { return sum; } + + MetricSettings getMetricDefinition() { + return metricSettings; + } + + public DoubleHistogram getHistogram() { + return histogram; + } + + @Override + public String toString() { + StringBuilder buf = new StringBuilder(); + buf.append(this.getClass().getName()).append(": "); + buf.append("outputFormat=").append(outputFormat).append(", "); + if (count > 0 && outputFormat == AssumedType.GAUGE) { + buf.append("max=").append(max).append(", "); + buf.append("min=").append(min).append(", "); + buf.append("sum=").append(sum).append(", "); + } + if (histogram != null) { + buf.append("histogram=").append(histogram).append(", "); + } + if (metricSettings != null) { + buf.append("metricSettings=").append(metricSettings).append(", "); + } + buf.append("current=").append(current).append(", "); + buf.append("count=").append(count); + return buf.toString(); + } + +} diff --git a/container-core/src/main/java/com/yahoo/metrics/simple/Value.java b/container-core/src/main/java/com/yahoo/metrics/simple/Value.java new file mode 100644 index 00000000000..fd4113a5e22 --- /dev/null +++ b/container-core/src/main/java/com/yahoo/metrics/simple/Value.java @@ -0,0 +1,246 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.metrics.simple; + +/** + * Wrapper for dimension values. + * + * @author steinar + */ +public abstract class Value { + private static final String UNSUPPORTED_VALUE_TYPE = "Unsupported value type."; + + /** + * Marker for the type of the contained value of a Value instance. + */ + public enum Discriminator { + LONG, DOUBLE, STRING; + } + + /** + * Get the long wrapped by a Value if one exists. + * + * @throws UnsupportedOperationException if LONG is not returned by {{@link #getType()}. + */ + public long longValue() throws UnsupportedOperationException { + throw new UnsupportedOperationException(UNSUPPORTED_VALUE_TYPE); + } + + /** + * Get the double wrapped by a Value if one exists. + * + * @throws UnsupportedOperationException if DOUBLE is not returned by {{@link #getType()}. + */ + public double doubleValue() throws UnsupportedOperationException { + throw new UnsupportedOperationException(UNSUPPORTED_VALUE_TYPE); + } + + /** + * Get the string wrapped by a Value if one exists. + * + * @throws UnsupportedOperationException if STRING is not returned by {{@link #getType()}. + */ + public String stringValue() throws UnsupportedOperationException { + throw new UnsupportedOperationException(UNSUPPORTED_VALUE_TYPE); + } + + /** + * Show the (single) supported standard type representation of a Value instance. + */ + public abstract Discriminator getType(); + + private static class LongValue extends Value { + private final long value; + + LongValue(long value) { + this.value = value; + } + + @Override + public long longValue() { + return value; + } + + @Override + public Discriminator getType() { + return Discriminator.LONG; + } + + @Override + public int hashCode() { + final int prime = 31; + int result = 1; + result = prime * result + (int) (value ^ (value >>> 32)); + return result; + } + + @Override + public boolean equals(Object obj) { + if (this == obj) { + return true; + } + if (obj == null) { + return false; + } + if (getClass() != obj.getClass()) { + return false; + } + LongValue other = (LongValue) obj; + if (value != other.value) { + return false; + } + return true; + } + + @Override + public String toString() { + StringBuilder builder = new StringBuilder(); + builder.append("LongValue [value=").append(value).append("]"); + return builder.toString(); + } + } + + private static class DoubleValue extends Value { + private final double value; + + DoubleValue(double value) { + this.value = value; + } + + @Override + public double doubleValue() { + return value; + } + + @Override + public Discriminator getType() { + return Discriminator.DOUBLE; + } + + @Override + public int hashCode() { + final int prime = 31; + int result = 1; + long temp; + temp = Double.doubleToLongBits(value); + result = prime * result + (int) (temp ^ (temp >>> 32)); + return result; + } + + @Override + public boolean equals(Object obj) { + if (this == obj) { + return true; + } + if (obj == null) { + return false; + } + if (getClass() != obj.getClass()) { + return false; + } + DoubleValue other = (DoubleValue) obj; + if (Double.doubleToLongBits(value) != Double.doubleToLongBits(other.value)) { + return false; + } + return true; + } + + @Override + public String toString() { + StringBuilder builder = new StringBuilder(); + builder.append("DoubleValue [value=").append(value).append("]"); + return builder.toString(); + } + } + + private static class StringValue extends Value { + private final String value; + + StringValue(String value) { + this.value = value; + } + + @Override + public String stringValue() { + return value; + } + + @Override + public Discriminator getType() { + return Discriminator.STRING; + } + + @Override + public int hashCode() { + final int prime = 31; + int result = 1; + result = prime * result + ((value == null) ? 0 : value.hashCode()); + return result; + } + + @Override + public boolean equals(Object obj) { + if (this == obj) { + return true; + } + if (obj == null) { + return false; + } + if (getClass() != obj.getClass()) { + return false; + } + StringValue other = (StringValue) obj; + if (value == null) { + if (other.value != null) { + return false; + } + } else if (!value.equals(other.value)) { + return false; + } + return true; + } + + @Override + public String toString() { + StringBuilder builder = new StringBuilder(); + builder.append("StringValue [value=").append(value).append("]"); + return builder.toString(); + } + } + + /** + * Helper method to wrap a long as a Value. The instance returned may or may + * not be unique. + * + * @param value + * the value to wrap + * @return an immutable wrapper + */ + public static Value of(long value) { + return new LongValue(value); + } + + /** + * Helper method to wrap a double as a Value. The instance returned may or + * may not be unique. + * + * @param value + * the value to wrap + * @return an immutable wrapper + * */ + public static Value of(double value) { + return new DoubleValue(value); + } + + /** + * Helper method to wrap a string as a Value. The instance returned may or + * may not be unique. + * + * @param value + * the value to wrap + * @return an immutable wrapper + */ + public static Value of(String value) { + return new StringValue(value); + } + +} diff --git a/container-core/src/main/java/com/yahoo/metrics/simple/jdisc/JdiscMetricsFactory.java b/container-core/src/main/java/com/yahoo/metrics/simple/jdisc/JdiscMetricsFactory.java new file mode 100644 index 00000000000..30102c43919 --- /dev/null +++ b/container-core/src/main/java/com/yahoo/metrics/simple/jdisc/JdiscMetricsFactory.java @@ -0,0 +1,60 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.metrics.simple.jdisc; + +import java.io.PrintStream; +import java.util.logging.Logger; + +import com.yahoo.container.jdisc.MetricConsumerFactory; +import com.yahoo.container.jdisc.state.MetricSnapshot; +import com.yahoo.container.jdisc.state.SnapshotProvider; +import com.yahoo.jdisc.application.MetricConsumer; +import com.yahoo.metrics.simple.Bucket; +import com.yahoo.metrics.simple.MetricReceiver; + +/** + * A factory for all the JDisc API classes. + * + * @author Steinar Knutsen + */ +public class JdiscMetricsFactory implements MetricConsumerFactory, SnapshotProvider { + + private static final Logger log = Logger.getLogger(JdiscMetricsFactory.class.getName()); + private final SimpleMetricConsumer metricInstance; + private final MetricReceiver metricReceiver; + + public JdiscMetricsFactory(MetricReceiver receiver) { + this.metricReceiver = receiver; + this.metricInstance = new SimpleMetricConsumer(receiver); + } + + @Override + public MetricConsumer newInstance() { + // the underlying implementation is thread safe anyway to allow for stand-alone use + return metricInstance; + } + + + @Override + public MetricSnapshot latestSnapshot() { + Bucket curr = metricReceiver.getSnapshot(); + if (curr == null) { + log.warning("no snapshot from instance of " + metricReceiver.getClass()); + return null; + } else { + SnapshotConverter converter = new SnapshotConverter(curr); + return converter.convert(); + } + } + + @Override + public void histogram(PrintStream output) { + Bucket curr = metricReceiver.getSnapshot(); + if (curr == null) { + log.warning("no snapshot from instance of " + metricReceiver.getClass()); + } else { + SnapshotConverter converter = new SnapshotConverter(curr); + converter.outputHistograms(output); + } + } + +} diff --git a/container-core/src/main/java/com/yahoo/metrics/simple/jdisc/SimpleMetricConsumer.java b/container-core/src/main/java/com/yahoo/metrics/simple/jdisc/SimpleMetricConsumer.java new file mode 100644 index 00000000000..ee5f18e78d3 --- /dev/null +++ b/container-core/src/main/java/com/yahoo/metrics/simple/jdisc/SimpleMetricConsumer.java @@ -0,0 +1,54 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.metrics.simple.jdisc; + +import java.util.HashMap; +import java.util.Map; + +import com.yahoo.jdisc.Metric.Context; +import com.yahoo.jdisc.application.MetricConsumer; +import com.yahoo.metrics.simple.Identifier; +import com.yahoo.metrics.simple.Measurement; +import com.yahoo.metrics.simple.Point; +import com.yahoo.metrics.simple.MetricReceiver; +import com.yahoo.metrics.simple.Sample; +import com.yahoo.metrics.simple.UntypedMetric.AssumedType; + +/** + * The single user facing part of the JDisc interfaces of simple metrics. + * + * @author Steinar Knutsen + */ +public class SimpleMetricConsumer implements MetricConsumer { + + private final MetricReceiver receiver; + + public SimpleMetricConsumer(MetricReceiver receiver) { + this.receiver = receiver; + } + + @Override + public void set(String key, Number val, Context ctx) { + receiver.update(new Sample(new Measurement(val), new Identifier(key, getSimpleCoordinate(ctx)), AssumedType.GAUGE)); + } + + @Override + public void add(String key, Number val, Context ctx) { + receiver.update(new Sample(new Measurement(val), new Identifier(key, getSimpleCoordinate(ctx)), AssumedType.COUNTER)); + } + + private Point getSimpleCoordinate(Context ctx) { + if (ctx instanceof Point) { + return (Point) ctx; + } else { + return null; + } + } + + @Override + public Context createContext(Map<String, ?> properties) { + if (properties == null) + properties = new HashMap<>(); + return new Point(properties); + } + +} diff --git a/container-core/src/main/java/com/yahoo/metrics/simple/jdisc/SnapshotConverter.java b/container-core/src/main/java/com/yahoo/metrics/simple/jdisc/SnapshotConverter.java new file mode 100644 index 00000000000..495062e38f8 --- /dev/null +++ b/container-core/src/main/java/com/yahoo/metrics/simple/jdisc/SnapshotConverter.java @@ -0,0 +1,227 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.metrics.simple.jdisc; + +import java.io.PrintStream; +import java.util.*; +import java.util.concurrent.TimeUnit; +import java.util.logging.Logger; + +import org.HdrHistogram.DoubleHistogram; + +import com.yahoo.collections.Tuple2; +import com.yahoo.container.jdisc.state.*; +import com.yahoo.metrics.simple.Bucket; +import com.yahoo.metrics.simple.Identifier; +import com.yahoo.metrics.simple.Point; +import com.yahoo.metrics.simple.UntypedMetric; +import com.yahoo.metrics.simple.Value; +import com.yahoo.text.JSON; + +/** + * Convert simple metrics snapshots into jdisc state snapshots. + * + * @author arnej27959 + */ +class SnapshotConverter { + + private static Logger log = Logger.getLogger(SnapshotConverter.class.getName()); + + final Bucket snapshot; + final Map<Point, Map<String, MetricValue>> perPointData = new HashMap<>(); + private static final char[] DIGITS = new char[] { '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', 'A', 'B', 'C', 'D', 'E', 'F' }; + + public SnapshotConverter(Bucket snapshot) { + this.snapshot = snapshot; + } + + static MetricDimensions convert(Point p) { + if (p == null) { + return StateMetricContext.newInstance(null); + } + List<String> dimensions = p.dimensions(); + List<Value> location = p.location(); + Map<String, Object> pointWrapper = new HashMap<>(dimensions.size()); + for (int i = 0; i < dimensions.size(); ++i) { + pointWrapper.put(dimensions.get(i), valueAsString(location.get(i))); + } + return StateMetricContext.newInstance(pointWrapper); + } + + // TODO: just a compatibility wrapper, should be removed ASAP + private static Object valueAsString(Value value) { + switch (value.getType()) { + case STRING: + return value.stringValue(); + case LONG: + return value.longValue(); + case DOUBLE: + return value.doubleValue(); + default: + throw new IllegalStateException("simplemetrics impl is out of sync with itself, please file a ticket."); + } + } + + static MetricValue convert(UntypedMetric val) { + if (val.isCounter()) { + return CountMetric.newInstance(val.getCount()); + } else { + if (val.getHistogram() == null) { + return GaugeMetric.newInstance(val.getLast(), val.getMax(), val.getMin(), val.getSum(), val.getCount()); + } else { + return GaugeMetric.newInstance(val.getLast(), val.getMax(), val.getMin(), val.getSum(), val.getCount(), + Optional.of(buildPercentileList(val.getHistogram()))); + } + } + } + + private static List<Tuple2<String, Double>> buildPercentileList(DoubleHistogram histogram) { + List<Tuple2<String, Double>> prefixAndValues = new ArrayList<>(2); + prefixAndValues.add(new Tuple2<>("95", histogram.getValueAtPercentile(95.0d))); + prefixAndValues.add(new Tuple2<>("99", histogram.getValueAtPercentile(99.0d))); + return prefixAndValues; + } + + MetricSnapshot convert() { + for (Map.Entry<Identifier, UntypedMetric> entry : snapshot.entrySet()) { + Identifier ident = entry.getKey(); + getMap(ident.getLocation()).put(ident.getName(), convert(entry.getValue())); + } + Map<MetricDimensions, MetricSet> data = new HashMap<>(); + for (Map.Entry<Point, Map<String, MetricValue>> entry : perPointData.entrySet()) { + MetricDimensions key = convert(entry.getKey()); + MetricSet newval = new MetricSet(entry.getValue()); + MetricSet old = data.get(key); + if (old != null) { + // should not happen, this is bad + // TODO: consider merging the two MetricSet instances + log.warning("losing MetricSet when converting for: "+entry.getKey()); + } else { + data.put(key, newval); + } + } + return new MetricSnapshot(snapshot.getFromMillis(), + snapshot.getToMillis(), + TimeUnit.MILLISECONDS, + data); + } + + private Map<String, MetricValue> getMap(Point point) { + if (point == null) { + point = Point.emptyPoint(); + } + if (! perPointData.containsKey(point)) { + perPointData.put(point, new HashMap<>()); + } + return perPointData.get(point); + } + + void outputHistograms(PrintStream output) { + boolean gotHistogram = false; + for (Map.Entry<Identifier, UntypedMetric> entry : snapshot.entrySet()) { + if (entry.getValue().getHistogram() == null) { + continue; + } + gotHistogram = true; + DoubleHistogram histogram = entry.getValue().getHistogram(); + Identifier id = entry.getKey(); + String metricIdentifier = getIdentifierString(id); + output.println("# start of metric " + metricIdentifier); + histogram.outputPercentileDistribution(output, 4, 1.0d, true); + output.println("# end of metric " + metricIdentifier); + } + if (!gotHistogram) { + output.println("# No histograms currently available."); + } + } + + private String getIdentifierString(Identifier id) { + StringBuilder buffer = new StringBuilder(); + Point location = id.getLocation(); + buffer.append(id.getName()); + if (location != null) { + buffer.append(", dimensions: { "); + Iterator<String> dimensions = location.dimensions().iterator(); + Iterator<Value> values = location.location().iterator(); + boolean firstDimension = true; + while (dimensions.hasNext() && values.hasNext()) { + + if (firstDimension) { + firstDimension = false; + } else { + buffer.append(", "); + } + serializeSingleDimension(buffer, dimensions.next(), values.next()); + } + buffer.append(" }"); + } + return buffer.toString(); + + } + + private void serializeSingleDimension(StringBuilder buffer, final String dimensionName, Value dimensionValue) { + buffer.append('"'); + escape(dimensionName, buffer); + buffer.append("\": "); + switch (dimensionValue.getType()) { + case LONG: + buffer.append(Long.toString(dimensionValue.longValue())); + break; + case DOUBLE: + buffer.append(Double.toString(dimensionValue.doubleValue())); + break; + case STRING: + buffer.append('"'); + escape(dimensionValue.stringValue(), buffer); + buffer.append('"'); + break; + default: + buffer.append("\"Unknown type for this dimension, this is a bug.\""); + break; + } + } + + private void escape(final String in, final StringBuilder target) { + for (final char c : in.toCharArray()) { + switch (c) { + case ('"'): + target.append("\\\""); + break; + case ('\\'): + target.append("\\\\"); + break; + case ('\b'): + target.append("\\b"); + break; + case ('\f'): + target.append("\\f"); + break; + case ('\n'): + target.append("\\n"); + break; + case ('\r'): + target.append("\\r"); + break; + case ('\t'): + target.append("\\t"); + break; + default: + if (c < 32) { + target.append("\\u").append(fourDigitHexString(c)); + } else { + target.append(c); + } + break; + } + } + } + + private static char[] fourDigitHexString(final char c) { + final char[] hex = new char[4]; + int in = ((c) & 0xFFFF); + for (int i = 3; i >= 0; --i) { + hex[i] = DIGITS[in & 0xF]; + in >>>= 4; + } + return hex; + } +} diff --git a/container-core/src/main/java/com/yahoo/metrics/simple/jdisc/package-info.java b/container-core/src/main/java/com/yahoo/metrics/simple/jdisc/package-info.java new file mode 100644 index 00000000000..d191a5764c0 --- /dev/null +++ b/container-core/src/main/java/com/yahoo/metrics/simple/jdisc/package-info.java @@ -0,0 +1,10 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +/** + * JDisc metrics API for simple metrics implementation. + * + * @author <a href="mailto:steinar@yahoo-inc.com">Steinar Knutsen</a> + */ +@ExportPackage +package com.yahoo.metrics.simple.jdisc; + +import com.yahoo.osgi.annotation.ExportPackage; diff --git a/container-core/src/main/java/com/yahoo/metrics/simple/package-info.java b/container-core/src/main/java/com/yahoo/metrics/simple/package-info.java new file mode 100644 index 00000000000..9306c7c59db --- /dev/null +++ b/container-core/src/main/java/com/yahoo/metrics/simple/package-info.java @@ -0,0 +1,33 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +/** + * A metrics API with declarable metric, and also an implementation of the + * JDisc Metrics API where the newest state is made continously available. + * + * <p> + * Users should have an instance of {@link com.yahoo.metrics.simple.MetricReceiver} + * injected in the constructor where needed, then declare metrics as instances + * of {@link com.yahoo.metrics.simple.Counter} and + * {@link com.yahoo.metrics.simple.Gauge} using + * {@link com.yahoo.metrics.simple.MetricReceiver#declareCounter(String)}, + * {@link com.yahoo.metrics.simple.MetricReceiver#declareCounter(String, Point)}, + * {@link com.yahoo.metrics.simple.MetricReceiver#declareGauge(String)}, + * {@link com.yahoo.metrics.simple.MetricReceiver#declareGauge(String, Point)}, or + * {@link com.yahoo.metrics.simple.MetricReceiver#declareGauge(String, java.util.Optional, MetricSettings)}. + * </p> + * + * <p> + * Clients input data through the API in {@link com.yahoo.metrics.simple.MetricReceiver}, + * while the internal work is done by {@link com.yahoo.metrics.simple.MetricAggregator}. + * Initialization is done top-down from {@link com.yahoo.metrics.simple.MetricManager}. + * The link between calls to MetricReceiver and MetricAggregator is the role of + * {@link com.yahoo.metrics.simple.MetricUpdater}. + * </p> + * + * @author Steinar Knutsen + */ +@PublicApi +@ExportPackage +package com.yahoo.metrics.simple; + +import com.yahoo.api.annotations.PublicApi; +import com.yahoo.osgi.annotation.ExportPackage; diff --git a/container-core/src/main/java/com/yahoo/metrics/simple/runtime/MetricProperties.java b/container-core/src/main/java/com/yahoo/metrics/simple/runtime/MetricProperties.java new file mode 100644 index 00000000000..9c3ecec10fc --- /dev/null +++ b/container-core/src/main/java/com/yahoo/metrics/simple/runtime/MetricProperties.java @@ -0,0 +1,20 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.metrics.simple.runtime; + +import java.nio.file.Path; +import java.nio.file.Paths; + +/** + * Constants used by Vespa to make the simple metrics implementation available + * to other components. + * + * @author Steinar Knutsen + */ +public final class MetricProperties { + + private MetricProperties() { + } + + public static final String BUNDLE_SYMBOLIC_NAME = "simplemetrics"; + +} diff --git a/container-core/src/main/java/com/yahoo/metrics/simple/runtime/package-info.java b/container-core/src/main/java/com/yahoo/metrics/simple/runtime/package-info.java new file mode 100644 index 00000000000..e7c7cd166eb --- /dev/null +++ b/container-core/src/main/java/com/yahoo/metrics/simple/runtime/package-info.java @@ -0,0 +1,12 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +/** + * Settings and properties used for setting up the simple metrics library in a + * container. + * + * @author <a href="mailto:steinar@yahoo-inc.com">Steinar Knutsen</a> + */ +@ExportPackage +package com.yahoo.metrics.simple.runtime; + +import com.yahoo.osgi.annotation.ExportPackage; + diff --git a/container-core/src/main/java/com/yahoo/osgi/provider/model/ComponentModel.java b/container-core/src/main/java/com/yahoo/osgi/provider/model/ComponentModel.java new file mode 100644 index 00000000000..8c501963db3 --- /dev/null +++ b/container-core/src/main/java/com/yahoo/osgi/provider/model/ComponentModel.java @@ -0,0 +1,50 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.osgi.provider.model; + +import com.yahoo.component.ComponentId; +import com.yahoo.component.ComponentSpecification; +import com.yahoo.container.bundle.BundleInstantiationSpecification; + +/** + * Describes how a component should be created. + * + * Immutable + * + * @author gjoranv + */ +public class ComponentModel { + + public final BundleInstantiationSpecification bundleInstantiationSpec; + public final String configId; // only used in the container, null when used in the model + + public ComponentModel(BundleInstantiationSpecification bundleInstantiationSpec, String configId) { + if (bundleInstantiationSpec == null) + throw new IllegalArgumentException("Null bundle instantiation spec!"); + + this.bundleInstantiationSpec = bundleInstantiationSpec; + this.configId = configId; + } + + public ComponentModel(String idSpec, String classSpec, String bundleSpec, String configId) { + this(BundleInstantiationSpecification.getFromStrings(idSpec, classSpec, bundleSpec), configId); + } + + // For vespamodel + public ComponentModel(BundleInstantiationSpecification bundleInstantiationSpec) { + this(bundleInstantiationSpec, null); + } + + // For vespamodel + public ComponentModel(String idSpec, String classSpec, String bundleSpec) { + this(BundleInstantiationSpecification.getFromStrings(idSpec, classSpec, bundleSpec)); + } + + public ComponentId getComponentId() { + return bundleInstantiationSpec.id; + } + + public ComponentSpecification getClassId() { + return bundleInstantiationSpec.classId; + } + +} diff --git a/container-core/src/main/java/com/yahoo/osgi/provider/model/package-info.java b/container-core/src/main/java/com/yahoo/osgi/provider/model/package-info.java new file mode 100644 index 00000000000..f930f56ae4a --- /dev/null +++ b/container-core/src/main/java/com/yahoo/osgi/provider/model/package-info.java @@ -0,0 +1,5 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +@ExportPackage +package com.yahoo.osgi.provider.model; + +import com.yahoo.osgi.annotation.ExportPackage; diff --git a/container-core/src/main/java/com/yahoo/processing/test/ProcessorLibrary.java b/container-core/src/main/java/com/yahoo/processing/test/ProcessorLibrary.java index 5f6201c6f2d..fc8904bee7f 100644 --- a/container-core/src/main/java/com/yahoo/processing/test/ProcessorLibrary.java +++ b/container-core/src/main/java/com/yahoo/processing/test/ProcessorLibrary.java @@ -169,7 +169,6 @@ public class ProcessorLibrary { List<FutureResponse> futureResponses = new ArrayList<>(chains.size()); for (Chain<? extends Processor> chain : chains) { - futureResponses.add(new AsyncExecution(chain, execution).process(request.clone())); } AsyncExecution.waitForAll(futureResponses, 1000); diff --git a/container-core/src/main/java/com/yahoo/restapi/ErrorResponse.java b/container-core/src/main/java/com/yahoo/restapi/ErrorResponse.java index d3e81a10720..1885a0c970c 100644 --- a/container-core/src/main/java/com/yahoo/restapi/ErrorResponse.java +++ b/container-core/src/main/java/com/yahoo/restapi/ErrorResponse.java @@ -5,6 +5,7 @@ import com.yahoo.slime.Cursor; import com.yahoo.slime.Slime; import static com.yahoo.jdisc.Response.Status.BAD_REQUEST; +import static com.yahoo.jdisc.Response.Status.CONFLICT; import static com.yahoo.jdisc.Response.Status.FORBIDDEN; import static com.yahoo.jdisc.Response.Status.INTERNAL_SERVER_ERROR; import static com.yahoo.jdisc.Response.Status.METHOD_NOT_ALLOWED; @@ -24,7 +25,8 @@ public class ErrorResponse extends SlimeJsonResponse { FORBIDDEN, METHOD_NOT_ALLOWED, INTERNAL_SERVER_ERROR, - UNAUTHORIZED + UNAUTHORIZED, + CONFLICT } public ErrorResponse(int statusCode, String errorType, String message) { @@ -63,4 +65,8 @@ public class ErrorResponse extends SlimeJsonResponse { return new ErrorResponse(METHOD_NOT_ALLOWED, errorCodes.METHOD_NOT_ALLOWED.name(), message); } + public static ErrorResponse conflict(String message) { + return new ErrorResponse(CONFLICT, errorCodes.CONFLICT.name(), message); + } + } diff --git a/container-core/src/main/java/com/yahoo/restapi/RestApi.java b/container-core/src/main/java/com/yahoo/restapi/RestApi.java index df05723ac14..6f5bf298de3 100644 --- a/container-core/src/main/java/com/yahoo/restapi/RestApi.java +++ b/container-core/src/main/java/com/yahoo/restapi/RestApi.java @@ -22,6 +22,7 @@ public interface RestApi { static RouteBuilder route(String pathPattern) { return new RestApiImpl.RouteBuilderImpl(pathPattern); } HttpResponse handleRequest(HttpRequest request); + ObjectMapper jacksonJsonMapper(); interface Builder { Builder setObjectMapper(ObjectMapper mapper); diff --git a/container-core/src/main/java/com/yahoo/restapi/RestApiException.java b/container-core/src/main/java/com/yahoo/restapi/RestApiException.java index ac3aa647b87..d9da320499f 100644 --- a/container-core/src/main/java/com/yahoo/restapi/RestApiException.java +++ b/container-core/src/main/java/com/yahoo/restapi/RestApiException.java @@ -40,8 +40,11 @@ public class RestApiException extends RuntimeException { public int statusCode() { return statusCode; } public HttpResponse response() { return response; } - public static class NotFoundException extends RestApiException { - public NotFoundException() { super(ErrorResponse::notFoundError, "Not Found", null); } + public static class NotFound extends RestApiException { + public NotFound() { this(null, null); } + public NotFound(Throwable cause) { this(cause.getMessage(), cause); } + public NotFound(String message) { this(message, null); } + public NotFound(String message, Throwable cause) { super(ErrorResponse::notFoundError, message, cause); } } public static class MethodNotAllowed extends RestApiException { @@ -52,12 +55,14 @@ public class RestApiException extends RuntimeException { } public static class BadRequest extends RestApiException { - public BadRequest(String message) { super(ErrorResponse::badRequest, message, null); } + public BadRequest(String message) { this(message, null); } + public BadRequest(Throwable cause) { this(cause.getMessage(), cause); } public BadRequest(String message, Throwable cause) { super(ErrorResponse::badRequest, message, cause); } } public static class InternalServerError extends RestApiException { - public InternalServerError(String message) { super(ErrorResponse::internalServerError, message, null); } + public InternalServerError(String message) { this(message, null); } + public InternalServerError(Throwable cause) { this(cause.getMessage(), cause); } public InternalServerError(String message, Throwable cause) { super(ErrorResponse::internalServerError, message, cause); } } @@ -65,4 +70,10 @@ public class RestApiException extends RuntimeException { public Forbidden(String message) { super(ErrorResponse::forbidden, message, null); } public Forbidden(String message, Throwable cause) { super(ErrorResponse::forbidden, message, cause); } } + + public static class Conflict extends RestApiException { + public Conflict() { this("Conflict", null); } + public Conflict(String message) { this(message, null); } + public Conflict(String message, Throwable cause) { super(ErrorResponse::conflict, message, cause); } + } } diff --git a/container-core/src/main/java/com/yahoo/restapi/RestApiImpl.java b/container-core/src/main/java/com/yahoo/restapi/RestApiImpl.java index e6c6d7ccb62..8ba94f9aca9 100644 --- a/container-core/src/main/java/com/yahoo/restapi/RestApiImpl.java +++ b/container-core/src/main/java/com/yahoo/restapi/RestApiImpl.java @@ -70,6 +70,8 @@ class RestApiImpl implements RestApi { } } + @Override public ObjectMapper jacksonJsonMapper() { return jacksonJsonMapper; } + private HttpResponse dispatchToRoute(Route route, RequestContextImpl context) { HandlerHolder<?> resolvedHandler = resolveHandler(context, route); RequestMapperHolder<?> resolvedRequestMapper = resolveRequestMapper(resolvedHandler); @@ -142,7 +144,7 @@ class RestApiImpl implements RestApi { private static Route createDefaultRoute() { RouteBuilder routeBuilder = new RouteBuilderImpl("{*}") .defaultHandler(context -> { - throw new RestApiException.NotFoundException(); + throw new RestApiException.NotFound(); }); return ((RouteBuilderImpl)routeBuilder).build(); } @@ -347,7 +349,10 @@ class RestApiImpl implements RestApi { @Override public ObjectMapper jacksonJsonMapper() { return jacksonJsonMapper; } @Override public UriBuilder uriBuilder() { URI uri = request.getUri(); - return new UriBuilder(uri.getScheme() + "://" + uri.getHost() + ':' + uri.getPort()); + int uriPort = uri.getPort(); + return uriPort != -1 + ? new UriBuilder(uri.getScheme() + "://" + uri.getHost() + ':' + uriPort) + : new UriBuilder(uri.getScheme() + "://" + uri.getHost()); } private class PathParametersImpl implements RestApi.RequestContext.PathParameters { diff --git a/container-core/src/main/java/com/yahoo/restapi/RestApiRequestHandler.java b/container-core/src/main/java/com/yahoo/restapi/RestApiRequestHandler.java index 9fe813903dd..c501ad8c804 100644 --- a/container-core/src/main/java/com/yahoo/restapi/RestApiRequestHandler.java +++ b/container-core/src/main/java/com/yahoo/restapi/RestApiRequestHandler.java @@ -4,6 +4,9 @@ package com.yahoo.restapi; import com.yahoo.container.jdisc.HttpRequest; import com.yahoo.container.jdisc.HttpResponse; import com.yahoo.container.jdisc.LoggingRequestHandler; +import com.yahoo.jdisc.Metric; + +import java.util.concurrent.Executor; /** * @author bjorncs @@ -25,12 +28,26 @@ public abstract class RestApiRequestHandler<T extends RestApiRequestHandler<T>> this.restApi = provider.createRestApi((T)this); } + /** + * @see #RestApiRequestHandler(Context, RestApiProvider) + */ + @SuppressWarnings("unchecked") + protected RestApiRequestHandler(Executor executor, Metric metric, RestApiProvider<T> provider) { + super(executor, metric); + this.restApi = provider.createRestApi((T)this); + } + protected RestApiRequestHandler(LoggingRequestHandler.Context context, RestApi restApi) { super(context); this.restApi = restApi; } + protected RestApiRequestHandler(Executor executor, Metric metric, RestApi restApi) { + super(executor, metric); + this.restApi = restApi; + } + @Override public final HttpResponse handle(HttpRequest request) { return restApi.handleRequest(request); } - protected RestApi restApi() { return restApi; } + public RestApi restApi() { return restApi; } } diff --git a/container-core/src/main/java/com/yahoo/restapi/RestApiTestDriver.java b/container-core/src/main/java/com/yahoo/restapi/RestApiTestDriver.java new file mode 100644 index 00000000000..7dc5b710bbe --- /dev/null +++ b/container-core/src/main/java/com/yahoo/restapi/RestApiTestDriver.java @@ -0,0 +1,96 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.restapi; + +import com.fasterxml.jackson.core.type.TypeReference; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.yahoo.container.jdisc.HttpRequest; +import com.yahoo.container.jdisc.HttpResponse; +import com.yahoo.container.jdisc.LoggingRequestHandler; +import com.yahoo.jdisc.http.server.jetty.testutils.TestDriver; +import com.yahoo.jdisc.test.MockMetric; + +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.InputStream; +import java.util.OptionalInt; +import java.util.concurrent.Executors; + +import static com.yahoo.yolean.Exceptions.uncheck; + +/** + * Test driver for {@link RestApi} + * + * @author bjorncs + */ +public class RestApiTestDriver implements AutoCloseable { + + private final RestApiRequestHandler<?> handler; + private final TestDriver testDriver; + + private RestApiTestDriver(Builder builder) { + this.handler = builder.handler; + this.testDriver = builder.jdiscHttpServer ? TestDriver.newBuilder().withRequestHandler(builder.handler).build() : null; + } + + public static Builder newBuilder(RestApiRequestHandler<?> handler) { return new Builder(handler); } + + @FunctionalInterface public interface RestApiRequestHandlerFactory { RestApiRequestHandler<?> create(LoggingRequestHandler.Context context); } + public static Builder newBuilder(RestApiRequestHandlerFactory factory) { return new Builder(factory); } + + public static LoggingRequestHandler.Context createHandlerTestContext() { + return new LoggingRequestHandler.Context(Executors.newSingleThreadExecutor(), new MockMetric()); + } + + public OptionalInt listenPort() { + return testDriver != null ? OptionalInt.of(testDriver.server().getListenPort()) : OptionalInt.empty(); + } + + public RestApiRequestHandler<?> handler() { return handler; } + public RestApi restApi() { return handler.restApi(); } + public ObjectMapper jacksonJsonMapper() { return handler.restApi().jacksonJsonMapper(); } + + public HttpResponse executeRequest(HttpRequest request) { return handler.handle(request); } + + public InputStream requestContentOf(Object jacksonEntity) { + ByteArrayOutputStream out = new ByteArrayOutputStream(); + uncheck(() -> handler.restApi().jacksonJsonMapper().writeValue(out, jacksonEntity)); + return new ByteArrayInputStream(out.toByteArray()); + } + + public <T> T parseJacksonResponseContent(HttpResponse response, TypeReference<T> type) { + return uncheck(() -> handler.restApi().jacksonJsonMapper().readValue(responseData(response), type)); + } + + public <T> T parseJacksonResponseContent(HttpResponse response, Class<T> type) { + return uncheck(() -> handler.restApi().jacksonJsonMapper().readValue(responseData(response), type)); + } + + private static byte[] responseData(HttpResponse response) { + ByteArrayOutputStream out = new ByteArrayOutputStream(); + uncheck(() -> response.render(out)); + return out.toByteArray(); + } + + @Override + public void close() throws Exception { + if (testDriver != null) { + testDriver.close(); + } + } + + public static class Builder { + private final RestApiRequestHandler<?> handler; + private boolean jdiscHttpServer = false; + + private Builder(RestApiRequestHandler<?> handler) { + this.handler = handler; + } + + private Builder(RestApiRequestHandlerFactory factory) { this(factory.create(createHandlerTestContext())); } + + public Builder withJdiscHttpServer() { this.jdiscHttpServer = true; return this; } + + public RestApiTestDriver build() { return new RestApiTestDriver(this); } + } + +} diff --git a/container-core/src/main/resources/configdefinitions/application-bundles.def b/container-core/src/main/resources/configdefinitions/application-bundles.def new file mode 100644 index 00000000000..7e03b1e3ac8 --- /dev/null +++ b/container-core/src/main/resources/configdefinitions/application-bundles.def @@ -0,0 +1,5 @@ +# Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package=com.yahoo.container.di.config + +# References to user bundles to install. +bundles[] file diff --git a/container-core/src/main/resources/configdefinitions/container.components.def b/container-core/src/main/resources/configdefinitions/container.components.def new file mode 100644 index 00000000000..f27abc2fa5a --- /dev/null +++ b/container-core/src/main/resources/configdefinitions/container.components.def @@ -0,0 +1,23 @@ +# Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +namespace=container + +## A list of components. Components depending on other components may use this to +## get its list of components injected. + +## A component +components[].id string +## The component id used by this component to subscribe to its configs (if any) +components[].configId reference default=":parent:" + +## The id of the class to instantiate for this component. +components[].classId string default="" + +## The symbolic name of the Osgi bundle this component is located in. +## Assumed to be the same as the classid if not set. +components[].bundle string default="" + +## The component id of the component to inject to this component +components[].inject[].id string + +## The name to use for the injected component when injected to this component +components[].inject[].name string default="" diff --git a/container-core/src/main/resources/configdefinitions/container.core.access-log.def b/container-core/src/main/resources/configdefinitions/container.core.access-log.def index 69058b3d8da..e6052b7068c 100644 --- a/container-core/src/main/resources/configdefinitions/container.core.access-log.def +++ b/container-core/src/main/resources/configdefinitions/container.core.access-log.def @@ -21,3 +21,6 @@ fileHandler.compressionFormat enum {GZIP, ZSTD} default=GZIP # Max queue length of file handler fileHandler.queueSize int default=10000 + +# Buffer size for the output stream has a default of 256k +fileHandler.bufferSize int default=262144 diff --git a/container-core/src/main/resources/configdefinitions/container.di.config.jersey-bundles.def b/container-core/src/main/resources/configdefinitions/container.di.config.jersey-bundles.def new file mode 100644 index 00000000000..a226420274d --- /dev/null +++ b/container-core/src/main/resources/configdefinitions/container.di.config.jersey-bundles.def @@ -0,0 +1,8 @@ +# Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +namespace=container.di.config + +# The SymbolicName[:Version] of the Jersey bundles +bundles[].spec string + +# The packages to scan for Jersey resources +bundles[].packages[] string diff --git a/container-core/src/main/resources/configdefinitions/container.di.config.jersey-injection.def b/container-core/src/main/resources/configdefinitions/container.di.config.jersey-injection.def new file mode 100644 index 00000000000..9f5be59abbd --- /dev/null +++ b/container-core/src/main/resources/configdefinitions/container.di.config.jersey-injection.def @@ -0,0 +1,5 @@ +# Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +namespace=container.di.config + +inject[].instance string +inject[].forClass string diff --git a/container-core/src/main/resources/configdefinitions/container.logging.connection-log.def b/container-core/src/main/resources/configdefinitions/container.logging.connection-log.def index 65b632c9008..cb2145cd01c 100644 --- a/container-core/src/main/resources/configdefinitions/container.logging.connection-log.def +++ b/container-core/src/main/resources/configdefinitions/container.logging.connection-log.def @@ -8,4 +8,7 @@ cluster string logDirectoryName string default="qrs" # Max queue length of file handler -queueSize int default=10000
\ No newline at end of file +queueSize int default=10000 + +# Buffer size for the output stream has a default of 256k +bufferSize int default=262144 diff --git a/container-core/src/main/resources/configdefinitions/jdisc.http.jdisc.http.connector.def b/container-core/src/main/resources/configdefinitions/jdisc.http.jdisc.http.connector.def index 055e5ad62d2..cb1e366f843 100644 --- a/container-core/src/main/resources/configdefinitions/jdisc.http.jdisc.http.connector.def +++ b/container-core/src/main/resources/configdefinitions/jdisc.http.jdisc.http.connector.def @@ -125,3 +125,6 @@ maxRequestsPerConnection int default=0 # Maximum number of seconds a connection can live before it's marked as non-persistent. Set to '0' to disable. maxConnectionLife double default=0.0 + +# Enable HTTP/2 (in addition to HTTP/1.1 using ALPN) +http2Enabled bool default=false diff --git a/container-core/src/main/resources/configdefinitions/metrics.manager.def b/container-core/src/main/resources/configdefinitions/metrics.manager.def new file mode 100644 index 00000000000..6446e0df8b6 --- /dev/null +++ b/container-core/src/main/resources/configdefinitions/metrics.manager.def @@ -0,0 +1,5 @@ +# Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +namespace=metrics + +reportPeriodSeconds int default=60 +pointsToKeepPerMetric int default=100 diff --git a/container-core/src/main/resources/configdefinitions/platform-bundles.def b/container-core/src/main/resources/configdefinitions/platform-bundles.def new file mode 100644 index 00000000000..a30a846b565 --- /dev/null +++ b/container-core/src/main/resources/configdefinitions/platform-bundles.def @@ -0,0 +1,5 @@ +# Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package=com.yahoo.container.di.config + +# Paths to platform bundles to install. +bundlePaths[] string |