diff options
32 files changed, 547 insertions, 302 deletions
diff --git a/config-model-api/src/main/java/com/yahoo/config/application/api/ValidationOverrides.java b/config-model-api/src/main/java/com/yahoo/config/application/api/ValidationOverrides.java index 11f9add6b25..441ef273a6f 100644 --- a/config-model-api/src/main/java/com/yahoo/config/application/api/ValidationOverrides.java +++ b/config-model-api/src/main/java/com/yahoo/config/application/api/ValidationOverrides.java @@ -66,6 +66,11 @@ public class ValidationOverrides { return false; } + public static String toAllowMessage(ValidationId id) { + return "To allow this add <allow until='yyyy-mm-dd'>" + id + "</allow> to validation-overrides.xml" + + ", see https://docs.vespa.ai/documentation/reference/validation-overrides.html"; + } + /** Returns the XML form of this, or null if it was not created by fromXml, nor is empty */ public String xmlForm() { return xmlForm; } @@ -155,7 +160,9 @@ public class ValidationOverrides { /** Returns "validationId: message" */ @Override - public String getMessage() { return validationId + ": " + super.getMessage(); } + public String getMessage() { + return validationId + ": " + super.getMessage() + ". " + toAllowMessage(validationId); + } } diff --git a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/ClusterSizeReductionValidatorTest.java b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/ClusterSizeReductionValidatorTest.java index 765acf9e27b..4c3583ba0ae 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/ClusterSizeReductionValidatorTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/ClusterSizeReductionValidatorTest.java @@ -1,6 +1,8 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.model.application.validation.change; +import com.yahoo.config.application.api.ValidationId; +import com.yahoo.config.application.api.ValidationOverrides; import com.yahoo.config.model.api.ConfigChangeAction; import com.yahoo.config.model.api.ConfigChangeRefeedAction; import com.yahoo.vespa.model.VespaModel; @@ -33,7 +35,8 @@ public class ClusterSizeReductionValidatorTest { fail("Expected exception due to cluster size reduction"); } catch (IllegalArgumentException expected) { - assertEquals("cluster-size-reduction: Size reduction in 'default' is too large. Current size: 30, new size: 14. New size must be at least 50% of the current size", + assertEquals("cluster-size-reduction: Size reduction in 'default' is too large. Current size: 30, new size: 14. New size must be at least 50% of the current size. " + + ValidationOverrides.toAllowMessage(ValidationId.clusterSizeReduction), Exceptions.toMessageString(expected)); } } diff --git a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/ContentClusterRemovalValidatorTest.java b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/ContentClusterRemovalValidatorTest.java index 25ad6dbc620..ee58ca67b02 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/ContentClusterRemovalValidatorTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/ContentClusterRemovalValidatorTest.java @@ -1,6 +1,8 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.model.application.validation.change; +import com.yahoo.config.application.api.ValidationId; +import com.yahoo.config.application.api.ValidationOverrides; import com.yahoo.vespa.model.VespaModel; import com.yahoo.vespa.model.application.validation.ValidationTester; import com.yahoo.yolean.Exceptions; @@ -24,7 +26,8 @@ public class ContentClusterRemovalValidatorTest { fail("Expected exception due to content cluster id change"); } catch (IllegalArgumentException expected) { - assertEquals("content-cluster-removal: Content cluster 'contentClusterId' is removed. This will cause loss of all data in this cluster", + assertEquals("content-cluster-removal: Content cluster 'contentClusterId' is removed. This will cause loss of all data in this cluster. " + + ValidationOverrides.toAllowMessage(ValidationId.contentClusterRemoval), Exceptions.toMessageString(expected)); } } diff --git a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/ContentTypeRemovalValidatorTest.java b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/ContentTypeRemovalValidatorTest.java index a52c6d7c7a2..ca45520711e 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/ContentTypeRemovalValidatorTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/ContentTypeRemovalValidatorTest.java @@ -1,6 +1,8 @@ // Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.model.application.validation.change; +import com.yahoo.config.application.api.ValidationId; +import com.yahoo.config.application.api.ValidationOverrides; import com.yahoo.vespa.model.VespaModel; import com.yahoo.vespa.model.application.validation.ValidationTester; import com.yahoo.yolean.Exceptions; @@ -28,7 +30,8 @@ public class ContentTypeRemovalValidatorTest { } catch (IllegalArgumentException expected) { assertEquals("content-type-removal: Type 'music' is removed in content cluster 'test'. " + - "This will cause loss of all data of this type", + "This will cause loss of all data of this type. " + + ValidationOverrides.toAllowMessage(ValidationId.contentTypeRemoval), Exceptions.toMessageString(expected)); } } diff --git a/config/src/main/java/com/yahoo/config/subscription/impl/JRTConfigRequester.java b/config/src/main/java/com/yahoo/config/subscription/impl/JRTConfigRequester.java index 88af414e28d..243c9e932a8 100644 --- a/config/src/main/java/com/yahoo/config/subscription/impl/JRTConfigRequester.java +++ b/config/src/main/java/com/yahoo/config/subscription/impl/JRTConfigRequester.java @@ -30,10 +30,10 @@ import com.yahoo.vespa.config.protocol.Trace; * as context, and puts the requests objects on a queue on the subscription, * for handling by the user thread. * - * @author vegardh - * @since 5.1 + * @author Vegard Havdal */ public class JRTConfigRequester implements RequestWaiter { + private static final Logger log = Logger.getLogger(JRTConfigRequester.class.getName()); public static final ConfigSourceSet defaultSourceSet = ConfigSourceSet.createDefault(); private static final int TRACELEVEL = 6; diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileDistributionImpl.java b/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileDistributionImpl.java index 2db89c2e8ed..36d76bbfc79 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileDistributionImpl.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileDistributionImpl.java @@ -10,7 +10,6 @@ import com.yahoo.jrt.Spec; import com.yahoo.jrt.StringArray; import com.yahoo.jrt.Supervisor; import com.yahoo.jrt.Target; -import com.yahoo.jrt.Transport; import com.yahoo.log.LogLevel; import com.yahoo.vespa.defaults.Defaults; @@ -24,11 +23,12 @@ import java.util.logging.Logger; public class FileDistributionImpl implements FileDistribution { private final static Logger log = Logger.getLogger(FileDistributionImpl.class.getName()); - private final Supervisor supervisor = new Supervisor(new Transport()); + private final Supervisor supervisor; private final File fileReferencesDir; - public FileDistributionImpl(ConfigserverConfig configserverConfig) { + public FileDistributionImpl(ConfigserverConfig configserverConfig, Supervisor supervisor) { this.fileReferencesDir = new File(Defaults.getDefaults().underVespaHome(configserverConfig.fileReferencesDir())); + this.supervisor = supervisor; } @Override diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/ConfigServerMaintenance.java b/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/ConfigServerMaintenance.java index c6a390caf86..2a53f9ee45c 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/ConfigServerMaintenance.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/ConfigServerMaintenance.java @@ -51,7 +51,7 @@ public class ConfigServerMaintenance extends AbstractComponent { this.defaultInterval = Duration.ofMinutes(configserverConfig.maintainerIntervalMinutes()); // TODO: Want job control or feature flag to control when to run this, for now use a very // long interval to avoid running the maintainer - this.tenantsMaintainerInterval = isCd || isTest + this.tenantsMaintainerInterval = isCd || isTest || configserverConfig.region().equals("us-central-1") ? defaultInterval : Duration.ofMinutes(configserverConfig.tenantsMaintainerIntervalMinutes()); } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/FileDistributionMaintainer.java b/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/FileDistributionMaintainer.java index 2664a0bde8c..1d16283d938 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/FileDistributionMaintainer.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/FileDistributionMaintainer.java @@ -31,9 +31,10 @@ public class FileDistributionMaintainer extends Maintainer { @Override protected void maintain() { - // TODO: For now only deletes files in CD system + // TODO: Delete files in all zones boolean deleteFiles = (SystemName.from(configserverConfig.system()) == SystemName.cd) - || Environment.from(configserverConfig.environment()).isTest(); + || Environment.from(configserverConfig.environment()).isTest() + || configserverConfig.region().equals("us-central-1"); applicationRepository.deleteUnusedFiledistributionReferences(fileReferencesDir, deleteFiles); } } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/session/FileDistributionFactory.java b/configserver/src/main/java/com/yahoo/vespa/config/server/session/FileDistributionFactory.java index 8394494adca..15bc3c1fb46 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/session/FileDistributionFactory.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/session/FileDistributionFactory.java @@ -3,6 +3,8 @@ package com.yahoo.vespa.config.server.session; import com.google.inject.Inject; import com.yahoo.cloud.config.ConfigserverConfig; +import com.yahoo.jrt.Supervisor; +import com.yahoo.jrt.Transport; import com.yahoo.vespa.config.server.filedistribution.FileDistributionImpl; import com.yahoo.vespa.config.server.filedistribution.FileDistributionProvider; @@ -17,6 +19,7 @@ import java.io.File; public class FileDistributionFactory { private final ConfigserverConfig configserverConfig; + private final Supervisor supervisor = new Supervisor(new Transport()); @Inject public FileDistributionFactory(ConfigserverConfig configserverConfig) { @@ -24,7 +27,7 @@ public class FileDistributionFactory { } public FileDistributionProvider createProvider(File applicationPackage) { - return new FileDistributionProvider(applicationPackage, new FileDistributionImpl(configserverConfig)); + return new FileDistributionProvider(applicationPackage, new FileDistributionImpl(configserverConfig, supervisor)); } } diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/ConfigServerBootstrapTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/ConfigServerBootstrapTest.java index c0c3683bebb..992d46d3115 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/ConfigServerBootstrapTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/ConfigServerBootstrapTest.java @@ -47,9 +47,9 @@ public class ConfigServerBootstrapTest { VipStatus vipStatus = new VipStatus(); ConfigServerBootstrap bootstrap = new ConfigServerBootstrap(tester.applicationRepository(), rpcServer, versionState, createStateMonitor(), vipStatus); assertFalse(vipStatus.isInRotation()); - waitUntil(() -> bootstrap.status() == StateMonitor.Status.up, "failed waiting for status 'up'"); waitUntil(rpcServer::isRunning, "failed waiting for Rpc server running"); - assertTrue(vipStatus.isInRotation()); + waitUntil(() -> bootstrap.status() == StateMonitor.Status.up, "failed waiting for status 'up'"); + waitUntil(vipStatus::isInRotation, "failed waiting for server to be in rotation"); bootstrap.deconstruct(); assertEquals(StateMonitor.Status.down, bootstrap.status()); diff --git a/container-jersey2/pom.xml b/container-jersey2/pom.xml index 76ff21dc028..c5ed7d872bf 100644 --- a/container-jersey2/pom.xml +++ b/container-jersey2/pom.xml @@ -53,10 +53,6 @@ <groupId>org.ow2.asm</groupId> <artifactId>asm</artifactId> </dependency> - <dependency> - <groupId>org.scala-lang</groupId> - <artifactId>scala-library</artifactId> - </dependency> </dependencies> <build> <plugins> diff --git a/container-jersey2/src/main/java/com/yahoo/container/servlet/jersey/ComponentGraphProvider.java b/container-jersey2/src/main/java/com/yahoo/container/servlet/jersey/ComponentGraphProvider.java new file mode 100644 index 00000000000..7ff9646cb27 --- /dev/null +++ b/container-jersey2/src/main/java/com/yahoo/container/servlet/jersey/ComponentGraphProvider.java @@ -0,0 +1,73 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.container.servlet.jersey; + +import com.yahoo.container.di.config.ResolveDependencyException; +import com.yahoo.container.di.config.RestApiContext; +import com.yahoo.container.jaxrs.annotation.Component; +import org.glassfish.hk2.api.Injectee; +import org.glassfish.hk2.api.InjectionResolver; +import org.glassfish.hk2.api.ServiceHandle; + +import javax.inject.Singleton; + +import java.lang.reflect.Type; +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; +import java.util.stream.Collectors; + +/** + * Resolves jdisc container components for jersey 2 components. + * + * @author Tony Vaagenes + * @author ollivir + */ +@Singleton // jersey2 requirement: InjectionResolvers must be in the Singleton scope +public class ComponentGraphProvider implements InjectionResolver<Component> { + private Collection<RestApiContext.Injectable> injectables; + + public ComponentGraphProvider(Collection<RestApiContext.Injectable> injectables) { + this.injectables = injectables; + } + + @Override + public Object resolve(Injectee injectee, ServiceHandle<?> root) { + Class<?> wantedClass; + Type type = injectee.getRequiredType(); + if (type instanceof Class) { + wantedClass = (Class<?>) type; + } else { + throw new UnsupportedOperationException("Only classes are supported, got " + type); + } + + List<RestApiContext.Injectable> componentsWithMatchingType = new ArrayList<>(); + for (RestApiContext.Injectable injectable : injectables) { + if (wantedClass.isInstance(injectable.instance)) { + componentsWithMatchingType.add(injectable); + } + } + + if (componentsWithMatchingType.size() == 1) { + return componentsWithMatchingType.get(0).instance; + } else { + String injectionDescription = "class '" + wantedClass + "' to inject into Jersey resource/provider '" + + injectee.getInjecteeClass() + "')"; + if (componentsWithMatchingType.size() > 1) { + String ids = componentsWithMatchingType.stream().map(c -> c.id.toString()).collect(Collectors.joining(",")); + throw new ResolveDependencyException("Multiple components found of " + injectionDescription + ": " + ids); + } else { + throw new ResolveDependencyException("Could not find a component of " + injectionDescription + "."); + } + } + } + + @Override + public boolean isMethodParameterIndicator() { + return true; + } + + @Override + public boolean isConstructorParameterIndicator() { + return true; + } +} diff --git a/container-jersey2/src/main/java/com/yahoo/container/servlet/jersey/JerseyApplication.java b/container-jersey2/src/main/java/com/yahoo/container/servlet/jersey/JerseyApplication.java new file mode 100644 index 00000000000..4c4e43bc8d5 --- /dev/null +++ b/container-jersey2/src/main/java/com/yahoo/container/servlet/jersey/JerseyApplication.java @@ -0,0 +1,25 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.container.servlet.jersey; + +import javax.ws.rs.core.Application; + +import java.util.Collection; +import java.util.HashSet; +import java.util.Set; + +/** + * @author Tony Vaagenes + * @author ollivir + */ +public class JerseyApplication extends Application { + private Set<Class<?>> classes; + + public JerseyApplication(Collection<Class<?>> resourcesAndProviderClasses) { + this.classes = new HashSet<>(resourcesAndProviderClasses); + } + + @Override + public Set<Class<?>> getClasses() { + return classes; + } +} diff --git a/container-jersey2/src/main/java/com/yahoo/container/servlet/jersey/JerseyServletProvider.java b/container-jersey2/src/main/java/com/yahoo/container/servlet/jersey/JerseyServletProvider.java new file mode 100644 index 00000000000..1dbe410ba54 --- /dev/null +++ b/container-jersey2/src/main/java/com/yahoo/container/servlet/jersey/JerseyServletProvider.java @@ -0,0 +1,118 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.container.servlet.jersey; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.datatype.jdk8.Jdk8Module; +import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule; +import com.fasterxml.jackson.jaxrs.json.JacksonJaxbJsonProvider; +import com.yahoo.container.di.componentgraph.Provider; +import com.yahoo.container.di.config.RestApiContext; +import com.yahoo.container.di.config.RestApiContext.BundleInfo; +import com.yahoo.container.jaxrs.annotation.Component; +import org.eclipse.jetty.servlet.ServletHolder; +import org.glassfish.hk2.api.InjectionResolver; +import org.glassfish.hk2.api.TypeLiteral; +import org.glassfish.hk2.utilities.Binder; +import org.glassfish.hk2.utilities.binding.AbstractBinder; +import org.glassfish.jersey.media.multipart.MultiPartFeature; +import org.glassfish.jersey.server.ResourceConfig; +import org.glassfish.jersey.servlet.ServletContainer; +import org.objectweb.asm.ClassReader; + +import java.io.IOException; +import java.io.InputStream; +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; +import java.util.Optional; + +import static com.yahoo.container.servlet.jersey.util.ResourceConfigUtil.registerComponent; + +/** + * @author Tony Vaagenes + * @author ollivir + */ +public class JerseyServletProvider implements Provider<ServletHolder> { + private final ServletHolder jerseyServletHolder; + + public JerseyServletProvider(RestApiContext restApiContext) { + this.jerseyServletHolder = new ServletHolder(new ServletContainer(resourceConfig(restApiContext))); + } + + private ResourceConfig resourceConfig(RestApiContext restApiContext) { + final ResourceConfig resourceConfig = ResourceConfig + .forApplication(new JerseyApplication(resourcesAndProviders(restApiContext.getBundles()))); + + registerComponent(resourceConfig, componentInjectorBinder(restApiContext)); + registerComponent(resourceConfig, jacksonDatatypeJdk8Provider()); + resourceConfig.register(MultiPartFeature.class); + + return resourceConfig; + } + + private static Collection<Class<?>> resourcesAndProviders(Collection<BundleInfo> bundles) { + final List<Class<?>> ret = new ArrayList<>(); + + for (BundleInfo bundle : bundles) { + for (String classEntry : bundle.getClassEntries()) { + Optional<String> className = detectResourceOrProvider(bundle.classLoader, classEntry); + className.ifPresent(cname -> ret.add(loadClass(bundle.symbolicName, bundle.classLoader, cname))); + } + } + return ret; + } + + private static Optional<String> detectResourceOrProvider(ClassLoader bundleClassLoader, String classEntry) { + try (InputStream inputStream = getResourceAsStream(bundleClassLoader, classEntry)) { + ResourceOrProviderClassVisitor visitor = ResourceOrProviderClassVisitor.visit(new ClassReader(inputStream)); + return visitor.getJerseyClassName(); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + + private static InputStream getResourceAsStream(ClassLoader bundleClassLoader, String classEntry) { + InputStream is = bundleClassLoader.getResourceAsStream(classEntry); + if (is == null) { + throw new RuntimeException("No entry " + classEntry + " in bundle " + bundleClassLoader); + } else { + return is; + } + } + + private static Class<?> loadClass(String bundleSymbolicName, ClassLoader classLoader, String className) { + try { + return classLoader.loadClass(className); + } catch (Exception e) { + throw new RuntimeException("Failed loading class " + className + " from bundle " + bundleSymbolicName, e); + } + } + + private static Binder componentInjectorBinder(RestApiContext restApiContext) { + final ComponentGraphProvider componentGraphProvider = new ComponentGraphProvider(restApiContext.getInjectableComponents()); + final TypeLiteral<InjectionResolver<Component>> componentAnnotationType = new TypeLiteral<InjectionResolver<Component>>() { + }; + + return new AbstractBinder() { + @Override + public void configure() { + bind(componentGraphProvider).to(componentAnnotationType); + } + }; + } + + private static JacksonJaxbJsonProvider jacksonDatatypeJdk8Provider() { + JacksonJaxbJsonProvider provider = new JacksonJaxbJsonProvider(); + provider.setMapper(new ObjectMapper().registerModule(new Jdk8Module()).registerModule(new JavaTimeModule())); + return provider; + } + + @Override + public ServletHolder get() { + return jerseyServletHolder; + } + + @Override + public void deconstruct() { + } +} diff --git a/container-jersey2/src/main/java/com/yahoo/container/servlet/jersey/ResourceOrProviderClassVisitor.java b/container-jersey2/src/main/java/com/yahoo/container/servlet/jersey/ResourceOrProviderClassVisitor.java new file mode 100644 index 00000000000..7cb47ac6118 --- /dev/null +++ b/container-jersey2/src/main/java/com/yahoo/container/servlet/jersey/ResourceOrProviderClassVisitor.java @@ -0,0 +1,103 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.container.servlet.jersey; + +import org.objectweb.asm.AnnotationVisitor; +import org.objectweb.asm.ClassReader; +import org.objectweb.asm.ClassVisitor; +import org.objectweb.asm.Opcodes; +import org.objectweb.asm.Type; + +import javax.ws.rs.Path; +import javax.ws.rs.ext.Provider; + +import java.util.HashSet; +import java.util.Optional; +import java.util.Set; + +/** + * @author Tony Vaagenes + * @author ollivir + */ +public class ResourceOrProviderClassVisitor extends ClassVisitor { + private String className = null; + private boolean isPublic = false; + private boolean isAbstract = false; + + private boolean isInnerClass = false; + private boolean isStatic = false; + + private boolean isAnnotated = false; + + public ResourceOrProviderClassVisitor() { + super(Opcodes.ASM6); + } + + public Optional<String> getJerseyClassName() { + if (isJerseyClass()) { + return Optional.of(getClassName()); + } else { + return Optional.empty(); + } + } + + public boolean isJerseyClass() { + return isAnnotated && isPublic && !isAbstract && (!isInnerClass || isStatic); + } + + public String getClassName() { + assert (className != null); + return org.objectweb.asm.Type.getObjectType(className).getClassName(); + } + + @Override + public void visit(int version, int access, String name, String signature, String superName, String[] interfaces) { + isPublic = isPublic(access); + className = name; + isAbstract = isAbstract(access); + } + + @Override + public void visitInnerClass(String name, String outerName, String innerName, int access) { + assert (className != null); + + if (name.equals(className)) { + isInnerClass = true; + isStatic = isStatic(access); + } + } + + @Override + public AnnotationVisitor visitAnnotation(String desc, boolean visible) { + isAnnotated |= annotationClassDescriptors.contains(desc); + return null; + } + + private static Set<String> annotationClassDescriptors = new HashSet<>(); + + static { + annotationClassDescriptors.add(Type.getDescriptor(Path.class)); + annotationClassDescriptors.add(Type.getDescriptor(Provider.class)); + } + + private static boolean isPublic(int access) { + return isSet(Opcodes.ACC_PUBLIC, access); + } + + private static boolean isStatic(int access) { + return isSet(Opcodes.ACC_STATIC, access); + } + + private static boolean isAbstract(int access) { + return isSet(Opcodes.ACC_ABSTRACT, access); + } + + private static boolean isSet(int bits, int access) { + return (access & bits) == bits; + } + + public static ResourceOrProviderClassVisitor visit(ClassReader classReader) { + ResourceOrProviderClassVisitor visitor = new ResourceOrProviderClassVisitor(); + classReader.accept(visitor, ClassReader.SKIP_DEBUG | ClassReader.SKIP_CODE | ClassReader.SKIP_FRAMES); + return visitor; + } +} diff --git a/container-jersey2/src/main/scala/com/yahoo/container/servlet/jersey/ComponentGraphProvider.scala b/container-jersey2/src/main/scala/com/yahoo/container/servlet/jersey/ComponentGraphProvider.scala deleted file mode 100644 index cabde3680a4..00000000000 --- a/container-jersey2/src/main/scala/com/yahoo/container/servlet/jersey/ComponentGraphProvider.scala +++ /dev/null @@ -1,40 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.container.servlet.jersey - -import javax.inject.Singleton - -import com.yahoo.container.di.config.{ResolveDependencyException, RestApiContext} -import com.yahoo.container.jaxrs.annotation.Component -import org.glassfish.hk2.api.{ServiceHandle, Injectee, InjectionResolver} - -/** - * Resolves jdisc container components for jersey 2 components. - * Similar to Gjoran's ComponentGraphProvider for jersey 1. - * @author tonytv - */ -@Singleton //jersey2 requirement: InjectionResolvers must be in the Singleton scope -class ComponentGraphProvider(injectables: Traversable[RestApiContext.Injectable]) extends InjectionResolver[Component] { - override def resolve(injectee: Injectee, root: ServiceHandle[_]): AnyRef = { - val wantedClass = injectee.getRequiredType match { - case c: Class[_] => c - case unsupported => throw new UnsupportedOperationException("Only classes are supported, got " + unsupported) - } - - val componentsWithMatchingType = injectables.filter{ injectable => - wantedClass.isInstance(injectable.instance) } - - val injectionDescription = - s"class '$wantedClass' to inject into Jersey resource/provider '${injectee.getInjecteeClass}')" - - if (componentsWithMatchingType.size > 1) - throw new ResolveDependencyException(s"Multiple components found of $injectionDescription: " + - componentsWithMatchingType.map(_.id).mkString(",")) - - componentsWithMatchingType.headOption.map(_.instance).getOrElse { - throw new ResolveDependencyException(s"Could not find a component of $injectionDescription.") - } - } - - override def isMethodParameterIndicator: Boolean = true - override def isConstructorParameterIndicator: Boolean = true -} diff --git a/container-jersey2/src/main/scala/com/yahoo/container/servlet/jersey/JerseyApplication.scala b/container-jersey2/src/main/scala/com/yahoo/container/servlet/jersey/JerseyApplication.scala deleted file mode 100644 index eea41003984..00000000000 --- a/container-jersey2/src/main/scala/com/yahoo/container/servlet/jersey/JerseyApplication.scala +++ /dev/null @@ -1,16 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.container.servlet.jersey - -import javax.ws.rs.core.Application - -import scala.collection.JavaConverters._ - -/** - * @author tonytv - */ -class JerseyApplication(resourcesAndProviderClasses: Set[Class[_]]) extends Application { - private val classes: java.util.Set[Class[_]] = resourcesAndProviderClasses.asJava - - override def getClasses = classes - override def getSingletons = super.getSingletons -} diff --git a/container-jersey2/src/main/scala/com/yahoo/container/servlet/jersey/JerseyServletProvider.scala b/container-jersey2/src/main/scala/com/yahoo/container/servlet/jersey/JerseyServletProvider.scala deleted file mode 100644 index f0eff54dc16..00000000000 --- a/container-jersey2/src/main/scala/com/yahoo/container/servlet/jersey/JerseyServletProvider.scala +++ /dev/null @@ -1,109 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.container.servlet.jersey - -import java.io.{IOException, InputStream} - -import com.fasterxml.jackson.databind.ObjectMapper -import com.fasterxml.jackson.datatype.jdk8.Jdk8Module -import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule -import com.fasterxml.jackson.jaxrs.json.JacksonJaxbJsonProvider -import com.yahoo.container.di.componentgraph.Provider -import com.yahoo.container.di.config.RestApiContext -import com.yahoo.container.di.config.RestApiContext.BundleInfo -import com.yahoo.container.jaxrs.annotation.Component -import com.yahoo.container.servlet.jersey.util.ResourceConfigUtil.registerComponent -import org.eclipse.jetty.servlet.ServletHolder -import org.glassfish.hk2.api.{InjectionResolver, TypeLiteral} -import org.glassfish.hk2.utilities.Binder -import org.glassfish.hk2.utilities.binding.AbstractBinder -import org.glassfish.jersey.media.multipart.MultiPartFeature -import org.glassfish.jersey.server.ResourceConfig -import org.glassfish.jersey.servlet.ServletContainer -import org.objectweb.asm.ClassReader - -import scala.collection.JavaConverters._ -import scala.util.control.Exception - - -/** - * @author tonytv - */ -class JerseyServletProvider(restApiContext: RestApiContext) extends Provider[ServletHolder] { - private val jerseyServletHolder = new ServletHolder(new ServletContainer(resourceConfig(restApiContext))) - - private def resourceConfig(restApiContext: RestApiContext) = { - val resourceConfig = ResourceConfig.forApplication( - new JerseyApplication(resourcesAndProviders(restApiContext.getBundles.asScala))) - - registerComponent(resourceConfig, componentInjectorBinder(restApiContext)) - registerComponent(resourceConfig, jacksonDatatypeJdk8Provider) - resourceConfig.register(classOf[MultiPartFeature]) - - resourceConfig - } - - def resourcesAndProviders(bundles: Traversable[BundleInfo]) = - (for { - bundle <- bundles.view - classEntry <- bundle.getClassEntries.asScala - className <- detectResourceOrProvider(bundle.classLoader, classEntry) - } yield loadClass(bundle.symbolicName, bundle.classLoader, className)).toSet - - - def detectResourceOrProvider(bundleClassLoader: ClassLoader, classEntry: String): Option[String] = { - using(getResourceAsStream(bundleClassLoader, classEntry)) { inputStream => - val visitor = ResourceOrProviderClassVisitor.visit(new ClassReader(inputStream)) - visitor.getJerseyClassName - } - } - - private def getResourceAsStream(bundleClassLoader: ClassLoader, classEntry: String) = { - bundleClassLoader.getResourceAsStream(classEntry) match { - case null => throw new RuntimeException(s"No entry $classEntry in bundle $bundleClassLoader") - case stream => stream - } - - } - - def using[T <: InputStream, R](stream: T)(f: T => R): R = { - try { - f(stream) - } finally { - Exception.ignoring(classOf[IOException]) { - stream.close() - } - } - } - - def loadClass(bundleSymbolicName: String, classLoader: ClassLoader, className: String) = { - try { - classLoader.loadClass(className) - } catch { - case e: Exception => throw new RuntimeException(s"Failed loading class $className from bundle $bundleSymbolicName", e) - } - } - - def componentInjectorBinder(restApiContext: RestApiContext): Binder = { - val componentGraphProvider = new ComponentGraphProvider(restApiContext.getInjectableComponents.asScala) - val componentAnnotationType = new TypeLiteral[InjectionResolver[Component]] {} - - new AbstractBinder { - override def configure() { - bind(componentGraphProvider).to(componentAnnotationType) - } - } - } - - def jacksonDatatypeJdk8Provider: JacksonJaxbJsonProvider = { - val provider = new JacksonJaxbJsonProvider() - provider.setMapper( - new ObjectMapper() - .registerModule(new Jdk8Module) - .registerModule(new JavaTimeModule)) - provider - } - - override def get() = jerseyServletHolder - override def deconstruct() {} -} - diff --git a/container-jersey2/src/main/scala/com/yahoo/container/servlet/jersey/ResourceOrProviderClassVisitor.scala b/container-jersey2/src/main/scala/com/yahoo/container/servlet/jersey/ResourceOrProviderClassVisitor.scala deleted file mode 100644 index 52674026c25..00000000000 --- a/container-jersey2/src/main/scala/com/yahoo/container/servlet/jersey/ResourceOrProviderClassVisitor.scala +++ /dev/null @@ -1,74 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.container.servlet.jersey - -import javax.ws.rs.Path -import javax.ws.rs.ext.Provider - -import org.objectweb.asm.{ClassVisitor, Opcodes, Type, AnnotationVisitor, ClassReader} - - -/** - * @author tonytv - */ -class ResourceOrProviderClassVisitor private () extends ClassVisitor(Opcodes.ASM6) { - private var className: String = null - private var isPublic: Boolean = false - private var isAbstract = false - - private var isInnerClass: Boolean = false - private var isStatic: Boolean = false - - private var isAnnotated: Boolean = false - - def getJerseyClassName: Option[String] = { - if (isJerseyClass) Some(getClassName) - else None - } - - def isJerseyClass: Boolean = { - isAnnotated && isPublic && !isAbstract && - (!isInnerClass || isStatic) - } - - def getClassName = { - assert (className != null) - Type.getObjectType(className).getClassName - } - - override def visit(version: Int, access: Int, name: String, signature: String, superName: String, interfaces: Array[String]) { - isPublic = ResourceOrProviderClassVisitor.isPublic(access) - className = name - isAbstract = ResourceOrProviderClassVisitor.isAbstract(access) - } - - override def visitInnerClass(name: String, outerName: String, innerName: String, access: Int) { - assert (className != null) - - if (name == className) { - isInnerClass = true - isStatic = ResourceOrProviderClassVisitor.isStatic(access) - } - } - - override def visitAnnotation(desc: String, visible: Boolean): AnnotationVisitor = { - isAnnotated |= ResourceOrProviderClassVisitor.annotationClassDescriptors(desc) - null - } -} - - -object ResourceOrProviderClassVisitor { - val annotationClassDescriptors = Set(classOf[Path], classOf[Provider]) map Type.getDescriptor - - def isPublic = isSet(Opcodes.ACC_PUBLIC) _ - def isStatic = isSet(Opcodes.ACC_STATIC) _ - def isAbstract = isSet(Opcodes.ACC_ABSTRACT) _ - - private def isSet(bits: Int)(access: Int): Boolean = (access & bits) == bits - - def visit(classReader: ClassReader): ResourceOrProviderClassVisitor = { - val visitor = new ResourceOrProviderClassVisitor - classReader.accept(visitor, ClassReader.SKIP_DEBUG | ClassReader.SKIP_CODE | ClassReader.SKIP_FRAMES) - visitor - } -} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java index 23cc1d31b06..f0e278c3e6d 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java @@ -5,6 +5,7 @@ import com.google.common.collect.ImmutableList; import com.yahoo.component.Version; import com.yahoo.config.application.api.DeploymentSpec; import com.yahoo.config.application.api.ValidationId; +import com.yahoo.config.application.api.ValidationOverrides; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.Environment; import com.yahoo.config.provision.TenantName; @@ -409,7 +410,8 @@ public class ApplicationController { .collect(Collectors.joining(", ")) + ", but does not include " + (deploymentsToRemove.size() > 1 ? "these zones" : "this zone") + - " in deployment.xml"); + " in deployment.xml. " + + ValidationOverrides.toAllowMessage(ValidationId.deploymentRemoval)); LockedApplication applicationWithRemoval = application; for (Deployment deployment : deploymentsToRemove) diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java index 0de153fc3f9..c24c8693688 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java @@ -3,6 +3,7 @@ package com.yahoo.vespa.hosted.controller; import com.yahoo.component.Version; import com.yahoo.config.application.api.ValidationId; +import com.yahoo.config.application.api.ValidationOverrides; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.ApplicationName; import com.yahoo.config.provision.Environment; @@ -179,7 +180,9 @@ public class ControllerTest { fail("Expected exception due to illegal production deployment removal"); } catch (IllegalArgumentException e) { - assertEquals("deployment-removal: application 'tenant1.app1' is deployed in corp-us-east-1, but does not include this zone in deployment.xml", e.getMessage()); + assertEquals("deployment-removal: application 'tenant1.app1' is deployed in corp-us-east-1, but does not include this zone in deployment.xml. " + + ValidationOverrides.toAllowMessage(ValidationId.deploymentRemoval), + e.getMessage()); } assertNotNull("Zone was not removed", applications.require(app1.id()).deployments().get(productionCorpUsEast1.zone(main).get())); diff --git a/docprocs/src/test/java/com/yahoo/docprocs/indexing/DocumentScriptTestCase.java b/docprocs/src/test/java/com/yahoo/docprocs/indexing/DocumentScriptTestCase.java index 5b1a4412b41..419b60432c4 100644 --- a/docprocs/src/test/java/com/yahoo/docprocs/indexing/DocumentScriptTestCase.java +++ b/docprocs/src/test/java/com/yahoo/docprocs/indexing/DocumentScriptTestCase.java @@ -1,11 +1,13 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.docprocs.indexing; +import com.yahoo.document.ArrayDataType; import com.yahoo.document.DataType; import com.yahoo.document.Document; import com.yahoo.document.DocumentType; import com.yahoo.document.DocumentUpdate; import com.yahoo.document.Field; +import com.yahoo.document.MapDataType; import com.yahoo.document.StructDataType; import com.yahoo.document.annotation.SpanTree; import com.yahoo.document.annotation.SpanTrees; @@ -16,6 +18,7 @@ import com.yahoo.document.datatypes.StringFieldValue; import com.yahoo.document.datatypes.Struct; import com.yahoo.document.datatypes.WeightedSet; import com.yahoo.document.fieldpathupdate.AssignFieldPathUpdate; +import com.yahoo.document.fieldpathupdate.FieldPathUpdate; import com.yahoo.document.update.FieldUpdate; import com.yahoo.document.update.MapValueUpdate; import com.yahoo.document.update.ValueUpdate; @@ -30,6 +33,7 @@ import org.junit.Test; import java.util.ArrayList; import java.util.Arrays; +import java.util.Collections; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; @@ -157,6 +161,82 @@ public class DocumentScriptTestCase { assertSpanTrees(str, "mySpanTree"); } + private class FieldPathFixture { + final DocumentType type; + final StructDataType structType; + final DataType structMap; + final DataType structArray; + + FieldPathFixture() { + type = newDocumentType(); + structType = new StructDataType("mystruct"); + structType.addField(new Field("title", DataType.STRING)); + structType.addField(new Field("rating", DataType.INT)); + structArray = new ArrayDataType(structType); + type.addField(new Field("structarray", structArray)); + structMap = new MapDataType(DataType.STRING, structType); + type.addField(new Field("structmap", structMap)); + type.addField(new Field("structfield", structType)); + } + + DocumentUpdate executeWithUpdate(String fieldName, FieldPathUpdate updateIn) { + DocumentUpdate update = new DocumentUpdate(type, "doc:scheme:"); + update.addFieldPathUpdate(updateIn); + return newScript(type, fieldName).execute(ADAPTER_FACTORY, update); + } + + FieldPathUpdate executeWithUpdateAndExpectFieldPath(String fieldName, FieldPathUpdate updateIn) { + DocumentUpdate update = executeWithUpdate(fieldName, updateIn); + assertEquals(1, update.getFieldPathUpdates().size()); + return update.getFieldPathUpdates().get(0); + } + } + + @Test + public void array_field_path_updates_survive_indexing_scripts() { + FieldPathFixture f = new FieldPathFixture(); + + Struct newElemValue = new Struct(f.structType); + newElemValue.setFieldValue("title", "iron moose 2, the moosening"); + + FieldPathUpdate updated = f.executeWithUpdateAndExpectFieldPath("structarray", new AssignFieldPathUpdate(f.type, "structarray[10]", newElemValue)); + + assertTrue(updated instanceof AssignFieldPathUpdate); + AssignFieldPathUpdate assignUpdate = (AssignFieldPathUpdate)updated; + assertEquals("structarray[10]", assignUpdate.getOriginalFieldPath()); + assertEquals(newElemValue, assignUpdate.getFieldValue()); + } + + @Test + public void map_field_path_updates_survive_indexing_scripts() { + FieldPathFixture f = new FieldPathFixture(); + + Struct newElemValue = new Struct(f.structType); + newElemValue.setFieldValue("title", "iron moose 3, moose in new york"); + + FieldPathUpdate updated = f.executeWithUpdateAndExpectFieldPath("structmap", new AssignFieldPathUpdate(f.type, "structmap{foo}", newElemValue)); + + assertTrue(updated instanceof AssignFieldPathUpdate); + AssignFieldPathUpdate assignUpdate = (AssignFieldPathUpdate)updated; + assertEquals("structmap{foo}", assignUpdate.getOriginalFieldPath()); + assertEquals(newElemValue, assignUpdate.getFieldValue()); + } + + @Test + public void nested_struct_fieldpath_update_is_not_converted_to_regular_field_value_update() { + FieldPathFixture f = new FieldPathFixture(); + + StringFieldValue newTitleValue = new StringFieldValue("iron moose 4, moose with a vengeance"); + DocumentUpdate update = f.executeWithUpdate("structfield", new AssignFieldPathUpdate(f.type, "structfield.title", newTitleValue)); + + assertEquals(1, update.getFieldPathUpdates().size()); + assertEquals(0, update.getFieldUpdates().size()); + assertTrue(update.getFieldPathUpdates().get(0) instanceof AssignFieldPathUpdate); + AssignFieldPathUpdate assignUpdate = (AssignFieldPathUpdate)update.getFieldPathUpdates().get(0); + assertEquals("structfield.title", assignUpdate.getOriginalFieldPath()); + assertEquals(newTitleValue, assignUpdate.getFieldValue()); + } + private static FieldValue processDocument(FieldValue fieldValue) { DocumentType docType = new DocumentType("myDocumentType"); docType.addField("myField", fieldValue.getDataType()); @@ -184,11 +264,15 @@ public class DocumentScriptTestCase { return update.getFieldUpdate("myField").getValueUpdate(0); } + private static DocumentScript newScript(DocumentType docType, String fieldName) { + return new DocumentScript(docType.getName(), Collections.singletonList(fieldName), + new StatementExpression(new InputExpression(fieldName), + new IndexExpression(fieldName))); + } + private static DocumentScript newScript(DocumentType docType) { String fieldName = docType.getFields().iterator().next().getName(); - return new DocumentScript(docType.getName(), Arrays.asList(fieldName), - new StatementExpression(new InputExpression(fieldName), - new IndexExpression(fieldName))); + return newScript(docType, fieldName); } private static StringFieldValue newString(String... spanTrees) { @@ -210,6 +294,7 @@ public class DocumentScriptTestCase { DocumentType type = new DocumentType("documentType"); type.addField("documentField", DataType.STRING); type.addField("extraField", DataType.STRING); + return type; } diff --git a/document/src/main/java/com/yahoo/document/datatypes/Array.java b/document/src/main/java/com/yahoo/document/datatypes/Array.java index e37a32f28f4..01326bcea62 100644 --- a/document/src/main/java/com/yahoo/document/datatypes/Array.java +++ b/document/src/main/java/com/yahoo/document/datatypes/Array.java @@ -290,7 +290,8 @@ public final class Array<T extends FieldValue> extends CollectionFieldValue<T> i if (pos < fieldPath.size()) { switch (fieldPath.get(pos).getType()) { case ARRAY_INDEX: - return iterateSubset(fieldPath.get(pos).getLookupIndex(), fieldPath.get(pos).getLookupIndex(), fieldPath, null, pos + 1, handler); + final int elemIndex = fieldPath.get(pos).getLookupIndex(); + return iterateSubset(elemIndex, elemIndex, fieldPath, null, pos + 1, handler); case VARIABLE: { FieldPathIteratorHandler.IndexValue val = handler.getVariables().get(fieldPath.get(pos).getVariableName()); if (val != null) { diff --git a/fat-model-dependencies/pom.xml b/fat-model-dependencies/pom.xml index 1415ca6e5aa..0011d108b98 100644 --- a/fat-model-dependencies/pom.xml +++ b/fat-model-dependencies/pom.xml @@ -16,13 +16,6 @@ <groupId>com.yahoo.vespa</groupId> <artifactId>config-model</artifactId> <version>${project.version}</version> - <exclusions> - <exclusion> - <!-- Large, and installed separately as part of Vespa --> - <groupId>org.tensorflow</groupId> - <artifactId>libtensorflow_jni</artifactId> - </exclusion> - </exclusions> </dependency> <dependency> <groupId>com.yahoo.vespa</groupId> diff --git a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/FieldPathUpdateHelper.java b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/FieldPathUpdateHelper.java index 171c6a8eb9a..5c170fe147e 100644 --- a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/FieldPathUpdateHelper.java +++ b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/FieldPathUpdateHelper.java @@ -20,19 +20,10 @@ public abstract class FieldPathUpdateHelper { if (!(update instanceof AssignFieldPathUpdate)) { return false; } - for (FieldPathEntry entry : update.getFieldPath()) { - switch (entry.getType()) { - case STRUCT_FIELD: - case MAP_ALL_KEYS: - case MAP_ALL_VALUES: - continue; - case ARRAY_INDEX: - case MAP_KEY: - case VARIABLE: - return false; - } - } - return true; + // Only consider field path updates that touch a top-level field as 'complete', + // as these may be converted to regular field value updates. + return ((update.getFieldPath().size() == 1) + && update.getFieldPath().get(0).getType() == FieldPathEntry.Type.STRUCT_FIELD); } public static void applyUpdate(FieldPathUpdate update, Document doc) { diff --git a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/IdentityFieldPathUpdateAdapter.java b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/IdentityFieldPathUpdateAdapter.java new file mode 100644 index 00000000000..42c9bd8c10c --- /dev/null +++ b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/IdentityFieldPathUpdateAdapter.java @@ -0,0 +1,68 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.indexinglanguage; + +import com.yahoo.document.DataType; +import com.yahoo.document.Document; +import com.yahoo.document.DocumentUpdate; +import com.yahoo.document.FieldPath; +import com.yahoo.document.datatypes.FieldValue; +import com.yahoo.document.fieldpathupdate.FieldPathUpdate; +import com.yahoo.vespa.indexinglanguage.expressions.Expression; +import com.yahoo.vespa.indexinglanguage.expressions.FieldValueAdapter; + +/** + * No-op update adapter which simply passes through the input update unchanged. + * I.e. getOutput() will return a DocumentUpdate containing only the FieldPathUpdate + * the IdentityFieldPathUpdateAdapter was created with. All other applicable calls are + * forwarded to the provided DocumentAdapter instance. + * + * This removes the need for a potentially lossy round-trip of update -> synthetic document -> update. + */ +public class IdentityFieldPathUpdateAdapter implements UpdateAdapter { + + private final FieldPathUpdate update; + private final DocumentAdapter fwdAdapter; + + public IdentityFieldPathUpdateAdapter(FieldPathUpdate update, DocumentAdapter fwdAdapter) { + this.update = update; + this.fwdAdapter = fwdAdapter; + } + + @Override + public DocumentUpdate getOutput() { + Document doc = fwdAdapter.getFullOutput(); + DocumentUpdate upd = new DocumentUpdate(doc.getDataType(), doc.getId()); + upd.addFieldPathUpdate(update); + return upd; + } + + @Override + public Expression getExpression(Expression expression) { + return expression; + } + + @Override + public FieldValue getInputValue(String fieldName) { + return fwdAdapter.getInputValue(fieldName); + } + + @Override + public FieldValue getInputValue(FieldPath fieldPath) { + return fwdAdapter.getInputValue(fieldPath); + } + + @Override + public FieldValueAdapter setOutputValue(Expression exp, String fieldName, FieldValue fieldValue) { + return fwdAdapter.setOutputValue(exp, fieldName, fieldValue); + } + + @Override + public DataType getInputType(Expression exp, String fieldName) { + return fwdAdapter.getInputType(exp, fieldName); + } + + @Override + public void tryOutputType(Expression exp, String fieldName, DataType valueType) { + fwdAdapter.tryOutputType(exp, fieldName, valueType); + } +} diff --git a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/SimpleAdapterFactory.java b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/SimpleAdapterFactory.java index 2ad09dfbdc4..509bdcaa32d 100644 --- a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/SimpleAdapterFactory.java +++ b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/SimpleAdapterFactory.java @@ -49,10 +49,12 @@ public class SimpleAdapterFactory implements AdapterFactory { Document complete = new Document(docType, upd.getId()); for (FieldPathUpdate fieldUpd : upd) { if (FieldPathUpdateHelper.isComplete(fieldUpd)) { + // A 'complete' field path update is basically a regular top-level field update + // in wolf's clothing. Convert it to a regular field update to be friendlier + // towards the search core backend. FieldPathUpdateHelper.applyUpdate(fieldUpd, complete); } else { - Document partial = FieldPathUpdateHelper.newPartialDocument(docId, fieldUpd); - ret.add(new FieldPathUpdateAdapter(newDocumentAdapter(partial, true), fieldUpd)); + ret.add(new IdentityFieldPathUpdateAdapter(fieldUpd, newDocumentAdapter(complete, true))); } } for (FieldUpdate fieldUpd : upd.getFieldUpdates()) { diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/StorageMaintainer.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/StorageMaintainer.java index 7f2d1f1eff7..a7bf22591d4 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/StorageMaintainer.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/StorageMaintainer.java @@ -276,8 +276,8 @@ public class StorageMaintainer { */ public void handleCoreDumpsForContainer(ContainerName containerName, NodeSpec node, boolean force) { // Sample number of coredumps on the host - try { - numberOfCoredumpsOnHost.sample(Files.list(environment.pathInNodeAdminToDoneCoredumps()).count()); + try (Stream<Path> files = Files.list(environment.pathInNodeAdminToDoneCoredumps())) { + numberOfCoredumpsOnHost.sample(files.count()); } catch (IOException e) { // Ignore for now - this is either test or a misconfiguration } diff --git a/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/maintainer/CoredumpHandler.java b/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/maintainer/CoredumpHandler.java index 99dfdb48334..63c74c17dd5 100644 --- a/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/maintainer/CoredumpHandler.java +++ b/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/maintainer/CoredumpHandler.java @@ -72,7 +72,7 @@ class CoredumpHandler { FileHelper.deleteDirectories(doneCoredumpsPath, Duration.ofDays(10), Optional.empty()); } - private void handleNewCoredumps() throws IOException { + private void handleNewCoredumps() { Path processingCoredumps = enqueueCoredumps(); processAndReportCoredumps(processingCoredumps); } @@ -82,12 +82,12 @@ class CoredumpHandler { * Moves a coredump to a new directory under the processing/ directory. Limit to only processing * one coredump at the time, starting with the oldest. */ - Path enqueueCoredumps() throws IOException { + Path enqueueCoredumps() { Path processingCoredumpsPath = coredumpsPath.resolve(PROCESSING_DIRECTORY_NAME); processingCoredumpsPath.toFile().mkdirs(); - if (Files.list(processingCoredumpsPath).count() > 0) return processingCoredumpsPath; + if (!FileHelper.listContentsOfDirectory(processingCoredumpsPath).isEmpty()) return processingCoredumpsPath; - Files.list(coredumpsPath) + FileHelper.listContentsOfDirectory(coredumpsPath).stream() .filter(path -> path.toFile().isFile() && ! path.getFileName().toString().startsWith(".")) .min((Comparator.comparingLong(o -> o.toFile().lastModified()))) .ifPresent(coredumpPath -> { @@ -101,10 +101,10 @@ class CoredumpHandler { return processingCoredumpsPath; } - void processAndReportCoredumps(Path processingCoredumpsPath) throws IOException { + void processAndReportCoredumps(Path processingCoredumpsPath) { doneCoredumpsPath.toFile().mkdirs(); - Files.list(processingCoredumpsPath) + FileHelper.listContentsOfDirectory(processingCoredumpsPath).stream() .filter(path -> path.toFile().isDirectory()) .forEach(coredumpDirectory -> { try { @@ -130,7 +130,7 @@ class CoredumpHandler { String collectMetadata(Path coredumpDirectory, Map<String, Object> nodeAttributes) throws IOException { Path metadataPath = coredumpDirectory.resolve(METADATA_FILE_NAME); if (!Files.exists(metadataPath)) { - Path coredumpPath = Files.list(coredumpDirectory).findFirst() + Path coredumpPath = FileHelper.listContentsOfDirectory(coredumpDirectory).stream().findFirst() .orElseThrow(() -> new RuntimeException("No coredump file found in processing directory " + coredumpDirectory)); Map<String, Object> metadata = coreCollector.collect(coredumpPath, installStatePath); metadata.putAll(nodeAttributes); diff --git a/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/maintainer/FileHelper.java b/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/maintainer/FileHelper.java index ae872042853..7b93e7ad98d 100644 --- a/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/maintainer/FileHelper.java +++ b/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/maintainer/FileHelper.java @@ -2,6 +2,7 @@ package com.yahoo.vespa.hosted.node.maintainer; import java.io.IOException; +import java.io.UncheckedIOException; import java.nio.file.Files; import java.nio.file.LinkOption; import java.nio.file.NoSuchFileException; @@ -63,7 +64,7 @@ public class FileHelper { throw new IllegalArgumentException("Number of files to keep must be a positive number"); } - List<Path> pathsInDeleteDir = Files.list(basePath) + List<Path> pathsInDeleteDir = listContentsOfDirectory(basePath).stream() .filter(Files::isRegularFile) .sorted(Comparator.comparing(FileHelper::getLastModifiedTime)) .skip(nMostRecentToKeep) @@ -153,13 +154,16 @@ public class FileHelper { return pattern == null || pattern.matcher(path.getFileName().toString()).find(); } - static List<Path> listContentsOfDirectory(Path basePath) { + /** + * @return list all files in a directory, returns empty list if directory does not exist + */ + public static List<Path> listContentsOfDirectory(Path basePath) { try (Stream<Path> directoryStream = Files.list(basePath)) { return directoryStream.collect(Collectors.toList()); } catch (NoSuchFileException ignored) { return Collections.emptyList(); } catch (IOException e) { - throw new RuntimeException("Failed to list contents of directory " + basePath.toAbsolutePath(), e); + throw new UncheckedIOException("Failed to list contents of directory " + basePath.toAbsolutePath(), e); } } @@ -167,7 +171,7 @@ public class FileHelper { try { return Files.getLastModifiedTime(path, LinkOption.NOFOLLOW_LINKS); } catch (IOException e) { - throw new RuntimeException("Failed to get last modified time of " + path.toAbsolutePath(), e); + throw new UncheckedIOException("Failed to get last modified time of " + path.toAbsolutePath(), e); } } } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/GroupPreparer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/GroupPreparer.java index 93704d244b5..d31b4438a38 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/GroupPreparer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/GroupPreparer.java @@ -9,7 +9,6 @@ import com.yahoo.transaction.Mutex; import com.yahoo.vespa.hosted.provision.Node; import com.yahoo.vespa.hosted.provision.NodeRepository; -import java.time.Clock; import java.util.List; /** @@ -67,6 +66,7 @@ public class GroupPreparer { allocation.offer(prioritizer.prioritize()); if (! allocation.fullfilled()) throw new OutOfCapacityException("Could not satisfy " + requestedNodes + " for " + cluster + + " in " + application.toShortString() + outOfCapacityDetails(allocation)); // Extend reservation for already reserved nodes diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DockerProvisioningTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DockerProvisioningTest.java index e1b1d74c6d0..2cabee98c0d 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DockerProvisioningTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DockerProvisioningTest.java @@ -160,13 +160,13 @@ public class DockerProvisioningTest { assertEquals(setOf("host1", "host2"), hostsOf(tester.getNodes(application1, Node.State.active))); try { - ApplicationId application2 = tester.makeApplicationId(); + ApplicationId application2 = ApplicationId.from("tenant1", "app1", "default"); prepareAndActivate(application2, 3, false, tester); fail("Expected allocation failure"); } catch (Exception e) { assertEquals("No room for 3 nodes as 2 of 4 hosts are exclusive", - "Could not satisfy request for 3 nodes of flavor 'dockerSmall' for container cluster 'myContainer' group 0 6.39: Not enough nodes available due to host exclusivity constraints.", + "Could not satisfy request for 3 nodes of flavor 'dockerSmall' for container cluster 'myContainer' group 0 6.39 in tenant1.app1: Not enough nodes available due to host exclusivity constraints.", e.getMessage()); } |