diff options
53 files changed, 357 insertions, 247 deletions
diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index e1cb205e5a5..8e95abfe059 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -12,9 +12,8 @@ In addition to the [public Travis build](https://travis-ci.org/vespa-engine/vesp we have a large acceptance and performance test suite which is also run continuously. We plan to add this to the open source code base later. -All pull requests are reviewed by a member of the -[Vespa committers](https://github.com/orgs/vespa-engine/teams/vespa/members) team, regardless -of who made it. If you want to become a committer, making some quality contributions is the way to start. +All pull requests are reviewed by a member of the Vespa Committers team. +If you want to become a committer, making some quality contributions is the way to start. ## Versioning Vespa uses semantic versioning - see diff --git a/build_settings.cmake b/build_settings.cmake index 578484fc0a7..425a2eddda7 100644 --- a/build_settings.cmake +++ b/build_settings.cmake @@ -23,14 +23,8 @@ set(C_WARN_OPTS "-Winline -Wuninitialized -Werror -Wall -W -Wchar-subscripts -Wc # Note: this is not a union of C_WARN_OPTS, since CMAKE_CXX_FLAGS already includes CMAKE_C_FLAGS, which in turn includes C_WARN_OPTS transitively set(CXX_SPECIFIC_WARN_OPTS "-Wsuggest-override -Wnon-virtual-dtor") -# Select C++ ABI -if(DEFINED VESPA_CXX_ABI_FLAGS) -else() - set (VESPA_CXX_ABI_FLAGS "-D_GLIBCXX_USE_CXX11_ABI=0") -endif() - # C and C++ compiler flags -set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -g -O3 ${C_WARN_OPTS} -fPIC ${VESPA_CXX_ABI_FLAGS} -DBOOST_DISABLE_ASSERTS -march=westmere -mtune=intel ${EXTRA_C_FLAGS}") +set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -g -O3 -fno-omit-frame-pointer ${C_WARN_OPTS} -fPIC ${VESPA_CXX_ABI_FLAGS} -DBOOST_DISABLE_ASSERTS ${VESPA_CPU_ARCH_FLAGS} -mtune=intel ${EXTRA_C_FLAGS}") set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${CMAKE_C_FLAGS} ${CXX_SPECIFIC_WARN_OPTS} -std=c++1z -fvisibility-inlines-hidden -fdiagnostics-color=auto ${EXTRA_CXX_FLAGS}") # Linker flags diff --git a/config/src/main/java/com/yahoo/vespa/config/LZ4PayloadCompressor.java b/config/src/main/java/com/yahoo/vespa/config/LZ4PayloadCompressor.java index 8714aa0e4f6..4e65ec130ac 100644 --- a/config/src/main/java/com/yahoo/vespa/config/LZ4PayloadCompressor.java +++ b/config/src/main/java/com/yahoo/vespa/config/LZ4PayloadCompressor.java @@ -9,9 +9,9 @@ import net.jpountz.lz4.LZ4Factory; * Wrapper for LZ4 compression that selects compression level based on properties. * * @author lulf - * @since 5.19 */ public class LZ4PayloadCompressor { + private static final LZ4Factory lz4Factory = LZ4Factory.safeInstance(); private static final String VESPA_CONFIG_PROTOCOL_COMPRESSION_LEVEL = "VESPA_CONFIG_PROTOCOL_COMPRESSION_LEVEL"; private static final int compressionLevel = getCompressionLevel(); @@ -36,4 +36,5 @@ public class LZ4PayloadCompressor { private LZ4Compressor getCompressor() { return (compressionLevel < 7) ? lz4Factory.fastCompressor() : lz4Factory.highCompressor(); } + } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/modelfactory/ModelsBuilder.java b/configserver/src/main/java/com/yahoo/vespa/config/server/modelfactory/ModelsBuilder.java index 6bd21e9cfb6..af4d998c347 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/modelfactory/ModelsBuilder.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/modelfactory/ModelsBuilder.java @@ -7,6 +7,7 @@ import com.yahoo.config.model.api.HostProvisioner; import com.yahoo.config.model.api.ModelContext; import com.yahoo.config.model.api.ModelFactory; import com.yahoo.config.provision.ApplicationId; +import com.yahoo.config.provision.ApplicationLockException; import com.yahoo.config.provision.OutOfCapacityException; import com.yahoo.config.provision.AllocatedHosts; import com.yahoo.config.provision.Rotation; @@ -87,7 +88,7 @@ public abstract class ModelsBuilder<MODELRESULT extends ModelResult> { if (allApplicationModels.size() > 0 && allApplicationModels.get(0).getModel().skipOldConfigModels(now)) break; } - catch (OutOfCapacityException e) { + catch (OutOfCapacityException | ApplicationLockException e) { // Don't wrap this exception, and don't try to load other model versions as this is (most likely) // caused by the state of the system, not the model version/application combination throw e; diff --git a/configserver/src/main/sh/start-configserver b/configserver/src/main/sh/start-configserver index 3b168b14c41..7d3f305a107 100755 --- a/configserver/src/main/sh/start-configserver +++ b/configserver/src/main/sh/start-configserver @@ -136,6 +136,7 @@ fixddir $bundlecachedir vespa-run-as-vespa-user vespa-runserver -s configserver -r 30 -p $pidfile -- \ java \ -Xms128m -Xmx2048m \ + -XX:+PreserveFramePointer \ -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=${VESPA_HOME}/var/crash \ -XX:OnOutOfMemoryError='kill -9 %p' \ $jvmargs \ diff --git a/container-dependencies-enforcer/pom.xml b/container-dependencies-enforcer/pom.xml index 4720135b609..2c608280fbe 100644 --- a/container-dependencies-enforcer/pom.xml +++ b/container-dependencies-enforcer/pom.xml @@ -5,13 +5,24 @@ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> <modelVersion>4.0.0</modelVersion> - <groupId>com.yahoo.vespa</groupId> + <parent> + <groupId>com.yahoo.vespa</groupId> + <artifactId>parent</artifactId> + <version>6-SNAPSHOT</version> + </parent> + <artifactId>container-dependencies-enforcer</artifactId> <version>6-SNAPSHOT</version> <packaging>pom</packaging> <dependencies> <dependency> + <groupId>junit</groupId> + <artifactId>junit</artifactId> + <version>4.11</version> + <scope>test</scope> + </dependency> + <dependency> <groupId>com.yahoo.vespa</groupId> <artifactId>application</artifactId> <version>${project.version}</version> @@ -57,64 +68,67 @@ </excludes> <includes> <include>com.yahoo.vespa</include> - <include>aopalliance:aopalliance:1.0:jar:provided</include> - <include>com.fasterxml.jackson.core:jackson-annotations:2.8.3:jar:provided</include> - <include>com.fasterxml.jackson.core:jackson-core:2.8.3:jar:provided</include> - <include>com.fasterxml.jackson.core:jackson-databind:2.8.3:jar:provided</include> - <include>com.fasterxml.jackson.datatype:jackson-datatype-jdk8:2.8.3:jar:provided</include> - <include>com.fasterxml.jackson.jaxrs:jackson-jaxrs-base:2.5.4:jar:provided</include> - <include>com.fasterxml.jackson.jaxrs:jackson-jaxrs-json-provider:2.5.4:jar:provided</include> - <include>com.fasterxml.jackson.module:jackson-module-jaxb-annotations:2.5.4:jar:provided</include> - <include>com.google.code.findbugs:annotations:1.3.9:jar:provided</include> - <include>com.google.code.findbugs:jsr305:1.3.9:jar:provided</include> - <include>com.google.guava:guava:18.0:jar:provided</include> - <include>com.google.inject.extensions:guice-assistedinject:3.0:jar:provided</include> - <include>com.google.inject.extensions:guice-multibindings:3.0:jar:provided</include> - <include>com.google.inject:guice:3.0:jar:provided:no_aop</include> - <include>commons-codec:commons-codec:1.4:jar:provided</include> - <include>commons-daemon:commons-daemon:1.0.3:jar:provided</include> - <include>commons-logging:commons-logging:1.1.1:jar:provided</include> - <include>javax.annotation:javax.annotation-api:1.2:jar:provided</include> - <include>javax.inject:javax.inject:1:jar:provided</include> - <include>javax.servlet:javax.servlet-api:3.1.0:jar:provided</include> - <include>javax.validation:validation-api:1.1.0.Final:jar:provided</include> - <include>javax.ws.rs:javax.ws.rs-api:2.0.1:jar:provided</include> - <include>net.jcip:jcip-annotations:1.0:jar:provided</include> - <include>net.jpountz.lz4:lz4:1.3.0:jar:provided</include> - <include>org.apache.felix:org.apache.felix.framework:4.2.1:jar:provided</include> - <include>org.apache.felix:org.apache.felix.log:1.0.1:jar:provided</include> - <include>org.apache.felix:org.apache.felix.main:4.2.1:jar:provided</include> - <include>org.apache.httpcomponents:httpclient:4.3.6:jar:provided</include> - <include>org.apache.httpcomponents:httpcore:4.3.3:jar:provided</include> - <include>org.eclipse.jetty:jetty-http:9.4.6.v20170531:jar:provided</include> - <include>org.eclipse.jetty:jetty-io:9.4.6.v20170531:jar:provided</include> - <include>org.eclipse.jetty:jetty-util:9.4.6.v20170531:jar:provided</include> - <include>org.glassfish.hk2.external:aopalliance-repackaged:2.5.0-b05:jar:provided</include> - <include>org.glassfish.hk2.external:javax.inject:2.5.0-b05:jar:provided</include> - <include>org.glassfish.hk2:hk2-api:2.5.0-b05:jar:provided</include> - <include>org.glassfish.hk2:hk2-locator:2.5.0-b05:jar:provided</include> - <include>org.glassfish.hk2:hk2-utils:2.5.0-b05:jar:provided</include> - <include>org.glassfish.hk2:osgi-resource-locator:1.0.1:jar:provided</include> - <include>org.glassfish.jersey.bundles.repackaged:jersey-guava:2.23.2:jar:provided</include> - <include>org.glassfish.jersey.containers:jersey-container-servlet-core:2.23.2:jar:provided</include> - <include>org.glassfish.jersey.containers:jersey-container-servlet:2.23.2:jar:provided</include> - <include>org.glassfish.jersey.core:jersey-client:2.23.2:jar:provided</include> - <include>org.glassfish.jersey.core:jersey-common:2.23.2:jar:provided</include> - <include>org.glassfish.jersey.core:jersey-server:2.23.2:jar:provided</include> - <include>org.glassfish.jersey.ext:jersey-entity-filtering:2.23.2:jar:provided</include> - <include>org.glassfish.jersey.ext:jersey-proxy-client:2.23.2:jar:provided</include> - <include>org.glassfish.jersey.media:jersey-media-jaxb:2.23.2:jar:provided</include> - <include>org.glassfish.jersey.media:jersey-media-json-jackson:2.23.2:jar:provided</include> - <include>org.glassfish.jersey.media:jersey-media-multipart:2.23.2:jar:provided</include> - <include>org.javassist:javassist:3.20.0-GA:jar:provided</include> - <include>org.json:json:20090211:jar:provided</include> - <include>org.jvnet.mimepull:mimepull:1.9.6:jar:provided</include> - <include>org.scala-lang.modules:scala-parser-combinators_2.11:1.0.1:jar:provided</include> - <include>org.slf4j:jcl-over-slf4j:1.7.5:jar:provided</include> - <include>org.slf4j:log4j-over-slf4j:1.7.5:jar:provided</include> - <include>org.slf4j:slf4j-api:1.7.5:jar:provided</include> - <include>org.slf4j:slf4j-jdk14:1.7.5:jar:provided</include> - <include>xml-apis:xml-apis:1.4.01:jar:provided</include> + <include>aopalliance:aopalliance:[1.0]:jar:provided</include> + <include>com.fasterxml.jackson.core:jackson-annotations:[${jackson2.version}]:jar:provided</include> + <include>com.fasterxml.jackson.core:jackson-core:[${jackson2.version}]:jar:provided</include> + <include>com.fasterxml.jackson.core:jackson-databind:[${jackson2.version}]:jar:provided</include> + <include>com.fasterxml.jackson.datatype:jackson-datatype-jdk8:[${jackson2.version}]:jar:provided</include> + + <!-- Use version range for jax deps, because jersey and junit affect the versions. --> + <include>com.fasterxml.jackson.jaxrs:jackson-jaxrs-base:[2.5.4, ${jackson2.version}]:jar:provided</include> + <include>com.fasterxml.jackson.jaxrs:jackson-jaxrs-json-provider:[2.5.4, ${jackson2.version}]:jar:provided</include> + <include>com.fasterxml.jackson.module:jackson-module-jaxb-annotations:[2.5.4, ${jackson2.version}]:jar:provided</include> + + <include>com.google.code.findbugs:annotations:[1.3.9]:jar:provided</include> + <include>com.google.code.findbugs:jsr305:[1.3.9]:jar:provided</include> + <include>com.google.guava:guava:[18.0]:jar:provided</include> + <include>com.google.inject.extensions:guice-assistedinject:[3.0]:jar:provided</include> + <include>com.google.inject.extensions:guice-multibindings:[3.0]:jar:provided</include> + <include>com.google.inject:guice:[3.0]:jar:provided:no_aop</include> + <include>commons-codec:commons-codec:[1.4]:jar:provided</include> + <include>commons-daemon:commons-daemon:[1.0.3]:jar:provided</include> + <include>commons-logging:commons-logging:[1.1.1]:jar:provided</include> + <include>javax.annotation:javax.annotation-api:[1.2]:jar:provided</include> + <include>javax.inject:javax.inject:[1]:jar:provided</include> + <include>javax.servlet:javax.servlet-api:[3.1.0]:jar:provided</include> + <include>javax.validation:validation-api:[1.1.0.Final]:jar:provided</include> + <include>javax.ws.rs:javax.ws.rs-api:[${javax.ws.rs-api.version}]:jar:provided</include> + <include>net.jcip:jcip-annotations:[1.0]:jar:provided</include> + <include>net.jpountz.lz4:lz4:[1.3.0]:jar:provided</include> + <include>org.apache.felix:org.apache.felix.framework:[4.2.1]:jar:provided</include> + <include>org.apache.felix:org.apache.felix.log:[1.0.1]:jar:provided</include> + <include>org.apache.felix:org.apache.felix.main:[4.2.1]:jar:provided</include> + <include>org.apache.httpcomponents:httpclient:[4.3.6]:jar:provided</include> + <include>org.apache.httpcomponents:httpcore:[4.3.3]:jar:provided</include> + <include>org.eclipse.jetty:jetty-http:[${jetty.version}]:jar:provided</include> + <include>org.eclipse.jetty:jetty-io:[${jetty.version}]:jar:provided</include> + <include>org.eclipse.jetty:jetty-util:[${jetty.version}]:jar:provided</include> + <include>org.glassfish.hk2.external:aopalliance-repackaged:[2.5.0-b05]:jar:provided</include> + <include>org.glassfish.hk2.external:javax.inject:[2.5.0-b05]:jar:provided</include> + <include>org.glassfish.hk2:hk2-api:[2.5.0-b05]:jar:provided</include> + <include>org.glassfish.hk2:hk2-locator:[2.5.0-b05]:jar:provided</include> + <include>org.glassfish.hk2:hk2-utils:[2.5.0-b05]:jar:provided</include> + <include>org.glassfish.hk2:osgi-resource-locator:[1.0.1]:jar:provided</include> + <include>org.glassfish.jersey.bundles.repackaged:jersey-guava:[${jersey2.version}]:jar:provided</include> + <include>org.glassfish.jersey.containers:jersey-container-servlet-core:[${jersey2.version}]:jar:provided</include> + <include>org.glassfish.jersey.containers:jersey-container-servlet:[${jersey2.version}]:jar:provided</include> + <include>org.glassfish.jersey.core:jersey-client:[${jersey2.version}]:jar:provided</include> + <include>org.glassfish.jersey.core:jersey-common:[${jersey2.version}]:jar:provided</include> + <include>org.glassfish.jersey.core:jersey-server:[${jersey2.version}]:jar:provided</include> + <include>org.glassfish.jersey.ext:jersey-entity-filtering:[${jersey2.version}]:jar:provided</include> + <include>org.glassfish.jersey.ext:jersey-proxy-client:[${jersey2.version}]:jar:provided</include> + <include>org.glassfish.jersey.media:jersey-media-jaxb:[${jersey2.version}]:jar:provided</include> + <include>org.glassfish.jersey.media:jersey-media-json-jackson:[${jersey2.version}]:jar:provided</include> + <include>org.glassfish.jersey.media:jersey-media-multipart:[${jersey2.version}]:jar:provided</include> + <include>org.javassist:javassist:[3.20.0-GA]:jar:provided</include> + <include>org.json:json:[20090211]:jar:provided</include> + <include>org.jvnet.mimepull:mimepull:[1.9.6]:jar:provided</include> + <include>org.scala-lang.modules:scala-parser-combinators_2.11:[1.0.1]:jar:provided</include> + <include>org.slf4j:jcl-over-slf4j:[1.7.5]:jar:provided</include> + <include>org.slf4j:log4j-over-slf4j:[1.7.5]:jar:provided</include> + <include>org.slf4j:slf4j-api:[1.7.5]:jar:provided</include> + <include>org.slf4j:slf4j-jdk14:[1.7.5]:jar:provided</include> + <include>xml-apis:xml-apis:[1.4.01]:jar:provided</include> </includes> </bannedDependencies> </rules> diff --git a/container-disc/src/main/sh/vespa-start-container-daemon.sh b/container-disc/src/main/sh/vespa-start-container-daemon.sh index 33f61c3660c..6f0c97f6177 100755 --- a/container-disc/src/main/sh/vespa-start-container-daemon.sh +++ b/container-disc/src/main/sh/vespa-start-container-daemon.sh @@ -186,6 +186,7 @@ exec_jsvc () { configure_preload exec $numactlcmd $envcmd $jsvc_binary_name \ -Dconfig.id="${VESPA_CONFIG_ID}" \ + -XX:+PreserveFramePointer \ ${jsvc_opts} \ ${memory_options} \ ${jvm_gcopts} \ @@ -258,6 +259,7 @@ maybe_use_jsvc exec $numactlcmd $envcmd java \ -Dconfig.id="${VESPA_CONFIG_ID}" \ + -XX:+PreserveFramePointer \ ${memory_options} \ ${jvm_gcopts} \ -XX:MaxJavaStackTraceDepth=-1 \ diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java index 43dd2b6e226..ccfcc0e4b0d 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java @@ -278,7 +278,7 @@ public class ApplicationController { Version version; if (options.deployCurrentVersion) version = application.currentVersion(controller, zone); - else if (canDeployDirectlyTo(zone, options)) + else if (application.deploymentJobs().isSelfTriggering()) // legacy mode: let the client decide version = options.vespaVersion.map(Version::new).orElse(controller.systemVersion()); else if ( ! application.deploying().isPresent() && ! zone.environment().isManuallyDeployed()) return unexpectedDeployment(applicationId, zone, applicationPackage); @@ -286,8 +286,10 @@ public class ApplicationController { version = application.currentDeployVersion(controller, zone); // Ensure that the deploying change is tested - if (! canDeployDirectlyTo(zone, options) && - ! application.deploymentJobs().isDeployableTo(zone.environment(), application.deploying())) + // FIXME: For now only for non-self-triggering applications - VESPA-8418 + if ( ! application.deploymentJobs().isSelfTriggering() && + ! zone.environment().isManuallyDeployed() && + ! application.deploymentJobs().isDeployableTo(zone.environment(), application.deploying())) throw new IllegalArgumentException("Rejecting deployment of " + application + " to " + zone + " as pending " + application.deploying().get() + " is untested"); @@ -303,7 +305,7 @@ public class ApplicationController { application = application.withProjectId(options.screwdriverBuildJob.get().screwdriverId.value()); if (application.deploying().isPresent() && application.deploying().get() instanceof Change.ApplicationChange) application = application.withDeploying(Optional.of(Change.ApplicationChange.of(revision))); - if ( ! triggeredWith(revision, application, jobType) && !canDeployDirectlyTo(zone, options) && jobType != null) { + if ( ! triggeredWith(revision, application, jobType) && !zone.environment().isManuallyDeployed() && jobType != null) { // Triggering information is used to store which changes were made or attempted // - For self-triggered applications we don't have any trigger information, so we add it here. // - For all applications, we don't have complete control over which revision is actually built, @@ -588,11 +590,6 @@ public class ApplicationController { return curator.lock(application, Duration.ofMinutes(10)); } - /** Returns whether a direct deployment to given zone is allowed */ - private static boolean canDeployDirectlyTo(Zone zone, DeployOptions options) { - return !options.screwdriverBuildJob.isPresent() || zone.environment().isManuallyDeployed(); - } - private static final class ApplicationRotation { private final ImmutableSet<String> cnames; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTrigger.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTrigger.java index 8826ee5bf67..d2af5fc50cb 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTrigger.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTrigger.java @@ -285,11 +285,6 @@ public class DeploymentTrigger { return application; } - // Ignore applications that are not associated with a project - if (!application.deploymentJobs().projectId().isPresent()) { - return application; - } - log.info(String.format("Triggering %s for %s, %s: %s", jobType, application, application.deploying().map(d -> "deploying " + d).orElse("restarted deployment"), cause)); diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ConfigServerClientMock.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ConfigServerClientMock.java index 13525969358..6018c99206e 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ConfigServerClientMock.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ConfigServerClientMock.java @@ -51,17 +51,8 @@ public class ConfigServerClientMock extends AbstractComponent implements ConfigS /** The exception to throw on the next prepare run, or null to continue normally */ private RuntimeException prepareException = null; - private Optional<Version> lastPrepareVersion = Optional.empty(); - - /** The version given in the previous prepare call, or empty if no call has been made */ - public Optional<Version> lastPrepareVersion() { - return lastPrepareVersion; - } - - /** Return map of applications that may have been activated */ - public Map<ApplicationId, Boolean> activated() { - return Collections.unmodifiableMap(applicationActivated); - } + /** The version given in the previous prepare call, or null if no call has been made */ + public Optional<Version> lastPrepareVersion = null; @Override public PreparedApplication prepare(DeploymentId deployment, DeployOptions deployOptions, Set<String> rotationCnames, Set<Rotation> rotations, byte[] content) { @@ -210,5 +201,4 @@ public class ConfigServerClientMock extends AbstractComponent implements ConfigS ? endpoints.get(endpoint) : result; } - } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java index 61cc84a07ba..91fca0d37d1 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java @@ -17,10 +17,16 @@ import com.yahoo.vespa.curator.Lock; import com.yahoo.vespa.hosted.controller.api.Tenant; import com.yahoo.vespa.hosted.controller.api.application.v4.model.DeployOptions; import com.yahoo.vespa.hosted.controller.api.application.v4.model.EndpointStatus; +import com.yahoo.vespa.hosted.controller.api.application.v4.model.GitRevision; +import com.yahoo.vespa.hosted.controller.api.application.v4.model.ScrewdriverBuildJob; import com.yahoo.vespa.hosted.controller.api.identifiers.AthensDomain; import com.yahoo.vespa.hosted.controller.api.identifiers.DeploymentId; +import com.yahoo.vespa.hosted.controller.api.identifiers.GitBranch; +import com.yahoo.vespa.hosted.controller.api.identifiers.GitCommit; +import com.yahoo.vespa.hosted.controller.api.identifiers.GitRepository; import com.yahoo.vespa.hosted.controller.api.identifiers.Property; import com.yahoo.vespa.hosted.controller.api.identifiers.PropertyId; +import com.yahoo.vespa.hosted.controller.api.identifiers.ScrewdriverId; import com.yahoo.vespa.hosted.controller.api.identifiers.TenantId; import com.yahoo.vespa.hosted.controller.api.identifiers.UserGroup; import com.yahoo.vespa.hosted.controller.api.integration.BuildService.BuildJob; @@ -211,7 +217,7 @@ public class ControllerTest { app1 = applications.require(app1.id()); assertEquals("First deployment gets system version", systemVersion, app1.deployedVersion().get()); - assertEquals(systemVersion, tester.configServer().lastPrepareVersion().get()); + assertEquals(systemVersion, tester.configServer().lastPrepareVersion.get()); // Unexpected deployment tester.deploy(productionUsWest1, app1, applicationPackage); @@ -234,7 +240,7 @@ public class ControllerTest { app1 = applications.require(app1.id()); assertEquals("Application change preserves version", systemVersion, app1.deployedVersion().get()); - assertEquals(systemVersion, tester.configServer().lastPrepareVersion().get()); + assertEquals(systemVersion, tester.configServer().lastPrepareVersion.get()); // A deployment to the new region gets the same version applicationPackage = new ApplicationPackageBuilder() @@ -245,7 +251,7 @@ public class ControllerTest { tester.deployAndNotify(app1, applicationPackage, true, productionUsEast3); app1 = applications.require(app1.id()); assertEquals("Application change preserves version", systemVersion, app1.deployedVersion().get()); - assertEquals(systemVersion, tester.configServer().lastPrepareVersion().get()); + assertEquals(systemVersion, tester.configServer().lastPrepareVersion.get()); // Version upgrade changes system version Change.VersionChange change = new Change.VersionChange(newSystemVersion); @@ -257,7 +263,7 @@ public class ControllerTest { app1 = applications.require(app1.id()); assertEquals("Version upgrade changes version", newSystemVersion, app1.deployedVersion().get()); - assertEquals(newSystemVersion, tester.configServer().lastPrepareVersion().get()); + assertEquals(newSystemVersion, tester.configServer().lastPrepareVersion.get()); } /** Adds a new version, higher than the current system version, makes it the system version and returns it */ @@ -527,6 +533,44 @@ public class ControllerTest { } @Test + public void testLegacyDeployments() { + // Setup system + DeploymentTester tester = new DeploymentTester(); + ApplicationController applications = tester.controller().applications(); + ApplicationPackage applicationPackage = new ApplicationPackageBuilder() + .environment(Environment.prod) + .region("us-east-3") + .build(); + Version systemVersion = tester.controller().versionStatus().systemVersion().get().versionNumber(); + + Application app1 = tester.createApplication("application1", "tenant1", 1, 1L); + applications.store(app1.with(app1.deploymentJobs().asSelfTriggering(true)), applications.lock(app1.id())); + + // Scenario: App already on 6.0, Upgrade to 6.1 (systemversion) + Zone prodZone = new Zone(Environment.prod, RegionName.from("us-east-3")); + Zone stagingZone = new Zone(Environment.staging, RegionName.from("us-east-3")); + Version existingVersion = Version.fromString("6.0"); + + // Add deployment on existing version + legacyDeploy(tester.controller(), app1, applicationPackage, prodZone, Optional.of(existingVersion), false); + + // Add dev/perf deployment on old version to verify that this does not affect Initialize staging step. VESPA-8469 + Version devVersion = Version.fromString("5.0"); + legacyDeploy(tester.controller(), app1, applicationPackage, new Zone(Environment.dev, RegionName.from("us-east-1")), Optional.of(devVersion), false); + legacyDeploy(tester.controller(), app1, applicationPackage, new Zone(Environment.perf, RegionName.from("us-east-3")), Optional.of(devVersion), false); + + // Initialize staging on existing version + legacyDeploy(tester.controller(), app1, applicationPackage, stagingZone, Optional.of(systemVersion), true); + app1 = applications.require(app1.id()); + assertEquals(existingVersion, app1.currentDeployVersion(tester.controller(), stagingZone)); + + // Upgrade to the new version in staging + legacyDeploy(tester.controller(), app1, applicationPackage, stagingZone, Optional.of(systemVersion), false); + app1 = applications.require(app1.id()); + assertEquals(systemVersion, app1.currentDeployVersion(tester.controller(), stagingZone)); + } + + @Test public void testDeployUntestedChangeFails() { ControllerTester tester = new ControllerTester(); ApplicationController applications = tester.controller().applications();TenantId tenant = tester.createTenant("tenant1", "domain1", 11L); @@ -543,6 +587,16 @@ public class ControllerTest { } } + private void legacyDeploy(Controller controller, Application application, ApplicationPackage applicationPackage, Zone zone, Optional<Version> version, boolean deployCurrentVersion) { + ScrewdriverId app1ScrewdriverId = new ScrewdriverId(String.valueOf(application.deploymentJobs().projectId().get())); + GitRevision app1RevisionId = new GitRevision(new GitRepository("repo"), new GitBranch("master"), new GitCommit("commit1")); + controller.applications().deployApplication(application.id(), + zone, + applicationPackage, + new DeployOptions(Optional.of(new ScrewdriverBuildJob(app1ScrewdriverId, app1RevisionId)), version, false, deployCurrentVersion)); + + } + @Test public void testCleanupOfStaleDeploymentData() throws IOException { DeploymentTester tester = new DeploymentTester(); @@ -625,31 +679,4 @@ public class ControllerTest { assertEquals("fake-global-rotation-tenant1.app1", record.get().value()); } - @Test - public void testDeployWithoutProjectId() { - DeploymentTester tester = new DeploymentTester(); - tester.controllerTester().zoneRegistry().setSystem(SystemName.cd); - ApplicationPackage applicationPackage = new ApplicationPackageBuilder() - .environment(Environment.prod) - .region("cd-us-central-1") - .build(); - - // Create application - Application app = tester.createApplication("app1", "tenant1", 1, 2L); - - // Direct deploy is allowed when project ID is missing - Zone zone = new Zone(Environment.prod, RegionName.from("cd-us-central-1")); - // Same options as used in our integration tests - DeployOptions options = new DeployOptions(Optional.empty(), Optional.empty(), false, - false); - tester.controller().applications().deployApplication(app.id(), zone, applicationPackage, options); - - assertTrue("Application deployed and activated", - tester.controllerTester().configServer().activated().getOrDefault(app.id(), false)); - - assertTrue("No job status added", - tester.applications().require(app.id()).deploymentJobs().jobStatus().isEmpty()); - - } - } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/PolledBuildSystemTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/PolledBuildSystemTest.java index c869bd90924..779af370ff4 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/PolledBuildSystemTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/PolledBuildSystemTest.java @@ -2,9 +2,12 @@ package com.yahoo.vespa.hosted.controller.deployment; import com.yahoo.config.provision.ApplicationId; +import com.yahoo.config.provision.Environment; +import com.yahoo.vespa.curator.mock.MockCurator; +import com.yahoo.vespa.hosted.controller.ControllerTester; import com.yahoo.vespa.hosted.controller.api.integration.BuildService.BuildJob; -import com.yahoo.vespa.hosted.controller.application.ApplicationPackage; import com.yahoo.vespa.hosted.controller.application.DeploymentJobs.JobType; +import com.yahoo.vespa.hosted.controller.persistence.CuratorDb; import com.yahoo.vespa.hosted.controller.persistence.MockCuratorDb; import org.junit.Test; import org.junit.runner.RunWith; @@ -34,16 +37,15 @@ public class PolledBuildSystemTest { @Test public void throttle_capacity_constrained_jobs() { - DeploymentTester tester = new DeploymentTester(); + ControllerTester tester = new ControllerTester(); BuildSystem buildSystem = new PolledBuildSystem(tester.controller(), new MockCuratorDb()); - int fooProjectId = 1; - int barProjectId = 2; - ApplicationPackage applicationPackage = new ApplicationPackageBuilder() - .region("us-west-1") - .build(); - ApplicationId foo = tester.createAndDeploy("app1", fooProjectId, applicationPackage).id(); - ApplicationId bar = tester.createAndDeploy("app2", barProjectId, applicationPackage).id(); + long fooProjectId = 1; + long barProjectId = 2; + ApplicationId foo = tester.createAndDeploy("tenant1", "domain1", "app1", + Environment.prod, fooProjectId).id(); + ApplicationId bar = tester.createAndDeploy("tenant2", "domain2", "app2", + Environment.prod, barProjectId).id(); // Trigger jobs in capacity constrained environment buildSystem.addJob(foo, jobType, false); diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentExpirerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentExpirerTest.java index ef0b05f9bb2..ee4f3631b54 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentExpirerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentExpirerTest.java @@ -1,22 +1,16 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.maintenance; +import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.Environment; import com.yahoo.config.provision.RegionName; import com.yahoo.config.provision.Zone; -import com.yahoo.vespa.hosted.controller.Application; -import com.yahoo.vespa.hosted.controller.application.ApplicationPackage; -import com.yahoo.vespa.hosted.controller.application.Deployment; -import com.yahoo.vespa.hosted.controller.deployment.ApplicationPackageBuilder; -import com.yahoo.vespa.hosted.controller.deployment.DeploymentTester; +import com.yahoo.vespa.hosted.controller.ControllerTester; import com.yahoo.vespa.hosted.controller.persistence.MockCuratorDb; -import org.junit.Before; import org.junit.Test; import java.io.IOException; import java.time.Duration; -import java.util.List; -import java.util.stream.Collectors; import static org.junit.Assert.assertEquals; @@ -25,53 +19,28 @@ import static org.junit.Assert.assertEquals; */ public class DeploymentExpirerTest { - private DeploymentTester tester; - - @Before - public void before() { - tester = new DeploymentTester(); - } - @Test public void testDeploymentExpiry() throws IOException, InterruptedException { - tester.controllerTester().zoneRegistry().setDeploymentTimeToLive( - new Zone(Environment.dev, RegionName.from("us-east-1")), - Duration.ofDays(14) - ); + ControllerTester tester = new ControllerTester(); + tester.zoneRegistry().setDeploymentTimeToLive(new Zone(Environment.dev, RegionName.from("us-east-1")), Duration.ofDays(14)); DeploymentExpirer expirer = new DeploymentExpirer(tester.controller(), Duration.ofDays(10), tester.clock(), new JobControl(new MockCuratorDb())); - Application devApp = tester.createApplication("app1", "tenant1", 123L, 1L); - Application prodApp = tester.createApplication("app2", "tenant2", 456L, 2L); - - // Deploy dev - tester.controllerTester().deploy(devApp, tester.controllerTester().toZone(Environment.dev)); + ApplicationId devApp = tester.createAndDeploy("tenant1", "domain1", "app1", Environment.dev, 123).id(); + ApplicationId prodApp = tester.createAndDeploy("tenant2", "domain2", "app2", Environment.prod, 456).id(); - // Deploy prod - ApplicationPackage prodAppPackage = new ApplicationPackageBuilder() - .region("us-west-1") - .build(); - tester.deployCompletely(prodApp, prodAppPackage); - - assertEquals(1, permanentDeployments(devApp).size()); - assertEquals(1, permanentDeployments(prodApp).size()); + assertEquals(1, tester.controller().applications().get(devApp).get().deployments().size()); + assertEquals(1, tester.controller().applications().get(prodApp).get().deployments().size()); // Not expired at first expirer.maintain(); - assertEquals(1, permanentDeployments(devApp).size()); - assertEquals(1, permanentDeployments(prodApp).size()); + assertEquals(1, tester.controller().applications().get(devApp).get().deployments().size()); + assertEquals(1, tester.controller().applications().get(prodApp).get().deployments().size()); // The dev application is removed tester.clock().advance(Duration.ofDays(15)); expirer.maintain(); - assertEquals(0, permanentDeployments(devApp).size()); - assertEquals(1, permanentDeployments(prodApp).size()); - } - - private List<Deployment> permanentDeployments(Application application) { - return tester.controller().applications().get(application.id()).get().deployments().values().stream() - .filter(deployment -> deployment.zone().environment() != Environment.test && - deployment.zone().environment() != Environment.staging) - .collect(Collectors.toList()); + assertEquals(0, tester.controller().applications().get(devApp).get().deployments().size()); + assertEquals(1, tester.controller().applications().get(prodApp).get().deployments().size()); } } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/UpgraderTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/UpgraderTest.java index 6709098ba06..ccdfc8d042e 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/UpgraderTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/UpgraderTest.java @@ -57,7 +57,7 @@ public class UpgraderTest { assertEquals("New system version: Should upgrade Canaries", 2, tester.buildSystem().jobs().size()); tester.completeUpgrade(canary0, version, "canary"); - assertEquals(version, tester.configServer().lastPrepareVersion().get()); + assertEquals(version, tester.configServer().lastPrepareVersion.get()); tester.updateVersionStatus(version); tester.upgrader().maintain(); @@ -107,7 +107,7 @@ public class UpgraderTest { assertEquals("New system version: Should upgrade Canaries", 2, tester.buildSystem().jobs().size()); tester.completeUpgrade(canary0, version, "canary"); - assertEquals(version, tester.configServer().lastPrepareVersion().get()); + assertEquals(version, tester.configServer().lastPrepareVersion.get()); tester.updateVersionStatus(version); tester.upgrader().maintain(); @@ -187,7 +187,7 @@ public class UpgraderTest { assertEquals("New system version: Should upgrade Canaries", 2, tester.buildSystem().jobs().size()); tester.completeUpgrade(canary0, version, "canary"); - assertEquals(version, tester.configServer().lastPrepareVersion().get()); + assertEquals(version, tester.configServer().lastPrepareVersion.get()); tester.updateVersionStatus(version); tester.upgrader().maintain(); diff --git a/dist/vespa.spec b/dist/vespa.spec index eae6a0b3817..a3e0ec4711f 100644 --- a/dist/vespa.spec +++ b/dist/vespa.spec @@ -90,7 +90,6 @@ Requires: zookeeper >= 3.4.9 %define _extra_link_directory /opt/vespa-libtorrent/lib;/opt/vespa-cppunit/lib %define _extra_include_directory /opt/vespa-libtorrent/include;/opt/vespa-cppunit/include %define _vespa_boost_lib_suffix %{nil} -%define _vespa_cxx_abi_flags -D_GLIBCXX_USE_CXX11_ABI=1 %endif Requires: java-1.8.0-openjdk Requires: openssl @@ -122,7 +121,6 @@ cmake3 -DCMAKE_INSTALL_PREFIX=%{_prefix} \ -DCMAKE_INSTALL_RPATH="%{_prefix}/lib64%{?_extra_link_directory:;%{_extra_link_directory}};/usr/lib/jvm/java-1.8.0/jre/lib/amd64/server" \ %{?_vespa_llvm_version:-DVESPA_LLVM_VERSION="%{_vespa_llvm_version}"} \ %{?_vespa_boost_lib_suffix:-DVESPA_BOOST_LIB_SUFFIX="%{_vespa_boost_lib_suffix}"} \ - %{?_vespa_cxx_abi_flags:-DVESPA_CXX_ABI_FLAGS="%{_vespa_cxx_abi_flags}"} \ . make %{_smp_mflags} diff --git a/searchlib/src/tests/fef/parameter/parameter_test.cpp b/searchlib/src/tests/fef/parameter/parameter_test.cpp index 5fa0633f56e..2cff534d289 100644 --- a/searchlib/src/tests/fef/parameter/parameter_test.cpp +++ b/searchlib/src/tests/fef/parameter/parameter_test.cpp @@ -9,6 +9,7 @@ LOG_SETUP("parameter_test"); using namespace search::fef::test; using CollectionType = search::fef::FieldInfo::CollectionType; +using DataType = search::fef::FieldInfo::DataType; namespace search { namespace fef { @@ -135,6 +136,7 @@ ParameterTest::testValidator() IndexEnvironmentBuilder builder(env); builder.addField(FieldType::INDEX, CollectionType::SINGLE, "foo") .addField(FieldType::ATTRIBUTE, CollectionType::SINGLE, "bar") + .addField(FieldType::ATTRIBUTE, CollectionType::SINGLE, DataType::TENSOR, "tbar") .addField(FieldType::INDEX, CollectionType::ARRAY, "afoo") .addField(FieldType::INDEX, CollectionType::WEIGHTEDSET, "wfoo") .addField(FieldType::INDEX, CollectionType::SINGLE, "hybrid"); @@ -156,6 +158,8 @@ ParameterTest::testValidator() EXPECT_TRUE(validate(env, SL().add("baz"), PDS().desc().feature())); EXPECT_TRUE(validate(env, SL().add("123"), PDS().desc().number())); EXPECT_TRUE(validate(env, SL().add("baz"), PDS().desc().string())); + EXPECT_TRUE(validate(env, SL().add("tbar"), PDS().desc().attributeField(ParameterCollection::ANY))); + EXPECT_TRUE(validate(env, SL().add("tbar"), PDS().desc().attribute(ParameterCollection::ANY))); // first fail but second pass EXPECT_TRUE(validate(env, SL().add("baz"), PDS().desc().field().desc().string())); @@ -180,6 +184,8 @@ ParameterTest::testValidator() EXPECT_FALSE(validate(env, SL().add("hybrid"), PDS().desc().attributeField(ParameterCollection::ANY))); EXPECT_FALSE(validate(env, SL().add("12a"), PDS().desc().number())); EXPECT_FALSE(validate(env, SL().add("a12"), PDS().desc().number())); + EXPECT_FALSE(validate(env, SL().add("tbar"), PDS().desc().attributeField(ParameterDataTypeSet::normalTypeSet(), ParameterCollection::ANY))); + EXPECT_FALSE(validate(env, SL().add("tbar"), PDS().desc().attribute(ParameterDataTypeSet::normalTypeSet(), ParameterCollection::ANY))); // test repeat PDS d1 = PDS().desc().field().repeat(); diff --git a/searchlib/src/vespa/searchlib/features/attributematchfeature.cpp b/searchlib/src/vespa/searchlib/features/attributematchfeature.cpp index 2ff53951d8b..27c7b77ec26 100644 --- a/searchlib/src/vespa/searchlib/features/attributematchfeature.cpp +++ b/searchlib/src/vespa/searchlib/features/attributematchfeature.cpp @@ -6,6 +6,7 @@ #include <vespa/searchlib/fef/featurenamebuilder.h> #include <vespa/searchlib/fef/fieldinfo.h> #include <vespa/searchlib/fef/properties.h> +#include <vespa/searchlib/fef/parameterdescriptions.h> #include <vespa/searchcommon/attribute/attributecontent.h> #include <vespa/log/log.h> @@ -271,7 +272,8 @@ AttributeMatchBlueprint::visitDumpFeatures(const IIndexEnvironment &env, { for (uint32_t i = 0; i < env.getNumFields(); ++i) { const FieldInfo * field = env.getField(i); - if (field->type() == FieldType::ATTRIBUTE) { + if (field->type() == FieldType::ATTRIBUTE && + ParameterDataTypeSet::normalTypeSet().allowedType(field->get_data_type())) { FeatureNameBuilder fnb; fnb.baseName(getBaseName()).parameter(field->name()); visitor.visitDumpFeature(fnb.buildName()); diff --git a/searchlib/src/vespa/searchlib/features/attributematchfeature.h b/searchlib/src/vespa/searchlib/features/attributematchfeature.h index 707572abf9e..3cdc6e322a3 100644 --- a/searchlib/src/vespa/searchlib/features/attributematchfeature.h +++ b/searchlib/src/vespa/searchlib/features/attributematchfeature.h @@ -98,7 +98,7 @@ public: void visitDumpFeatures(const fef::IIndexEnvironment & env, fef::IDumpFeatureVisitor & visitor) const override; fef::Blueprint::UP createInstance() const override; fef::ParameterDescriptions getDescriptions() const override { - return fef::ParameterDescriptions().desc().attributeField(fef::ParameterCollection::ANY); + return fef::ParameterDescriptions().desc().attributeField(fef::ParameterDataTypeSet::normalTypeSet(), fef::ParameterCollection::ANY); } bool setup(const fef::IIndexEnvironment & env, const fef::ParameterList & params) override; diff --git a/searchlib/src/vespa/searchlib/fef/parameterdescriptions.h b/searchlib/src/vespa/searchlib/fef/parameterdescriptions.h index d0e29cfa318..59dd38cf0cd 100644 --- a/searchlib/src/vespa/searchlib/fef/parameterdescriptions.h +++ b/searchlib/src/vespa/searchlib/fef/parameterdescriptions.h @@ -2,6 +2,7 @@ #pragma once +#include <vespa/searchcommon/common/datatype.h> #include <vector> #include <cstddef> @@ -37,16 +38,71 @@ struct ParameterCollection { }; }; +/* + * A set of accepted data types for a parameter. + */ +class ParameterDataTypeSet +{ +public: + using DataType = search::index::schema::DataType; + +private: + uint32_t _typeMask; + + static uint32_t asMask(DataType dataType) { + return (1u << static_cast<unsigned int>(dataType)); + } + static uint32_t normalTypesMask() { + return (asMask(DataType::UINT1) | + asMask(DataType::UINT2) | + asMask(DataType::UINT4) | + asMask(DataType::INT8) | + asMask(DataType::INT16) | + asMask(DataType::INT32) | + asMask(DataType::INT64) | + asMask(DataType::FLOAT) | + asMask(DataType::DOUBLE) | + asMask(DataType::STRING) | + asMask(DataType::RAW)); + } + static uint32_t allTypesMask() { + return (normalTypesMask() | + asMask(DataType::BOOLEANTREE) | + asMask(DataType::TENSOR) | + asMask(DataType::REFERENCE)); + } + ParameterDataTypeSet(uint32_t typeMask) + : _typeMask(typeMask) + { + } +public: + ParameterDataTypeSet() + : ParameterDataTypeSet(allTypesMask()) + { + } + static ParameterDataTypeSet normalTypeSet() { + return ParameterDataTypeSet(normalTypesMask()); + } + bool allowedType(DataType dataType) const { + return ((asMask(dataType) & _typeMask) != 0); + } +}; + /** * The description of a single parameter within a single * ParameterDescription object. **/ struct ParamDescItem { ParameterType::Enum type; + ParameterDataTypeSet dataTypeSet; ParameterCollection::Enum collection; ParamDescItem(ParameterType::Enum t, ParameterCollection::Enum c) - : type(t), collection(c) {} + : type(t), dataTypeSet(), collection(c) {} + ParamDescItem(ParameterType::Enum t, + ParameterDataTypeSet dts, + ParameterCollection::Enum c) + : type(t), dataTypeSet(dts), collection(c) {} }; /** @@ -101,6 +157,9 @@ private: Description & getCurrent() { return _descriptions.back(); } void addParameter(const ParamDescItem ¶m); + void addParameter(ParameterType::Enum type, ParameterDataTypeSet dataTypeSet, ParameterCollection::Enum collection) { + addParameter(ParamDescItem(type, dataTypeSet, collection)); + } void addParameter(ParameterType::Enum type, ParameterCollection::Enum collection) { addParameter(ParamDescItem(type, collection)); } @@ -141,6 +200,10 @@ public: addParameter(ParameterType::ATTRIBUTE_FIELD, collection); return *this; } + ParameterDescriptions & attributeField(ParameterDataTypeSet dataTypeSet, ParameterCollection::Enum collection) { + addParameter(ParameterType::ATTRIBUTE_FIELD, dataTypeSet, collection); + return *this; + } /** * Adds an attribute parameter to the current description. */ @@ -148,6 +211,10 @@ public: addParameter(ParameterType::ATTRIBUTE, collection); return *this; } + ParameterDescriptions & attribute(ParameterDataTypeSet dataTypeSet, ParameterCollection::Enum collection) { + addParameter(ParameterType::ATTRIBUTE, dataTypeSet, collection); + return *this; + } /** * Adds a feature parameter to the current description. */ diff --git a/searchlib/src/vespa/searchlib/fef/parametervalidator.cpp b/searchlib/src/vespa/searchlib/fef/parametervalidator.cpp index 62c9efc6739..dda5ec0b719 100644 --- a/searchlib/src/vespa/searchlib/fef/parametervalidator.cpp +++ b/searchlib/src/vespa/searchlib/fef/parametervalidator.cpp @@ -25,6 +25,10 @@ bool checkCollectionType(ParameterCollection::Enum accept, CollectionType actual return false; } +bool checkDataType(ParameterDataTypeSet accept, search::index::schema::DataType actual) { + return accept.allowedType(actual); +} + class ValidateException { public: @@ -50,7 +54,9 @@ ParameterValidator::Result & ParameterValidator::Result::operator=(const Result ParameterValidator::Result::~Result() { } void -ParameterValidator::validateField(ParameterType::Enum type, ParameterCollection::Enum collection, +ParameterValidator::validateField(ParameterType::Enum type, + ParameterDataTypeSet dataTypeSet, + ParameterCollection::Enum collection, size_t i, Result & result) { const FieldInfo * field = _indexEnv.getFieldByName(_params[i]); @@ -74,6 +80,10 @@ ParameterValidator::validateField(ParameterType::Enum type, ParameterCollection: i, _params[i].c_str())); } } + if (!checkDataType(dataTypeSet, field->get_data_type())) { + throw ValidateException(make_string("Param[%zu]: field '%s' has inappropriate data type", + i, _params[i].c_str())); + } if (!checkCollectionType(collection, field->collection())) { throw ValidateException(make_string("Param[%zu]: field '%s' has inappropriate collection type", i, _params[i].c_str())); @@ -116,7 +126,7 @@ ParameterValidator::validate(const ParameterDescriptions::Description & desc) case ParameterType::INDEX_FIELD: case ParameterType::ATTRIBUTE_FIELD: case ParameterType::ATTRIBUTE: - validateField(type, param.collection, i, result); + validateField(type, param.dataTypeSet, param.collection, i, result); break; case ParameterType::NUMBER: validateNumber(type, i, result); diff --git a/searchlib/src/vespa/searchlib/fef/parametervalidator.h b/searchlib/src/vespa/searchlib/fef/parametervalidator.h index 25dc5296988..6bde59641da 100644 --- a/searchlib/src/vespa/searchlib/fef/parametervalidator.h +++ b/searchlib/src/vespa/searchlib/fef/parametervalidator.h @@ -59,7 +59,7 @@ private: const StringVector & _params; const ParameterDescriptions & _descs; - void validateField(ParameterType::Enum type, ParameterCollection::Enum collection, + void validateField(ParameterType::Enum type, ParameterDataTypeSet dataTypeSet, ParameterCollection::Enum collection, size_t i, Result & result); void validateNumber(ParameterType::Enum type, size_t i, Result & result); Result validate(const ParameterDescriptions::Description & desc); diff --git a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/FeedClient.java b/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/FeedClient.java index ed62c507a4b..299541a5a2d 100644 --- a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/FeedClient.java +++ b/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/FeedClient.java @@ -17,7 +17,6 @@ import java.util.concurrent.atomic.AtomicInteger; * * @author dybis * @see FeedClientFactory - * @since 5.29.0 */ public interface FeedClient extends AutoCloseable { @@ -25,6 +24,7 @@ public interface FeedClient extends AutoCloseable { * Streams a document to cluster(s). If the pipeline and buffers are full, this call will be blocking. * Documents might time out before they are sent. Failed documents are not retried. * Don't call stream() after close is called. + * * @param documentId Document id of the document. * @param documentData The document data as JSON or XML (as specified when using the factory to create the API) */ @@ -34,6 +34,7 @@ public interface FeedClient extends AutoCloseable { * Streams a document to cluster(s). If the pipeline and buffers are full, this call will be blocking. * Documents might time out before they are sent. Failed documents are not retried. * Don't call stream() after close is called. + * * @param documentId Document id of the document. * @param documentData The document data as JSON or XML (as specified when using the factory to create the API) * @param context Any context, will be accessible in the result of the callback. @@ -47,7 +48,7 @@ public interface FeedClient extends AutoCloseable { * AtomicInteger for counters and follow general guides for thread-safe programming. * There is an example implementation in class SimpleLoggerResultCallback. */ - static interface ResultCallback { + interface ResultCallback { void onCompletion(String docId, Result documentResult); } @@ -60,17 +61,19 @@ public interface FeedClient extends AutoCloseable { /** * Returns stats about the cluster. + * * @return JSON string with information about cluster. */ - public String getStatsAsJson(); + String getStatsAsJson(); /** * Utility function that takes an array of JSON documents and calls the FeedClient for each element. + * * @param inputStream This can be a very large stream. The outer element is an array (of document operations). * @param feedClient The feedClient that will receive the document operations. * @param numSent increased per document sent to API (but no waiting for results). */ - public static void feedJson(InputStream inputStream, FeedClient feedClient, AtomicInteger numSent) { + static void feedJson(InputStream inputStream, FeedClient feedClient, AtomicInteger numSent) { JsonReader.read(inputStream, feedClient, numSent); } @@ -78,15 +81,17 @@ public interface FeedClient extends AutoCloseable { * Utility function that takes an array of XML documents and calls the FeedClient for each element. * The XML document has to be formatted with line space on each line (like "regular" XML, but stricter * than the specifications of XML). + * * @param inputStream This can be a very large stream. * @param feedClient The feedClient that will receive the document operations. * @param numSent increased per document sent to API (but no waiting for results). */ - public static void feedXml(InputStream inputStream, FeedClient feedClient, AtomicInteger numSent) { + static void feedXml(InputStream inputStream, FeedClient feedClient, AtomicInteger numSent) { try { XmlFeedReader.read(inputStream, feedClient, numSent); } catch (Exception e) { throw new RuntimeException(e); } } + } diff --git a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/FeedClientFactory.java b/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/FeedClientFactory.java index aba16e9c9a4..cfa9ce670f6 100644 --- a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/FeedClientFactory.java +++ b/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/FeedClientFactory.java @@ -9,6 +9,7 @@ import static com.yahoo.vespa.http.client.SessionFactory.createTimeoutExecutor; /** * Factory for creating FeedClient. + * * @author dybis */ public class FeedClientFactory { diff --git a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/Result.java b/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/Result.java index ed2ed08e4ac..138be61de80 100644 --- a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/Result.java +++ b/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/Result.java @@ -16,8 +16,7 @@ import java.util.List; * but may contain more than one Result.Detail instances, as these pertains to a * single endpoint, and a Result may wrap data for multiple endpoints. * - * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a> - * @since 5.1.20 + * @author Einar M R Rosenvinge */ // This should be an interface, but in order to be binary compatible during refactoring we made it abstract. public class Result { diff --git a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/Session.java b/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/Session.java index 0c85c025793..de39eabd3ee 100644 --- a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/Session.java +++ b/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/Session.java @@ -13,11 +13,11 @@ import java.util.concurrent.BlockingQueue; * * A {@link SessionFactory} is provided to instantiate Sessions. * - * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a> - * @since 5.1.20 + * @author Einar M R Rosenvinge * @see SessionFactory */ public interface Session extends AutoCloseable { + /** * Returns an OutputStream that can be used to write ONE operation, identified by the * given document ID. The data format must match the @@ -33,7 +33,7 @@ public interface Session extends AutoCloseable { * @param documentId the unique ID identifying this operation in the system * @return an OutputStream to write the operation payload into */ - public OutputStream stream(CharSequence documentId); + OutputStream stream(CharSequence documentId); /** * Returns {@link Result}s for all operations enqueued by {@link #stream(CharSequence)}. @@ -44,7 +44,7 @@ public interface Session extends AutoCloseable { * @return a blocking queue for retrieving results * @see Result */ - public BlockingQueue<Result> results(); + BlockingQueue<Result> results(); /** * Closes this Session. All resources are freed, persistent connections are closed and @@ -52,11 +52,12 @@ public interface Session extends AutoCloseable { * * @throws RuntimeException in cases where underlying resources throw on shutdown/close */ - public void close(); + void close(); /** * Returns stats about the cluster. * @return JSON string with information about cluster. */ - public String getStatsAsJson(); + String getStatsAsJson(); + } diff --git a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/SessionFactory.java b/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/SessionFactory.java index 25241d249ff..2e47e45dff0 100644 --- a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/SessionFactory.java +++ b/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/SessionFactory.java @@ -13,8 +13,7 @@ import java.util.concurrent.ThreadFactory; /** * Factory for creating {@link Session} instances. * - * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a> - * @since 5.1.20 + * @author Einar M R Rosenvinge */ public final class SessionFactory { @@ -77,4 +76,5 @@ public final class SessionFactory { return t; } } + } diff --git a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/SimpleLoggerResultCallback.java b/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/SimpleLoggerResultCallback.java index 8729644e5ea..85e6baec9f7 100644 --- a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/SimpleLoggerResultCallback.java +++ b/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/SimpleLoggerResultCallback.java @@ -13,6 +13,7 @@ import java.util.concurrent.atomic.AtomicInteger; * "Result received: 34 (1 failed so far, 2003 sent, success rate 1999.23 docs/sec)." * On each failure it will print the Result object content. If tracing is enabled, it will print trace messages to * std err as well. + * * @author dybis */ public class SimpleLoggerResultCallback implements FeedClient.ResultCallback { @@ -99,4 +100,5 @@ public class SimpleLoggerResultCallback implements FeedClient.ResultCallback { } } } + } diff --git a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/config/Cluster.java b/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/config/Cluster.java index fb928e2dacd..557aeedb4c1 100644 --- a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/config/Cluster.java +++ b/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/config/Cluster.java @@ -11,11 +11,11 @@ import java.util.List; /** * A set of {@link Endpoint} instances. Construct using {@link Cluster.Builder}. * - * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a> - * @since 5.1.20 + * @author Einar M R Rosenvinge */ @Immutable public final class Cluster { + /** * Builder for {@link Cluster}. */ @@ -74,5 +74,4 @@ public final class Cluster { return "Cluster " + endpoints + ", route " + route; } - } diff --git a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/config/ConnectionParams.java b/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/config/ConnectionParams.java index 6d6c1ddaa49..3fe42b21e93 100644 --- a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/config/ConnectionParams.java +++ b/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/config/ConnectionParams.java @@ -21,9 +21,8 @@ import java.util.concurrent.TimeUnit; * the Session to the Vespa clusters. * This class is immutable * and has no public constructor - to instantiate one, use a {@link Builder}. - - * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a> - * @since 5.1.20 + * + * @author Einar M R Rosenvinge */ @Immutable public final class ConnectionParams { diff --git a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/config/Endpoint.java b/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/config/Endpoint.java index b044c359712..de7bfb5bb19 100644 --- a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/config/Endpoint.java +++ b/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/config/Endpoint.java @@ -9,8 +9,7 @@ import java.io.Serializable; * Represents an endpoint, in most cases a JDisc container * in a Vespa cluster configured with <code>document-api</code>. * - * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a> - * @since 5.1.20 + * @author Einar M R Rosenvinge */ @Immutable public final class Endpoint implements Serializable { diff --git a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/config/FeedParams.java b/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/config/FeedParams.java index 46487f8431a..4f5b30444a9 100644 --- a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/config/FeedParams.java +++ b/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/config/FeedParams.java @@ -11,8 +11,7 @@ import java.util.concurrent.TimeUnit; * when creating {@link com.yahoo.vespa.http.client.Session}s. This class is immutable * and has no public constructor - to instantiate one, use a {@link Builder}. - * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a> - * @since 5.1.20 + * @author Einar M R Rosenvinge */ @Immutable public final class FeedParams { @@ -327,4 +326,5 @@ public final class FeedParams { public long getLocalQueueTimeOut() { return localQueueTimeOut; } + } diff --git a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/config/SessionParams.java b/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/config/SessionParams.java index 4f1f76766d7..9a8640d09e2 100644 --- a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/config/SessionParams.java +++ b/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/config/SessionParams.java @@ -14,13 +14,13 @@ import java.util.List; * when creating {@link com.yahoo.vespa.http.client.Session}s. This class is immutable * and has no public constructor - to instantiate one, use a {@link Builder}. * - * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a> - * @since 5.1.20 + * @author Einar M R Rosenvinge * @see com.yahoo.vespa.http.client.SessionFactory * @see Builder */ @Immutable public final class SessionParams { + /** * Interface for handling serious errors with connection. */ @@ -180,4 +180,5 @@ public final class SessionParams { public ErrorReporter getErrorReport() { return errorReport; } + } diff --git a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/api/FeedClientImpl.java b/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/api/FeedClientImpl.java index 11cea32dd37..a16d992324d 100644 --- a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/api/FeedClientImpl.java +++ b/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/api/FeedClientImpl.java @@ -17,6 +17,7 @@ import java.util.concurrent.TimeUnit; /** * Implementation of FeedClient. It is a thin layer on top of multiClusterHandler and multiClusterResultAggregator. + * * @author dybis */ public class FeedClientImpl implements FeedClient { diff --git a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/api/MultiClusterSessionOutputStream.java b/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/api/MultiClusterSessionOutputStream.java index 129b67b13b8..bf55a46277d 100644 --- a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/api/MultiClusterSessionOutputStream.java +++ b/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/api/MultiClusterSessionOutputStream.java @@ -9,9 +9,11 @@ import java.io.IOException; /** * Class for wiring up the Session API. It is the return value of stream() in the Session API. + * * @author dybis */ class MultiClusterSessionOutputStream extends ByteArrayOutputStream { + private final CharSequence documentId; private final OperationProcessor operationProcessor; private final Object context; @@ -31,4 +33,5 @@ class MultiClusterSessionOutputStream extends ByteArrayOutputStream { operationProcessor.sendDocument(document); super.close(); } + } diff --git a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/communication/ApacheGatewayConnection.java b/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/communication/ApacheGatewayConnection.java index 01c36c7df28..420f64d4bf3 100644 --- a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/communication/ApacheGatewayConnection.java +++ b/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/communication/ApacheGatewayConnection.java @@ -43,7 +43,8 @@ import java.util.logging.Logger; import java.util.zip.GZIPOutputStream; /** - * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a> + * @author Einar M R Rosenvinge + * * @since 5.1.20 */ @Beta diff --git a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/communication/ByteBufferInputStream.java b/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/communication/ByteBufferInputStream.java index 044055937d0..56a525502a5 100644 --- a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/communication/ByteBufferInputStream.java +++ b/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/communication/ByteBufferInputStream.java @@ -10,7 +10,8 @@ import java.util.ArrayDeque; import java.util.Deque; /** - * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a> + * @author Einar M R Rosenvinge + * * @since 5.1.20 */ @Beta diff --git a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/communication/ClusterConnection.java b/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/communication/ClusterConnection.java index fa9c1a0a65d..fb57c63dbf2 100644 --- a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/communication/ClusterConnection.java +++ b/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/communication/ClusterConnection.java @@ -22,8 +22,7 @@ import java.util.concurrent.ScheduledThreadPoolExecutor; import java.util.concurrent.TimeUnit; /** - * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a> - * @since 5.1.20 + * @author Einar M R Rosenvinge */ @Beta public class ClusterConnection implements AutoCloseable { diff --git a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/communication/DocumentQueue.java b/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/communication/DocumentQueue.java index b9be94e03f1..671e6f07dbe 100644 --- a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/communication/DocumentQueue.java +++ b/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/communication/DocumentQueue.java @@ -13,10 +13,13 @@ import java.util.Optional; import java.util.concurrent.TimeUnit; /** - * Document queue that only gives you document operations on documents for which there are no already in flight operations for. + * Document queue that only gives you document operations on documents for which there are no + * already in flight operations for. + * * @author dybis */ class DocumentQueue { + private final Deque<Document> queue; private final int maxSize; private boolean closed = false; @@ -119,4 +122,5 @@ class DocumentQueue { return Optional.empty(); } } + } diff --git a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/communication/DryRunGatewayConnection.java b/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/communication/DryRunGatewayConnection.java index 058cc8411c0..23ab5e36e14 100644 --- a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/communication/DryRunGatewayConnection.java +++ b/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/communication/DryRunGatewayConnection.java @@ -57,4 +57,5 @@ public class DryRunGatewayConnection implements GatewayConnection { @Override public void close() { } + } diff --git a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/communication/EndpointIOException.java b/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/communication/EndpointIOException.java index f75a5b122d7..ae15f6ec22b 100644 --- a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/communication/EndpointIOException.java +++ b/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/communication/EndpointIOException.java @@ -7,9 +7,11 @@ import java.io.IOException; /** * Class for throwing exception from endpoint. + * * @author dybis */ public class EndpointIOException extends IOException { + private final Endpoint endpoint; private static final long serialVersionUID = 29335813211L; @@ -21,4 +23,5 @@ public class EndpointIOException extends IOException { public Endpoint getEndpoint() { return endpoint; } + } diff --git a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/communication/EndpointResultQueue.java b/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/communication/EndpointResultQueue.java index 405ecf8ade2..98dd067b7c5 100644 --- a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/communication/EndpointResultQueue.java +++ b/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/communication/EndpointResultQueue.java @@ -18,11 +18,11 @@ import java.util.concurrent.TimeUnit; import java.util.logging.Logger; /** - * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a> - * @since 5.1.22 + * @author Einar M R Rosenvinge */ @Beta class EndpointResultQueue { + private static Logger log = Logger.getLogger(EndpointResultQueue.class.getName()); private final OperationProcessor operationProcessor; private final Map<String, TimerFuture> futureByOperation = new HashMap<>(); @@ -58,8 +58,7 @@ class EndpointResultQueue { resultReceived(result, clusterId, true); } - private synchronized void resultReceived( - EndpointResult result, int clusterId, boolean duplicateGivesWarning) { + private synchronized void resultReceived(EndpointResult result, int clusterId, boolean duplicateGivesWarning) { operationProcessor.resultReceived(result, clusterId); TimerFuture timerFuture = futureByOperation.remove(result.getOperationId()); @@ -133,4 +132,5 @@ class EndpointResultQueue { return future; } } + } diff --git a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/communication/GatewayConnection.java b/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/communication/GatewayConnection.java index 8fe17d30a5d..3e5bdfe3056 100644 --- a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/communication/GatewayConnection.java +++ b/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/communication/GatewayConnection.java @@ -9,6 +9,7 @@ import java.io.InputStream; import java.util.List; public interface GatewayConnection { + InputStream writeOperations(List<Document> docs) throws ServerResponseException, IOException; InputStream drain() throws ServerResponseException, IOException; @@ -20,4 +21,5 @@ public interface GatewayConnection { void handshake() throws ServerResponseException, IOException; void close(); + } diff --git a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/communication/GatewayThrottler.java b/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/communication/GatewayThrottler.java index fc495249c30..cc637904553 100644 --- a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/communication/GatewayThrottler.java +++ b/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/communication/GatewayThrottler.java @@ -10,6 +10,7 @@ import java.util.Random; * @author dybis */ public class GatewayThrottler { + private long backOffTimeMs = 0; private final long maxSleepTimeMs; private static Random random = new Random(); @@ -42,4 +43,5 @@ public class GatewayThrottler { Double result = expected * factor; return result.intValue(); } + } diff --git a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/communication/IOThread.java b/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/communication/IOThread.java index 2d4dc26395a..7874dcb24ab 100644 --- a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/communication/IOThread.java +++ b/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/communication/IOThread.java @@ -23,11 +23,12 @@ import java.util.logging.Logger; /** * Class for handling asynchronous feeding of new documents and processing of results. - * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a> - * @since 5.1.20 + * + * @author Einar M R Rosenvinge */ @Beta class IOThread implements Runnable, AutoCloseable { + private static Logger log = Logger.getLogger(IOThread.class.getName()); private final Endpoint endpoint; private final GatewayConnection client; @@ -403,4 +404,5 @@ class IOThread implements Runnable, AutoCloseable { resultQueue.failOperation(endpointResult, clusterId); } } + } diff --git a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/operationProcessor/ConcurrentDocumentOperationBlocker.java b/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/operationProcessor/ConcurrentDocumentOperationBlocker.java index 5709032f2d7..366675d083c 100644 --- a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/operationProcessor/ConcurrentDocumentOperationBlocker.java +++ b/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/operationProcessor/ConcurrentDocumentOperationBlocker.java @@ -5,6 +5,7 @@ import java.util.concurrent.Semaphore; /** * A semaphore that can be re-sized. + * * @author dybis */ final public class ConcurrentDocumentOperationBlocker { @@ -64,4 +65,5 @@ final public class ConcurrentDocumentOperationBlocker { super.reducePermits(reduction); } } + } diff --git a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/operationProcessor/DocumentSendInfo.java b/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/operationProcessor/DocumentSendInfo.java index 13399486714..54eac939b9f 100644 --- a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/operationProcessor/DocumentSendInfo.java +++ b/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/operationProcessor/DocumentSendInfo.java @@ -12,6 +12,7 @@ import java.util.Map; * This class is NOT thread-safe by design. */ class DocumentSendInfo { + private final Document document; private final Map<Integer, Result.Detail> detailByClusterId = new HashMap<>(); // This is lazily populated as normal cases does not require retries. @@ -68,4 +69,5 @@ class DocumentSendInfo { Document getDocument() { return document; } + } diff --git a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/operationProcessor/EndPointResultFactory.java b/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/operationProcessor/EndPointResultFactory.java index eb6572d4dbb..129e3ca245b 100644 --- a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/operationProcessor/EndPointResultFactory.java +++ b/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/operationProcessor/EndPointResultFactory.java @@ -18,11 +18,11 @@ import java.util.List; import java.util.logging.Logger; /** - * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a> - * @since 5.1.20 + * @author Einar M R Rosenvinge */ @Beta public final class EndPointResultFactory { + private static Logger log = Logger.getLogger(EndPointResultFactory.class.getName()); private static final String EMPTY_MESSAGE = "-"; diff --git a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/operationProcessor/IncompleteResultsThrottler.java b/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/operationProcessor/IncompleteResultsThrottler.java index 8721bfb53fa..6ecbb56888e 100644 --- a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/operationProcessor/IncompleteResultsThrottler.java +++ b/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/operationProcessor/IncompleteResultsThrottler.java @@ -30,6 +30,7 @@ import java.util.Random; * @author dybis */ public class IncompleteResultsThrottler { + private final ConcurrentDocumentOperationBlocker blocker = new ConcurrentDocumentOperationBlocker(); private final int maxInFlightValue; private final int minInFlightValue; @@ -180,4 +181,5 @@ public class IncompleteResultsThrottler { tryBoostingSizeIfMinValueOverSeveralCycles(size), maxInFlightValue)); blocker.setMaxConcurrency(maxInFlightNow); } + } diff --git a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/operationProcessor/OperationProcessor.java b/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/operationProcessor/OperationProcessor.java index 605830a3205..7cea767f9ba 100644 --- a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/operationProcessor/OperationProcessor.java +++ b/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/operationProcessor/OperationProcessor.java @@ -29,11 +29,12 @@ import java.util.logging.Logger; /** * Merges several endpointResult into one Result and does the callback. + * * @author dybis - * @since 5.1.20 */ @Beta public class OperationProcessor { + private static final Logger log = Logger.getLogger(OperationProcessor.class.getName()); private final Map<String, DocumentSendInfo> docSendInfoByOperationId = new HashMap<>(); private final ArrayListMultimap<String, Document> blockedDocumentsByDocumentId = ArrayListMultimap.create(); @@ -191,13 +192,10 @@ public class OperationProcessor { final Result result = process(endpointResult, clusterId); if (result != null) { - try { - resultCallback.onCompletion(result.getDocumentId(), result); - if (traceToStderr && result.hasLocalTrace()) { - System.err.println(result.toString()); - } - } finally { - incompleteResultsThrottler.resultReady(result.isSuccess()); + incompleteResultsThrottler.resultReady(result.isSuccess()); + resultCallback.onCompletion(result.getDocumentId(), result); + if (traceToStderr && result.hasLocalTrace()) { + System.err.println(result.toString()); } } } @@ -244,12 +242,10 @@ public class OperationProcessor { try { clusterConnection.post(document); } catch (EndpointIOException eio) { - resultReceived( - EndPointResultFactory.createError( - eio.getEndpoint(), - document.getOperationId(), - eio), - clusterConnection.getClusterId()); + resultReceived(EndPointResultFactory.createError(eio.getEndpoint(), + document.getOperationId(), + eio), + clusterConnection.getClusterId()); } } @@ -298,4 +294,5 @@ public class OperationProcessor { throw new RuntimeException("Did not manage to shut down retry threads. Please report problem."); } } + } diff --git a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/operationProcessor/OperationStats.java b/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/operationProcessor/OperationStats.java index b407cc67e9e..a28c2f9805f 100644 --- a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/operationProcessor/OperationStats.java +++ b/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/operationProcessor/OperationStats.java @@ -14,6 +14,7 @@ import java.io.Writer; import java.util.List; public class OperationStats { + private static JsonFactory jsonFactory = new JsonFactory(); private final String sessionParamsAsXmlString; private List<ClusterConnection> clusters; @@ -65,4 +66,5 @@ public class OperationStats { return "{ \"Error\" : \""+ e.getMessage() + "\"}"; } } + } diff --git a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/runner/CommandLineArguments.java b/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/runner/CommandLineArguments.java index 598c64a20a6..23fb788f116 100644 --- a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/runner/CommandLineArguments.java +++ b/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/runner/CommandLineArguments.java @@ -19,6 +19,7 @@ import java.util.concurrent.TimeUnit; /** * Commandline interface for the binary. + * * @author dybis */ @Beta @@ -28,6 +29,7 @@ public class CommandLineArguments { /** * Creates a CommandLineArguments instance and populates it with data. + * * @param args array of arguments. * @return null on failure or if help option is set to true. */ @@ -223,4 +225,5 @@ public class CommandLineArguments { } return builder.build(); } + } diff --git a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/runner/FormatInputStream.java b/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/runner/FormatInputStream.java index 302a4ce03f5..36cdf18e102 100644 --- a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/runner/FormatInputStream.java +++ b/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/runner/FormatInputStream.java @@ -21,6 +21,7 @@ import java.util.Optional; * @author valerijf */ public class FormatInputStream { + private InputStream inputStream; private Format format; @@ -100,4 +101,5 @@ public class FormatInputStream { public enum Format { JSON, XML } + } diff --git a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/runner/Runner.java b/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/runner/Runner.java index 164c2524483..0983d893ecc 100644 --- a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/runner/Runner.java +++ b/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/runner/Runner.java @@ -14,11 +14,11 @@ import java.util.Optional; import java.util.concurrent.atomic.AtomicInteger; /** - * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a> + * @author Einar M R Rosenvinge * @author dybis - * @since 5.1.20 */ public class Runner { + /** * Feed data from inputFile to session. * @param feedClient where to send data to |