diff options
36 files changed, 143 insertions, 130 deletions
diff --git a/component/src/main/java/com/yahoo/component/ComponentSpecification.java b/component/src/main/java/com/yahoo/component/ComponentSpecification.java index c2c31c97f8b..9662eee0f66 100644 --- a/component/src/main/java/com/yahoo/component/ComponentSpecification.java +++ b/component/src/main/java/com/yahoo/component/ComponentSpecification.java @@ -43,8 +43,7 @@ public final class ComponentSpecification { } public ComponentSpecification(String name, VersionSpecification versionSpecification, ComponentId namespace) { - spec = new Spec<>(new VersionHandler(), - name, versionSpecification, namespace); + spec = new Spec<>(new VersionHandler(), name, versionSpecification, namespace); stringValue = spec.createStringValue(); } diff --git a/component/src/main/java/com/yahoo/component/Spec.java b/component/src/main/java/com/yahoo/component/Spec.java index 7911b53d23c..a3259484b4a 100644 --- a/component/src/main/java/com/yahoo/component/Spec.java +++ b/component/src/main/java/com/yahoo/component/Spec.java @@ -9,6 +9,7 @@ import static com.yahoo.container.util.Util.firstNonNull; * @author tonytv */ final class Spec<VERSION> { + private final VersionHandler<VERSION> versionHandler; interface VersionHandler<VERSION> { @@ -21,8 +22,7 @@ final class Spec<VERSION> { final ComponentId namespace; @SuppressWarnings("unchecked") - Spec(VersionHandler<VERSION> versionHandler, - String name, VERSION version, ComponentId namespace) { + Spec(VersionHandler<VERSION> versionHandler, String name, VERSION version, ComponentId namespace) { assert (name != null); validateName(name); diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/jersey/Jersey2Servlet.java b/config-model/src/main/java/com/yahoo/vespa/model/container/jersey/Jersey2Servlet.java index def9442f9e4..d7c9482cab4 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/jersey/Jersey2Servlet.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/jersey/Jersey2Servlet.java @@ -12,6 +12,7 @@ import com.yahoo.vespa.model.container.component.Servlet; * @author tonytv */ public class Jersey2Servlet extends Servlet { + public static final String BUNDLE = "container-jersey2"; public static final String CLASS = "com.yahoo.container.servlet.jersey.JerseyServletProvider"; @@ -26,10 +27,9 @@ public class Jersey2Servlet extends Servlet { } private static ComponentSpecification idSpecFromPath(String path) { - return new ComponentSpecification( - RestApi.idFromPath(path), - VersionSpecification.emptyVersionSpecification, - REST_API_NAMESPACE); + return new ComponentSpecification(RestApi.idFromPath(path), + VersionSpecification.emptyVersionSpecification, + REST_API_NAMESPACE); } } diff --git a/config-provisioning/src/main/java/com/yahoo/config/provision/ApplicationId.java b/config-provisioning/src/main/java/com/yahoo/config/provision/ApplicationId.java index bb6cd7c7a77..30454ae1a20 100644 --- a/config-provisioning/src/main/java/com/yahoo/config/provision/ApplicationId.java +++ b/config-provisioning/src/main/java/com/yahoo/config/provision/ApplicationId.java @@ -20,36 +20,6 @@ public final class ApplicationId implements Comparable<ApplicationId> { private final String stringValue; private final String serializedForm; - public static class Builder { - - private TenantName tenant; - private ApplicationName application; - private InstanceName instance; - - public Builder() { - this.tenant = TenantName.defaultName(); - this.application = null; - this.instance = InstanceName.defaultName(); - } - - public Builder tenant(TenantName ten) { this.tenant = ten; return this; } - public Builder tenant(String ten) { return tenant(TenantName.from(ten)); } - - public Builder applicationName(ApplicationName nam) { this.application = nam; return this; } - public Builder applicationName(String nam) { return applicationName(ApplicationName.from(nam)); } - - public Builder instanceName(InstanceName ins) { this.instance = ins; return this; } - public Builder instanceName(String ins) { return instanceName(InstanceName.from(ins)); } - - public ApplicationId build() { - if (application == null) { - throw new IllegalArgumentException("must set application name in builder"); - } - return ApplicationId.from(tenant, application, instance); - } - - } - public ApplicationId(ApplicationIdConfig config) { this(TenantName.from(config.tenant()), ApplicationName.from(config.application()), InstanceName.from(config.instance())); } @@ -62,8 +32,12 @@ public final class ApplicationId implements Comparable<ApplicationId> { this.serializedForm = toSerializedForm(); } - public static ApplicationId from(TenantName tenant, ApplicationName application, InstanceName instanceName) { - return new ApplicationId(tenant, application, instanceName); + public static ApplicationId from(TenantName tenant, ApplicationName application, InstanceName instance) { + return new ApplicationId(tenant, application, instance); + } + + public static ApplicationId from(String tenant, String application, String instance) { + return new ApplicationId(TenantName.from(tenant), ApplicationName.from(application), InstanceName.from(instance)); } /** Creates an application id from a string on the form application:environment:region:instance */ @@ -144,11 +118,41 @@ public final class ApplicationId implements Comparable<ApplicationId> { } /** Returns an application id where all fields are "*" */ - public static ApplicationId global() { // TODO: Sukk ... get rid of this + public static ApplicationId global() { return new Builder().tenant("*") .applicationName("*") .instanceName("*") .build(); } + public static class Builder { + + private TenantName tenant; + private ApplicationName application; + private InstanceName instance; + + public Builder() { + this.tenant = TenantName.defaultName(); + this.application = null; + this.instance = InstanceName.defaultName(); + } + + public Builder tenant(TenantName ten) { this.tenant = ten; return this; } + public Builder tenant(String ten) { return tenant(TenantName.from(ten)); } + + public Builder applicationName(ApplicationName nam) { this.application = nam; return this; } + public Builder applicationName(String nam) { return applicationName(ApplicationName.from(nam)); } + + public Builder instanceName(InstanceName ins) { this.instance = ins; return this; } + public Builder instanceName(String ins) { return instanceName(InstanceName.from(ins)); } + + public ApplicationId build() { + if (application == null) { + throw new IllegalArgumentException("must set application name in builder"); + } + return ApplicationId.from(tenant, application, instance); + } + + } + } diff --git a/config-provisioning/src/main/java/com/yahoo/config/provision/HostFilter.java b/config-provisioning/src/main/java/com/yahoo/config/provision/HostFilter.java index 8f3127f938f..d771f36cdc1 100644 --- a/config-provisioning/src/main/java/com/yahoo/config/provision/HostFilter.java +++ b/config-provisioning/src/main/java/com/yahoo/config/provision/HostFilter.java @@ -2,6 +2,7 @@ package com.yahoo.config.provision; import com.google.common.collect.ImmutableSet; +import com.yahoo.text.StringUtilities; import java.util.Collection; import java.util.Collections; @@ -86,20 +87,10 @@ public class HostFilter { /** Returns a host filter from three comma and-or space separated string lists. The strings may be null or empty. */ public static HostFilter from(String hostNames, String flavors, String clusterTypes, String clusterIds) { return new HostFilter( - split(hostNames), - split(flavors), - split(clusterTypes).stream().map(ClusterSpec.Type::from).collect(Collectors.toSet()), - split(clusterIds).stream().map(ClusterSpec.Id::from).collect(Collectors.toSet())); - } - - /** Splits a string on space and comma */ - public static Set<String> split(String s) { - if (s == null || s.isEmpty()) return Collections.emptySet(); - ImmutableSet.Builder<String> b = new ImmutableSet.Builder<>(); - for (String item : s.split("[\\s\\,]")) - if ( ! item.isEmpty()) - b.add(item); - return b.build(); + StringUtilities.split(hostNames), + StringUtilities.split(flavors), + StringUtilities.split(clusterTypes).stream().map(ClusterSpec.Type::from).collect(Collectors.toSet()), + StringUtilities.split(clusterIds).stream().map(ClusterSpec.Id::from).collect(Collectors.toSet())); } } diff --git a/docker-api/src/main/java/com/yahoo/vespa/hosted/dockerapi/Docker.java b/docker-api/src/main/java/com/yahoo/vespa/hosted/dockerapi/Docker.java index ba22ddf9272..663bb58cacc 100644 --- a/docker-api/src/main/java/com/yahoo/vespa/hosted/dockerapi/Docker.java +++ b/docker-api/src/main/java/com/yahoo/vespa/hosted/dockerapi/Docker.java @@ -57,6 +57,8 @@ public interface Docker { Optional<Container> getContainer(String hostname); + Optional<Container> getContainer(ContainerName containerName); + CompletableFuture<DockerImage> pullImageAsync(DockerImage image); boolean imageIsDownloaded(DockerImage image); diff --git a/docker-api/src/main/java/com/yahoo/vespa/hosted/dockerapi/DockerImpl.java b/docker-api/src/main/java/com/yahoo/vespa/hosted/dockerapi/DockerImpl.java index 00ace9a1958..fd4551d0f02 100644 --- a/docker-api/src/main/java/com/yahoo/vespa/hosted/dockerapi/DockerImpl.java +++ b/docker-api/src/main/java/com/yahoo/vespa/hosted/dockerapi/DockerImpl.java @@ -348,6 +348,14 @@ public class DockerImpl implements Docker { .findFirst(); } + @Override + public Optional<Container> getContainer(ContainerName containerName) { + return listAllContainers().stream() + .flatMap(this::asContainer) + .filter(c -> Objects.equals(containerName, c.name)) + .findFirst(); + } + private Stream<Container> asContainer(com.github.dockerjava.api.model.Container dockerClientContainer) { return inspectContainerCmd(dockerClientContainer.getId()) .map(response -> diff --git a/node-admin/.gitignore b/node-admin/.gitignore index 8f00331723c..750f12730e3 100644 --- a/node-admin/.gitignore +++ b/node-admin/.gitignore @@ -1 +1,4 @@ -vespa-local/Dockerfile +/dependencies +/Dockerfile +include/services.xml +/**/*.pyc diff --git a/node-admin/vespa-local/Dockerfile.template b/node-admin/Dockerfile.template index 53f8bfac18f..e69308deae3 100644 --- a/node-admin/vespa-local/Dockerfile.template +++ b/node-admin/Dockerfile.template @@ -15,19 +15,21 @@ FROM $NODE_ADMIN_FROM_IMAGE # Be aware that this Dockerfile is not being used in any pipelines and has no relation to production environments etc. # It is here for developers' convenience - it allows building and experimenting with node-admin locally. +CMD $VESPA_HOME/bin/start-services.sh + # Things for convenience. RUN yum install -y tcpdump -ADD root-bashrc /root/.bashrc +ADD include/root-bashrc /root/.bashrc # Make config-server aware of node flavor 'docker'. -ADD node-flavors.xml $VESPA_HOME/conf/configserver-app/node-flavors.xml +ADD include/node-flavors.xml $VESPA_HOME/conf/configserver-app/node-flavors.xml # Make config-server also listen to 4080 -ADD http-server.xml $VESPA_HOME/conf/configserver-app/hosted-vespa/http-server.xml +ADD include/http-server.xml $VESPA_HOME/conf/configserver-app/hosted-vespa/http-server.xml # Entrypoint for running config server in a container. -ADD start-config-server.sh $VESPA_HOME/bin/start-config-server.sh +ADD include/start-config-server.sh $VESPA_HOME/bin/start-config-server.sh # Included in base image, but here overridden with local modifications. # TODO: Update the source instead. -ADD start-services.sh $VESPA_HOME/bin/start-services.sh +ADD include/start-services.sh $VESPA_HOME/bin/start-services.sh diff --git a/node-admin/vespa-local/http-server.xml b/node-admin/include/http-server.xml index f77d392de97..f77d392de97 100644 --- a/node-admin/vespa-local/http-server.xml +++ b/node-admin/include/http-server.xml diff --git a/node-admin/vespa-local/node-flavors.xml b/node-admin/include/node-flavors.xml index 7d22975994a..7d22975994a 100644 --- a/node-admin/vespa-local/node-flavors.xml +++ b/node-admin/include/node-flavors.xml diff --git a/node-admin/vespa-local/nodectl-instance.sh b/node-admin/include/nodectl-instance.sh index b40ce44e33a..b40ce44e33a 100755 --- a/node-admin/vespa-local/nodectl-instance.sh +++ b/node-admin/include/nodectl-instance.sh diff --git a/node-admin/vespa-local/root-bashrc b/node-admin/include/root-bashrc index fda1752c85f..fda1752c85f 100644 --- a/node-admin/vespa-local/root-bashrc +++ b/node-admin/include/root-bashrc diff --git a/node-admin/vespa-local/start-config-server.sh b/node-admin/include/start-config-server.sh index a592ec3cbd3..a592ec3cbd3 100755 --- a/node-admin/vespa-local/start-config-server.sh +++ b/node-admin/include/start-config-server.sh diff --git a/node-admin/vespa-local/start-services.sh b/node-admin/include/start-services.sh index 0fe5c1c0724..0fe5c1c0724 100755 --- a/node-admin/vespa-local/start-services.sh +++ b/node-admin/include/start-services.sh diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/docker/DockerOperationsImpl.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/docker/DockerOperationsImpl.java index 8edfb6c13f6..971fe235f70 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/docker/DockerOperationsImpl.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/docker/DockerOperationsImpl.java @@ -281,10 +281,11 @@ public class DockerOperationsImpl implements DockerOperations { @Override public void executeCommandInNetworkNamespace(ContainerName containerName, String[] command) { final PrefixLogger logger = PrefixLogger.getNodeAgentLogger(DockerOperationsImpl.class, containerName); - final Integer containerPid = getContainer(containerName.asString()) + final Integer containerPid = docker.getContainer(containerName) .filter(container -> container.state.isRunning()) .map(container -> container.pid) - .orElseThrow(() -> new RuntimeException("PID not found for container: " + containerName.asString())); + .orElseThrow(() -> new RuntimeException("PID not found for container with name: " + + containerName.asString())); final List<String> wrappedCommand = new LinkedList<>(); wrappedCommand.add("sudo"); diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/docker/DockerOperationsImplTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/docker/DockerOperationsImplTest.java index 5eaeb927c53..8d867a16c05 100644 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/docker/DockerOperationsImplTest.java +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/docker/DockerOperationsImplTest.java @@ -123,10 +123,10 @@ public class DockerOperationsImplTest { "-nvL"), capturedArgs); } - private Container makeContainer(String hostname, Container.State state, int pid) { - final Container container = new Container(hostname, new DockerImage("mock"), - new ContainerName(hostname), state, pid); - when(dockerOperations.getContainer(eq(hostname))).thenReturn(Optional.of(container)); + private Container makeContainer(String name, Container.State state, int pid) { + final Container container = new Container(name + ".fqdn", new DockerImage("mock"), + new ContainerName(name), state, pid); + when(docker.getContainer(eq(container.name))).thenReturn(Optional.of(container)); return container; } } diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/docker/LocalZoneUtils.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/docker/LocalZoneUtils.java index 1153b38d361..65ef3c4f3bf 100644 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/docker/LocalZoneUtils.java +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/docker/LocalZoneUtils.java @@ -123,9 +123,9 @@ public class LocalZoneUtils { } public static void buildVespaLocalDockerImage(Docker docker, DockerImage vespaBaseImage) throws IOException { - Path dockerfilePath = PROJECT_ROOT.resolve("node-admin/vespa-local/Dockerfile"); + Path dockerfilePath = PROJECT_ROOT.resolve("node-admin/Dockerfile"); - Path dockerfileTemplatePath = Paths.get("node-admin/vespa-local/Dockerfile.template"); + Path dockerfileTemplatePath = Paths.get("node-admin/Dockerfile.template"); String dockerfileTemplate = new String(Files.readAllBytes(dockerfileTemplatePath)) .replaceAll("\\$NODE_ADMIN_FROM_IMAGE", vespaBaseImage.asString()) .replaceAll("\\$VESPA_HOME", Defaults.getDefaults().vespaHome()); diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/DockerMock.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/DockerMock.java index ac0e24d8c71..75c25b3b93e 100644 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/DockerMock.java +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/DockerMock.java @@ -104,6 +104,13 @@ public class DockerMock implements Docker { } @Override + public Optional<Container> getContainer(ContainerName containerName) { + synchronized (monitor) { + return containers.stream().filter(container -> container.name.equals(containerName)).findFirst(); + } + } + + @Override public CompletableFuture<DockerImage> pullImageAsync(DockerImage image) { synchronized (monitor) { callOrderVerifier.add("pullImageAsync with " + image); diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeFailer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeFailer.java index 61381073334..df373902eb6 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeFailer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeFailer.java @@ -73,9 +73,8 @@ public class NodeFailer extends Maintainer { // Ready nodes updateNodeLivenessEventsForReadyNodes(); for (Node node : readyNodesWhichAreDead( )) { - // ready Docker hosts and nodes do not run Vespa services, so skip those - if (node.flavor().getType() == Flavor.Type.DOCKER_CONTAINER || node.type() == NodeType.host) - continue; + // Docker hosts and nodes do not run Vespa services + if (node.flavor().getType() == Flavor.Type.DOCKER_CONTAINER || node.type() == NodeType.host) continue; nodeRepository().fail(node.hostname()); } for (Node node : readyNodesWithHardwareFailure()) diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/filter/ApplicationFilter.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/filter/ApplicationFilter.java index b728b862686..f17fb3ff8ca 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/filter/ApplicationFilter.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/filter/ApplicationFilter.java @@ -8,6 +8,7 @@ import com.yahoo.config.provision.ClusterMembership; import com.yahoo.config.provision.HostFilter; import com.yahoo.config.provision.InstanceName; import com.yahoo.config.provision.TenantName; +import com.yahoo.text.StringUtilities; import com.yahoo.vespa.hosted.provision.Node; import java.util.Collections; @@ -47,7 +48,7 @@ public class ApplicationFilter extends NodeFilter { } public static ApplicationFilter from(String applicationIds, NodeFilter next) { - return new ApplicationFilter(HostFilter.split(applicationIds).stream().map(ApplicationFilter::toApplicationId).collect(Collectors.toSet()), next); + return new ApplicationFilter(StringUtilities.split(applicationIds).stream().map(ApplicationFilter::toApplicationId).collect(Collectors.toSet()), next); } public static ApplicationId toApplicationId(String applicationIdString) { diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/filter/NodeTypeFilter.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/filter/NodeTypeFilter.java index 65fd9dd9d4a..de091aebb68 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/filter/NodeTypeFilter.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/filter/NodeTypeFilter.java @@ -3,6 +3,7 @@ package com.yahoo.vespa.hosted.provision.node.filter; import com.google.common.collect.ImmutableSet; import com.yahoo.config.provision.HostFilter; import com.yahoo.config.provision.NodeType; +import com.yahoo.text.StringUtilities; import com.yahoo.vespa.hosted.provision.Node; import java.util.Collections; @@ -36,7 +37,7 @@ public class NodeTypeFilter extends NodeFilter { /** Returns a node filter which matches a comma or space-separated list of types */ public static NodeTypeFilter from(String types, NodeFilter next) { - return new NodeTypeFilter(HostFilter.split(types).stream().map(NodeType::valueOf).collect(Collectors.toSet()), next); + return new NodeTypeFilter(StringUtilities.split(types).stream().map(NodeType::valueOf).collect(Collectors.toSet()), next); } } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/filter/ParentHostFilter.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/filter/ParentHostFilter.java index edf0463835a..2eaf4c4ad39 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/filter/ParentHostFilter.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/filter/ParentHostFilter.java @@ -3,6 +3,7 @@ package com.yahoo.vespa.hosted.provision.node.filter; import com.google.common.collect.ImmutableSet; import com.yahoo.config.provision.HostFilter; +import com.yahoo.text.StringUtilities; import com.yahoo.vespa.hosted.provision.Node; import java.util.Objects; @@ -35,7 +36,7 @@ public class ParentHostFilter extends NodeFilter { /** Returns a copy of the given filter which only matches for the given parent */ public static ParentHostFilter from(String parentNames, NodeFilter filter) { - return new ParentHostFilter(HostFilter.split(parentNames).stream().collect(Collectors.toSet()), filter); + return new ParentHostFilter(StringUtilities.split(parentNames).stream().collect(Collectors.toSet()), filter); } } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/filter/StateFilter.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/filter/StateFilter.java index fa5972e59e1..a3468cf3dfd 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/filter/StateFilter.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/filter/StateFilter.java @@ -8,6 +8,7 @@ import com.yahoo.config.provision.ClusterMembership; import com.yahoo.config.provision.HostFilter; import com.yahoo.config.provision.InstanceName; import com.yahoo.config.provision.TenantName; +import com.yahoo.text.StringUtilities; import com.yahoo.vespa.hosted.provision.Node; import java.util.Collections; @@ -45,7 +46,7 @@ public class StateFilter extends NodeFilter { /** Returns a node filter which matches a comma or space-separated list of states */ public static StateFilter from(String states, NodeFilter next) { - return new StateFilter(HostFilter.split(states).stream().map(Node.State::valueOf).collect(Collectors.toSet()), next); + return new StateFilter(StringUtilities.split(states).stream().map(Node.State::valueOf).collect(Collectors.toSet()), next); } } @@ -1149,7 +1149,6 @@ <module>metrics</module> <module>node-repository</module> <module>node-admin</module> - <module>node-admin-maintenance</module> <module>orchestrator-restapi</module> <module>orchestrator</module> <module>persistence</module> diff --git a/searchcore/src/vespa/searchcore/fdispatch/common/timestat.h b/searchcore/src/vespa/searchcore/fdispatch/common/timestat.h index 66cab230589..ab22fd18e61 100644 --- a/searchcore/src/vespa/searchcore/fdispatch/common/timestat.h +++ b/searchcore/src/vespa/searchcore/fdispatch/common/timestat.h @@ -1,9 +1,9 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -// Copyright (C) 2003 Fast Search & Transfer ASA -// Copyright (C) 2003 Overture Services Norway AS + #pragma once +#include <cstdint> class FastS_TimeStatTotals { diff --git a/searchcore/src/vespa/searchcore/fdispatch/program/searchadapter.cpp b/searchcore/src/vespa/searchcore/fdispatch/program/searchadapter.cpp index 317e357d96a..94f9240e83f 100644 --- a/searchcore/src/vespa/searchcore/fdispatch/program/searchadapter.cpp +++ b/searchcore/src/vespa/searchcore/fdispatch/program/searchadapter.cpp @@ -1,12 +1,6 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> -#include <vespa/log/log.h> -LOG_SETUP(".fdispatch.searchadapter"); #include "searchadapter.h" -#include <vespa/searchlib/common/packets.h> -#include <vespa/searchlib/fef/queryproperties.h> -#include <vespa/searchlib/parsequery/simplequerystack.h> #include <vespa/searchcore/fdispatch/search/datasetcollection.h> #include <vespa/searchcore/fdispatch/search/dataset_base.h> #include <vespa/searchcore/fdispatch/search/nodemanager.h> @@ -60,6 +54,8 @@ SearchAdapter::createReply() } r.coverage = SearchReply::Coverage(_searchInfo->_activeDocs, _searchInfo->_coverageDocs); + r.coverage.setSoonActive(_searchInfo->_soonActiveDocs); + r.coverage.setDegradeReason(_searchInfo->_degradeReason); FastS_hitresult *hitbuf = _queryResult->_hitbuf; r.hits.resize(hitcnt); diff --git a/searchcore/src/vespa/searchcore/fdispatch/search/dataset_base.cpp b/searchcore/src/vespa/searchcore/fdispatch/search/dataset_base.cpp index 4bc535a3f29..34c8e75c5c0 100644 --- a/searchcore/src/vespa/searchcore/fdispatch/search/dataset_base.cpp +++ b/searchcore/src/vespa/searchcore/fdispatch/search/dataset_base.cpp @@ -1,21 +1,10 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -// Copyright (C) 1998-2003 Fast Search & Transfer ASA -// Copyright (C) 2003 Overture Services Norway AS - -#include <vespa/fastos/fastos.h> -#include <vespa/log/log.h> -LOG_SETUP(".search.dataset_base"); -#include <vespa/searchlib/common/fslimits.h> -#include <vespa/searchcore/util/log.h> -#include <vespa/searchcore/fdispatch/search/configdesc.h> -#include <vespa/searchcore/fdispatch/common/search.h> -#include <vespa/vespalib/util/atomic.h> -#include <vespa/searchcore/fdispatch/common/queryperf.h> - -#include <vespa/searchcore/fdispatch/search/datasetcollection.h> -#include <vespa/searchcore/fdispatch/search/engine_base.h> -#include <vespa/searchcore/fdispatch/search/dataset_base.h> -#include <vespa/searchcore/fdispatch/search/nodemanager.h> + +#include "dataset_base.h" +#include "configdesc.h" +#include "datasetcollection.h" +#include "engine_base.h" +#include "nodemanager.h" //-------------------------------------------------------------------------- diff --git a/searchcore/src/vespa/searchcore/fdispatch/search/dataset_base.h b/searchcore/src/vespa/searchcore/fdispatch/search/dataset_base.h index 425712cf1c7..13201210b79 100644 --- a/searchcore/src/vespa/searchcore/fdispatch/search/dataset_base.h +++ b/searchcore/src/vespa/searchcore/fdispatch/search/dataset_base.h @@ -8,6 +8,8 @@ #include <vespa/searchcore/fdispatch/common/timestat.h> #include <vespa/searchcore/util/log.h> #include <atomic> +#include <vespa/fastos/time.h> +#include <vespa/fastos/cond.h> class FastS_TimeKeeper; diff --git a/searchcore/src/vespa/searchcore/fdispatch/search/datasetcollection.cpp b/searchcore/src/vespa/searchcore/fdispatch/search/datasetcollection.cpp index baa324bae0b..af296cadf1c 100644 --- a/searchcore/src/vespa/searchcore/fdispatch/search/datasetcollection.cpp +++ b/searchcore/src/vespa/searchcore/fdispatch/search/datasetcollection.cpp @@ -1,20 +1,11 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -// Copyright (C) 1998-2003 Fast Search & Transfer ASA -// Copyright (C) 2003 Overture Services Norway AS -#include <vespa/fastos/fastos.h> -#include <vespa/log/log.h> -#include <vespa/fnet/fnet.h> -#include <vespa/fastlib/io/bufferedfile.h> - -#include <vespa/searchcore/util/log.h> -#include <vespa/searchlib/common/fslimits.h> +#include "datasetcollection.h" +#include "fnet_dataset.h" #include <vespa/searchcore/fdispatch/common/search.h> -#include <vespa/searchlib/parsequery/simplequerystack.h> -#include <vespa/searchcore/fdispatch/search/fnet_dataset.h> -#include <vespa/searchcore/fdispatch/search/datasetcollection.h> - +#include <vespa/fnet/fnet.h> +#include <vespa/log/log.h> LOG_SETUP(".search.datasetcollection"); FastS_DataSetBase * diff --git a/searchcore/src/vespa/searchcore/fdispatch/search/engine_base.cpp b/searchcore/src/vespa/searchcore/fdispatch/search/engine_base.cpp index 718d270ccd4..8332b42b1df 100644 --- a/searchcore/src/vespa/searchcore/fdispatch/search/engine_base.cpp +++ b/searchcore/src/vespa/searchcore/fdispatch/search/engine_base.cpp @@ -1,17 +1,11 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -// Copyright (C) 1998-2003 Fast Search & Transfer ASA -// Copyright (C) 2003 Overture Services Norway AS -#include <vespa/fastos/fastos.h> +#include "engine_base.h" +#include "configdesc.h" +#include "plain_dataset.h" + #include <vespa/log/log.h> LOG_SETUP(".search.engine_base"); -#include <vespa/searchcore/util/log.h> -#include <vespa/vespalib/util/atomic.h> - -#include <vespa/searchcore/fdispatch/search/configdesc.h> -#include <vespa/searchcore/fdispatch/search/plain_dataset.h> -#include <vespa/searchcore/fdispatch/search/engine_base.h> -#include <vespa/searchcore/fdispatch/common/appcontext.h> //--------------------------------------------------------------------------- diff --git a/searchcore/src/vespa/searchcore/fdispatch/search/poss_count.h b/searchcore/src/vespa/searchcore/fdispatch/search/poss_count.h index 0ba99117d4f..d7b0b565ebd 100644 --- a/searchcore/src/vespa/searchcore/fdispatch/search/poss_count.h +++ b/searchcore/src/vespa/searchcore/fdispatch/search/poss_count.h @@ -2,6 +2,7 @@ #pragma once +#include <cstdint> struct PossCount { bool valid; diff --git a/searchlib/src/tests/engine/transportserver/transportserver_test.cpp b/searchlib/src/tests/engine/transportserver/transportserver_test.cpp index 8daf6fd6f98..9a5e4f85d88 100644 --- a/searchlib/src/tests/engine/transportserver/transportserver_test.cpp +++ b/searchlib/src/tests/engine/transportserver/transportserver_test.cpp @@ -210,6 +210,7 @@ TEST("test SearchReply::Coverage set and get") { EXPECT_EQUAL(7u, c.setActive(7).getActive()); EXPECT_EQUAL(9u, c.setSoonActive(9).getSoonActive()); EXPECT_EQUAL(19u, c.setCovered(19).getCovered()); + EXPECT_EQUAL(5u, c.setDegradeReason(5).getDegradeReason()); EXPECT_EQUAL(1u, SearchReply::Coverage().degradeMatchPhase().getDegradeReason()); EXPECT_EQUAL(2u, SearchReply::Coverage().degradeTimeout().getDegradeReason()); EXPECT_EQUAL(4u, SearchReply::Coverage().degradeAdaptiveTimeout().getDegradeReason()); diff --git a/searchlib/src/vespa/searchlib/engine/searchreply.h b/searchlib/src/vespa/searchlib/engine/searchreply.h index d01e677d218..b10fde5c9da 100644 --- a/searchlib/src/vespa/searchlib/engine/searchreply.h +++ b/searchlib/src/vespa/searchlib/engine/searchreply.h @@ -38,10 +38,11 @@ public: uint64_t getCovered() const { return _covered; } uint64_t getActive() const { return _active; } uint64_t getSoonActive() const { return _soonActive; } - uint64_t getDegradeReason() const { return _degradeReason; } + uint32_t getDegradeReason() const { return _degradeReason; } Coverage & setCovered(uint64_t v) { _covered = v; return *this; } Coverage & setActive(uint64_t v) { _active = v; return *this; } Coverage & setSoonActive(uint64_t v) { _soonActive = v; return *this; } + Coverage & setDegradeReason(uint32_t v) { _degradeReason = v; return *this; } Coverage & degradeMatchPhase() { _degradeReason |= MATCH_PHASE; return *this; } Coverage & degradeTimeout() { _degradeReason |= TIMEOUT; return *this; } Coverage & degradeAdaptiveTimeout() { _degradeReason |= ADAPTIVE_TIMEOUT; return *this; } diff --git a/vespajlib/src/main/java/com/yahoo/text/StringUtilities.java b/vespajlib/src/main/java/com/yahoo/text/StringUtilities.java index d65681e8f5b..975e2955d3a 100644 --- a/vespajlib/src/main/java/com/yahoo/text/StringUtilities.java +++ b/vespajlib/src/main/java/com/yahoo/text/StringUtilities.java @@ -1,18 +1,23 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.text; +import com.google.common.collect.ImmutableSet; + import java.nio.charset.Charset; +import java.util.Collections; import java.util.List; import java.io.ByteArrayOutputStream; +import java.util.Set; /** * Escapes strings into and out of a format where they only contain printable characters. * * Need to duplicate escape / unescape of strings as we have in C++ for java version of system states. * - * @author <a href="mailto:humbe@yahoo-inc.com">Haakon Humberset</a> + * @author Haakon Humberset */ public class StringUtilities { + private static Charset UTF8 = Charset.forName("utf8"); private static byte toHex(int val) { return (byte) (val < 10 ? '0' + val : 'a' + (val - 10)); } @@ -201,4 +206,15 @@ public class StringUtilities { public static String quote(Object object) { return "'" + object.toString() + "'"; } + + /** Splits a string on both space and comma */ + public static Set<String> split(String s) { + if (s == null || s.isEmpty()) return Collections.emptySet(); + ImmutableSet.Builder<String> b = new ImmutableSet.Builder<>(); + for (String item : s.split("[\\s\\,]")) + if ( ! item.isEmpty()) + b.add(item); + return b.build(); + } + } diff --git a/zkfacade/src/main/java/com/yahoo/vespa/zookeeper/RestrictedServerCnxnFactory.java b/zkfacade/src/main/java/com/yahoo/vespa/zookeeper/RestrictedServerCnxnFactory.java index bababa9a25c..972cf577ab3 100644 --- a/zkfacade/src/main/java/com/yahoo/vespa/zookeeper/RestrictedServerCnxnFactory.java +++ b/zkfacade/src/main/java/com/yahoo/vespa/zookeeper/RestrictedServerCnxnFactory.java @@ -2,6 +2,7 @@ package com.yahoo.vespa.zookeeper; import com.google.common.collect.ImmutableSet; import com.yahoo.net.HostName; +import com.yahoo.text.StringUtilities; import org.apache.zookeeper.server.NIOServerCnxn; import org.apache.zookeeper.server.NIOServerCnxnFactory; @@ -58,7 +59,9 @@ public class RestrictedServerCnxnFactory extends NIOServerCnxnFactory { /** Returns the allowed client host names. If the list is empty any host is allowed. */ private ImmutableSet<String> findAllowedZooKeeperClients() { - // Environment has precedence. Note that this allows setting restrict to "" to turn off client restriction + // Environment has precedence. Note that + // - if this is set to "", client restriction is disabled + // - this environment variable is a public API - do not change String environmentAllowedZooKeeperClients = System.getenv("vespa_zkfacade__restrict"); if (environmentAllowedZooKeeperClients != null) return ImmutableSet.copyOf(toHostnameSet(environmentAllowedZooKeeperClients)); @@ -67,9 +70,9 @@ public class RestrictedServerCnxnFactory extends NIOServerCnxnFactory { return ZooKeeperServer.getAllowedClientHostnames(); } - private Set<String> toHostnameSet(String commaSeparatedString) { + private Set<String> toHostnameSet(String hosatnamesString) { Set<String> hostnames = new HashSet<>(); - for (String hostname : commaSeparatedString.split(",")) { + for (String hostname : StringUtilities.split(hosatnamesString)) { if ( ! hostname.trim().isEmpty()) hostnames.add(hostname.trim()); } |