diff options
38 files changed, 499 insertions, 142 deletions
diff --git a/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/instanceconfirmation/InstanceConfirmationResource.java b/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/instanceconfirmation/InstanceConfirmationResource.java index 4c71cb7855d..7a4e82d134b 100644 --- a/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/instanceconfirmation/InstanceConfirmationResource.java +++ b/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/instanceconfirmation/InstanceConfirmationResource.java @@ -16,7 +16,7 @@ import java.util.logging.Logger; /** * @author bjorncs */ -@Path("/{path: instance|refresh}") +@Path("/instance") public class InstanceConfirmationResource { private static final Logger log = Logger.getLogger(InstanceConfirmationResource.class.getName()); @@ -32,6 +32,7 @@ public class InstanceConfirmationResource { @Consumes(MediaType.APPLICATION_JSON) @Produces(MediaType.APPLICATION_JSON) public InstanceConfirmation confirmInstance(InstanceConfirmation instanceConfirmation) { + log.log(LogLevel.DEBUG, instanceConfirmation.toString()); if (!instanceValidator.isValidInstance(instanceConfirmation)) { log.log(LogLevel.ERROR, "Invalid instance: " + instanceConfirmation); throw new ForbiddenException("Instance is invalid"); diff --git a/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/instanceconfirmation/InstanceRefreshResource.java b/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/instanceconfirmation/InstanceRefreshResource.java new file mode 100644 index 00000000000..53972a39c97 --- /dev/null +++ b/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/instanceconfirmation/InstanceRefreshResource.java @@ -0,0 +1,44 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.athenz.instanceproviderservice.instanceconfirmation; + +import com.google.inject.Inject; +import com.yahoo.container.jaxrs.annotation.Component; +import com.yahoo.log.LogLevel; + +import javax.ws.rs.Consumes; +import javax.ws.rs.ForbiddenException; +import javax.ws.rs.POST; +import javax.ws.rs.Path; +import javax.ws.rs.Produces; +import javax.ws.rs.core.MediaType; +import java.util.logging.Logger; + +/** + * ZTS calls this resource when it's requested to refresh an instance certificate + * + * @author bjorncs + */ +@Path("/refresh") +public class InstanceRefreshResource { + + private static final Logger log = Logger.getLogger(InstanceRefreshResource.class.getName()); + + private final InstanceValidator instanceValidator; + + @Inject + public InstanceRefreshResource(@Component InstanceValidator instanceValidator) { + this.instanceValidator = instanceValidator; + } + + @POST + @Consumes(MediaType.APPLICATION_JSON) + @Produces(MediaType.APPLICATION_JSON) + public InstanceConfirmation confirmInstanceRefresh(InstanceConfirmation instanceConfirmation) { + log.log(LogLevel.DEBUG, instanceConfirmation.toString()); + if (!instanceValidator.isValidRefresh(instanceConfirmation)) { + log.log(LogLevel.ERROR, "Invalid instance refresh: " + instanceConfirmation); + throw new ForbiddenException("Instance is invalid"); + } + return instanceConfirmation; + } +} diff --git a/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/instanceconfirmation/InstanceValidator.java b/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/instanceconfirmation/InstanceValidator.java index b75f7d05394..dcaf50c1c04 100644 --- a/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/instanceconfirmation/InstanceValidator.java +++ b/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/instanceconfirmation/InstanceValidator.java @@ -61,6 +61,17 @@ public class InstanceValidator { return false; } + // TODO Add actual validation. Cannot reuse isValidInstance as identity document is not part of the refresh request. + // We'll have to perform some validation on the instance id and other fields of the attribute map. + // Separate between tenant and node certificate as well. + public boolean isValidRefresh(InstanceConfirmation confirmation) { + log.log(LogLevel.INFO, () -> String.format("Accepting refresh for instance with identity '%s', provider '%s', instanceId '%s'.", + new AthenzService(confirmation.domain, confirmation.service).getFullName(), + confirmation.provider, + confirmation.attributes.get("sanDNS").toString())); + return true; + } + // If/when we dont care about logging exactly whats wrong, this can be simplified // TODO Use identity type to determine if this check should be performed boolean isSameIdentityAsInServicesXml(ApplicationId applicationId, String domain, String service) { diff --git a/configdefinitions/src/vespa/configserver.def b/configdefinitions/src/vespa/configserver.def index 6833c00da8e..daaf8906d24 100644 --- a/configdefinitions/src/vespa/configserver.def +++ b/configdefinitions/src/vespa/configserver.def @@ -67,3 +67,4 @@ sleepTimeWhenRedeployingFails long default=30 # Feature Flags (poor man's feature flags, to be overridden in configserver-config.xml if needed) deleteApplicationLegacy bool default=false +buildMinimalSetOfConfigModels bool default=true diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java b/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java index 935c9faec9b..4380cf5f047 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java @@ -15,8 +15,10 @@ import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.Environment; import com.yahoo.config.provision.HostFilter; import com.yahoo.config.provision.Provisioner; +import com.yahoo.config.provision.RegionName; import com.yahoo.config.provision.SystemName; import com.yahoo.config.provision.TenantName; +import com.yahoo.config.provision.Zone; import com.yahoo.container.jdisc.HttpResponse; import com.yahoo.io.IOUtils; import com.yahoo.log.LogLevel; @@ -290,10 +292,16 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye * @throws RuntimeException if the delete transaction fails. This method is exception safe. */ public boolean delete(ApplicationId applicationId) { + Zone zone = new Zone(Environment.from(configserverConfig.environment()), RegionName.from(configserverConfig.region())); + List<String> hostedZonesToUseDeleteApplication = + Arrays.asList("dev.us-east-1", "dev.corp-us-east-1", "test.us-east-1", + "prod.corp-us-east-1", "prod.aws-us-east-1a", "prod.aws-us-west-1b"); + boolean useDeleteApplicationLegacyInThisZone = !hostedZonesToUseDeleteApplication.contains(zone.toString()); + // TODO: Use deleteApplication() in all zones if (configserverConfig.deleteApplicationLegacy() || - (configserverConfig.hostedVespa() && SystemName.from(configserverConfig.system()) == SystemName.main - && !Arrays.asList("corp-us-east-1", "aws-us-east-1a").contains(configserverConfig.region()))) { + (configserverConfig.hostedVespa() && SystemName.from(configserverConfig.system()) == SystemName.main && + useDeleteApplicationLegacyInThisZone)) { return deleteApplicationLegacy(applicationId); } else { return deleteApplication(applicationId); @@ -306,7 +314,7 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye * @return true if the application was found and deleted, false if it was not present * @throws RuntimeException if the delete transaction fails. This method is exception safe. */ - public boolean deleteApplication(ApplicationId applicationId) { + boolean deleteApplication(ApplicationId applicationId) { Tenant tenant = tenantRepository.getTenant(applicationId.tenant()); if (tenant == null) return false; @@ -553,6 +561,9 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye return session.getSessionId(); } + public void deleteOldSessions() { + listApplications().forEach(app -> tenantRepository.getTenant(app.tenant()).getLocalSessionRepo().purgeOldSessions()); + } // ---------------- Tenant operations ---------------------------------------------------------------- diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/ConfigServerBootstrap.java b/configserver/src/main/java/com/yahoo/vespa/config/server/ConfigServerBootstrap.java index 3584e614071..50052ac856c 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/ConfigServerBootstrap.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/ConfigServerBootstrap.java @@ -28,7 +28,6 @@ import java.util.concurrent.Executors; public class ConfigServerBootstrap extends AbstractComponent implements Runnable { private static final java.util.logging.Logger log = java.util.logging.Logger.getLogger(ConfigServerBootstrap.class.getName()); - private static final ExecutorService rpcServerExecutor = Executors.newSingleThreadExecutor(new DaemonThreadFactory("config server RPC server")); private static final String vipStatusClusterIdentifier = "configserver"; enum MainThread {START, DO_NOT_START} @@ -43,6 +42,7 @@ public class ConfigServerBootstrap extends AbstractComponent implements Runnable private final Duration maxDurationOfRedeployment; private final Duration sleepTimeWhenRedeployingFails; private final RedeployingApplicationsFails exitIfRedeployingApplicationsFails; + private final ExecutorService rpcServerExecutor; // The tenants object is injected so that all initial requests handlers are // added to the rpc server before it starts answering rpc requests. @@ -66,6 +66,7 @@ public class ConfigServerBootstrap extends AbstractComponent implements Runnable this.maxDurationOfRedeployment = Duration.ofSeconds(applicationRepository.configserverConfig().maxDurationOfBootstrap()); this.sleepTimeWhenRedeployingFails = Duration.ofSeconds(applicationRepository.configserverConfig().sleepTimeWhenRedeployingFails()); this.exitIfRedeployingApplicationsFails = exitIfRedeployingApplicationsFails; + rpcServerExecutor = Executors.newSingleThreadExecutor(new DaemonThreadFactory("config server RPC server")); initializing(); // Initially take server out of rotation if (mainThread == MainThread.START) start(); diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/ConfigServerMaintenance.java b/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/ConfigServerMaintenance.java index 1f8e8ff6fe8..c1fc484e23c 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/ConfigServerMaintenance.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/ConfigServerMaintenance.java @@ -10,11 +10,19 @@ import com.yahoo.vespa.curator.Curator; import java.time.Duration; +/** + * Maintenance jobs of the config server. + * Each maintenance job is a singleton instance of its implementing class, created and owned by this, + * and running its own dedicated thread. + * + * @author hmusum + */ public class ConfigServerMaintenance extends AbstractComponent { private final TenantsMaintainer tenantsMaintainer; private final ZooKeeperDataMaintainer zooKeeperDataMaintainer; private final FileDistributionMaintainer fileDistributionMaintainer; + private final SessionsMaintainer sessionsMaintainer; @SuppressWarnings("unused") // instantiated by Dependency Injection public ConfigServerMaintenance(ConfigserverConfig configserverConfig, @@ -25,6 +33,7 @@ public class ConfigServerMaintenance extends AbstractComponent { tenantsMaintainer = new TenantsMaintainer(applicationRepository, curator, defaults.tenantsMaintainerInterval); zooKeeperDataMaintainer = new ZooKeeperDataMaintainer(applicationRepository, curator, defaults.defaultInterval); fileDistributionMaintainer = new FileDistributionMaintainer(applicationRepository, curator, defaults.defaultInterval, configserverConfig); + sessionsMaintainer = new SessionsMaintainer(applicationRepository, curator, defaults.defaultInterval); } @Override @@ -32,6 +41,7 @@ public class ConfigServerMaintenance extends AbstractComponent { tenantsMaintainer.deconstruct(); zooKeeperDataMaintainer.deconstruct(); fileDistributionMaintainer.deconstruct(); + sessionsMaintainer.deconstruct(); } /* diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/FileDistributionMaintainer.java b/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/FileDistributionMaintainer.java index 4f45b200111..a5d4aa226a6 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/FileDistributionMaintainer.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/FileDistributionMaintainer.java @@ -9,7 +9,13 @@ import com.yahoo.vespa.defaults.Defaults; import java.io.File; import java.time.Duration; -// Note: Unit test is in ApplicationRepositoryTest +/** + * Removes unused file references from disk + * <p> + * Note: Unit test is in ApplicationRepositoryTest + * + * @author hmusum + */ public class FileDistributionMaintainer extends Maintainer { private final ApplicationRepository applicationRepository; diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/Maintainer.java b/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/Maintainer.java index ae000385dfd..513f70e4187 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/Maintainer.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/Maintainer.java @@ -16,6 +16,11 @@ import java.util.concurrent.TimeUnit; import java.util.logging.Level; import java.util.logging.Logger; +/** + * A maintainer is some job which runs at a fixed interval to perform some maintenance task in the config server. + * + * @author hmusum + */ public abstract class Maintainer extends AbstractComponent implements Runnable { protected static final Logger log = Logger.getLogger(Maintainer.class.getName()); diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/SessionsMaintainer.java b/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/SessionsMaintainer.java new file mode 100644 index 00000000000..53f2553161f --- /dev/null +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/SessionsMaintainer.java @@ -0,0 +1,26 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config.server.maintenance; + +import com.yahoo.vespa.config.server.ApplicationRepository; +import com.yahoo.vespa.curator.Curator; + +import java.time.Duration; + +/** + * Removes inactive sessions + * <p> + * Note: Unit test is in ApplicationRepositoryTest + * + * @author hmusum + */ +public class SessionsMaintainer extends Maintainer { + + SessionsMaintainer(ApplicationRepository applicationRepository, Curator curator, Duration interval) { + super(applicationRepository, curator, interval); + } + + @Override + protected void maintain() { + applicationRepository.deleteOldSessions(); + } +} diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/TenantsMaintainer.java b/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/TenantsMaintainer.java index ad3b1f9b1ea..86bd4e8db8e 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/TenantsMaintainer.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/TenantsMaintainer.java @@ -7,6 +7,11 @@ import com.yahoo.vespa.curator.Curator; import java.time.Duration; import java.time.Instant; +/** + * Removes unused tenants (has no applications and was created more than 7 days ago) + * + * @author hmusum + */ public class TenantsMaintainer extends Maintainer { private final Duration ttlForUnusedTenant; @@ -21,7 +26,6 @@ public class TenantsMaintainer extends Maintainer { } @Override - // Delete unused tenants that were created more than ttlForUnusedTenant ago protected void maintain() { applicationRepository.deleteUnusedTenants(ttlForUnusedTenant, Instant.now()); } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/ZooKeeperDataMaintainer.java b/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/ZooKeeperDataMaintainer.java index c906423d991..625665312fd 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/ZooKeeperDataMaintainer.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/ZooKeeperDataMaintainer.java @@ -1,5 +1,5 @@ // Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - package com.yahoo.vespa.config.server.maintenance; +package com.yahoo.vespa.config.server.maintenance; import com.yahoo.path.Path; import com.yahoo.vespa.config.server.ApplicationRepository; @@ -9,6 +9,8 @@ import java.time.Duration; /** * Removes unused zookeeper data (for now only data used by old file distribution code is removed) + * + * @author hmusum */ public class ZooKeeperDataMaintainer extends Maintainer { diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/modelfactory/ActivatedModelsBuilder.java b/configserver/src/main/java/com/yahoo/vespa/config/server/modelfactory/ActivatedModelsBuilder.java index 2955d948d69..c18d3c7fe48 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/modelfactory/ActivatedModelsBuilder.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/modelfactory/ActivatedModelsBuilder.java @@ -46,21 +46,19 @@ public class ActivatedModelsBuilder extends ModelsBuilder<Application> { private final long appGeneration; private final SessionZooKeeperClient zkClient; private final PermanentApplicationPackage permanentApplicationPackage; - private final ConfigserverConfig configserverConfig; private final ConfigDefinitionRepo configDefinitionRepo; private final Metrics metrics; private final Curator curator; private final DeployLogger logger; public ActivatedModelsBuilder(TenantName tenant, long appGeneration, SessionZooKeeperClient zkClient, GlobalComponentRegistry globalComponentRegistry) { - super(globalComponentRegistry.getModelFactoryRegistry(), - globalComponentRegistry.getHostProvisioner().isPresent(), + super(globalComponentRegistry.getModelFactoryRegistry(), + globalComponentRegistry.getConfigserverConfig(), globalComponentRegistry.getZone()); this.tenant = tenant; this.appGeneration = appGeneration; this.zkClient = zkClient; this.permanentApplicationPackage = globalComponentRegistry.getPermanentApplicationPackage(); - this.configserverConfig = globalComponentRegistry.getConfigserverConfig(); this.configDefinitionRepo = globalComponentRegistry.getConfigDefinitionRepo(); this.metrics = globalComponentRegistry.getMetrics(); this.curator = globalComponentRegistry.getCurator(); diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/modelfactory/ModelsBuilder.java b/configserver/src/main/java/com/yahoo/vespa/config/server/modelfactory/ModelsBuilder.java index 090e7aac494..7bb77fdcbff 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/modelfactory/ModelsBuilder.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/modelfactory/ModelsBuilder.java @@ -2,6 +2,7 @@ package com.yahoo.vespa.config.server.modelfactory; import com.google.common.util.concurrent.UncheckedTimeoutException; +import com.yahoo.cloud.config.ConfigserverConfig; import com.yahoo.config.application.api.ApplicationPackage; import com.yahoo.config.model.api.HostProvisioner; import com.yahoo.config.model.api.ModelFactory; @@ -10,6 +11,7 @@ import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.ApplicationLockException; import com.yahoo.config.provision.Environment; import com.yahoo.config.provision.OutOfCapacityException; +import com.yahoo.config.provision.SystemName; import com.yahoo.config.provision.Version; import com.yahoo.config.provision.Zone; import com.yahoo.lang.SettableOptional; @@ -42,15 +44,17 @@ public abstract class ModelsBuilder<MODELRESULT extends ModelResult> { private static final Logger log = Logger.getLogger(ModelsBuilder.class.getName()); private final ModelFactoryRegistry modelFactoryRegistry; + protected final ConfigserverConfig configserverConfig; /** True if we are running in hosted mode */ private final boolean hosted; private final Zone zone; - protected ModelsBuilder(ModelFactoryRegistry modelFactoryRegistry, boolean hosted, Zone zone) { + protected ModelsBuilder(ModelFactoryRegistry modelFactoryRegistry, ConfigserverConfig configserverConfig, Zone zone) { this.modelFactoryRegistry = modelFactoryRegistry; - this.hosted = hosted; + this.configserverConfig = configserverConfig; + this.hosted = configserverConfig.hostedVespa(); this.zone = zone; } @@ -143,13 +147,16 @@ public abstract class ModelsBuilder<MODELRESULT extends ModelResult> { allApplicationVersions.add(latestModelVersion); // TODO: Enable for all zones - // Note: Cannot be enabled for prod zones yet, due to an issue with how AccessControlValidator works - if (Arrays.asList(Environment.dev, Environment.test, Environment.staging).contains(zone().environment())) + if (configserverConfig.buildMinimalSetOfConfigModels() && + (SystemName.from(configserverConfig.system()) == SystemName.cd || + Arrays.asList(Environment.dev, Environment.test, Environment.staging).contains(zone().environment()) || + Arrays.asList("corp-us-east-1", "ap-southeast-1").contains(zone().region().value()))) versions = keepThoseUsedOn(allocatedHosts.get(), versions); - // Make sure we build wanted version if we are building models for this major version + // Make sure we build wanted version if we are building models for this major version and we are on hosted vespa + // If not on hosted vespa, we do not want to try to build this version, since we have only one version (the latest) Version wantedVersion = Version.fromIntValues(wantedNodeVespaVersion.getMajor(), wantedNodeVespaVersion.getMinor(), wantedNodeVespaVersion.getMicro()); - if (wantedVersion.getMajor() == latest.getMajor()) + if (hosted && wantedVersion.getMajor() == latest.getMajor()) versions.add(wantedVersion); // TODO: We use the allocated hosts from the newest version when building older model versions. diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/modelfactory/PreparedModelsBuilder.java b/configserver/src/main/java/com/yahoo/vespa/config/server/modelfactory/PreparedModelsBuilder.java index ba093ca9db1..56bdd432d90 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/modelfactory/PreparedModelsBuilder.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/modelfactory/PreparedModelsBuilder.java @@ -1,6 +1,7 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.config.server.modelfactory; +import com.yahoo.cloud.config.ConfigserverConfig; import com.yahoo.config.application.api.ApplicationPackage; import com.yahoo.config.application.api.DeployLogger; import com.yahoo.config.model.api.ConfigChangeAction; @@ -64,8 +65,9 @@ public class PreparedModelsBuilder extends ModelsBuilder<PreparedModelsBuilder.P DeployLogger logger, PrepareParams params, Optional<ApplicationSet> currentActiveApplicationSet, - ModelContext.Properties properties) { - super(modelFactoryRegistry, properties.hostedVespa(), properties.zone()); + ModelContext.Properties properties, + ConfigserverConfig configserverConfig) { + super(modelFactoryRegistry, configserverConfig, properties.zone()); this.permanentApplicationPackage = permanentApplicationPackage; this.configDefinitionRepo = configDefinitionRepo; diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/session/LocalSessionRepo.java b/configserver/src/main/java/com/yahoo/vespa/config/server/session/LocalSessionRepo.java index dbb581eb16d..acfd81b33bf 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/session/LocalSessionRepo.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/session/LocalSessionRepo.java @@ -1,7 +1,6 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.config.server.session; -import com.yahoo.concurrent.ThreadFactoryFactory; import com.yahoo.log.LogLevel; import com.yahoo.path.Path; import com.yahoo.transaction.NestedTransaction; @@ -13,13 +12,10 @@ import com.yahoo.vespa.curator.Curator; import java.io.File; import java.io.FilenameFilter; import java.time.Clock; -import java.time.Duration; import java.util.ArrayList; import java.util.HashMap; import java.util.List; import java.util.Map; -import java.util.concurrent.ScheduledExecutorService; -import java.util.concurrent.ScheduledThreadPoolExecutor; import java.util.concurrent.TimeUnit; import java.util.logging.Logger; @@ -32,11 +28,7 @@ public class LocalSessionRepo extends SessionRepo<LocalSession> { private static final Logger log = Logger.getLogger(LocalSessionRepo.class.getName()); private static final FilenameFilter sessionApplicationsFilter = (dir, name) -> name.matches("\\d+"); - private static final Duration delay = Duration.ofMinutes(5); - // One executor for all instances of this class - private static final ScheduledExecutorService purgeOldSessionsExecutor = - new ScheduledThreadPoolExecutor(1, ThreadFactoryFactory.getDaemonThreadFactory("purge-old-sessions")); private final Map<Long, LocalSessionStateWatcher> sessionStateWatchers = new HashMap<>(); private final long sessionLifetime; // in seconds private final Clock clock; @@ -46,7 +38,6 @@ public class LocalSessionRepo extends SessionRepo<LocalSession> { Clock clock, long sessionLifeTime, Curator curator) { this(clock, curator, sessionLifeTime); loadSessions(tenantFileSystemDirs.sessionsPath(), loader); - purgeOldSessionsExecutor.scheduleWithFixedDelay(this::purgeOldSessions, delay.getSeconds(), delay.getSeconds(), TimeUnit.SECONDS); } // Constructor public only for testing @@ -85,7 +76,6 @@ public class LocalSessionRepo extends SessionRepo<LocalSession> { } } - // public for testing public void purgeOldSessions() { log.log(LogLevel.DEBUG, "Purging old sessions"); try { diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionPreparer.java b/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionPreparer.java index 6a240806004..49287669a06 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionPreparer.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionPreparer.java @@ -169,7 +169,8 @@ public class SessionPreparer { logger, params, currentActiveApplicationSet, - properties); + properties, + configserverConfig); } void checkTimeout(String step) { diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/ApplicationRepositoryTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/ApplicationRepositoryTest.java index 18e2525f61a..b0320dad88b 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/ApplicationRepositoryTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/ApplicationRepositoryTest.java @@ -1,11 +1,14 @@ // Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.config.server; +import com.google.common.io.Files; +import com.yahoo.cloud.config.ConfigserverConfig; import com.yahoo.component.Version; import com.yahoo.component.Vtag; import com.yahoo.config.model.application.provider.FilesApplicationPackage; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.ApplicationName; +import com.yahoo.config.provision.Deployment; import com.yahoo.config.provision.Environment; import com.yahoo.config.provision.InstanceName; import com.yahoo.config.provision.Provisioner; @@ -13,6 +16,7 @@ import com.yahoo.config.provision.TenantName; import com.yahoo.io.IOUtils; import com.yahoo.test.ManualClock; import com.yahoo.text.Utf8; +import com.yahoo.vespa.config.server.deploy.DeployTester; import com.yahoo.vespa.config.server.http.SessionHandlerTest; import com.yahoo.vespa.config.server.http.v2.PrepareResult; import com.yahoo.vespa.config.server.session.LocalSession; @@ -31,12 +35,16 @@ import java.io.IOException; import java.time.Clock; import java.time.Duration; import java.time.Instant; +import java.util.ArrayList; +import java.util.Collection; import java.util.Collections; +import java.util.Optional; import java.util.Set; import static org.hamcrest.CoreMatchers.is; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotEquals; import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertNull; import static org.junit.Assert.assertThat; @@ -234,6 +242,45 @@ public class ApplicationRepositoryTest { assertFalse(applicationRepository.deleteApplicationLegacy(applicationId())); } + @Test + public void testDeletingInactiveSessions() { + ManualClock clock = new ManualClock(Instant.now()); + ConfigserverConfig configserverConfig = new ConfigserverConfig(new ConfigserverConfig.Builder() + .configServerDBDir(Files.createTempDir() + .getAbsolutePath()) + .configDefinitionsDir(Files.createTempDir() + .getAbsolutePath()) + .sessionLifetime(60)); + DeployTester tester = new DeployTester(configserverConfig, clock); + tester.deployApp("src/test/apps/app", "myapp", Instant.now()); // session 2 (numbering starts at 2) + + clock.advance(Duration.ofSeconds(10)); + Optional<Deployment> deployment2 = tester.redeployFromLocalActive(); + + assertTrue(deployment2.isPresent()); + deployment2.get().activate(); // session 3 + long activeSessionId = tester.tenant().getApplicationRepo().getSessionIdForApplication(tester.applicationId()); + + clock.advance(Duration.ofSeconds(10)); + Optional<com.yahoo.config.provision.Deployment> deployment3 = tester.redeployFromLocalActive(); + assertTrue(deployment3.isPresent()); + deployment3.get().prepare(); // session 4 (not activated) + + LocalSession deployment3session = ((com.yahoo.vespa.config.server.deploy.Deployment) deployment3.get()).session(); + assertNotEquals(activeSessionId, deployment3session); + // No change to active session id + assertEquals(activeSessionId, tester.tenant().getApplicationRepo().getSessionIdForApplication(tester.applicationId())); + assertEquals(3, tester.tenant().getLocalSessionRepo().listSessions().size()); + + clock.advance(Duration.ofHours(1)); // longer than session lifetime + + // All sessions except 3 should be removed after the call to deleteOldSessions + tester.applicationRepository().deleteOldSessions(); + final Collection<LocalSession> sessions = tester.tenant().getLocalSessionRepo().listSessions(); + assertEquals(1, sessions.size()); + assertEquals(3, new ArrayList<>(sessions).get(0).getSessionId()); + } + private PrepareResult prepareAndActivateApp(File application) throws IOException { FilesApplicationPackage appDir = FilesApplicationPackage.fromFile(application); ApplicationId applicationId = applicationId(); diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/ConfigServerBootstrapTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/ConfigServerBootstrapTest.java index 74d8c508b26..293b8e1e8eb 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/ConfigServerBootstrapTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/ConfigServerBootstrapTest.java @@ -2,15 +2,27 @@ package com.yahoo.vespa.config.server; import com.yahoo.cloud.config.ConfigserverConfig; +import com.yahoo.config.model.api.ModelFactory; +import com.yahoo.config.model.provision.Host; +import com.yahoo.config.model.provision.Hosts; import com.yahoo.config.model.provision.InMemoryProvisioner; +import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.ClusterSpec; +import com.yahoo.config.provision.Environment; +import com.yahoo.config.provision.RegionName; +import com.yahoo.config.provision.Version; +import com.yahoo.config.provision.Zone; import com.yahoo.container.handler.VipStatus; import com.yahoo.container.jdisc.config.HealthMonitorConfig; import com.yahoo.container.jdisc.state.StateMonitor; import com.yahoo.jdisc.core.SystemTimer; +import com.yahoo.path.Path; +import com.yahoo.text.Utf8; import com.yahoo.vespa.config.server.deploy.DeployTester; import com.yahoo.vespa.config.server.rpc.RpcServer; import com.yahoo.vespa.config.server.version.VersionState; +import com.yahoo.vespa.curator.Curator; +import com.yahoo.vespa.curator.mock.MockCurator; import org.junit.Rule; import org.junit.Test; import org.junit.rules.TemporaryFolder; @@ -18,8 +30,13 @@ import org.junit.rules.TemporaryFolder; import java.io.File; import java.io.IOException; import java.nio.file.Paths; +import java.time.Clock; import java.time.Duration; import java.time.Instant; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.Optional; import java.util.function.BooleanSupplier; import static org.junit.Assert.assertEquals; @@ -97,6 +114,41 @@ public class ConfigServerBootstrapTest { assertEquals(StateMonitor.Status.initializing, bootstrap.status()); assertFalse(rpcServer.isRunning()); assertFalse(vipStatus.isInRotation()); + + bootstrap.deconstruct(); + } + + // Tests that we do not try to create the config model version stored in zookeeper when not on hosted vespa, since + // we are then only able to create the latest version + @Test + public void testBootstrapNonHostedOneConfigModel() throws Exception { + ConfigserverConfig configserverConfig = createConfigserverConfigNonHosted(temporaryFolder); + List<ModelFactory> modelFactories = Collections.singletonList(DeployTester.createModelFactory(Version.fromString("1.2.3"))); + List<Host> hosts = Arrays.asList(createHost("host1", "1.2.3"), createHost("host2", "1.2.3"), createHost("host3", "1.2.3")); + InMemoryProvisioner provisioner = new InMemoryProvisioner(new Hosts(hosts), true); + Curator curator = new MockCurator(); + DeployTester tester = new DeployTester(modelFactories, configserverConfig, + Clock.systemUTC(), new Zone(Environment.dev, RegionName.defaultName()), + provisioner, curator); + ApplicationId app = tester.deployApp("src/test/apps/app/", "myApp", "1.2.3", Instant.now()); + + File versionFile = temporaryFolder.newFile(); + VersionState versionState = new VersionState(versionFile); + assertTrue(versionState.isUpgraded()); + + // Ugly hack, but I see no other way of doing it: + // Manipulate application version in zookeeper so that it is an older version than the model we know, which is + // the case when upgrading on non-hosted installations + curator.set(Path.fromString("/config/v2/tenants/" + app.tenant().value() + "/sessions/2/version"), Utf8.toBytes("1.2.2")); + + RpcServer rpcServer = createRpcServer(configserverConfig); + VipStatus vipStatus = new VipStatus(); + ConfigServerBootstrap bootstrap = new ConfigServerBootstrap(tester.applicationRepository(), rpcServer, versionState, + createStateMonitor(), vipStatus); + + waitUntil(rpcServer::isRunning, "failed waiting for Rpc server running"); + waitUntil(() -> bootstrap.status() == StateMonitor.Status.up, "failed waiting for status 'up'"); + waitUntil(vipStatus::isInRotation, "failed waiting for server to be in rotation"); } private void waitUntil(BooleanSupplier booleanSupplier, String messageIfWaitingFails) throws InterruptedException { @@ -120,15 +172,28 @@ public class ConfigServerBootstrapTest { } private static ConfigserverConfig createConfigserverConfig(TemporaryFolder temporaryFolder) throws IOException { + return createConfigserverConfig(temporaryFolder, true); + } + + private static ConfigserverConfig createConfigserverConfigNonHosted(TemporaryFolder temporaryFolder) throws IOException { + return createConfigserverConfig(temporaryFolder, false); + } + + private static ConfigserverConfig createConfigserverConfig(TemporaryFolder temporaryFolder, boolean hosted) throws IOException { return new ConfigserverConfig(new ConfigserverConfig.Builder() .configServerDBDir(temporaryFolder.newFolder("serverdb").getAbsolutePath()) .configDefinitionsDir(temporaryFolder.newFolder("configdefinitions").getAbsolutePath()) - .hostedVespa(true) - .multitenant(true) + .hostedVespa(hosted) + .multitenant(hosted) .maxDurationOfBootstrap(1) /* seconds */ .sleepTimeWhenRedeployingFails(0)); /* seconds */ } + private Host createHost(String hostname, String version) { + return new Host(hostname, Collections.emptyList(), Optional.empty(), Optional.of(com.yahoo.component.Version.fromString(version))); + } + + public static class MockRpc extends com.yahoo.vespa.config.server.rpc.MockRpc { volatile boolean isRunning = false; diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/DeployTester.java b/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/DeployTester.java index 8685065d001..b70947de0a6 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/DeployTester.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/DeployTester.java @@ -108,8 +108,13 @@ public class DeployTester { } public DeployTester(List<ModelFactory> modelFactories, ConfigserverConfig configserverConfig, Clock clock, Zone zone, HostProvisioner provisioner) { + this(modelFactories, configserverConfig, clock, zone, provisioner, new MockCurator()); + } + + public DeployTester(List<ModelFactory> modelFactories, ConfigserverConfig configserverConfig, Clock clock, Zone zone, + HostProvisioner provisioner, Curator curator) { this.clock = clock; - TestComponentRegistry componentRegistry = createComponentRegistry(new MockCurator(), Metrics.createTestMetrics(), + TestComponentRegistry componentRegistry = createComponentRegistry(curator, Metrics.createTestMetrics(), modelFactories, configserverConfig, clock, zone, provisioner); try { diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/RedeployTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/RedeployTest.java index 9bc9a93e9fa..c6f4df74049 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/RedeployTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/RedeployTest.java @@ -1,28 +1,21 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.config.server.deploy; -import com.google.common.io.Files; -import com.yahoo.cloud.config.ConfigserverConfig; import com.yahoo.config.model.api.ModelFactory; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.ApplicationName; import com.yahoo.config.provision.InstanceName; import com.yahoo.config.provision.Version; -import com.yahoo.test.ManualClock; -import com.yahoo.vespa.config.server.session.LocalSession; import org.junit.Test; import java.time.Clock; -import java.time.Duration; import java.time.Instant; import java.util.ArrayList; -import java.util.Collection; import java.util.List; import java.util.Optional; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertNotEquals; import static org.junit.Assert.assertTrue; /** @@ -61,43 +54,4 @@ public class RedeployTest { assertFalse(tester.redeployFromLocalActive(id).isPresent()); } - @Test - public void testPurgingOfOldNonActiveDeployments() { - ManualClock clock = new ManualClock(Instant.now()); - ConfigserverConfig configserverConfig = new ConfigserverConfig(new ConfigserverConfig.Builder() - .configServerDBDir(Files.createTempDir() - .getAbsolutePath()) - .configDefinitionsDir(Files.createTempDir() - .getAbsolutePath()) - .sessionLifetime(60)); - DeployTester tester = new DeployTester(configserverConfig, clock); - tester.deployApp("src/test/apps/app", "myapp", Instant.now()); // session 2 (numbering starts at 2) - - clock.advance(Duration.ofSeconds(10)); - Optional<com.yahoo.config.provision.Deployment> deployment2 = tester.redeployFromLocalActive(); - - assertTrue(deployment2.isPresent()); - deployment2.get().activate(); // session 3 - long activeSessionId = tester.tenant().getApplicationRepo().getSessionIdForApplication(tester.applicationId()); - - clock.advance(Duration.ofSeconds(10)); - Optional<com.yahoo.config.provision.Deployment> deployment3 = tester.redeployFromLocalActive(); - assertTrue(deployment3.isPresent()); - deployment3.get().prepare(); // session 4 (not activated) - - LocalSession deployment3session = ((Deployment) deployment3.get()).session(); - assertNotEquals(activeSessionId, deployment3session); - // No change to active session id - assertEquals(activeSessionId, tester.tenant().getApplicationRepo().getSessionIdForApplication(tester.applicationId())); - assertEquals(3, tester.tenant().getLocalSessionRepo().listSessions().size()); - - clock.advance(Duration.ofHours(1)); // longer than session lifetime - - // All sessions except 3 should be removed after the call to purgeOldSessions - tester.tenant().getLocalSessionRepo().purgeOldSessions(); - final Collection<LocalSession> sessions = tester.tenant().getLocalSessionRepo().listSessions(); - assertEquals(1, sessions.size()); - assertEquals(3, new ArrayList<>(sessions).get(0).getSessionId()); - } - } diff --git a/container-dependencies-enforcer/pom.xml b/container-dependencies-enforcer/pom.xml index f693517c844..9929f267c47 100644 --- a/container-dependencies-enforcer/pom.xml +++ b/container-dependencies-enforcer/pom.xml @@ -51,7 +51,6 @@ <plugin> <groupId>org.apache.maven.plugins</groupId> <artifactId>maven-enforcer-plugin</artifactId> - <version>3.0.0-M1</version> <executions> <execution> <!-- To allow running 'mvn enforcer:enforce' from the command line --> @@ -88,7 +87,9 @@ <include>com.google.inject.extensions:guice-assistedinject:[${guice.version}]:jar:provided</include> <include>com.google.inject.extensions:guice-multibindings:[${guice.version}]:jar:provided</include> <include>com.google.inject:guice:[${guice.version}]:jar:provided:no_aop</include> - <include>com.sun.activation:javax.activation:jar[1.2.0]:jar:provided</include> + <include>com.sun.activation:javax.activation:[1.2.0]:jar:provided</include> + <include>com.sun.xml.bind:jaxb-core:[${jaxb.version}]:jar:provided</include> + <include>com.sun.xml.bind:jaxb-impl:[${jaxb.version}]:jar:provided</include> <include>commons-codec:commons-codec:[1.4]:jar:provided</include> <include>commons-daemon:commons-daemon:[1.0.3]:jar:provided</include> <include>commons-logging:commons-logging:[1.1.1]:jar:provided</include> diff --git a/container-dependency-versions/pom.xml b/container-dependency-versions/pom.xml index 195e66902db..ec4e9eae5fb 100644 --- a/container-dependency-versions/pom.xml +++ b/container-dependency-versions/pom.xml @@ -186,7 +186,7 @@ <version>${javax.ws.rs-api.version}</version> </dependency> - <!-- TODO: upgrade jaxb-api artifacts to >=2.3.0. + <!-- TODO: upgrade jaxb artifacts to >=2.3.0. See https://stackoverflow.com/questions/50237516/proper-fix-for-java-10-complaining-about-illegal-reflection-access-by-jaxb-impl --> <dependency> <groupId>javax.xml.bind</groupId> diff --git a/container-search-gui/src/main/resources/gui/_includes/index.html b/container-search-gui/src/main/resources/gui/_includes/index.html index 85de6c68cf4..ca3302dd404 100644 --- a/container-search-gui/src/main/resources/gui/_includes/index.html +++ b/container-search-gui/src/main/resources/gui/_includes/index.html @@ -188,18 +188,17 @@ <SCRIPT language="javascript"> - method = "POST"; + const CONFIG = $.getJSON('/querybuilder/gui_variables.json', function(data){window.CONFIG = data;}); + method = "POST"; var number = 0; var childno = {}; var json = JSON.parse("{}"); var searchApiReference = null; - window.onload = function() {addNewRow(); getSearchApiReference();}; var possible = ["yql", "hits", "offset", "queryProfile", "nocache", "groupingSessionCache", "searchChain", "timeout", "trace","tracelevel","traceLevel", "", , "model", "ranking", "collapse","collapsesize","collapsesize","presentation", "pos", "streaming", "rules", "recall", "user", "nocachewrite", "metrics"]; var usedProps = []; - var hasChilds = ["model", "ranking", "trace","tracelevel", "ranking.matchPhase", "ranking.matchPhase.diversity", "presentation","collapse", "pos", "streaming", "rules", "metrics"]; var removedIndexes = [0]; var childrenProps = { "model" : ["defaultIndex", "encoding", "language", "queryString", "restrict", "searchPath", "sources", "type"], @@ -216,6 +215,19 @@ "rules" : ["off", "rulebase"] }; + window.onload = function() { + // Adding variables from configuration file + if (window.CONFIG.hasOwnProperty("featurename")){ + childrenProps["ranking.features"] = window.CONFIG.featurename; + } + if (window.CONFIG.hasOwnProperty("propertyname")){ + childrenProps["ranking.properties"] = window.CONFIG.propertyname; + } + + addNewRow(); + getSearchApiReference(); + }; + var stringType = ["yql", "queryProfile", "searchChain", "model.defaultIndex", "model.encoding", "model.language", "model.queryString", "model.searchPath", "model.type", "ranking.features", "ranking.profile", "ranking.properties", "ranking.sorting", "ranking.matchPhase.diversity.attribute", "ranking.matchPhase.attribute","pos.attribute", "presentation.summary", "collapse.summary", @@ -518,7 +530,7 @@ showInformation(no, key); findUsedProps(); var fullKey = getFullName(no, key); - if(contains(hasChilds, fullKey)){ + if(fullKey in childrenProps){ var input = document.getElementById("v"+no); if ($(input).is("textarea")){ editAreaLoader.delete_instance(window.yqlID); @@ -611,14 +623,14 @@ } } if (!validKey(no, key)){ - var keyInput = document.getElementById("i"+no); - keyInput.style = "border-width: 1px; border-color: red;" + //var keyInput = document.getElementById("i"+no); + //keyInput.style = "border-width: 1px; border-color: red;" var valueInput = document.getElementById("v"+no); - valueInput.placeholder = "Invalid parameter"; + valueInput.placeholder = "Possible invalid parameter"; } if (validKey(no, key)){ var keyInput = document.getElementById("i"+no); - keyInput.style = "border-width: 0px;" + //keyInput.style = "border-width: 0px;" } } @@ -665,15 +677,15 @@ generateJSON(); } - function removeChild(no){ + function removeChildRow(no){ var a = "" + no; - var parentIndexes; + var parentNo = a.split(".").slice(0, -1).join("."); var div = document.getElementById(no); - var bigdiv = document.getElementById(a.split(".").slice(0, -1).join(".")); + var bigdiv = document.getElementById(parentNo); bigdiv.removeChild(div); - removedIndexes.push(parseInt(no)); + removedIndexes.push(no); findUsedProps(); - updateFields(); + updateChildrenFields(parentNo); generateJSON(); } @@ -703,7 +715,7 @@ var newButton = document.createElement("button"); newButton.id = "b"+temp; newButton.innerHTML = "-"; - newButton.onclick = function(){removeChild(temp);}; + newButton.onclick = function(){removeChildRow(temp);}; newButton.classList.add("removeRow"); var br = document.createElement("br"); br.id = "br"+temp @@ -753,17 +765,17 @@ parentKey = document.getElementById("i"+parentNo).value; var temp = parseInt(childno[parentNo]); while (temp > 0){ - if (!contains(removedIndexes, temp)){ + if (!contains(removedIndexes, parentNo+"."+temp)){ var datalist = document.getElementById("prop"+parentNo+"."+temp); - setTimeout(function(){ datalist.innerHTML = ""; var list = childrenProps[getFullName(parentNo,parentKey)]; list.forEach(function(item){ - var option = document.createElement("option"); - option.value = item; - datalist.appendChild(option); + if(!contains(usedProps,getFullName(parentNo,parentKey)+"."+item)){ + var option = document.createElement("option"); + option.value = item; + datalist.appendChild(option); + } }); - }, 30); } temp -= 1; } @@ -851,14 +863,37 @@ } function addUsedProps(no, thresh){ - var temp = no; - while (temp > thresh){ - if (!contains(removedIndexes, temp)){ - var key = document.getElementById("i"+temp).value; - usedProps.push(key); - } - temp--; - } + var temp = no; + if (countDots(""+temp) > 0){ + var list = (""+no).split("."); + var child_id = list[list.length-1]; + } else{ + var child_id = (""+no) + } + while (child_id > thresh){ + if (!contains(removedIndexes, temp)){ + var key = document.getElementById("i"+temp).value; + usedProps.push(getFullName(temp, key)); + if (childno[temp] != -1 && childno[temp] != 0){ + var temp2 = ""+temp+"."+childno[temp]; + var thresh2 = 0; + addUsedProps(temp2, thresh2); + } + } + let dots = countDots(""+temp); + if (dots > 0){ + var temp2 = temp.split(".").slice(0, -1); + var newNo = temp.split(".")[temp2.length]; + newNo = parseInt(newNo)-1; + temp2.push(newNo); + temp = temp2.join("."); + child_id--; + } + if (dots == 0){ + temp = parseInt(temp)-1; + child_id--; + } + } } function startSending(){ @@ -933,7 +968,7 @@ function changeInformation(no,key){ if (key===""){return "Choose a parameter for information"} - if (contains(hasChilds, getFullName(no,key))){return "Add parameters under the parent ''" + getFullName(no,key) + "'"} + if (getFullName(no,key) in childrenProps){return "Add parameters under the parent ''" + getFullName(no,key) + "'"} var refId = getFullName(no,key); var ref = searchApiReference; var tabletext = stripMyHTML(ref.getElementById(refId).nextElementSibling.outerHTML); diff --git a/container-search-gui/src/main/resources/gui/gui_variables.json b/container-search-gui/src/main/resources/gui/gui_variables.json new file mode 100644 index 00000000000..1b5f411611a --- /dev/null +++ b/container-search-gui/src/main/resources/gui/gui_variables.json @@ -0,0 +1,4 @@ +{ + "propertyname" : ["propertyname"], + "featurename" : ["featurename"] +} diff --git a/jdisc_core/pom.xml b/jdisc_core/pom.xml index 6b22aa4a985..efa4e62affd 100644 --- a/jdisc_core/pom.xml +++ b/jdisc_core/pom.xml @@ -22,6 +22,14 @@ <artifactId>jaxb-api</artifactId> </dependency> <dependency> + <groupId>com.sun.xml.bind</groupId> + <artifactId>jaxb-core</artifactId> + </dependency> + <dependency> + <groupId>com.sun.xml.bind</groupId> + <artifactId>jaxb-impl</artifactId> + </dependency> + <dependency> <groupId>com.sun.activation</groupId> <artifactId>javax.activation</artifactId> </dependency> @@ -240,6 +248,8 @@ <argument>${project.build.directory}/dependency/config-lib.jar</argument> <argument>${project.build.directory}/dependency/yolean.jar</argument> <argument>${project.build.directory}/dependency/jaxb-api.jar</argument> + <argument>${project.build.directory}/dependency/jaxb-core.jar</argument> + <argument>${project.build.directory}/dependency/jaxb-impl.jar</argument> <argument>${project.build.directory}/dependency/javax.activation.jar</argument> </arguments> </configuration> diff --git a/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/server/jetty/HttpRequestDispatch.java b/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/server/jetty/HttpRequestDispatch.java index db7670b604a..0a20e4f4757 100644 --- a/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/server/jetty/HttpRequestDispatch.java +++ b/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/server/jetty/HttpRequestDispatch.java @@ -124,7 +124,6 @@ class HttpRequestDispatch { } boolean reportedError = false; - parent.metricReporter.contentSize((int) parent.jettyRequest.getContentRead()); if (error != null) { if (error instanceof CompletionException && error.getCause() instanceof EofException) { diff --git a/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/server/jetty/ServletRequestReader.java b/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/server/jetty/ServletRequestReader.java index 47ef20e181e..a80f5bb3c14 100644 --- a/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/server/jetty/ServletRequestReader.java +++ b/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/server/jetty/ServletRequestReader.java @@ -45,6 +45,8 @@ class ServletRequestReader implements ReadListener { private final Executor executor; private final MetricReporter metricReporter; + private int bytesRead; + /** * Rules: * 1. If state != State.READING, then numberOfOutstandingUserCalls must not increase @@ -134,6 +136,7 @@ class ServletRequestReader implements ReadListener { int bytesReceived = buf.remaining(); requestContentChannel.write(buf, writeCompletionHandler); metricReporter.successfulRead(bytesReceived); + bytesRead += bytesReceived; } catch (final Throwable t) { finishedFuture.completeExceptionally(t); } finally { @@ -184,6 +187,7 @@ class ServletRequestReader implements ReadListener { private void doneReading() { final boolean shouldCloseRequestContentChannel; + int bytesRead; synchronized (monitor) { if (state != State.READING) { return; @@ -195,11 +199,14 @@ class ServletRequestReader implements ReadListener { if (shouldCloseRequestContentChannel) { state = State.REQUEST_CONTENT_CLOSED; } + bytesRead = this.bytesRead; } if (shouldCloseRequestContentChannel) { closeCompletionHandler_noThrow(); } + + metricReporter.contentSize(bytesRead); } private void closeCompletionHandler_noThrow() { diff --git a/maven-plugins/pom.xml b/maven-plugins/pom.xml index 0aabe630a63..546dfdafcd9 100644 --- a/maven-plugins/pom.xml +++ b/maven-plugins/pom.xml @@ -21,5 +21,7 @@ <module>../bundle-plugin</module> <module>../configgen</module> <module>../config-class-plugin</module> + <module>../application-deploy-plugin</module> + <module>../vespa-application-maven-plugin</module> </modules> </project> diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/identity/AthenzCredentialsMaintainer.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/identity/AthenzCredentialsMaintainer.java index ff85c49bb13..a8403b8b10d 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/identity/AthenzCredentialsMaintainer.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/identity/AthenzCredentialsMaintainer.java @@ -7,6 +7,7 @@ import com.yahoo.vespa.athenz.api.AthenzService; import com.yahoo.vespa.athenz.client.zts.DefaultZtsClient; import com.yahoo.vespa.athenz.client.zts.InstanceIdentity; import com.yahoo.vespa.athenz.client.zts.ZtsClient; +import com.yahoo.vespa.athenz.client.zts.ZtsClientException; import com.yahoo.vespa.athenz.identity.ServiceIdentityProvider; import com.yahoo.vespa.athenz.identityprovider.api.EntityBindingsMapper; import com.yahoo.vespa.athenz.identityprovider.api.IdentityDocumentClient; @@ -142,6 +143,8 @@ public class AthenzCredentialsMaintainer { log.info(String.format("Deleted private key file (path=%s)", privateKeyFile)); if (Files.deleteIfExists(certificateFile)) log.info(String.format("Deleted certificate file (path=%s)", certificateFile)); + if (Files.deleteIfExists(identityDocumentFile)) + log.info(String.format("Deleted identity document file (path=%s)", certificateFile)); } catch (IOException e) { throw new UncheckedIOException(e); } @@ -201,6 +204,12 @@ public class AthenzCredentialsMaintainer { csr); writePrivateKeyAndCertificate(keyPair.getPrivate(), instanceIdentity.certificate()); log.info("Instance successfully refreshed and credentials written to file"); + } catch (ZtsClientException e) { + // TODO Find out why certificate was revoked and hopefully remove this workaround + if (e.getErrorCode() == 403 && e.getDescription().startsWith("Certificate revoked")) { + log.error("Certificate cannot be refreshed as it is revoked by ZTS - re-registering the instance now", e); + registerIdentity(); + } } catch (IOException e) { throw new UncheckedIOException(e); } diff --git a/parent/pom.xml b/parent/pom.xml index 5cef496a941..56da33a9c93 100644 --- a/parent/pom.xml +++ b/parent/pom.xml @@ -83,7 +83,7 @@ <artifactId>maven-bundle-plugin</artifactId> <version>3.5.0</version> <configuration> - <!-- TODO: this should be a temporary workaround until bundle-plugin understands java 10 --> + <!-- TODO: remove when bundle-plugin understands java 10, https://issues.apache.org/jira/browse/FELIX-5879 --> <instructions> <_noee>true</_noee> </instructions> @@ -129,6 +129,11 @@ </plugin> <plugin> <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-enforcer-plugin</artifactId> + <version>3.0.0-M1</version> + </plugin> + <plugin> + <groupId>org.apache.maven.plugins</groupId> <artifactId>maven-install-plugin</artifactId> <version>2.5.2</version> <configuration> @@ -25,7 +25,6 @@ <modules> <module>application</module> - <module>application-deploy-plugin</module> <module>application-model</module> <module>application-preprocessor</module> <module>athenz-identity-provider-service</module> @@ -120,7 +119,6 @@ <module>vespaclient-core</module> <module>vespaclient-container-plugin</module> <module>vespaclient-java</module> - <module>vespa-application-maven-plugin</module> <module>vespa-athenz</module> <module>vespa-documentgen-plugin</module> <module>vespa_feed_perf</module> diff --git a/searchlib/src/tests/fef/termfieldmodel/termfieldmodel_test.cpp b/searchlib/src/tests/fef/termfieldmodel/termfieldmodel_test.cpp index 01c72497246..e689e613886 100644 --- a/searchlib/src/tests/fef/termfieldmodel/termfieldmodel_test.cpp +++ b/searchlib/src/tests/fef/termfieldmodel/termfieldmodel_test.cpp @@ -192,6 +192,50 @@ TEST("term field model") { testInvalidId(); } +TEST("append positions") { + TermFieldMatchData tfmd; + tfmd.setFieldId(123); + EXPECT_EQUAL(0u, tfmd.size()); + EXPECT_EQUAL(1u, tfmd.capacity()); + tfmd.reset(7); + EXPECT_EQUAL(0u, tfmd.size()); + EXPECT_EQUAL(1u, tfmd.capacity()); + TermFieldMatchDataPosition pos(0x01020304, 0x10203040, 0x11223344, 0x12345678); + tfmd.appendPosition(pos); + EXPECT_EQUAL(1u, tfmd.size()); + EXPECT_EQUAL(1u, tfmd.capacity()); + EXPECT_EQUAL(0x01020304u, tfmd.begin()->getElementId()); + EXPECT_EQUAL(0x10203040u, tfmd.begin()->getPosition()); + EXPECT_EQUAL(0x11223344, tfmd.begin()->getElementWeight()); + EXPECT_EQUAL(0x12345678u, tfmd.begin()->getElementLen()); + tfmd.reset(11); + EXPECT_EQUAL(0u, tfmd.size()); + EXPECT_EQUAL(1u, tfmd.capacity()); + TermFieldMatchDataPosition pos2(0x21020304, 0x20203040, 0x21223344, 0x22345678); + tfmd.appendPosition(pos); + tfmd.appendPosition(pos2); + EXPECT_EQUAL(2u, tfmd.size()); + EXPECT_EQUAL(2u, tfmd.capacity()); + TermFieldMatchDataPosition pos3(0x31020304, 0x30203040, 0x31223344, 0x32345678); + tfmd.appendPosition(pos3); + EXPECT_EQUAL(3u, tfmd.size()); + EXPECT_EQUAL(4u, tfmd.capacity()); + EXPECT_EQUAL(0x01020304u, tfmd.begin()->getElementId()); + EXPECT_EQUAL(0x10203040u, tfmd.begin()->getPosition()); + EXPECT_EQUAL(0x11223344, tfmd.begin()->getElementWeight()); + EXPECT_EQUAL(0x12345678u, tfmd.begin()->getElementLen()); + + EXPECT_EQUAL(0x21020304u, tfmd.begin()[1].getElementId()); + EXPECT_EQUAL(0x20203040u, tfmd.begin()[1].getPosition()); + EXPECT_EQUAL(0x21223344, tfmd.begin()[1].getElementWeight()); + EXPECT_EQUAL(0x22345678u, tfmd.begin()[1].getElementLen()); + + EXPECT_EQUAL(0x31020304u, tfmd.begin()[2].getElementId()); + EXPECT_EQUAL(0x30203040u, tfmd.begin()[2].getPosition()); + EXPECT_EQUAL(0x31223344, tfmd.begin()[2].getElementWeight()); + EXPECT_EQUAL(0x32345678u, tfmd.begin()[2].getElementLen()); +} + TEST("Access subqueries") { State state; testSetup(state); diff --git a/searchlib/src/vespa/searchlib/fef/termfieldmatchdata.cpp b/searchlib/src/vespa/searchlib/fef/termfieldmatchdata.cpp index 7ff5dcf8b6c..c5c7d2dc574 100644 --- a/searchlib/src/vespa/searchlib/fef/termfieldmatchdata.cpp +++ b/searchlib/src/vespa/searchlib/fef/termfieldmatchdata.cpp @@ -40,7 +40,7 @@ TermFieldMatchData & TermFieldMatchData::operator = (const TermFieldMatchData & TermFieldMatchData::~TermFieldMatchData() { if (isRawScore()) { - } else if (isMultiPos()) { + } else if (allocated()) { delete [] _data._positions._positions; } else { getFixed()->~TermFieldMatchDataPosition(); @@ -79,21 +79,30 @@ constexpr size_t MAX_ELEMS = std::numeric_limits<uint16_t>::max(); void TermFieldMatchData::resizePositionVector(size_t sz) { + assert(allocated()); + assert(sz >= _sz); size_t newSize(std::min(MAX_ELEMS, std::max(1ul, sz))); TermFieldMatchDataPosition * n = new TermFieldMatchDataPosition[newSize]; + for (size_t i(0); i < _data._positions._allocated; i++) { + n[i] = _data._positions._positions[i]; + } + delete [] _data._positions._positions; + _data._positions._allocated = newSize; + _data._positions._positions = n; +} + +void +TermFieldMatchData::allocateVector() +{ + assert(_sz < 2); + assert(!allocated()); + size_t newSize = 2; + TermFieldMatchDataPosition * n = new TermFieldMatchDataPosition[newSize]; if (_sz > 0) { - if (isMultiPos()) { - for (size_t i(0); i < _data._positions._allocated; i++) { - n[i] = _data._positions._positions[i]; - } - delete [] _data._positions._positions; - } else { - assert(_sz == 1); - n[0] = *getFixed(); - _data._positions._maxElementLength = getFixed()->getElementLen(); - } + n[0] = *getFixed(); + _data._positions._maxElementLength = getFixed()->getElementLen(); } - _fieldId = _fieldId | 0x4000; + _fieldId = _fieldId | 0x4000; // set allocated() flag _data._positions._allocated = newSize; _data._positions._positions = n; } @@ -101,6 +110,7 @@ TermFieldMatchData::resizePositionVector(size_t sz) void TermFieldMatchData::appendPositionToAllocatedVector(const TermFieldMatchDataPosition &pos) { + assert(allocated()); if (__builtin_expect(_sz >= _data._positions._allocated, false)) { resizePositionVector(_sz*2); } diff --git a/searchlib/src/vespa/searchlib/fef/termfieldmatchdata.h b/searchlib/src/vespa/searchlib/fef/termfieldmatchdata.h index 018af889557..a0a37d9d66c 100644 --- a/searchlib/src/vespa/searchlib/fef/termfieldmatchdata.h +++ b/searchlib/src/vespa/searchlib/fef/termfieldmatchdata.h @@ -47,6 +47,7 @@ private: int32_t getElementWeight() const { return empty() ? 1 : allocated() ? getMultiple()->getElementWeight() : getFixed()->getElementWeight(); } uint32_t getMaxElementLength() const { return empty() ? 0 : allocated() ? _data._positions._maxElementLength : getFixed()->getElementLen(); } void appendPositionToAllocatedVector(const TermFieldMatchDataPosition &pos); + void allocateVector(); void resizePositionVector(size_t sz) __attribute__((noinline)); enum { FIELDID_MASK = 0x1fff}; @@ -66,6 +67,10 @@ public: size_t capacity() const { return allocated() ? _data._positions._allocated : 1; } void reservePositions(size_t sz) { if (sz > capacity()) { + if (!allocated()) { + allocateVector(); + if (sz <= capacity()) return; + } resizePositionVector(sz); } } @@ -224,11 +229,14 @@ public: * @param pos low-level occurrence information **/ TermFieldMatchData &appendPosition(const TermFieldMatchDataPosition &pos) { - if (isMultiPos() || (_sz > 0)) { - appendPositionToAllocatedVector(pos); - } else { + if (_sz == 0 && !allocated()) { _sz = 1; new (_data._position) TermFieldMatchDataPosition(pos); + } else { + if (!allocated()) { + allocateVector(); + } + appendPositionToAllocatedVector(pos); } return *this; } diff --git a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zts/DefaultZtsClient.java b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zts/DefaultZtsClient.java index 8c67c3386b7..8a94518cee7 100644 --- a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zts/DefaultZtsClient.java +++ b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zts/DefaultZtsClient.java @@ -10,10 +10,11 @@ import com.yahoo.vespa.athenz.api.AthenzRole; import com.yahoo.vespa.athenz.api.AthenzService; import com.yahoo.vespa.athenz.api.NToken; import com.yahoo.vespa.athenz.api.ZToken; +import com.yahoo.vespa.athenz.client.zts.bindings.ErrorResponseEntity; +import com.yahoo.vespa.athenz.client.zts.bindings.IdentityRefreshRequestEntity; import com.yahoo.vespa.athenz.client.zts.bindings.IdentityResponseEntity; import com.yahoo.vespa.athenz.client.zts.bindings.InstanceIdentityCredentials; import com.yahoo.vespa.athenz.client.zts.bindings.InstanceRefreshInformation; -import com.yahoo.vespa.athenz.client.zts.bindings.IdentityRefreshRequestEntity; import com.yahoo.vespa.athenz.client.zts.bindings.InstanceRegisterInformation; import com.yahoo.vespa.athenz.client.zts.bindings.RoleCertificateRequestEntity; import com.yahoo.vespa.athenz.client.zts.bindings.RoleCertificateResponseEntity; @@ -33,7 +34,6 @@ import org.apache.http.entity.StringEntity; import org.apache.http.impl.client.CloseableHttpClient; import org.apache.http.impl.client.DefaultHttpRequestRetryHandler; import org.apache.http.impl.client.HttpClientBuilder; -import org.apache.http.util.EntityUtils; import org.eclipse.jetty.http.HttpStatus; import javax.net.ssl.SSLContext; @@ -48,7 +48,6 @@ import java.util.List; import java.util.concurrent.locks.Lock; import java.util.concurrent.locks.ReadWriteLock; import java.util.concurrent.locks.ReentrantReadWriteLock; -import java.util.stream.Collectors; import static com.yahoo.vespa.athenz.tls.SignatureAlgorithm.SHA256_WITH_RSA; import static com.yahoo.vespa.athenz.tls.SubjectAlternativeName.Type.DNS_NAME; @@ -237,10 +236,8 @@ public class DefaultZtsClient implements ZtsClient { if (HttpStatus.isSuccess(response.getStatusLine().getStatusCode())) { return objectMapper.readValue(response.getEntity().getContent(), entityType); } else { - String message = EntityUtils.toString(response.getEntity()); - throw new ZtsClientException( - String.format("Unable to get identity. http code/message: %d/%s", - response.getStatusLine().getStatusCode(), message)); + ErrorResponseEntity errorEntity = objectMapper.readValue(response.getEntity().getContent(), ErrorResponseEntity.class); + throw new ZtsClientException(errorEntity.code, errorEntity.description); } } diff --git a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zts/ZtsClientException.java b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zts/ZtsClientException.java index 3d3696ad870..0b0d6914fea 100644 --- a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zts/ZtsClientException.java +++ b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zts/ZtsClientException.java @@ -8,11 +8,25 @@ package com.yahoo.vespa.athenz.client.zts; */ public class ZtsClientException extends RuntimeException { - public ZtsClientException(String message) { - super(message); + private final int errorCode; + private final String description; + + public ZtsClientException(int errorCode, String description) { + super(createMessage(errorCode, description)); + this.errorCode = errorCode; + this.description = description; + } + + public int getErrorCode() { + return errorCode; } - public ZtsClientException(String message, Throwable cause) { - super(message, cause); + public String getDescription() { + return description; } + + private static String createMessage(int code, String description) { + return String.format("Received error from ZTS: code=%d, message=\"%s\"", code, description); + } + } diff --git a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zts/bindings/ErrorResponseEntity.java b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zts/bindings/ErrorResponseEntity.java new file mode 100644 index 00000000000..431af084f9f --- /dev/null +++ b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zts/bindings/ErrorResponseEntity.java @@ -0,0 +1,23 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.athenz.client.zts.bindings; + +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.annotation.JsonProperty; + +/** + * @author bjorncs + */ +@JsonIgnoreProperties(ignoreUnknown = true) +public class ErrorResponseEntity { + + public final int code; + public final String description; + + @JsonCreator + public ErrorResponseEntity(@JsonProperty("code") int code, + @JsonProperty("message") String description) { + this.code = code; + this.description = description; + } +} |