diff options
51 files changed, 290 insertions, 225 deletions
diff --git a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/EventDiffCalculator.java b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/EventDiffCalculator.java index e0ad023b2e8..cadc065dd51 100644 --- a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/EventDiffCalculator.java +++ b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/EventDiffCalculator.java @@ -149,7 +149,7 @@ public class EventDiffCalculator { } else if (isMayHaveMergesPendingUpEdge(prevReason, currReason)) { events.add(createNodeEvent(info, "Node may have merges pending", params)); } else if (isMayHaveMergesPendingDownEdge(prevReason, currReason)) { - events.add(createNodeEvent(info, "Node no longer have merges pending", params)); + events.add(createNodeEvent(info, "Node no longer has merges pending", params)); } } } diff --git a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/EventDiffCalculatorTest.java b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/EventDiffCalculatorTest.java index 743fddcf48b..e7c4bbfcaa8 100644 --- a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/EventDiffCalculatorTest.java +++ b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/EventDiffCalculatorTest.java @@ -366,7 +366,7 @@ public class EventDiffCalculatorTest { assertThat(events, hasItem(allOf( eventForNode(storageNode(1)), nodeEventForBucketSpace("default"), - nodeEventWithDescription("Node no longer have merges pending")))); + nodeEventWithDescription("Node no longer has merges pending")))); } @Test diff --git a/config-model/src/main/java/com/yahoo/config/model/provision/Hosts.java b/config-model/src/main/java/com/yahoo/config/model/provision/Hosts.java index 542f50247a7..61a5db04ca7 100644 --- a/config-model/src/main/java/com/yahoo/config/model/provision/Hosts.java +++ b/config-model/src/main/java/com/yahoo/config/model/provision/Hosts.java @@ -77,7 +77,7 @@ public class Hosts { if ("localhost".equals(name)) { name = HostName.getLocalhost(); } - final List<String> hostAliases = VespaDomBuilder.getHostAliases(hostE.getChildNodes()); + List<String> hostAliases = VespaDomBuilder.getHostAliases(hostE.getChildNodes()); if (hostAliases.isEmpty()) { throw new IllegalArgumentException("No host aliases defined for host '" + name + "'"); } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/HostResource.java b/config-model/src/main/java/com/yahoo/vespa/model/HostResource.java index 72309d4c266..0f7c040bf02 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/HostResource.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/HostResource.java @@ -105,7 +105,7 @@ public class HostResource implements Comparable<HostResource> { if (wantedPort > 0) { if (service.getPortCount() < 1) { throw new RuntimeException(service + " wants baseport " + wantedPort + - ", but it has not reserved any ports, so it cannot name a desired baseport."); + ", but it has not reserved any ports, so it cannot name a desired baseport."); } if (service.requiresWantedPort() || canUseWantedPort(service, wantedPort, serviceBasePort)) serviceBasePort = wantedPort; diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/configserver/ConfigserverCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/container/configserver/ConfigserverCluster.java index 8cdf9ba4977..cdd90ae1a36 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/configserver/ConfigserverCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/configserver/ConfigserverCluster.java @@ -22,8 +22,7 @@ import java.util.Optional; /** * Represents a config server cluster. * - * @author lulf - * @since 5.15 + * @author Ulf Lilleengen */ public class ConfigserverCluster extends AbstractConfigProducer implements diff --git a/container-core/src/main/java/com/yahoo/container/Server.java b/container-core/src/main/java/com/yahoo/container/Server.java index c37cb2eeb2c..207050a8d88 100644 --- a/container-core/src/main/java/com/yahoo/container/Server.java +++ b/container-core/src/main/java/com/yahoo/container/Server.java @@ -63,12 +63,8 @@ public class Server { instance = new Server(); } - // TODO: Don't throw exception // TODO: Make independent of config - public void initialize(QrConfig config) throws Exception { - //TODO: Reenable - //mBeanServer = new MBeanServer(configId); - + public void initialize(QrConfig config) { localServerDiscriminator = config.discriminator(); container.setupFileAcquirer(config.filedistributor()); initRpcServer(config.rpc()); diff --git a/container-disc/src/main/java/com/yahoo/container/jdisc/ConfiguredApplication.java b/container-disc/src/main/java/com/yahoo/container/jdisc/ConfiguredApplication.java index bf696771b20..10510ff5679 100644 --- a/container-disc/src/main/java/com/yahoo/container/jdisc/ConfiguredApplication.java +++ b/container-disc/src/main/java/com/yahoo/container/jdisc/ConfiguredApplication.java @@ -11,6 +11,7 @@ import com.yahoo.config.ConfigInstance; import com.yahoo.config.subscription.ConfigInterruptedException; import com.yahoo.container.Container; import com.yahoo.container.QrConfig; +import com.yahoo.container.Server; import com.yahoo.container.core.ChainsConfig; import com.yahoo.container.core.config.HandlersConfigurerDi; import com.yahoo.container.di.config.Subscriber; @@ -121,7 +122,7 @@ public final class ConfiguredApplication implements Application { @Override public void start() { qrConfig = getConfig(QrConfig.class); - ContainerDiscApplication.hackToInitializeServer(qrConfig); + hackToInitializeServer(qrConfig); ContainerBuilder builder = createBuilderWithGuiceBindings(); configureComponents(builder.guiceModules().activate()); @@ -133,6 +134,16 @@ public final class ConfiguredApplication implements Application { portWatcher.start(); } + + private static void hackToInitializeServer(QrConfig config) { + try { + Server.get().initialize(config); + } catch (Exception e) { + log.log(LogLevel.ERROR, "Caught exception when initializing server. Exiting.", e); + Runtime.getRuntime().halt(1); + } + } + private <T extends ConfigInstance> T getConfig(Class<T> configClass) { Subscriber subscriber = subscriberFactory.getSubscriber( Collections.singleton(new ConfigKey<>(configClass, configId))); @@ -190,6 +201,7 @@ public final class ConfiguredApplication implements Application { ContainerBuilder builder = createBuilderWithGuiceBindings(); configurer.runOnceAndEnsureRegistryHackRun(builder.guiceModules().activate()); intitializeAndActivateContainer(builder); + if (qrConfig.restartOnDeploy()) break; } catch (ConfigInterruptedException | InterruptedException e) { break; } catch (Exception | LinkageError e) { // LinkageError: OSGi problems diff --git a/container-disc/src/main/java/com/yahoo/container/jdisc/ContainerDiscApplication.java b/container-disc/src/main/java/com/yahoo/container/jdisc/ContainerDiscApplication.java index f6ffb0c2ff6..0d2224f8c1c 100644 --- a/container-disc/src/main/java/com/yahoo/container/jdisc/ContainerDiscApplication.java +++ b/container-disc/src/main/java/com/yahoo/container/jdisc/ContainerDiscApplication.java @@ -1,30 +1,18 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.container.jdisc; - import com.google.inject.AbstractModule; import com.google.inject.Inject; -import com.yahoo.container.QrConfig; -import com.yahoo.container.Server; import com.yahoo.container.jdisc.messagebus.SessionCache; import com.yahoo.jrt.ListenFailedException; -import com.yahoo.log.LogLevel; - -import java.util.logging.Logger; - /** - * The application which sets up the jDisc container - * - * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a> + * TODO: The contents of this class can probably be moved into ConfiguredApplication */ public class ContainerDiscApplication { - private static final Logger log = Logger.getLogger(ContainerDiscApplication.class.getName()); - private SessionCache sessionCache; - @Inject public ContainerDiscApplication(String configId) throws ListenFailedException { sessionCache = new SessionCache(configId); @@ -39,13 +27,4 @@ public class ContainerDiscApplication { }; } - public static void hackToInitializeServer(QrConfig config) { - try { - Server.get().initialize(config); - } catch (Exception e) { - log.log(LogLevel.ERROR, "Caught exception when initializing server. Exiting.", e); - Runtime.getRuntime().halt(1); - } - } - } diff --git a/container-search/src/main/java/com/yahoo/vespa/streamingvisitors/VdsStreamingSearcher.java b/container-search/src/main/java/com/yahoo/vespa/streamingvisitors/VdsStreamingSearcher.java index 53bbe43d142..360d16080d0 100644 --- a/container-search/src/main/java/com/yahoo/vespa/streamingvisitors/VdsStreamingSearcher.java +++ b/container-search/src/main/java/com/yahoo/vespa/streamingvisitors/VdsStreamingSearcher.java @@ -26,12 +26,14 @@ import com.yahoo.prelude.fastsearch.VespaBackEndSearcher; import com.yahoo.search.Query; import com.yahoo.search.Result; import com.yahoo.processing.request.CompoundName; +import com.yahoo.search.result.Coverage; import com.yahoo.search.result.ErrorMessage; import com.yahoo.search.result.Relevance; import com.yahoo.search.searchchain.Execution; import com.yahoo.searchlib.aggregation.Grouping; import com.yahoo.vdslib.DocumentSummary; import com.yahoo.vdslib.SearchResult; +import com.yahoo.vdslib.VisitorStatistics; /** * The searcher which forwards queries to storage nodes using visiting. @@ -151,7 +153,9 @@ public class VdsStreamingSearcher extends VespaBackEndSearcher { ", returned hit count = ", hits.size(), ", summary count = ", summaryMap.size()); + VisitorStatistics visitStats = visitor.getStatistics(); result.setTotalHitCount(visitor.getTotalHitCount()); + result.setCoverage(new Coverage(visitStats.getDocumentsVisited(), visitStats.getDocumentsVisited())); query.trace(visitor.getStatistics().toString(), false, 2); query.getContext(true).setProperty(STREAMING_STATISTICS, visitor.getStatistics()); diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/ConfigServerClient.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/ConfigServerClient.java index 852629768c5..8f9fda4d72e 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/ConfigServerClient.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/ConfigServerClient.java @@ -29,6 +29,11 @@ public interface ConfigServerClient { PreparedApplication prepare(DeploymentId applicationInstance, DeployOptions deployOptions, Set<String> rotationCnames, Set<String> rotationNames, byte[] content); + // TODO: Remove default implementation + default PreparedApplication deploy(DeploymentId applicationInstance, DeployOptions deployOptions, Set<String> rotationCnames, Set<String> rotationNames, byte[] content) { + return prepare(applicationInstance, deployOptions, rotationCnames, rotationNames, content); + } + List<String> getNodeQueryHost(DeploymentId applicationInstance, String type) throws NoInstanceException; void restart(DeploymentId applicationInstance, Optional<Hostname> hostname) throws NoInstanceException; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java index 83ae5f5332f..c6fadb45161 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java @@ -8,6 +8,7 @@ import com.yahoo.config.application.api.DeploymentSpec; import com.yahoo.config.application.api.ValidationId; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.Environment; +import com.yahoo.config.provision.SystemName; import com.yahoo.config.provision.TenantName; import com.yahoo.vespa.athenz.api.NToken; import com.yahoo.vespa.curator.Lock; @@ -330,10 +331,17 @@ public class ApplicationController { // Carry out deployment options = withVersion(version, options); - ConfigServerClient.PreparedApplication preparedApplication = - configServer.prepare(new DeploymentId(applicationId, zone), options, cnames, rotationNames, - applicationPackage.zippedContent()); - preparedApplication.activate(); + + + ConfigServerClient.PreparedApplication preparedApplication; + DeploymentId deploymentId = new DeploymentId(applicationId, zone); + // TODO: Using deploy() only for user tenants in CD for now + if (controller.system().equals(SystemName.cd) && deploymentId.applicationId().tenant().value().startsWith(Tenant.userPrefix)) { + preparedApplication = configServer.deploy(deploymentId, options, cnames, rotationNames, applicationPackage.zippedContent()); + } else { + preparedApplication = configServer.prepare(deploymentId, options, cnames, rotationNames, applicationPackage.zippedContent()); + preparedApplication.activate(); + } application = application.withNewDeployment(zone, applicationVersion, version, clock.instant()); store(application); diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ConfigServerClientMock.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ConfigServerClientMock.java index 08f96195d2a..d0e378a74d5 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ConfigServerClientMock.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ConfigServerClientMock.java @@ -121,7 +121,52 @@ public class ConfigServerClientMock extends AbstractComponent implements ConfigS } }; } - + + @Override + public PreparedApplication deploy(DeploymentId deployment, DeployOptions deployOptions, Set<String> rotationCnames, + Set<String> rotationNames, byte[] content) { + lastPrepareVersion = deployOptions.vespaVersion.map(Version::new).orElse(null); + if (prepareException != null) { + RuntimeException prepareException = this.prepareException; + this.prepareException = null; + throw prepareException; + } + applicationActivated.put(deployment.applicationId(), false); + applicationInstances.put(deployment.applicationId(), UUID.randomUUID() + ":4080"); + + return new PreparedApplication() { + @Override + public void activate() { /* Nothing to do, done in */} + + @Override + public List<Log> messages() { + Log warning = new Log(); + warning.level = "WARNING"; + warning.time = 1; + warning.message = "The warning"; + + Log info = new Log(); + info.level = "INFO"; + info.time = 2; + info.message = "The info"; + + return Arrays.asList(warning, info); + } + + @Override + public PrepareResponse prepareResponse() { + applicationActivated.put(deployment.applicationId(), true); + + PrepareResponse prepareResponse = new PrepareResponse(); + prepareResponse.message = "foo"; + prepareResponse.configChangeActions = new ConfigChangeActions(Collections.emptyList(), + Collections.emptyList()); + prepareResponse.tenant = new TenantId("tenant"); + return prepareResponse; + } + }; + } + @Override public List<String> getNodeQueryHost(DeploymentId deployment, String type) { if (applicationInstances.containsKey(deployment.applicationId())) { diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java index 2dacadaaa3e..4d0a17a74f4 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java @@ -1,7 +1,6 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.restapi.application; -import com.google.common.base.Functions; import com.yahoo.application.container.handler.Request; import com.yahoo.application.container.handler.Response; import com.yahoo.component.Version; @@ -423,6 +422,34 @@ public class ApplicationApiTest extends ControllerContainerTest { new File("deploy-result.json")); } + + // Tests deployment to config server when using just on API call + // For now this depends on a switch in ApplicationController that does this for by- tenants in CD only + @Test + public void testDeployDirectlyUsingOneCallForDeploy() throws Exception { + // Setup + ContainerControllerTester controllerTester = new ContainerControllerTester(container, responseFiles); + ContainerTester tester = controllerTester.containerTester(); + tester.updateSystemVersion(); + UserId userId = new UserId("new_user"); + createAthenzDomainWithAdmin(ATHENZ_TENANT_DOMAIN, userId); + + // Create tenant + // PUT (create) the authenticated user + byte[] data = new byte[0]; + tester.assertResponse(request("/application/v4/user?user=new_user&domain=by", PUT) + .data(data) + .userIdentity(userId), // Normalized to by-new-user by API + new File("create-user-response.json")); + + // POST (deploy) an application to a dev zone + HttpEntity entity = createApplicationDeployData(applicationPackage, Optional.empty()); + tester.assertResponse(request("/application/v4/tenant/by-new-user/application/application1/environment/dev/region/cd-us-central-1/instance/default", POST) + .data(entity) + .userIdentity(userId), + new File("deploy-result.json")); + } + @Test public void testSortsDeploymentsAndJobs() throws Exception { // Setup diff --git a/documentapi/src/test/java/com/yahoo/documentapi/messagebus/protocol/test/TestFileUtil.java b/documentapi/src/test/java/com/yahoo/documentapi/messagebus/protocol/test/TestFileUtil.java index 722e42bc3a5..8bc6f56e574 100644 --- a/documentapi/src/test/java/com/yahoo/documentapi/messagebus/protocol/test/TestFileUtil.java +++ b/documentapi/src/test/java/com/yahoo/documentapi/messagebus/protocol/test/TestFileUtil.java @@ -4,26 +4,16 @@ package com.yahoo.documentapi.messagebus.protocol.test; import java.io.FileOutputStream; import java.io.IOException; import java.nio.charset.Charset; -import java.nio.file.FileSystems; import java.nio.file.Files; -import java.nio.file.Path; import java.nio.file.Paths; -import static java.nio.file.StandardCopyOption.ATOMIC_MOVE; - public class TestFileUtil { protected static final String DATA_PATH = "./test/crosslanguagefiles"; public static void writeToFile(String path, byte[] data) throws IOException { - // Write to a temporary file to avoid racing with cross-language tests reading the - // exact same file we're trying to write. - String tmpPath = path + ".tmp"; - try (FileOutputStream stream = new FileOutputStream(tmpPath)) { + try (FileOutputStream stream = new FileOutputStream(path)) { stream.write(data); } - // We make the assumption that all file systems we run these tests on support some form - // of atomic moving rather than "move by content copy". - Files.move(FileSystems.getDefault().getPath(tmpPath), FileSystems.getDefault().getPath(path), ATOMIC_MOVE); } /** diff --git a/documentapi/src/tests/messages/testbase.cpp b/documentapi/src/tests/messages/testbase.cpp index 9ccaa7abf2f..02c180853d4 100644 --- a/documentapi/src/tests/messages/testbase.cpp +++ b/documentapi/src/tests/messages/testbase.cpp @@ -175,18 +175,14 @@ TestBase::dump(const mbus::Blob& blob) const bool TestBase::writeFile(const string &filename, const mbus::Blob& blob) const { - std::string tmp_filename = filename + ".tmp"; - int file = open(tmp_filename.c_str(), O_WRONLY | O_CREAT | O_TRUNC, 0644); + int file = open(filename.c_str(), O_WRONLY | O_CREAT | O_TRUNC, 0644); if (file == -1) { return false; } - if (write(file, blob.data(), blob.size()) != static_cast<ssize_t>(blob.size())) { - throw vespalib::Exception("write failed"); + if (write(file, blob.data(), blob.size()) != (ssize_t)blob.size()) { + throw vespalib::Exception("write failed"); } close(file); - if (rename(tmp_filename.c_str(), filename.c_str()) != 0) { - throw vespalib::Exception("rename failed"); - } return true; } @@ -199,8 +195,8 @@ TestBase::readFile(const string &filename) const if (file != -1) { lseek(file, 0, SEEK_SET); if (read(file, blob.data(), len) != len) { - throw vespalib::Exception("read failed"); - } + throw vespalib::Exception("read failed"); + } close(file); } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeRepository.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeRepository.java index 0c256f9bee8..2ef79ec53dd 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeRepository.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeRepository.java @@ -477,22 +477,21 @@ public class NodeRepository extends AbstractComponent { } /* - * This method is used to enable a smooth rollout of dynamic docker flavor allocations. Once we have switch - * everything this can be simplified to only deleting the node. - * - * Should only be called by node-admin for docker containers + * This method is used by the REST API to handle readying nodes for new allocations. For docker containers this will + * remove the node from node repository, otherwise the node will be moved to state ready. */ - public List<Node> markNodeAvailableForNewAllocation(String hostname) { + public Node markNodeAvailableForNewAllocation(String hostname, Agent agent, String reason) { Node node = getNode(hostname).orElseThrow(() -> new NotFoundException("No node with hostname '" + hostname + "'")); - if (node.flavor().getType() != Flavor.Type.DOCKER_CONTAINER) { - throw new IllegalArgumentException( - "Cannot make " + hostname + " available for new allocation, must be a docker container node"); - } else if (node.state() != Node.State.dirty) { - throw new IllegalArgumentException( - "Cannot make " + hostname + " available for new allocation, must be in state dirty, but was in " + node.state()); + if (node.flavor().getType() == Flavor.Type.DOCKER_CONTAINER) { + if (node.state() != Node.State.dirty) { + throw new IllegalArgumentException( + "Cannot make " + hostname + " available for new allocation, must be in state dirty, but was in " + node.state()); + } + return removeRecursively(node, true).get(0); } - return removeRecursively(node, true); + if (node.state() == Node.State.ready) return node; + return setReady(Collections.singletonList(node), agent, reason).get(0); } /** diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/FailedExpirer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/FailedExpirer.java index 78552971769..fc6560b9f1b 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/FailedExpirer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/FailedExpirer.java @@ -48,14 +48,13 @@ import java.util.stream.Collectors; public class FailedExpirer extends Maintainer { private static final Logger log = Logger.getLogger(NodeRetirer.class.getName()); - - private static final Duration defaultExpiry = Duration.ofDays(4); // Grace period to allow recovery of data - private static final Duration containerExpiry = Duration.ofHours(1); // Stateless nodes, no data to recover private static final int maxAllowedFailures = 5; // Stop recycling nodes after this number of failures private final NodeRepository nodeRepository; private final Zone zone; private final Clock clock; + private final Duration defaultExpiry; // Grace period to allow recovery of data + private final Duration containerExpiry; // Stateless nodes, no data to recover public FailedExpirer(NodeRepository nodeRepository, Zone zone, Clock clock, Duration interval, JobControl jobControl) { @@ -63,6 +62,12 @@ public class FailedExpirer extends Maintainer { this.nodeRepository = nodeRepository; this.zone = zone; this.clock = clock; + if (zone.system() == SystemName.main) { + defaultExpiry = Duration.ofDays(4); + containerExpiry = Duration.ofHours(1); + } else { + defaultExpiry = containerExpiry = Duration.ofMinutes(30); + } } @Override @@ -72,7 +77,7 @@ public class FailedExpirer extends Maintainer { .filter(node -> node.allocation().isPresent() && node.allocation().get().membership().cluster().type() == ClusterSpec.Type.container) .collect(Collectors.toList()); - List<Node> remainingNodes = getExpiredNodes(zone.system() == SystemName.cd ? containerExpiry : defaultExpiry); + List<Node> remainingNodes = getExpiredNodes(defaultExpiry); remainingNodes.removeAll(containerNodes); recycle(containerNodes); recycle(remainingNodes); diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRepositoryMaintenance.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRepositoryMaintenance.java index f7d547f1076..18af498a38d 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRepositoryMaintenance.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRepositoryMaintenance.java @@ -9,7 +9,6 @@ import com.yahoo.config.provision.Environment; import com.yahoo.config.provision.HostLivenessTracker; import com.yahoo.config.provision.Zone; import com.yahoo.jdisc.Metric; -import com.yahoo.vespa.curator.Curator; import com.yahoo.vespa.hosted.provision.NodeRepository; import com.yahoo.vespa.hosted.provision.maintenance.retire.RetireIPv4OnlyNodes; import com.yahoo.vespa.hosted.provision.maintenance.retire.RetirementPolicy; @@ -51,15 +50,15 @@ public class NodeRepositoryMaintenance extends AbstractComponent { private final JobControl jobControl; @Inject - public NodeRepositoryMaintenance(NodeRepository nodeRepository, Deployer deployer, Curator curator, + public NodeRepositoryMaintenance(NodeRepository nodeRepository, Deployer deployer, HostLivenessTracker hostLivenessTracker, ServiceMonitor serviceMonitor, Zone zone, Orchestrator orchestrator, Metric metric, ConfigserverConfig configserverConfig) { - this(nodeRepository, deployer, curator, hostLivenessTracker, serviceMonitor, zone, Clock.systemUTC(), + this(nodeRepository, deployer, hostLivenessTracker, serviceMonitor, zone, Clock.systemUTC(), orchestrator, metric, configserverConfig); } - public NodeRepositoryMaintenance(NodeRepository nodeRepository, Deployer deployer, Curator curator, + public NodeRepositoryMaintenance(NodeRepository nodeRepository, Deployer deployer, HostLivenessTracker hostLivenessTracker, ServiceMonitor serviceMonitor, Zone zone, Clock clock, Orchestrator orchestrator, Metric metric, ConfigserverConfig configserverConfig) { @@ -127,8 +126,6 @@ public class NodeRepositoryMaintenance extends AbstractComponent { /** The time a node must be continuously nonresponsive before it is failed */ private final Duration failGrace; - private final Duration zooKeeperAccessMaintenanceInterval; - private final Duration reservationExpiry; private final Duration inactiveExpiry; private final Duration retiredExpiry; @@ -160,13 +157,11 @@ public class NodeRepositoryMaintenance extends AbstractComponent { if (environment.equals(Environment.prod)) { - zooKeeperAccessMaintenanceInterval = Duration.ofMinutes(1); reservationExpiry = Duration.ofMinutes(20); // same as deployment timeout inactiveExpiry = Duration.ofHours(4); // enough time for the application owner to discover and redeploy retiredInterval = Duration.ofMinutes(29); dirtyExpiry = Duration.ofHours(2); // enough time to clean the node } else { - zooKeeperAccessMaintenanceInterval = Duration.ofSeconds(10); reservationExpiry = Duration.ofMinutes(10); // Need to be long enough for deployment to be finished for all config model versions inactiveExpiry = Duration.ofSeconds(2); // support interactive wipe start over retiredInterval = Duration.ofMinutes(5); diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRetirer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRetirer.java index 30b5f6f737d..149e013687f 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRetirer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRetirer.java @@ -34,6 +34,7 @@ import java.util.stream.Stream; * @author freva */ public class NodeRetirer extends Maintainer { + public static final FlavorSpareChecker.SpareNodesPolicy SPARE_NODES_POLICY = flavorSpareCount -> flavorSpareCount.getNumReadyAmongReplacees() > 2; @@ -226,4 +227,5 @@ public class NodeRetirer extends Maintainer { Node::flavor, Collectors.groupingBy(Node::state, Collectors.counting()))); } + } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/RetiredExpirer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/RetiredExpirer.java index b06b5934713..7bc7d3ee732 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/RetiredExpirer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/RetiredExpirer.java @@ -87,7 +87,7 @@ public class RetiredExpirer extends Maintainer { /** * Checks if the node can be removed: * if the node is a docker host, it will only be removed if it has no children, - * or all its children are parked or failed + * or all its children are parked or failed. * Otherwise, a removal is allowed if either of these are true: * - The node has been in state {@link History.Event.Type#retired} for longer than {@link #retiredExpiry} * - Orchestrator allows it @@ -101,8 +101,8 @@ public class RetiredExpirer extends Maintainer { Optional<Instant> timeOfRetiredEvent = node.history().event(History.Event.Type.retired).map(History.Event::at); Optional<Instant> retireAfter = timeOfRetiredEvent.map(retiredEvent -> retiredEvent.plus(retiredExpiry)); - boolean shouldRetireNowBecauseExpried = retireAfter.map(time -> time.isBefore(clock.instant())).orElse(false); - if (shouldRetireNowBecauseExpried) { + boolean shouldRetireNowBecauseExpired = retireAfter.map(time -> time.isBefore(clock.instant())).orElse(false); + if (shouldRetireNowBecauseExpired) { return true; } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/NodesApiHandler.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/NodesApiHandler.java index c6667fd9d47..2f7d3120211 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/NodesApiHandler.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/NodesApiHandler.java @@ -6,7 +6,6 @@ import com.yahoo.config.provision.NodeType; import com.yahoo.container.jdisc.HttpRequest; import com.yahoo.container.jdisc.HttpResponse; import com.yahoo.container.jdisc.LoggingRequestHandler; -import com.yahoo.container.logging.AccessLog; import com.yahoo.io.IOUtils; import com.yahoo.slime.ArrayTraverser; import com.yahoo.slime.Inspector; @@ -107,8 +106,9 @@ public class NodesApiHandler extends LoggingRequestHandler { private HttpResponse handlePUT(HttpRequest request) { String path = request.getUri().getPath(); // Check paths to disallow illegal state changes - if (path.startsWith("/nodes/v2/state/ready/")) { - nodeRepository.setReady(lastElement(path), Agent.operator, "Readied through the nodes/v2 API"); + if (path.startsWith("/nodes/v2/state/ready/") || + path.startsWith("/nodes/v2/state/availablefornewallocations/")) { + nodeRepository.markNodeAvailableForNewAllocation(lastElement(path), Agent.operator, "Readied through the nodes/v2 API"); return new MessageResponse("Moved " + lastElement(path) + " to ready"); } else if (path.startsWith("/nodes/v2/state/failed/")) { @@ -129,12 +129,6 @@ public class NodesApiHandler extends LoggingRequestHandler { nodeRepository.reactivate(lastElement(path), Agent.operator, "Reactivated through nodes/v2 API"); return new MessageResponse("Moved " + lastElement(path) + " to active"); } - else if (path.startsWith("/nodes/v2/state/availablefornewallocations/")) { - String hostname = lastElement(path); - List<Node> available = nodeRepository.markNodeAvailableForNewAllocation(hostname); - return new MessageResponse("Marked following nodes as available for new allocation: " + - available.stream().map(Node::hostname).collect(Collectors.joining(", "))); - } throw new NotFoundException("Cannot put to path '" + path + "'"); } diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/RestApiTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/RestApiTest.java index dbf08597d6b..2e69867e9b1 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/RestApiTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/RestApiTest.java @@ -169,9 +169,12 @@ public class RestApiTest { "{\"message\":\"Moved test-container-1 to dirty\"}"); // ... and set it back to ready as if this was from the node-admin with the temporary state rest api - assertResponse(new Request("http://localhost:8080/nodes/v2/state/availablefornewallocations/test-container-1", + assertResponse(new Request("http://localhost:8080/nodes/v2/state/ready/test-container-1", new byte[0], Request.Method.PUT), - "{\"message\":\"Marked following nodes as available for new allocation: test-container-1\"}"); + "{\"message\":\"Moved test-container-1 to ready\"}"); + + assertResponse(new Request("http://localhost:8080/nodes/v2/node/test-container-1", new byte[0], Request.Method.GET), + 404, "{\"error-code\":\"NOT_FOUND\",\"message\":\"No node with hostname 'test-container-1'\"}"); // Put a host in failed and make sure it's children are also failed assertResponse(new Request("http://localhost:8080/nodes/v2/state/failed/dockerhost1.yahoo.com", new byte[0], Request.Method.PUT), diff --git a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/OrchestratorImpl.java b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/OrchestratorImpl.java index dbc526bdf03..095a7da8322 100644 --- a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/OrchestratorImpl.java +++ b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/OrchestratorImpl.java @@ -43,6 +43,7 @@ import java.util.stream.Collectors; * @author smorgrav */ public class OrchestratorImpl implements Orchestrator { + private static final Logger log = Logger.getLogger(OrchestratorImpl.class.getName()); private final Policy policy; diff --git a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/policy/HostedVespaPolicy.java b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/policy/HostedVespaPolicy.java index 1e9efa2e700..8e02f940127 100644 --- a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/policy/HostedVespaPolicy.java +++ b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/policy/HostedVespaPolicy.java @@ -23,6 +23,7 @@ import java.util.logging.Logger; */ public class HostedVespaPolicy implements Policy { + public static final String APPLICATION_SUSPENDED_CONSTRAINT = "application-suspended"; public static final String ENOUGH_SERVICES_UP_CONSTRAINT = "enough-services-up"; public static final String SET_NODE_STATE_CONSTRAINT = "controller-set-node-state"; @@ -125,4 +126,5 @@ public class HostedVespaPolicy implements Policy { ApplicationApi applicationApi = new ApplicationApiImpl(nodeGroup, hostStatusService, clusterControllerClientFactory); releaseSuspensionGrant(applicationApi); } + } diff --git a/searchlib/src/main/java/com/yahoo/searchlib/aggregation/FS4Hit.java b/searchlib/src/main/java/com/yahoo/searchlib/aggregation/FS4Hit.java index ad057e2c6e1..399ffd3128f 100644 --- a/searchlib/src/main/java/com/yahoo/searchlib/aggregation/FS4Hit.java +++ b/searchlib/src/main/java/com/yahoo/searchlib/aggregation/FS4Hit.java @@ -9,7 +9,7 @@ import com.yahoo.vespa.objects.Serializer; /** * This class represents a single hit from the fastserver4 backend * - * @author <a href="mailto:havardpe@yahoo-inc.com">Haavard Pettersen</a> + * @author havardpe */ public class FS4Hit extends Hit { diff --git a/searchlib/src/main/java/com/yahoo/searchlib/aggregation/hll/UniqueCountEstimator.java b/searchlib/src/main/java/com/yahoo/searchlib/aggregation/hll/UniqueCountEstimator.java index 59cdb85311f..1d0d0ed5fee 100644 --- a/searchlib/src/main/java/com/yahoo/searchlib/aggregation/hll/UniqueCountEstimator.java +++ b/searchlib/src/main/java/com/yahoo/searchlib/aggregation/hll/UniqueCountEstimator.java @@ -8,5 +8,7 @@ package com.yahoo.searchlib.aggregation.hll; * @author bjorncs */ public interface UniqueCountEstimator<T> { + long estimateCount(T sketch); + } diff --git a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/FeedClient.java b/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/FeedClient.java index fc6605eeba0..c5af7a49570 100644 --- a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/FeedClient.java +++ b/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/FeedClient.java @@ -17,6 +17,7 @@ import java.util.concurrent.atomic.AtomicInteger; * * @author dybis * @see FeedClientFactory + * @see com.yahoo.text.Text#stripInvalidCharacters(String) to remove invalid characters from string fields before feeding */ public interface FeedClient extends AutoCloseable { diff --git a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/SimpleLoggerResultCallback.java b/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/SimpleLoggerResultCallback.java index 85e6baec9f7..0b6917d9960 100644 --- a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/SimpleLoggerResultCallback.java +++ b/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/SimpleLoggerResultCallback.java @@ -32,6 +32,7 @@ public class SimpleLoggerResultCallback implements FeedClient.ResultCallback { /** * Constructor + * * @param sentDocumentCounter a counter that is increased outside this class, but can be nice to print here. * @param printStatsForEveryXDocument how often to print stats. */ diff --git a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/Document.java b/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/Document.java index d725f1df233..88d8d702d7c 100644 --- a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/Document.java +++ b/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/Document.java @@ -1,8 +1,6 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.http.client.core; -import com.google.common.annotations.Beta; - import java.io.IOException; import java.math.BigInteger; import java.nio.ByteBuffer; @@ -11,13 +9,11 @@ import java.nio.charset.CharacterCodingException; import java.nio.charset.StandardCharsets; import java.util.concurrent.ThreadLocalRandom; - /** -* @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a> -* @since 5.1.20 -*/ -@Beta + * @author Einar M R Rosenvinge + */ final public class Document { + private final String documentId; private final ByteBuffer data; private final long createTimeMillis = System.currentTimeMillis(); @@ -88,4 +84,5 @@ final public class Document { } return operationId; } + } diff --git a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/Encoder.java b/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/Encoder.java index 51488997ec6..135b2021a16 100644 --- a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/Encoder.java +++ b/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/Encoder.java @@ -1,8 +1,6 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.http.client.core; -import com.google.common.annotations.Beta; - /** * Simple encoding scheme to remove space, linefeed, control characters and * anything outside ISO 646.irv:1991 from strings. The scheme is supposed to be @@ -10,10 +8,10 @@ import com.google.common.annotations.Beta; * used as quoting characters, the output is by definition US-ASCII only * characters. * - * @author <a href="mailto:steinar@yahoo-inc.com">Steinar Knutsen</a> + * @author Steinar Knutsen */ -@Beta public final class Encoder { + /** * ISO 646.irv:1991 safe quoting into a StringBuilder instance. * @@ -23,10 +21,9 @@ public final class Encoder { * the destination buffer * @return the destination buffer given as input */ - public static StringBuilder encode(final String input, - final StringBuilder output) { + public static StringBuilder encode(String input, StringBuilder output) { for (int i = 0; i < input.length(); i = input.offsetByCodePoints(i, 1)) { - final int c = input.codePointAt(i); + int c = input.codePointAt(i); if (c <= '~') { if (c <= ' ') { encode(c, output); @@ -58,52 +55,46 @@ public final class Encoder { * @throws IllegalArgumentException * if the input string contains unexpected or invalid data */ - public static StringBuilder decode(final String input, - final StringBuilder output) { + public static StringBuilder decode(String input, StringBuilder output) { for (int i = 0; i < input.length(); i = input.offsetByCodePoints(i, 1)) { - final int c = input.codePointAt(i); + int c = input.codePointAt(i); if (c > '~') { - throw new IllegalArgumentException( - "Input contained character above printable ASCII."); + throw new IllegalArgumentException("Input contained character above printable ASCII."); } switch (c) { - case '{': - i = decode(input, i, output); - break; - default: - output.append((char) c); - break; + case '{': + i = decode(input, i, output); + break; + default: + output.append((char) c); + break; } } return output; } - private static int decode(final String input, final int offset, - final StringBuilder output) { + private static int decode(String input, int offset, StringBuilder output) { char c = 0; int end = offset; - final int start = offset + 1; + int start = offset + 1; int codePoint; while ('}' != c) { if (++end >= input.length()) { - throw new IllegalArgumentException( - "Unterminated quoted character or empty quoting."); + throw new IllegalArgumentException("Unterminated quoted character or empty quoting."); } c = input.charAt(end); } try { codePoint = Integer.parseInt(input.substring(start, end), 16); - } catch (final NumberFormatException e) { - throw new IllegalArgumentException("Unexpected quoted data: [" - + input.substring(start, end) + "]", e); + } catch (NumberFormatException e) { + throw new IllegalArgumentException("Unexpected quoted data: [" + input.substring(start, end) + "]", e); } if (Character.charCount(codePoint) > 1) { try { output.append(Character.toChars(codePoint)); - } catch (final IllegalArgumentException e) { - throw new IllegalArgumentException("Unexpected quoted data: [" - + input.substring(start, end) + "]", e); + } catch (IllegalArgumentException e) { + throw new IllegalArgumentException("Unexpected quoted data: [" + input.substring(start, end) + "]", e); } } else { output.append((char) codePoint); @@ -112,7 +103,7 @@ public final class Encoder { } - private static void encode(final int c, final StringBuilder output) { + private static void encode(int c, StringBuilder output) { output.append("{").append(Integer.toHexString(c)).append("}"); } diff --git a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/ErrorCode.java b/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/ErrorCode.java index 445ad5295c1..08926d529c5 100644 --- a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/ErrorCode.java +++ b/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/ErrorCode.java @@ -1,15 +1,12 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.http.client.core; -import com.google.common.annotations.Beta; - /** * Return types for the server. * * @author Einar M R Rosenvinge * @author Steinar Knutsen */ -@Beta public enum ErrorCode { OK(true, true), diff --git a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/Exceptions.java b/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/Exceptions.java index ce93aecac8e..1388effcec3 100644 --- a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/Exceptions.java +++ b/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/Exceptions.java @@ -1,14 +1,11 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.http.client.core; -import com.google.common.annotations.Beta; - /** * Helper methods for handling exceptions * * @author bratseth */ -@Beta public abstract class Exceptions { /** diff --git a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/Headers.java b/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/Headers.java index 50c96bafcf4..3791ddc5462 100644 --- a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/Headers.java +++ b/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/Headers.java @@ -1,15 +1,13 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.http.client.core; -import com.google.common.annotations.Beta; - /** * Wrapper for shared constants used by both client and server. * - * @author <a href="mailto:steinar@yahoo-inc.com">Steinar Knutsen</a> + * @author Steinar Knutsen */ -@Beta public final class Headers { + private Headers() { } diff --git a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/JsonReader.java b/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/JsonReader.java index 9b301351626..f5db2af216d 100644 --- a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/JsonReader.java +++ b/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/JsonReader.java @@ -13,6 +13,7 @@ import java.util.concurrent.atomic.AtomicInteger; /** * Reads a stream of json documents and sends them to feedClient. + * * @author dybis */ public class JsonReader { diff --git a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/OperationStatus.java b/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/OperationStatus.java index 7aec207e0ab..f3e2c8669e9 100644 --- a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/OperationStatus.java +++ b/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/OperationStatus.java @@ -1,18 +1,14 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.http.client.core; -import com.google.common.annotations.Beta; import com.google.common.base.Splitter; - import java.util.Iterator; - /** * Serialization/deserialization class for the result of a single document operation against Vespa. * * @author Steinar Knutsen */ -@Beta public final class OperationStatus { public static final String IS_CONDITION_NOT_MET = "IS-CONDITION-NOT-MET"; diff --git a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/ServerResponseException.java b/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/ServerResponseException.java index ef41a04f7c7..e3b0a0ab2cf 100644 --- a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/ServerResponseException.java +++ b/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/ServerResponseException.java @@ -1,15 +1,12 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.http.client.core; -import com.google.common.annotations.Beta; - /** * The request was not processed properly on the server. * * @author Einar M R Rosenvinge */ @SuppressWarnings("serial") -@Beta public class ServerResponseException extends Exception { private final int responseCode; diff --git a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/ThrottlePolicy.java b/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/ThrottlePolicy.java index 0239396cdd4..a4bd5d51496 100644 --- a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/ThrottlePolicy.java +++ b/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/ThrottlePolicy.java @@ -24,13 +24,8 @@ public class ThrottlePolicy { * @param messagesQueued if any messages where queued. * @return The new value to be used for max-in-flight (should be cropped externally to fit max/min values). */ - public int calcNewMaxInFlight( - final double maxPerformanceChange, - final int numOk, - final int previousNumOk, - final int previousMaxInFlight, - final int maxInFlightNow, - final boolean messagesQueued) { + public int calcNewMaxInFlight(double maxPerformanceChange, int numOk, int previousNumOk, int previousMaxInFlight, + int maxInFlightNow, boolean messagesQueued) { double difference = calculateRuleBasedDifference( maxPerformanceChange, numOk, previousNumOk, previousMaxInFlight, maxInFlightNow); @@ -56,12 +51,8 @@ public class ThrottlePolicy { return maxInFlightNow + delta; } - private static double calculateRuleBasedDifference( - final double maxPerformanceChange, - final double numOk, - final double previousNumOk, - final double previousMaxInFlight, - final double maxInFlightNow) { + private static double calculateRuleBasedDifference(double maxPerformanceChange, double numOk, double previousNumOk, + double previousMaxInFlight, double maxInFlightNow) { double difference = min( maxPerformanceChange, abs((numOk - previousNumOk) / safeDenominator(previousNumOk))); @@ -78,4 +69,5 @@ public class ThrottlePolicy { private static double safeDenominator(double x) { return x == 0.0 ? 1.0 : x; } + } diff --git a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/XmlFeedReader.java b/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/XmlFeedReader.java index 30070f89502..ba89ed550de 100644 --- a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/XmlFeedReader.java +++ b/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/XmlFeedReader.java @@ -4,7 +4,6 @@ package com.yahoo.vespa.http.client.core; import com.yahoo.vespa.http.client.FeedClient; import org.xml.sax.Attributes; import org.xml.sax.InputSource; -import org.xml.sax.SAXException; import org.xml.sax.ext.DefaultHandler2; import javax.xml.parsers.SAXParser; @@ -22,7 +21,7 @@ public class XmlFeedReader { // Static class. private XmlFeedReader() {} - public static void read(InputStream inputStream, FeedClient feedClient, AtomicInteger numSent) throws Exception{ + public static void read(InputStream inputStream, FeedClient feedClient, AtomicInteger numSent) throws Exception { SAXParserFactory parserFactor = SAXParserFactory.newInstance(); parserFactor.setValidating(false); @@ -46,6 +45,7 @@ public class XmlFeedReader { * Streams XML and sends each document operation to feeder. */ class SAXClientFeeder extends DefaultHandler2 { + public static final String CDATA_START = "<![CDATA["; public static final String CDATA_STOP = "]]>"; private final FeedClient feedClient; @@ -62,25 +62,23 @@ class SAXClientFeeder extends DefaultHandler2 { } @Override - public void startCDATA () throws SAXException { + public void startCDATA() { content.append(CDATA_START); isCData = true; } @Override - public void endCDATA () throws SAXException { + public void endCDATA() { content.append(CDATA_STOP); isCData = false; } @Override - public void comment(char[] ch, int start, int length) throws SAXException { - - } + public void comment(char[] ch, int start, int length) { } @SuppressWarnings("fallthrough") @Override - public void startElement(String uri, String localName, String qName, Attributes attributes) throws SAXException { + public void startElement(String uri, String localName, String qName, Attributes attributes) { switch(qName){ case "vespafeed": vespaIndent++; @@ -110,7 +108,7 @@ class SAXClientFeeder extends DefaultHandler2 { } @Override - public void endElement(String uri, String localName, String qName) throws SAXException { + public void endElement(String uri, String localName, String qName) { content.append("</") .append(qName) .append(">"); @@ -133,8 +131,7 @@ class SAXClientFeeder extends DefaultHandler2 { } @Override - public void characters (char buf [], int offset, int len) - throws SAXException { + public void characters (char buf [], int offset, int len) { if (isCData) { content.append(buf, offset, len); return; @@ -153,4 +150,5 @@ class SAXClientFeeder extends DefaultHandler2 { } } } + } diff --git a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/api/FeedClientImpl.java b/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/api/FeedClientImpl.java index 903c1ad4842..eaf869bdcdc 100644 --- a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/api/FeedClientImpl.java +++ b/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/api/FeedClientImpl.java @@ -53,7 +53,7 @@ public class FeedClientImpl implements FeedClient { charsetEncoder.onMalformedInput(CodingErrorAction.REPORT); charsetEncoder.onUnmappableCharacter(CodingErrorAction.REPORT); - final Document document = new Document(documentId, documentData, context); + Document document = new Document(documentId, documentData, context); operationProcessor.sendDocument(document); } diff --git a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/communication/ApacheGatewayConnection.java b/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/communication/ApacheGatewayConnection.java index 7cdd260285f..d48a86eea61 100644 --- a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/communication/ApacheGatewayConnection.java +++ b/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/communication/ApacheGatewayConnection.java @@ -1,7 +1,6 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.http.client.core.communication; -import com.google.common.annotations.Beta; import com.yahoo.component.Vtag; import com.yahoo.vespa.http.client.config.ConnectionParams; import com.yahoo.vespa.http.client.config.Endpoint; @@ -46,7 +45,6 @@ import java.util.zip.GZIPOutputStream; /** * @author Einar M R Rosenvinge */ -@Beta class ApacheGatewayConnection implements GatewayConnection { private static Logger log = Logger.getLogger(ApacheGatewayConnection.class.getName()); @@ -83,7 +81,7 @@ class ApacheGatewayConnection implements GatewayConnection { this.httpClientFactory = httpClientFactory; this.connectionParams = connectionParams; this.httpClient = null; - final boolean isJson = feedParams.getDataFormat() == FeedParams.DataFormat.JSON_UTF8; + boolean isJson = feedParams.getDataFormat() == FeedParams.DataFormat.JSON_UTF8; if (isJson) { startOfFeed = START_OF_FEED_JSON; endOfFeed = END_OF_FEED_JSON; @@ -379,6 +377,7 @@ class ApacheGatewayConnection implements GatewayConnection { * On re-connect we want to recreate the connection, hence we need a factory. */ public static class HttpClientFactory { + final ConnectionParams connectionParams; final boolean useSsl; diff --git a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/communication/ByteBufferInputStream.java b/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/communication/ByteBufferInputStream.java index 56a525502a5..d951666090a 100644 --- a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/communication/ByteBufferInputStream.java +++ b/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/communication/ByteBufferInputStream.java @@ -1,8 +1,6 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.http.client.core.communication; -import com.google.common.annotations.Beta; - import java.io.IOException; import java.io.InputStream; import java.nio.ByteBuffer; @@ -11,10 +9,7 @@ import java.util.Deque; /** * @author Einar M R Rosenvinge - * - * @since 5.1.20 */ -@Beta class ByteBufferInputStream extends InputStream { private final Deque<ByteBuffer> currentBuffers = new ArrayDeque<>(); diff --git a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/communication/ClusterConnection.java b/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/communication/ClusterConnection.java index cbc80466dd5..67011a24b45 100644 --- a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/communication/ClusterConnection.java +++ b/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/communication/ClusterConnection.java @@ -4,7 +4,6 @@ package com.yahoo.vespa.http.client.core.communication; import com.fasterxml.jackson.core.JsonFactory; import com.fasterxml.jackson.core.JsonGenerator; import com.fasterxml.jackson.databind.ObjectMapper; -import com.google.common.annotations.Beta; import com.yahoo.vespa.http.client.config.Cluster; import com.yahoo.vespa.http.client.config.ConnectionParams; import com.yahoo.vespa.http.client.config.Endpoint; @@ -24,7 +23,6 @@ import java.util.concurrent.TimeUnit; /** * @author Einar M R Rosenvinge */ -@Beta public class ClusterConnection implements AutoCloseable { private final OperationProcessor operationProcessor; diff --git a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/communication/EndpointResultQueue.java b/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/communication/EndpointResultQueue.java index c7731f5ab48..cd146cf0e87 100644 --- a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/communication/EndpointResultQueue.java +++ b/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/communication/EndpointResultQueue.java @@ -1,9 +1,6 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.http.client.core.communication; -import com.google.common.annotations.Beta; -import com.google.common.collect.ArrayListMultimap; -import com.google.common.collect.ListMultimap; import com.yahoo.vespa.http.client.FeedEndpointException; import com.yahoo.vespa.http.client.config.Endpoint; import com.yahoo.vespa.http.client.core.operationProcessor.EndPointResultFactory; @@ -11,7 +8,6 @@ import com.yahoo.vespa.http.client.core.EndpointResult; import com.yahoo.vespa.http.client.core.operationProcessor.OperationProcessor; import java.util.HashMap; -import java.util.List; import java.util.Map; import java.util.concurrent.ScheduledFuture; import java.util.concurrent.ScheduledThreadPoolExecutor; @@ -21,7 +17,6 @@ import java.util.logging.Logger; /** * @author Einar M R Rosenvinge */ -@Beta class EndpointResultQueue { private static Logger log = Logger.getLogger(EndpointResultQueue.class.getName()); @@ -112,7 +107,6 @@ class EndpointResultQueue { operationProcessor.resultReceived(endpointResult, clusterId); } - @Beta private class DocumentTimerTask implements Runnable { private final String operationId; @@ -126,7 +120,6 @@ class EndpointResultQueue { } } - @Beta private class TimerFuture { private final ScheduledFuture<?> future; diff --git a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/communication/IOThread.java b/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/communication/IOThread.java index 26dc1d1bd73..f3cb0b7bcf7 100644 --- a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/communication/IOThread.java +++ b/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/communication/IOThread.java @@ -1,7 +1,6 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.http.client.core.communication; -import com.google.common.annotations.Beta; import com.yahoo.vespa.http.client.FeedConnectException; import com.yahoo.vespa.http.client.FeedProtocolException; import com.yahoo.vespa.http.client.Result; @@ -29,7 +28,6 @@ import java.util.logging.Logger; * * @author Einar M R Rosenvinge */ -@Beta class IOThread implements Runnable, AutoCloseable { private static Logger log = Logger.getLogger(IOThread.class.getName()); diff --git a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/operationProcessor/EndPointResultFactory.java b/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/operationProcessor/EndPointResultFactory.java index 129e3ca245b..95df465c7ca 100644 --- a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/operationProcessor/EndPointResultFactory.java +++ b/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/operationProcessor/EndPointResultFactory.java @@ -1,7 +1,6 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.http.client.core.operationProcessor; -import com.google.common.annotations.Beta; import com.yahoo.vespa.http.client.Result; import com.yahoo.vespa.http.client.config.Endpoint; import com.yahoo.vespa.http.client.core.EndpointResult; @@ -20,7 +19,6 @@ import java.util.logging.Logger; /** * @author Einar M R Rosenvinge */ -@Beta public final class EndPointResultFactory { private static Logger log = Logger.getLogger(EndPointResultFactory.class.getName()); diff --git a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/operationProcessor/OperationProcessor.java b/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/operationProcessor/OperationProcessor.java index 4b95f367485..cff6ad2ed48 100644 --- a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/operationProcessor/OperationProcessor.java +++ b/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/operationProcessor/OperationProcessor.java @@ -1,7 +1,6 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.http.client.core.operationProcessor; -import com.google.common.annotations.Beta; import com.google.common.collect.ArrayListMultimap; import com.yahoo.vespa.http.client.FeedClient; import com.yahoo.vespa.http.client.FeedEndpointException; @@ -33,7 +32,6 @@ import java.util.logging.Logger; * * @author dybis */ -@Beta public class OperationProcessor { private static final Logger log = Logger.getLogger(OperationProcessor.class.getName()); diff --git a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/operationProcessor/OperationStats.java b/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/operationProcessor/OperationStats.java index a28c2f9805f..014bcf41951 100644 --- a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/operationProcessor/OperationStats.java +++ b/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/operationProcessor/OperationStats.java @@ -1,7 +1,6 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.http.client.core.operationProcessor; - import com.fasterxml.jackson.core.JsonFactory; import com.fasterxml.jackson.core.JsonGenerator; import com.fasterxml.jackson.databind.ObjectMapper; @@ -10,7 +9,6 @@ import com.yahoo.vespa.http.client.core.communication.ClusterConnection; import java.io.IOException; import java.io.StringWriter; -import java.io.Writer; import java.util.List; public class OperationStats { @@ -30,8 +28,8 @@ public class OperationStats { } private String generateSessionParamsAsXmlString(final SessionParams sessionParams) { - final ObjectMapper objectMapper = new ObjectMapper(); - final StringWriter stringWriter = new StringWriter(); + ObjectMapper objectMapper = new ObjectMapper(); + StringWriter stringWriter = new StringWriter(); try { JsonGenerator jsonGenerator = jsonFactory.createGenerator(stringWriter); objectMapper.writeValue(jsonGenerator, sessionParams); @@ -43,7 +41,7 @@ public class OperationStats { public String getStatsAsJson() { try { - final StringWriter stringWriter = new StringWriter(); + StringWriter stringWriter = new StringWriter(); JsonGenerator jsonGenerator = jsonFactory.createGenerator(stringWriter); jsonGenerator.writeStartObject(); jsonGenerator.writeArrayFieldStart("clusters"); diff --git a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/runner/CommandLineArguments.java b/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/runner/CommandLineArguments.java index 48d7916b34e..279735cdd08 100644 --- a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/runner/CommandLineArguments.java +++ b/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/runner/CommandLineArguments.java @@ -1,8 +1,6 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.http.client.runner; -import com.google.common.annotations.Beta; - import com.google.common.base.Splitter; import com.yahoo.vespa.http.client.config.Cluster; import com.yahoo.vespa.http.client.config.ConnectionParams; @@ -22,7 +20,6 @@ import java.util.concurrent.TimeUnit; * * @author dybis */ -@Beta @Command(name = "vespa-http-client", description = "This is a tool for feeding xml or json data to a Vespa application.") public class CommandLineArguments { diff --git a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/runner/Runner.java b/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/runner/Runner.java index 0983d893ecc..6ed6cae7346 100644 --- a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/runner/Runner.java +++ b/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/runner/Runner.java @@ -58,8 +58,8 @@ public class Runner { } - public static void main(String[] args) throws IOException, InterruptedException { - final CommandLineArguments commandLineArgs = CommandLineArguments.build(args); + public static void main(String[] args) throws IOException { + CommandLineArguments commandLineArgs = CommandLineArguments.build(args); if (commandLineArgs == null) { return; } @@ -73,10 +73,10 @@ public class Runner { int intervalOfLogging = commandLineArgs.getVerbose() ? commandLineArgs.getWhenVerboseEnabledPrintMessageForEveryXDocuments() : Integer.MAX_VALUE; - final AtomicInteger numSent = new AtomicInteger(0); - final SimpleLoggerResultCallback callback = new SimpleLoggerResultCallback(numSent, intervalOfLogging); + AtomicInteger numSent = new AtomicInteger(0); + SimpleLoggerResultCallback callback = new SimpleLoggerResultCallback(numSent, intervalOfLogging); - final FeedClient feedClient = FeedClientFactory.create( + FeedClient feedClient = FeedClientFactory.create( commandLineArgs.createSessionParams(formatInputStream.getFormat()== FormatInputStream.Format.JSON), callback); long sendTotalTimeMs = send( @@ -89,11 +89,12 @@ public class Runner { double transferTimeSec = ((double) sendTotalTimeMs) / 1000.0; System.err.println("Sent " + fileSizeMb + " MB in " + transferTimeSec + " seconds."); System.err.println("Speed: " + ((fileSizeMb / transferTimeSec) * 8.0) + " Mbits/sec, + HTTP overhead " + - "(not taking compression into account)"); + "(not taking compression into account)"); if (transferTimeSec > 0) { System.err.printf("Docs/sec %.3f%n\n", numSent.get() / transferTimeSec); } } callback.printProgress(); } + } diff --git a/vespajlib/src/main/java/com/yahoo/text/Text.java b/vespajlib/src/main/java/com/yahoo/text/Text.java index 0bdc2fb63bc..7748864ced5 100644 --- a/vespajlib/src/main/java/com/yahoo/text/Text.java +++ b/vespajlib/src/main/java/com/yahoo/text/Text.java @@ -90,19 +90,38 @@ public final class Text { * Validates that the given string value only contains text characters and * returns the first illegal code point if one is found. */ - public static OptionalInt validateTextString(String value) { - for (int i = 0; i < value.length(); i++) { - char theChar = value.charAt(i); - int codePoint = value.codePointAt(i); - if (Character.isHighSurrogate(theChar)) { - // Skip one char ahead, since codePointAt() consumes one more char in this case - ++i; - } - if (!Text.isTextCharacter(codePoint)) { + public static OptionalInt validateTextString(String string) { + for (int i = 0; i < string.length(); i++) { + int codePoint = string.codePointAt(i); + if ( ! Text.isTextCharacter(codePoint)) return OptionalInt.of(codePoint); - } + + if (Character.isHighSurrogate(string.charAt(i))) + ++i; // // codePointAt() consumes one more char in this case } return OptionalInt.empty(); } + /** + * Returns a string where any invalid characters in the input string is replaced by spaces + */ + public static String stripInvalidCharacters(String string) { + StringBuilder stripped = null; // lazy, as most string will not need stripping + for (int i = 0; i < string.length(); i++) { + int codePoint = string.codePointAt(i); + if ( ! Text.isTextCharacter(codePoint)) { + if (stripped == null) + stripped = new StringBuilder(string.substring(0, i)); + stripped.append(' '); + } + else if (stripped != null) { + stripped.appendCodePoint(codePoint); + } + + if (Character.isHighSurrogate(string.charAt(i))) + ++i; // // codePointAt() consumes one more char in this case + } + return stripped != null ? stripped.toString() : string; + } + } diff --git a/vespajlib/src/test/java/com/yahoo/text/TextTestCase.java b/vespajlib/src/test/java/com/yahoo/text/TextTestCase.java new file mode 100644 index 00000000000..0c1cf9b4b30 --- /dev/null +++ b/vespajlib/src/test/java/com/yahoo/text/TextTestCase.java @@ -0,0 +1,41 @@ +package com.yahoo.text; + +import org.junit.Test; + +import java.util.OptionalInt; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; + +public class TextTestCase { + + @Test + public void testValidateTextString() { + assertFalse(Text.validateTextString("valid").isPresent()); + assertEquals(OptionalInt.of(1), Text.validateTextString("text\u0001text\u0003")); + assertEquals(OptionalInt.of(917503), + Text.validateTextString(new StringBuilder().appendCodePoint(0xDFFFF).toString())); + assertEquals(OptionalInt.of(917503), + Text.validateTextString(new StringBuilder("foo").appendCodePoint(0xDFFFF).toString())); + assertEquals(OptionalInt.of(917503), + Text.validateTextString(new StringBuilder().appendCodePoint(0xDFFFF).append("foo").toString())); + assertEquals(OptionalInt.of(917503), + Text.validateTextString(new StringBuilder("foo").appendCodePoint(0xDFFFF).append("foo").toString())); + } + + @Test + public void testStripTextString() { + assertEquals("", Text.stripInvalidCharacters("")); + assertEquals("valid", Text.stripInvalidCharacters("valid")); + assertEquals("text text ", Text.stripInvalidCharacters("text\u0001text\u0003")); + assertEquals(" ", + Text.stripInvalidCharacters(new StringBuilder().appendCodePoint(0xDFFFF).toString())); + assertEquals("foo ", + Text.stripInvalidCharacters(new StringBuilder("foo").appendCodePoint(0xDFFFF).toString())); + assertEquals(" foo", + Text.stripInvalidCharacters(new StringBuilder().appendCodePoint(0xDFFFF).append("foo").toString())); + assertEquals("foo foo", + Text.stripInvalidCharacters(new StringBuilder("foo").appendCodePoint(0xDFFFF).append("foo").toString())); + } + +} |